1/*- 2 * Copyright (c) 2014 Andrew Turner 3 * Copyright (c) 2014-2015 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Andrew Turner 7 * under sponsorship from the FreeBSD Foundation 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 */ 31 32#include <machine/asm.h> 33__FBSDID("$FreeBSD$"); 34 35#include <machine/setjmp.h> 36#include <machine/param.h> 37#include <machine/vmparam.h> 38 39#include "assym.inc" 40 41.macro check_user_access user_arg, limit, bad_addr_func 42 ldr x7, =(\limit) 43 cmp x\user_arg, x7 44 b.cs \bad_addr_func 45.endm 46 47/* 48 * One of the fu* or su* functions failed, return -1. 49 */ 50ENTRY(fsu_fault) 51 SET_FAULT_HANDLER(xzr, x1) /* Reset the handler function */ 52 EXIT_USER_ACCESS_CHECK(w0, x1) 53fsu_fault_nopcb: 54 mov x0, #-1 55 ret 56END(fsu_fault) 57 58/* 59 * int swapueword8_llsc(volatile uint8_t *, uint8_t *) 60 */ 61ENTRY(swapueword8_llsc) 62 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 63 adr x6, fsu_fault /* Load the fault handler */ 64 SET_FAULT_HANDLER(x6, x4) /* And set it */ 65 ENTER_USER_ACCESS(w6, x4) 66 67 ldrb w7, [x1] 68 69 ldxrb w2, [x0] 70 stxrb w3, w7, [x0] 71 cbnz w3, 1f 72 73 strb w2, [x1] /* Stash old value in *val */ 74 751: EXIT_USER_ACCESS(w6) 76 SET_FAULT_HANDLER(xzr, x6) 77 mov w0, w3 78 ret 79END(swapueword8_llsc) 80 81/* 82 * int swapueword8_lse(volatile uint8_t *, uint8_t *) 83 */ 84ENTRY(swapueword8_lse) 85 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 86 adr x6, fsu_fault /* Load the fault handler */ 87 SET_FAULT_HANDLER(x6, x4) /* And set it */ 88 ENTER_USER_ACCESS(w6, x4) 89 90 ldrb w7, [x1] 91 92 .arch_extension lse 93 swpb w7, w2, [x0] 94 .arch_extension nolse 95 96 strb w2, [x1] /* Stash old value in *val */ 97 98 EXIT_USER_ACCESS(w6) 99 SET_FAULT_HANDLER(xzr, x6) 100 mov w0, #0 101 ret 102END(swapueword8_lse) 103 104/* 105 * int swapueword32_llsc(volatile uint32_t *, uint32_t *) 106 */ 107ENTRY(swapueword32_llsc) 108 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 109 adr x6, fsu_fault /* Load the fault handler */ 110 SET_FAULT_HANDLER(x6, x4) /* And set it */ 111 ENTER_USER_ACCESS(w6, x4) 112 113 ldr w7, [x1] 114 115 ldxr w2, [x0] /* Stash the old value in w2 */ 116 stxr w3, w7, [x0] /* Store new value */ 117 cbnz w3, 1f 118 119 str w2, [x1] /* Stash old value in *val */ 120 1211: EXIT_USER_ACCESS(w6) 122 SET_FAULT_HANDLER(xzr, x6) 123 mov w0, w3 124 ret 125END(swapueword32_llsc) 126 127/* 128 * int swapueword32_lse(volatile uint32_t *, uint32_t *) 129 */ 130ENTRY(swapueword32_lse) 131 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 132 adr x6, fsu_fault /* Load the fault handler */ 133 SET_FAULT_HANDLER(x6, x4) /* And set it */ 134 ENTER_USER_ACCESS(w6, x4) 135 136 ldr w7, [x1] 137 138 .arch_extension lse 139 swp w7, w2, [x0] 140 .arch_extension nolse 141 142 str w2, [x1] /* Stash old value in *val */ 143 144 EXIT_USER_ACCESS(w6) 145 SET_FAULT_HANDLER(xzr, x6) 146 mov w0, #0 147 ret 148END(swapueword32_llsc) 149 150/* 151 * int casueword32_llsc(volatile uint32_t *, uint32_t, uint32_t *, uint32_t) 152 */ 153ENTRY(casueword32_llsc) 154 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 155 adr x6, fsu_fault /* Load the fault handler */ 156 mov w5, #1 157 SET_FAULT_HANDLER(x6, x4) /* And set it */ 158 ENTER_USER_ACCESS(w6, x4) 159 ldxr w4, [x0] /* Load-exclusive the data */ 160 cmp w4, w1 /* Compare */ 161 b.ne 1f /* Not equal, exit */ 162 stxr w5, w3, [x0] /* Store the new data */ 1631: EXIT_USER_ACCESS(w6) 164 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 165 str w4, [x2] /* Store the read data */ 166 mov w0, w5 /* Result same as store status */ 167 ret /* Return */ 168END(casueword32_llsc) 169 170/* 171 * int casueword32_lse(volatile uint32_t *, uint32_t, uint32_t *, uint32_t) 172 */ 173ENTRY(casueword32_lse) 174 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 175 adr x6, fsu_fault /* Load the fault handler */ 176 SET_FAULT_HANDLER(x6, x4) /* And set it */ 177 ENTER_USER_ACCESS(w6, x4) 178 mov w7, w1 /* Back up the compare value */ 179 .arch_extension lse 180 cas w1, w3, [x0] /* Compare and Swap */ 181 .arch_extension nolse 182 cmp w1, w7 /* Check if successful */ 183 cset w0, ne /* Return 0 on success, 1 on failure */ 184 EXIT_USER_ACCESS(w6) 185 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 186 str w1, [x2] /* Store the read data */ 187 ret /* Return */ 188END(casueword32_lse) 189 190/* 191 * int casueword_llsc(volatile u_long *, u_long, u_long *, u_long) 192 */ 193ENTRY(casueword_llsc) 194 check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb 195 adr x6, fsu_fault /* Load the fault handler */ 196 mov w5, #1 197 SET_FAULT_HANDLER(x6, x4) /* And set it */ 198 ENTER_USER_ACCESS(w6, x4) 199 ldxr x4, [x0] /* Load-exclusive the data */ 200 cmp x4, x1 /* Compare */ 201 b.ne 1f /* Not equal, exit */ 202 stxr w5, x3, [x0] /* Store the new data */ 2031: EXIT_USER_ACCESS(w6) 204 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 205 str x4, [x2] /* Store the read data */ 206 mov w0, w5 /* Result same as store status */ 207 ret /* Return */ 208END(casueword_llsc) 209 210/* 211 * int casueword_lse(volatile u_long *, u_long, u_long *, u_long) 212 */ 213ENTRY(casueword_lse) 214 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 215 adr x6, fsu_fault /* Load the fault handler */ 216 SET_FAULT_HANDLER(x6, x4) /* And set it */ 217 ENTER_USER_ACCESS(w6, x4) 218 mov x7, x1 /* Back up the compare value */ 219 .arch_extension lse 220 cas x1, x3, [x0] /* Compare and Swap */ 221 .arch_extension nolse 222 cmp x1, x7 /* Check if successful */ 223 cset w0, ne /* Return 0 on success, 1 on failure */ 224 EXIT_USER_ACCESS(w6) 225 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 226 str x1, [x2] /* Store the read data */ 227 ret /* Return */ 228END(casueword_lse) 229 230.macro fsudata insn, ret_reg, user_arg 231 adr x7, fsu_fault /* Load the fault handler */ 232 SET_FAULT_HANDLER(x7, x6) /* And set it */ 233 \insn \ret_reg, [x\user_arg] /* Try accessing the data */ 234 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 235.endm 236 237/* 238 * int fubyte(volatile const void *) 239 */ 240ENTRY(fubyte) 241 check_user_access 0, (VM_MAXUSER_ADDRESS), fsu_fault_nopcb 242 fsudata ldtrb, w0, 0 243 ret /* Return */ 244END(fubyte) 245 246/* 247 * int fuword(volatile const void *) 248 */ 249ENTRY(fuword16) 250 check_user_access 0, (VM_MAXUSER_ADDRESS-1), fsu_fault_nopcb 251 fsudata ldtrh, w0, 0 252 ret /* Return */ 253END(fuword16) 254 255/* 256 * int32_t fueword32(volatile const void *, int32_t *) 257 */ 258ENTRY(fueword32) 259 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 260 fsudata ldtr, w0, 0 261 str w0, [x1] /* Save the data in kernel space */ 262 mov w0, #0 /* Success */ 263 ret /* Return */ 264END(fueword32) 265 266/* 267 * long fueword(volatile const void *, int64_t *) 268 * int64_t fueword64(volatile const void *, int64_t *) 269 */ 270EENTRY(fueword64) 271ENTRY(fueword) 272 check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb 273 fsudata ldtr, x0, 0 274 str x0, [x1] /* Save the data in kernel space */ 275 mov x0, #0 /* Success */ 276 ret /* Return */ 277END(fueword) 278EEND(fueword64) 279 280/* 281 * int subyte(volatile void *, int) 282 */ 283ENTRY(subyte) 284 check_user_access 0, (VM_MAXUSER_ADDRESS), fsu_fault_nopcb 285 fsudata sttrb, w1, 0 286 mov x0, #0 /* Success */ 287 ret /* Return */ 288END(subyte) 289 290/* 291 * int suword16(volatile void *, int) 292 */ 293ENTRY(suword16) 294 check_user_access 0, (VM_MAXUSER_ADDRESS-1), fsu_fault_nopcb 295 fsudata sttrh, w1, 0 296 mov x0, #0 /* Success */ 297 ret /* Return */ 298END(suword16) 299 300/* 301 * int suword32(volatile void *, int) 302 */ 303ENTRY(suword32) 304 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 305 fsudata sttr, w1, 0 306 mov x0, #0 /* Success */ 307 ret /* Return */ 308END(suword32) 309 310/* 311 * int suword(volatile void *, long) 312 */ 313EENTRY(suword64) 314ENTRY(suword) 315 check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb 316 fsudata sttr, x1, 0 317 mov x0, #0 /* Success */ 318 ret /* Return */ 319END(suword) 320EEND(suword64) 321 322ENTRY(setjmp) 323 /* Store the stack pointer */ 324 mov x8, sp 325 str x8, [x0], #8 326 327 /* Store the general purpose registers and lr */ 328 stp x19, x20, [x0], #16 329 stp x21, x22, [x0], #16 330 stp x23, x24, [x0], #16 331 stp x25, x26, [x0], #16 332 stp x27, x28, [x0], #16 333 stp x29, lr, [x0], #16 334 335 /* Return value */ 336 mov x0, #0 337 ret 338END(setjmp) 339 340ENTRY(longjmp) 341 /* Restore the stack pointer */ 342 ldr x8, [x0], #8 343 mov sp, x8 344 345 /* Restore the general purpose registers and lr */ 346 ldp x19, x20, [x0], #16 347 ldp x21, x22, [x0], #16 348 ldp x23, x24, [x0], #16 349 ldp x25, x26, [x0], #16 350 ldp x27, x28, [x0], #16 351 ldp x29, lr, [x0], #16 352 353 /* Load the return value */ 354 mov x0, x1 355 ret 356END(longjmp) 357 358/* 359 * pagezero, simple implementation 360 */ 361ENTRY(pagezero_simple) 362 add x1, x0, #PAGE_SIZE 363 3641: 365 stp xzr, xzr, [x0], #0x10 366 stp xzr, xzr, [x0], #0x10 367 stp xzr, xzr, [x0], #0x10 368 stp xzr, xzr, [x0], #0x10 369 cmp x0, x1 370 b.ne 1b 371 ret 372 373END(pagezero_simple) 374 375/* 376 * pagezero, cache assisted 377 */ 378ENTRY(pagezero_cache) 379 add x1, x0, #PAGE_SIZE 380 381 ldr x2, =dczva_line_size 382 ldr x2, [x2] 383 3841: 385 dc zva, x0 386 add x0, x0, x2 387 cmp x0, x1 388 b.ne 1b 389 ret 390 391END(pagezero_cache) 392