1/*- 2 * Copyright (c) 2014 Andrew Turner 3 * Copyright (c) 2014-2015 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Andrew Turner 7 * under sponsorship from the FreeBSD Foundation 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 */ 31 32#include <machine/asm.h> 33#include <machine/setjmp.h> 34#include <machine/param.h> 35#include <machine/vmparam.h> 36 37#include "assym.inc" 38 39.macro check_user_access user_arg, limit, bad_addr_func 40 ldr x7, =(\limit) 41 cmp x\user_arg, x7 42 b.cs \bad_addr_func 43.endm 44 45/* 46 * One of the fu* or su* functions failed, return -1. 47 */ 48ENTRY(fsu_fault) 49 SET_FAULT_HANDLER(xzr, x1) /* Reset the handler function */ 50 EXIT_USER_ACCESS_CHECK(w0, x1) 51fsu_fault_nopcb: 52 mov x0, #-1 53 ret 54END(fsu_fault) 55 56/* 57 * int swapueword8_llsc(volatile uint8_t *, uint8_t *) 58 */ 59ENTRY(swapueword8_llsc) 60 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 61 adr x6, fsu_fault /* Load the fault handler */ 62 SET_FAULT_HANDLER(x6, x4) /* And set it */ 63 ENTER_USER_ACCESS(w6, x4) 64 65 ldrb w7, [x1] 66 67 ldxrb w2, [x0] 68 stxrb w3, w7, [x0] 69 cbnz w3, 1f 70 71 strb w2, [x1] /* Stash old value in *val */ 72 731: EXIT_USER_ACCESS(w6) 74 SET_FAULT_HANDLER(xzr, x6) 75 mov w0, w3 76 ret 77END(swapueword8_llsc) 78 79/* 80 * int swapueword8_lse(volatile uint8_t *, uint8_t *) 81 */ 82ENTRY(swapueword8_lse) 83 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 84 adr x6, fsu_fault /* Load the fault handler */ 85 SET_FAULT_HANDLER(x6, x4) /* And set it */ 86 ENTER_USER_ACCESS(w6, x4) 87 88 ldrb w7, [x1] 89 90 .arch_extension lse 91 swpb w7, w2, [x0] 92 .arch_extension nolse 93 94 strb w2, [x1] /* Stash old value in *val */ 95 96 EXIT_USER_ACCESS(w6) 97 SET_FAULT_HANDLER(xzr, x6) 98 mov w0, #0 99 ret 100END(swapueword8_lse) 101 102/* 103 * int swapueword32_llsc(volatile uint32_t *, uint32_t *) 104 */ 105ENTRY(swapueword32_llsc) 106 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 107 adr x6, fsu_fault /* Load the fault handler */ 108 SET_FAULT_HANDLER(x6, x4) /* And set it */ 109 ENTER_USER_ACCESS(w6, x4) 110 111 ldr w7, [x1] 112 113 ldxr w2, [x0] /* Stash the old value in w2 */ 114 stxr w3, w7, [x0] /* Store new value */ 115 cbnz w3, 1f 116 117 str w2, [x1] /* Stash old value in *val */ 118 1191: EXIT_USER_ACCESS(w6) 120 SET_FAULT_HANDLER(xzr, x6) 121 mov w0, w3 122 ret 123END(swapueword32_llsc) 124 125/* 126 * int swapueword32_lse(volatile uint32_t *, uint32_t *) 127 */ 128ENTRY(swapueword32_lse) 129 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 130 adr x6, fsu_fault /* Load the fault handler */ 131 SET_FAULT_HANDLER(x6, x4) /* And set it */ 132 ENTER_USER_ACCESS(w6, x4) 133 134 ldr w7, [x1] 135 136 .arch_extension lse 137 swp w7, w2, [x0] 138 .arch_extension nolse 139 140 str w2, [x1] /* Stash old value in *val */ 141 142 EXIT_USER_ACCESS(w6) 143 SET_FAULT_HANDLER(xzr, x6) 144 mov w0, #0 145 ret 146END(swapueword32_llsc) 147 148/* 149 * int casueword32_llsc(volatile uint32_t *, uint32_t, uint32_t *, uint32_t) 150 */ 151ENTRY(casueword32_llsc) 152 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 153 adr x6, fsu_fault /* Load the fault handler */ 154 mov w5, #1 155 SET_FAULT_HANDLER(x6, x4) /* And set it */ 156 ENTER_USER_ACCESS(w6, x4) 157 ldxr w4, [x0] /* Load-exclusive the data */ 158 cmp w4, w1 /* Compare */ 159 b.ne 1f /* Not equal, exit */ 160 stxr w5, w3, [x0] /* Store the new data */ 1611: EXIT_USER_ACCESS(w6) 162 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 163 str w4, [x2] /* Store the read data */ 164 mov w0, w5 /* Result same as store status */ 165 ret /* Return */ 166END(casueword32_llsc) 167 168/* 169 * int casueword32_lse(volatile uint32_t *, uint32_t, uint32_t *, uint32_t) 170 */ 171ENTRY(casueword32_lse) 172 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 173 adr x6, fsu_fault /* Load the fault handler */ 174 SET_FAULT_HANDLER(x6, x4) /* And set it */ 175 ENTER_USER_ACCESS(w6, x4) 176 mov w7, w1 /* Back up the compare value */ 177 .arch_extension lse 178 cas w1, w3, [x0] /* Compare and Swap */ 179 .arch_extension nolse 180 cmp w1, w7 /* Check if successful */ 181 cset w0, ne /* Return 0 on success, 1 on failure */ 182 EXIT_USER_ACCESS(w6) 183 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 184 str w1, [x2] /* Store the read data */ 185 ret /* Return */ 186END(casueword32_lse) 187 188/* 189 * int casueword_llsc(volatile u_long *, u_long, u_long *, u_long) 190 */ 191ENTRY(casueword_llsc) 192 check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb 193 adr x6, fsu_fault /* Load the fault handler */ 194 mov w5, #1 195 SET_FAULT_HANDLER(x6, x4) /* And set it */ 196 ENTER_USER_ACCESS(w6, x4) 197 ldxr x4, [x0] /* Load-exclusive the data */ 198 cmp x4, x1 /* Compare */ 199 b.ne 1f /* Not equal, exit */ 200 stxr w5, x3, [x0] /* Store the new data */ 2011: EXIT_USER_ACCESS(w6) 202 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 203 str x4, [x2] /* Store the read data */ 204 mov w0, w5 /* Result same as store status */ 205 ret /* Return */ 206END(casueword_llsc) 207 208/* 209 * int casueword_lse(volatile u_long *, u_long, u_long *, u_long) 210 */ 211ENTRY(casueword_lse) 212 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 213 adr x6, fsu_fault /* Load the fault handler */ 214 SET_FAULT_HANDLER(x6, x4) /* And set it */ 215 ENTER_USER_ACCESS(w6, x4) 216 mov x7, x1 /* Back up the compare value */ 217 .arch_extension lse 218 cas x1, x3, [x0] /* Compare and Swap */ 219 .arch_extension nolse 220 cmp x1, x7 /* Check if successful */ 221 cset w0, ne /* Return 0 on success, 1 on failure */ 222 EXIT_USER_ACCESS(w6) 223 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 224 str x1, [x2] /* Store the read data */ 225 ret /* Return */ 226END(casueword_lse) 227 228.macro fsudata insn, ret_reg, user_arg 229 adr x7, fsu_fault /* Load the fault handler */ 230 SET_FAULT_HANDLER(x7, x6) /* And set it */ 231 \insn \ret_reg, [x\user_arg] /* Try accessing the data */ 232 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 233.endm 234 235/* 236 * int fubyte(volatile const void *) 237 */ 238ENTRY(fubyte) 239 check_user_access 0, (VM_MAXUSER_ADDRESS), fsu_fault_nopcb 240 fsudata ldtrb, w0, 0 241 ret /* Return */ 242END(fubyte) 243 244/* 245 * int fuword(volatile const void *) 246 */ 247ENTRY(fuword16) 248 check_user_access 0, (VM_MAXUSER_ADDRESS-1), fsu_fault_nopcb 249 fsudata ldtrh, w0, 0 250 ret /* Return */ 251END(fuword16) 252 253/* 254 * int32_t fueword32(volatile const void *, int32_t *) 255 */ 256ENTRY(fueword32) 257 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 258 fsudata ldtr, w0, 0 259 str w0, [x1] /* Save the data in kernel space */ 260 mov w0, #0 /* Success */ 261 ret /* Return */ 262END(fueword32) 263 264/* 265 * long fueword(volatile const void *, int64_t *) 266 * int64_t fueword64(volatile const void *, int64_t *) 267 */ 268EENTRY(fueword64) 269ENTRY(fueword) 270 check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb 271 fsudata ldtr, x0, 0 272 str x0, [x1] /* Save the data in kernel space */ 273 mov x0, #0 /* Success */ 274 ret /* Return */ 275END(fueword) 276EEND(fueword64) 277 278/* 279 * int subyte(volatile void *, int) 280 */ 281ENTRY(subyte) 282 check_user_access 0, (VM_MAXUSER_ADDRESS), fsu_fault_nopcb 283 fsudata sttrb, w1, 0 284 mov x0, #0 /* Success */ 285 ret /* Return */ 286END(subyte) 287 288/* 289 * int suword16(volatile void *, int) 290 */ 291ENTRY(suword16) 292 check_user_access 0, (VM_MAXUSER_ADDRESS-1), fsu_fault_nopcb 293 fsudata sttrh, w1, 0 294 mov x0, #0 /* Success */ 295 ret /* Return */ 296END(suword16) 297 298/* 299 * int suword32(volatile void *, int) 300 */ 301ENTRY(suword32) 302 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 303 fsudata sttr, w1, 0 304 mov x0, #0 /* Success */ 305 ret /* Return */ 306END(suword32) 307 308/* 309 * int suword(volatile void *, long) 310 */ 311EENTRY(suword64) 312ENTRY(suword) 313 check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb 314 fsudata sttr, x1, 0 315 mov x0, #0 /* Success */ 316 ret /* Return */ 317END(suword) 318EEND(suword64) 319 320ENTRY(setjmp) 321 /* Store the stack pointer */ 322 mov x8, sp 323 str x8, [x0], #8 324 325 /* Store the general purpose registers and lr */ 326 stp x19, x20, [x0], #16 327 stp x21, x22, [x0], #16 328 stp x23, x24, [x0], #16 329 stp x25, x26, [x0], #16 330 stp x27, x28, [x0], #16 331 stp x29, lr, [x0], #16 332 333 /* Return value */ 334 mov x0, #0 335 ret 336END(setjmp) 337 338ENTRY(longjmp) 339 /* Restore the stack pointer */ 340 ldr x8, [x0], #8 341 mov sp, x8 342 343 /* Restore the general purpose registers and lr */ 344 ldp x19, x20, [x0], #16 345 ldp x21, x22, [x0], #16 346 ldp x23, x24, [x0], #16 347 ldp x25, x26, [x0], #16 348 ldp x27, x28, [x0], #16 349 ldp x29, lr, [x0], #16 350 351 /* Load the return value */ 352 mov x0, x1 353 ret 354END(longjmp) 355 356/* 357 * pagezero, simple implementation 358 */ 359ENTRY(pagezero_simple) 360 add x1, x0, #PAGE_SIZE 361 3621: 363 stp xzr, xzr, [x0], #0x10 364 stp xzr, xzr, [x0], #0x10 365 stp xzr, xzr, [x0], #0x10 366 stp xzr, xzr, [x0], #0x10 367 cmp x0, x1 368 b.ne 1b 369 ret 370 371END(pagezero_simple) 372 373/* 374 * pagezero, cache assisted 375 */ 376ENTRY(pagezero_cache) 377 add x1, x0, #PAGE_SIZE 378 379 adrp x2, dczva_line_size 380 ldr x2, [x2, :lo12:dczva_line_size] 381 3821: 383 dc zva, x0 384 add x0, x0, x2 385 cmp x0, x1 386 b.ne 1b 387 ret 388 389END(pagezero_cache) 390