1/*- 2 * Copyright (c) 2014 Andrew Turner 3 * Copyright (c) 2014-2015 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Andrew Turner 7 * under sponsorship from the FreeBSD Foundation 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 */ 31 32#include <machine/asm.h> 33__FBSDID("$FreeBSD$"); 34 35#include <machine/setjmp.h> 36#include <machine/param.h> 37#include <machine/vmparam.h> 38 39#include "assym.inc" 40 41/* 42 * One of the fu* or su* functions failed, return -1. 43 */ 44ENTRY(fsu_fault) 45 SET_FAULT_HANDLER(xzr, x1) /* Reset the handler function */ 46 EXIT_USER_ACCESS_CHECK(w0, x1) 47fsu_fault_nopcb: 48 mov x0, #-1 49 ret 50END(fsu_fault) 51 52/* 53 * int casueword32(volatile uint32_t *, uint32_t, uint32_t *, uint32_t) 54 */ 55ENTRY(casueword32) 56 ldr x4, =(VM_MAXUSER_ADDRESS-3) 57 cmp x0, x4 58 b.cs fsu_fault_nopcb 59 adr x6, fsu_fault /* Load the fault handler */ 60 mov w5, #1 61 SET_FAULT_HANDLER(x6, x4) /* And set it */ 62 ENTER_USER_ACCESS(w6, x4) 631: ldxr w4, [x0] /* Load-exclusive the data */ 64 cmp w4, w1 /* Compare */ 65 b.ne 2f /* Not equal, exit */ 66 stxr w5, w3, [x0] /* Store the new data */ 672: EXIT_USER_ACCESS(w6) 68 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 69 str w4, [x2] /* Store the read data */ 70 mov w0, w5 /* Result same as store status */ 71 ret /* Return */ 72END(casueword32) 73 74/* 75 * int casueword(volatile u_long *, u_long, u_long *, u_long) 76 */ 77ENTRY(casueword) 78 ldr x4, =(VM_MAXUSER_ADDRESS-7) 79 cmp x0, x4 80 b.cs fsu_fault_nopcb 81 adr x6, fsu_fault /* Load the fault handler */ 82 mov w5, #1 83 SET_FAULT_HANDLER(x6, x4) /* And set it */ 84 ENTER_USER_ACCESS(w6, x4) 851: ldxr x4, [x0] /* Load-exclusive the data */ 86 cmp x4, x1 /* Compare */ 87 b.ne 2f /* Not equal, exit */ 88 stxr w5, x3, [x0] /* Store the new data */ 892: EXIT_USER_ACCESS(w6) 90 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 91 str x4, [x2] /* Store the read data */ 92 mov w0, w5 /* Result same as store status */ 93 ret /* Return */ 94END(casueword) 95 96/* 97 * int fubyte(volatile const void *) 98 */ 99ENTRY(fubyte) 100 ldr x1, =VM_MAXUSER_ADDRESS 101 cmp x0, x1 102 b.cs fsu_fault_nopcb 103 adr x6, fsu_fault /* Load the fault handler */ 104 SET_FAULT_HANDLER(x6, x1) /* And set it */ 105 ldtrb w0, [x0] /* Try loading the data */ 106 SET_FAULT_HANDLER(xzr, x1) /* Reset the fault handler */ 107 ret /* Return */ 108END(fubyte) 109 110/* 111 * int fuword(volatile const void *) 112 */ 113ENTRY(fuword16) 114 ldr x1, =(VM_MAXUSER_ADDRESS-1) 115 cmp x0, x1 116 b.cs fsu_fault_nopcb 117 adr x6, fsu_fault /* Load the fault handler */ 118 SET_FAULT_HANDLER(x6, x1) /* And set it */ 119 ldtrh w0, [x0] /* Try loading the data */ 120 SET_FAULT_HANDLER(xzr, x1) /* Reset the fault handler */ 121 ret /* Return */ 122END(fuword16) 123 124/* 125 * int32_t fueword32(volatile const void *, int32_t *) 126 */ 127ENTRY(fueword32) 128 ldr x2, =(VM_MAXUSER_ADDRESS-3) 129 cmp x0, x2 130 b.cs fsu_fault_nopcb 131 adr x6, fsu_fault /* Load the fault handler */ 132 SET_FAULT_HANDLER(x6, x2) /* And set it */ 133 ldtr w0, [x0] /* Try loading the data */ 134 SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ 135 str w0, [x1] /* Save the data in kernel space */ 136 mov w0, #0 /* Success */ 137 ret /* Return */ 138END(fueword32) 139 140/* 141 * long fueword(volatile const void *, int64_t *) 142 * int64_t fueword64(volatile const void *, int64_t *) 143 */ 144ENTRY(fueword) 145EENTRY(fueword64) 146 ldr x2, =(VM_MAXUSER_ADDRESS-7) 147 cmp x0, x2 148 b.cs fsu_fault_nopcb 149 adr x6, fsu_fault /* Load the fault handler */ 150 SET_FAULT_HANDLER(x6, x2) /* And set it */ 151 ldtr x0, [x0] /* Try loading the data */ 152 SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ 153 str x0, [x1] /* Save the data in kernel space */ 154 mov x0, #0 /* Success */ 155 ret /* Return */ 156EEND(fueword64) 157END(fueword) 158 159/* 160 * int subyte(volatile void *, int) 161 */ 162ENTRY(subyte) 163 ldr x2, =VM_MAXUSER_ADDRESS 164 cmp x0, x2 165 b.cs fsu_fault_nopcb 166 adr x6, fsu_fault /* Load the fault handler */ 167 SET_FAULT_HANDLER(x6, x2) /* And set it */ 168 sttrb w1, [x0] /* Try storing the data */ 169 SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ 170 mov x0, #0 /* Success */ 171 ret /* Return */ 172END(subyte) 173 174/* 175 * int suword16(volatile void *, int) 176 */ 177ENTRY(suword16) 178 ldr x2, =(VM_MAXUSER_ADDRESS-1) 179 cmp x0, x2 180 b.cs fsu_fault_nopcb 181 adr x6, fsu_fault /* Load the fault handler */ 182 SET_FAULT_HANDLER(x6, x2) /* And set it */ 183 sttrh w1, [x0] /* Try storing the data */ 184 SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ 185 mov x0, #0 /* Success */ 186 ret /* Return */ 187END(suword16) 188 189/* 190 * int suword32(volatile void *, int) 191 */ 192ENTRY(suword32) 193 ldr x2, =(VM_MAXUSER_ADDRESS-3) 194 cmp x0, x2 195 b.cs fsu_fault_nopcb 196 adr x6, fsu_fault /* Load the fault handler */ 197 SET_FAULT_HANDLER(x6, x2) /* And set it */ 198 sttr w1, [x0] /* Try storing the data */ 199 SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ 200 mov x0, #0 /* Success */ 201 ret /* Return */ 202END(suword32) 203 204/* 205 * int suword(volatile void *, long) 206 */ 207ENTRY(suword) 208EENTRY(suword64) 209 ldr x2, =(VM_MAXUSER_ADDRESS-7) 210 cmp x0, x2 211 b.cs fsu_fault_nopcb 212 adr x6, fsu_fault /* Load the fault handler */ 213 SET_FAULT_HANDLER(x6, x2) /* And set it */ 214 sttr x1, [x0] /* Try storing the data */ 215 SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ 216 mov x0, #0 /* Success */ 217 ret /* Return */ 218EEND(suword64) 219END(suword) 220 221ENTRY(setjmp) 222 /* Store the stack pointer */ 223 mov x8, sp 224 str x8, [x0], #8 225 226 /* Store the general purpose registers and lr */ 227 stp x19, x20, [x0], #16 228 stp x21, x22, [x0], #16 229 stp x23, x24, [x0], #16 230 stp x25, x26, [x0], #16 231 stp x27, x28, [x0], #16 232 stp x29, lr, [x0], #16 233 234 /* Return value */ 235 mov x0, #0 236 ret 237END(setjmp) 238 239ENTRY(longjmp) 240 /* Restore the stack pointer */ 241 ldr x8, [x0], #8 242 mov sp, x8 243 244 /* Restore the general purpose registers and lr */ 245 ldp x19, x20, [x0], #16 246 ldp x21, x22, [x0], #16 247 ldp x23, x24, [x0], #16 248 ldp x25, x26, [x0], #16 249 ldp x27, x28, [x0], #16 250 ldp x29, lr, [x0], #16 251 252 /* Load the return value */ 253 mov x0, x1 254 ret 255END(longjmp) 256 257/* 258 * pagezero, simple implementation 259 */ 260ENTRY(pagezero_simple) 261 add x1, x0, #PAGE_SIZE 262 2631: 264 stp xzr, xzr, [x0], #0x10 265 stp xzr, xzr, [x0], #0x10 266 stp xzr, xzr, [x0], #0x10 267 stp xzr, xzr, [x0], #0x10 268 cmp x0, x1 269 b.ne 1b 270 ret 271 272END(pagezero_simple) 273 274/* 275 * pagezero, cache assisted 276 */ 277ENTRY(pagezero_cache) 278 add x1, x0, #PAGE_SIZE 279 280 ldr x2, =dczva_line_size 281 ldr x2, [x2] 282 2831: 284 dc zva, x0 285 add x0, x0, x2 286 cmp x0, x1 287 b.ne 1b 288 ret 289 290END(pagezero_cache) 291