1/*- 2 * Copyright (c) 2015 The FreeBSD Foundation 3 * 4 * This software was developed by Andrew Turner under 5 * sponsorship from the FreeBSD Foundation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 */ 29 30#include <machine/asm.h> 31#include <sys/errno.h> 32 33#include <machine/param.h> 34#include <machine/vmparam.h> 35 36#include "assym.inc" 37 38.macro check_user_access user_arg, size_arg, bad_access_func 39 adds x6, x\user_arg, x\size_arg 40 b.cs \bad_access_func 41 ldr x7, =VM_MAXUSER_ADDRESS 42 cmp x6, x7 43 b.hi \bad_access_func 44.endm 45 46/* 47 * Fault handler for the copy{in,out} functions below. 48 */ 49ENTRY(copyio_fault) 50 SET_FAULT_HANDLER(xzr, x1) /* Clear the handler */ 51 EXIT_USER_ACCESS_CHECK(w0, x1) 52copyio_fault_nopcb: 53 mov x0, #EFAULT 54 ret 55END(copyio_fault) 56 57/* 58 * Copies from a kernel to user address 59 * 60 * int copyout(const void *kaddr, void *udaddr, size_t len) 61 */ 62ENTRY(copyout) 63 cbz x2, 1f 64 check_user_access 1, 2, copyio_fault_nopcb 65 66 b copycommon 67 681: mov x0, xzr /* return 0 */ 69 ret 70 71END(copyout) 72 73/* 74 * Copies from a user to kernel address 75 * 76 * int copyin(const void *uaddr, void *kdaddr, size_t len) 77 */ 78ENTRY(copyin) 79 cbz x2, 1f 80 check_user_access 0, 2, copyio_fault_nopcb 81 82 b copycommon 83 841: mov x0, xzr /* return 0 */ 85 ret 86 87END(copyin) 88 89/* 90 * Copies a string from a user to kernel address 91 * 92 * int copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done) 93 */ 94ENTRY(copyinstr) 95 mov x5, xzr /* count = 0 */ 96 mov w4, #1 /* If zero return faulure */ 97 cbz x2, 3f /* If len == 0 then skip loop */ 98 99 adr x6, copyio_fault /* Get the handler address */ 100 SET_FAULT_HANDLER(x6, x7) /* Set the handler */ 101 102 ldr x7, =VM_MAXUSER_ADDRESS 1031: cmp x0, x7 104 b.cs copyio_fault 105 ldtrb w4, [x0] /* Load from uaddr */ 106 add x0, x0, #1 /* Next char */ 107 strb w4, [x1], #1 /* Store in kaddr */ 108 add x5, x5, #1 /* count++ */ 109 cbz w4, 2f /* Break when NUL-terminated */ 110 sub x2, x2, #1 /* len-- */ 111 cbnz x2, 1b 112 1132: SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */ 114 115 1163: cbz x3, 4f /* Check if done != NULL */ 117 str x5, [x3] /* done = count */ 118 1194: mov w1, #ENAMETOOLONG /* Load ENAMETOOLONG to return if failed */ 120 cmp w4, #0 /* Check if we saved the NUL-terminator */ 121 csel w0, wzr, w1, eq /* If so return success, else failure */ 122 ret 123END(copyinstr) 124 125/* 126 * Local helper 127 * 128 * x0 - src pointer 129 * x1 - dst pointer 130 * x2 - size 131 * lr - the return address, so jump here instead of calling 132 * 133 * This function is optimized to minimize concurrent memory accesses. In 134 * present form it is suited for cores with a single memory prefetching 135 * unit. 136 * ARM64TODO: 137 * Consider using separate functions for each ARM64 core. Adding memory 138 * access interleaving might increase a total throughput on A57 or A72. 139 */ 140 .text 141 .align 4 142 .local copycommon 143 .type copycommon,@function 144 145copycommon: 146 adr x6, copyio_fault /* Get the handler address */ 147 SET_FAULT_HANDLER(x6, x7) /* Set the handler */ 148 ENTER_USER_ACCESS(w6, x7) 149 150 /* Check alignment */ 151 orr x3, x0, x1 152 ands x3, x3, 0x07 153 b.eq aligned 154 155 /* Unaligned is byte by byte copy */ 156byte_by_byte: 157 ldrb w3, [x0], #0x01 158 strb w3, [x1], #0x01 159 subs x2, x2, #0x01 160 b.ne byte_by_byte 161 b ending 162 163aligned: 164 cmp x2, #0x10 165 b.lt lead_out 166 cmp x2, #0x40 167 b.lt by_dwords_start 168 169 /* Block copy */ 170 lsr x15, x2, #0x06 171by_blocks: 172 ldp x3, x4, [x0], #0x10 173 ldp x5, x6, [x0], #0x10 174 ldp x7, x8, [x0], #0x10 175 ldp x9, x10, [x0], #0x10 176 stp x3, x4, [x1], #0x10 177 stp x5, x6, [x1], #0x10 178 stp x7, x8, [x1], #0x10 179 stp x9, x10, [x1], #0x10 180 181 subs x15, x15, #0x01 182 b.ne by_blocks 183 184 and x2, x2, #0x3f 185 186by_dwords_start: 187 lsr x15, x2, #0x04 188 cbz x15, lead_out 189by_dwords: 190 ldp x3, x4, [x0], #0x10 191 stp x3, x4, [x1], #0x10 192 subs x15, x15, #0x01 193 b.ne by_dwords 194 195 /* Less than 16 bytes to copy */ 196lead_out: 197 tbz x2, #0x03, last_word 198 ldr x3, [x0], #0x08 199 str x3, [x1], #0x08 200 201last_word: 202 tbz x2, #0x02, last_hword 203 ldr w3, [x0], #0x04 204 str w3, [x1], #0x04 205 206last_hword: 207 tbz x2, #0x01, last_byte 208 ldrh w3, [x0], #0x02 209 strh w3, [x1], #0x02 210 211last_byte: 212 tbz x2, #0x00, ending 213 ldrb w3, [x0] 214 strb w3, [x1] 215 216ending: 217 EXIT_USER_ACCESS_CHECK(w6, x7) 218 SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */ 219 220 mov x0, xzr /* return 0 */ 221 ret 222 .size copycommon, . - copycommon 223