1/*- 2 * Copyright (c) 2015 The FreeBSD Foundation 3 * 4 * This software was developed by Andrew Turner under 5 * sponsorship from the FreeBSD Foundation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 */ 29 30#include <machine/asm.h> 31__FBSDID("$FreeBSD$"); 32 33#include <sys/errno.h> 34 35#include <machine/vmparam.h> 36 37#include "assym.inc" 38 39.macro check_user_access user_arg, size_arg, bad_access_func 40 adds x6, x\user_arg, x\size_arg 41 b.cs \bad_access_func 42 ldr x7, =VM_MAXUSER_ADDRESS 43 cmp x6, x7 44 b.hi \bad_access_func 45.endm 46 47/* 48 * Fault handler for the copy{in,out} functions below. 49 */ 50ENTRY(copyio_fault) 51 SET_FAULT_HANDLER(xzr, x1) /* Clear the handler */ 52 EXIT_USER_ACCESS_CHECK(w0, x1) 53copyio_fault_nopcb: 54 mov x0, #EFAULT 55 ret 56END(copyio_fault) 57 58/* 59 * Copies from a kernel to user address 60 * 61 * int copyout(const void *kaddr, void *udaddr, size_t len) 62 */ 63ENTRY(copyout) 64 cbz x2, 1f 65 check_user_access 1, 2, copyio_fault_nopcb 66 67 b copycommon 68 691: mov x0, xzr /* return 0 */ 70 ret 71 72END(copyout) 73 74/* 75 * Copies from a user to kernel address 76 * 77 * int copyin(const void *uaddr, void *kdaddr, size_t len) 78 */ 79ENTRY(copyin) 80 cbz x2, 1f 81 check_user_access 0, 2, copyio_fault_nopcb 82 83 b copycommon 84 851: mov x0, xzr /* return 0 */ 86 ret 87 88END(copyin) 89 90/* 91 * Copies a string from a user to kernel address 92 * 93 * int copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done) 94 */ 95ENTRY(copyinstr) 96 mov x5, xzr /* count = 0 */ 97 mov w4, #1 /* If zero return faulure */ 98 cbz x2, 3f /* If len == 0 then skip loop */ 99 100 adr x6, copyio_fault /* Get the handler address */ 101 SET_FAULT_HANDLER(x6, x7) /* Set the handler */ 102 103 ldr x7, =VM_MAXUSER_ADDRESS 1041: cmp x0, x7 105 b.cs copyio_fault 106 ldtrb w4, [x0] /* Load from uaddr */ 107 add x0, x0, #1 /* Next char */ 108 strb w4, [x1], #1 /* Store in kaddr */ 109 add x5, x5, #1 /* count++ */ 110 cbz w4, 2f /* Break when NUL-terminated */ 111 sub x2, x2, #1 /* len-- */ 112 cbnz x2, 1b 113 1142: SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */ 115 116 1173: cbz x3, 4f /* Check if done != NULL */ 118 str x5, [x3] /* done = count */ 119 1204: mov w1, #ENAMETOOLONG /* Load ENAMETOOLONG to return if failed */ 121 cmp w4, #0 /* Check if we saved the NUL-terminator */ 122 csel w0, wzr, w1, eq /* If so return success, else failure */ 123 ret 124END(copyinstr) 125 126/* 127 * Local helper 128 * 129 * x0 - src pointer 130 * x1 - dst pointer 131 * x2 - size 132 * lr - the return address, so jump here instead of calling 133 * 134 * This function is optimized to minimize concurrent memory accesses. In 135 * present form it is suited for cores with a single memory prefetching 136 * unit. 137 * ARM64TODO: 138 * Consider using separate functions for each ARM64 core. Adding memory 139 * access interleaving might increase a total throughput on A57 or A72. 140 */ 141 .text 142 .align 4 143 .local copycommon 144 .type copycommon,@function 145 146copycommon: 147 adr x6, copyio_fault /* Get the handler address */ 148 SET_FAULT_HANDLER(x6, x7) /* Set the handler */ 149 ENTER_USER_ACCESS(w6, x7) 150 151 /* Check alignment */ 152 orr x3, x0, x1 153 ands x3, x3, 0x07 154 b.eq aligned 155 156 /* Unaligned is byte by byte copy */ 157byte_by_byte: 158 ldrb w3, [x0], #0x01 159 strb w3, [x1], #0x01 160 subs x2, x2, #0x01 161 b.ne byte_by_byte 162 b ending 163 164aligned: 165 cmp x2, #0x10 166 b.lt lead_out 167 cmp x2, #0x40 168 b.lt by_dwords_start 169 170 /* Block copy */ 171 lsr x15, x2, #0x06 172by_blocks: 173 ldp x3, x4, [x0], #0x10 174 ldp x5, x6, [x0], #0x10 175 ldp x7, x8, [x0], #0x10 176 ldp x9, x10, [x0], #0x10 177 stp x3, x4, [x1], #0x10 178 stp x5, x6, [x1], #0x10 179 stp x7, x8, [x1], #0x10 180 stp x9, x10, [x1], #0x10 181 182 subs x15, x15, #0x01 183 b.ne by_blocks 184 185 and x2, x2, #0x3f 186 187by_dwords_start: 188 lsr x15, x2, #0x04 189 cbz x15, lead_out 190by_dwords: 191 ldp x3, x4, [x0], #0x10 192 stp x3, x4, [x1], #0x10 193 subs x15, x15, #0x01 194 b.ne by_dwords 195 196 /* Less than 16 bytes to copy */ 197lead_out: 198 tbz x2, #0x03, last_word 199 ldr x3, [x0], #0x08 200 str x3, [x1], #0x08 201 202last_word: 203 tbz x2, #0x02, last_hword 204 ldr w3, [x0], #0x04 205 str w3, [x1], #0x04 206 207last_hword: 208 tbz x2, #0x01, last_byte 209 ldrh w3, [x0], #0x02 210 strh w3, [x1], #0x02 211 212last_byte: 213 tbz x2, #0x00, ending 214 ldrb w3, [x0] 215 strb w3, [x1] 216 217ending: 218 EXIT_USER_ACCESS_CHECK(w6, x7) 219 SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */ 220 221 mov x0, xzr /* return 0 */ 222 ret 223 .size copycommon, . - copycommon 224