1/*- 2 * Copyright (c) 2015 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * This software was developed by Andrew Turner under 6 * sponsorship from the FreeBSD Foundation. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 */ 30 31#include <machine/asm.h> 32__FBSDID("$FreeBSD$"); 33 34#include <sys/errno.h> 35 36#include <machine/vmparam.h> 37 38#include "assym.inc" 39 40/* 41 * Fault handler for the copy{in,out} functions below. 42 */ 43ENTRY(copyio_fault) 44 SET_FAULT_HANDLER(xzr, x1) /* Clear the handler */ 45 EXIT_USER_ACCESS_CHECK(w0, x1) 46copyio_fault_nopcb: 47 mov x0, #EFAULT 48 ret 49END(copyio_fault) 50 51/* 52 * Copies from a kernel to user address 53 * 54 * int copyout(const void *kaddr, void *udaddr, size_t len) 55 */ 56ENTRY(copyout) 57 cbz x2, 1f 58 adds x3, x1, x2 59 b.cs copyio_fault_nopcb 60 ldr x4, =VM_MAXUSER_ADDRESS 61 cmp x3, x4 62 b.hi copyio_fault_nopcb 63 64 b copycommon 65 661: mov x0, xzr /* return 0 */ 67 ret 68 69END(copyout) 70 71/* 72 * Copies from a user to kernel address 73 * 74 * int copyin(const void *uaddr, void *kdaddr, size_t len) 75 */ 76ENTRY(copyin) 77 cbz x2, 1f 78 adds x3, x0, x2 79 b.cs copyio_fault_nopcb 80 ldr x4, =VM_MAXUSER_ADDRESS 81 cmp x3, x4 82 b.hi copyio_fault_nopcb 83 84 b copycommon 85 861: mov x0, xzr /* return 0 */ 87 ret 88 89END(copyin) 90 91/* 92 * Copies a string from a user to kernel address 93 * 94 * int copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done) 95 */ 96ENTRY(copyinstr) 97 mov x5, xzr /* count = 0 */ 98 mov w4, #1 /* If zero return faulure */ 99 cbz x2, 3f /* If len == 0 then skip loop */ 100 101 adr x6, copyio_fault /* Get the handler address */ 102 SET_FAULT_HANDLER(x6, x7) /* Set the handler */ 103 ENTER_USER_ACCESS(w6, x7) 104 105 ldr x7, =VM_MAXUSER_ADDRESS 1061: cmp x0, x7 107 b.cs copyio_fault 108 ldtrb w4, [x0] /* Load from uaddr */ 109 add x0, x0, #1 /* Next char */ 110 strb w4, [x1], #1 /* Store in kaddr */ 111 add x5, x5, #1 /* count++ */ 112 cbz w4, 2f /* Break when NUL-terminated */ 113 sub x2, x2, #1 /* len-- */ 114 cbnz x2, 1b 115 1162: EXIT_USER_ACCESS(w6) 117 SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */ 118 119 1203: cbz x3, 4f /* Check if done != NULL */ 121 str x5, [x3] /* done = count */ 122 1234: mov w1, #ENAMETOOLONG /* Load ENAMETOOLONG to return if failed */ 124 cmp w4, #0 /* Check if we saved the NUL-terminator */ 125 csel w0, wzr, w1, eq /* If so return success, else failure */ 126 ret 127END(copyinstr) 128 129/* 130 * Local helper 131 * 132 * x0 - src pointer 133 * x1 - dst pointer 134 * x2 - size 135 * lr - the return address, so jump here instead of calling 136 * 137 * This function is optimized to minimize concurrent memory accesses. In 138 * present form it is suited for cores with a single memory prefetching 139 * unit. 140 * ARM64TODO: 141 * Consider using separate functions for each ARM64 core. Adding memory 142 * access interleaving might increase a total throughput on A57 or A72. 143 */ 144 .text 145 .align 4 146 .local copycommon 147 .type copycommon,@function 148 149copycommon: 150 adr x6, copyio_fault /* Get the handler address */ 151 SET_FAULT_HANDLER(x6, x7) /* Set the handler */ 152 ENTER_USER_ACCESS(w6, x7) 153 154 /* Check alignment */ 155 orr x3, x0, x1 156 ands x3, x3, 0x07 157 b.eq aligned 158 159 /* Unaligned is byte by byte copy */ 160byte_by_byte: 161 ldrb w3, [x0], #0x01 162 strb w3, [x1], #0x01 163 subs x2, x2, #0x01 164 b.ne byte_by_byte 165 b ending 166 167aligned: 168 cmp x2, #0x10 169 b.lt lead_out 170 cmp x2, #0x40 171 b.lt by_dwords_start 172 173 /* Block copy */ 174 lsr x15, x2, #0x06 175by_blocks: 176 ldp x3, x4, [x0], #0x10 177 ldp x5, x6, [x0], #0x10 178 ldp x7, x8, [x0], #0x10 179 ldp x9, x10, [x0], #0x10 180 stp x3, x4, [x1], #0x10 181 stp x5, x6, [x1], #0x10 182 stp x7, x8, [x1], #0x10 183 stp x9, x10, [x1], #0x10 184 185 subs x15, x15, #0x01 186 b.ne by_blocks 187 188 and x2, x2, #0x3f 189 190by_dwords_start: 191 lsr x15, x2, #0x04 192 cbz x15, lead_out 193by_dwords: 194 ldp x3, x4, [x0], #0x10 195 stp x3, x4, [x1], #0x10 196 subs x15, x15, #0x01 197 b.ne by_dwords 198 199 /* Less than 16 bytes to copy */ 200lead_out: 201 tbz x2, #0x03, last_word 202 ldr x3, [x0], #0x08 203 str x3, [x1], #0x08 204 205last_word: 206 tbz x2, #0x02, last_hword 207 ldr w3, [x0], #0x04 208 str w3, [x1], #0x04 209 210last_hword: 211 tbz x2, #0x01, last_byte 212 ldrh w3, [x0], #0x02 213 strh w3, [x1], #0x02 214 215last_byte: 216 tbz x2, #0x00, ending 217 ldrb w3, [x0] 218 strb w3, [x1] 219 220ending: 221 EXIT_USER_ACCESS_CHECK(w6, x7) 222 SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */ 223 224 mov x0, xzr /* return 0 */ 225 ret 226 .size copycommon, . - copycommon 227