1/* $OpenBSD: copy.S,v 1.20 2024/06/06 00:36:46 bluhm Exp $ */ 2/* $NetBSD: copy.S,v 1.1 2003/04/26 18:39:26 fvdl Exp $ */ 3 4/* 5 * Copyright (c) 2001 Wasabi Systems, Inc. 6 * All rights reserved. 7 * 8 * Written by Frank van der Linden for Wasabi Systems, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed for the NetBSD Project by 21 * Wasabi Systems, Inc. 22 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 23 * or promote products derived from this software without specific prior 24 * written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39#include "assym.h" 40 41#include <sys/errno.h> 42#include <sys/syscall.h> 43 44#include <machine/asm.h> 45#include <machine/codepatch.h> 46 47#define DECLARE_ONFAULT(x) \ 48 .pushsection .nofault.0, "a" ;\ 49 .quad x ;\ 50 .popsection 51 52/* 53 * Copy routines from and to userland, plus a few more. See the 54 * section 9 manpages for info. Some cases can be optimized more. 55 * 56 * I wonder if it's worthwhile to make these use SSE2 registers. 57 */ 58 59ENTRY(kcopy) 60 RETGUARD_SETUP(kcopy, r10) 61 movq CPUVAR(CURPCB),%rax 62 pushq PCB_ONFAULT(%rax) 63 leaq copy_fault(%rip),%r11 64 movq %r11, PCB_ONFAULT(%rax) 65 66 xchgq %rdi,%rsi 67 movq %rdx,%rcx 68 69 movq %rdi,%rax 70 subq %rsi,%rax 71 cmpq %rcx,%rax # overlapping? 72 jb 1f 73 shrq $3,%rcx # nope, copy forward by 64-bit words 74 rep 75 movsq 76 77 movq %rdx,%rcx 78 andl $7,%ecx # any bytes left? 79 rep 80 movsb 81 82 movq CPUVAR(CURPCB),%rdx 83 popq PCB_ONFAULT(%rdx) 84 xorq %rax,%rax 85 RETGUARD_CHECK(kcopy, r10) 86 ret 87 lfence 88 891: addq %rcx,%rdi # copy backward 90 addq %rcx,%rsi 91 std 92 andq $7,%rcx # any fractional bytes? 93 decq %rdi 94 decq %rsi 95 rep 96 movsb 97 movq %rdx,%rcx # copy remainder by 64-bit words 98 shrq $3,%rcx 99 subq $7,%rsi 100 subq $7,%rdi 101 rep 102 movsq 103 cld 104 105 movq CPUVAR(CURPCB),%rdx 106 popq PCB_ONFAULT(%rdx) 107 xorq %rax,%rax 108 RETGUARD_CHECK(kcopy, r10) 109 ret 110 lfence 111 112ENTRY(copyout) 113 RETGUARD_SETUP(kcopy, r10) 114 pushq $0 115 116 xchgq %rdi,%rsi 117 movq %rdx,%rax 118 119 movq %rdi,%rdx 120 addq %rax,%rdx 121 jc copy_fault 122 movq $VM_MAXUSER_ADDRESS,%r8 123 cmpq %r8,%rdx 124 ja copy_fault 125 126 movq CPUVAR(CURPCB),%rdx 127 leaq copy_fault(%rip),%r11 128 movq %r11,PCB_ONFAULT(%rdx) 129 SMAP_STAC 130 movq %rax,%rcx 131 shrq $3,%rcx 132 rep 133 movsq 134 movb %al,%cl 135 shrb $2,%cl 136 andb $1,%cl 137 rep 138 movsl 139 movb %al,%cl 140 andb $3,%cl 141 rep 142 movsb 143 SMAP_CLAC 144 popq PCB_ONFAULT(%rdx) 145 xorl %eax,%eax 146 RETGUARD_CHECK(kcopy, r10) 147 ret 148 lfence 149 150ENTRY(_copyin) 151 RETGUARD_SETUP(kcopy, r10) 152 movq CPUVAR(CURPCB),%rax 153 pushq $0 154 leaq copy_fault(%rip),%r11 155 movq %r11,PCB_ONFAULT(%rax) 156 SMAP_STAC 157 xchgq %rdi,%rsi 158 movq %rdx,%rax 159 160 movq %rsi,%rdx 161 addq %rax,%rdx 162 jc copy_fault 163 movq $VM_MAXUSER_ADDRESS,%r8 164 cmpq %r8,%rdx 165 ja copy_fault 166 1673: /* bcopy(%rsi, %rdi, %rax); */ 168 movq %rax,%rcx 169 shrq $3,%rcx 170 rep 171 movsq 172 movb %al,%cl 173 shrb $2,%cl 174 andb $1,%cl 175 rep 176 movsl 177 movb %al,%cl 178 andb $3,%cl 179 rep 180 movsb 181 182 SMAP_CLAC 183 movq CPUVAR(CURPCB),%rdx 184 popq PCB_ONFAULT(%rdx) 185 xorl %eax,%eax 186 RETGUARD_CHECK(kcopy, r10) 187 ret 188 lfence 189 190NENTRY(copy_fault) 191DECLARE_ONFAULT(copy_fault) 192 cld 193 SMAP_CLAC 194 movq CPUVAR(CURPCB),%rdx 195 popq PCB_ONFAULT(%rdx) 196 movl $EFAULT,%eax 197 RETGUARD_CHECK(kcopy, r10) 198 ret 199 lfence 200 201ENTRY(copyoutstr) 202 RETGUARD_SETUP(copyoutstr, r10) 203 xchgq %rdi,%rsi 204 movq %rdx,%r8 205 movq %rcx,%r9 206 2075: movq CPUVAR(CURPCB),%rax 208 leaq copystr_fault(%rip),%r11 209 movq %r11,PCB_ONFAULT(%rax) 210 SMAP_STAC 211 /* 212 * Get min(%rdx, VM_MAXUSER_ADDRESS-%rdi). 213 */ 214 movq $VM_MAXUSER_ADDRESS,%rax 215 subq %rdi,%rax 216 jbe copystr_fault /* die if CF == 1 || ZF == 1 */ 217 cmpq %rdx,%rax 218 jae 1f 219 movq %rax,%rdx 220 movq %rax,%r8 221 2221: incq %rdx 223 2241: decq %rdx 225 jz 2f 226 lodsb 227 stosb 228 testb %al,%al 229 jnz 1b 230 231 /* Success -- 0 byte reached. */ 232 decq %rdx 233 xorq %rax,%rax 234 jmp copystr_return 235 2362: /* rdx is zero -- return EFAULT or ENAMETOOLONG. */ 237 movq $VM_MAXUSER_ADDRESS,%r11 238 cmpq %r11,%rdi 239 jae copystr_fault 240 movl $ENAMETOOLONG,%eax 241 jmp copystr_return 242 243ENTRY(_copyinstr) 244 RETGUARD_SETUP(copyoutstr, r10) 245 xchgq %rdi,%rsi 246 movq %rdx,%r8 247 movq %rcx,%r9 248 249 movq CPUVAR(CURPCB),%rcx 250 leaq copystr_fault(%rip),%r11 251 movq %r11,PCB_ONFAULT(%rcx) 252 SMAP_STAC 253 254 /* 255 * Get min(%rdx, VM_MAXUSER_ADDRESS-%rsi). 256 */ 257 movq $VM_MAXUSER_ADDRESS,%rax 258 subq %rsi,%rax 259 jbe copystr_fault /* die if CF == 1 || ZF == 1 */ 260 cmpq %rdx,%rax 261 jae 1f 262 movq %rax,%rdx 263 movq %rax,%r8 264 2651: incq %rdx 266 2671: decq %rdx 268 jz 2f 269 lodsb 270 stosb 271 testb %al,%al 272 jnz 1b 273 274 /* Success -- 0 byte reached. */ 275 decq %rdx 276 xorq %rax,%rax 277 jmp copystr_return 278 2792: /* edx is zero -- return EFAULT or ENAMETOOLONG. */ 280 movq $VM_MAXUSER_ADDRESS,%r11 281 cmpq %r11,%rsi 282 jae copystr_fault 283 movl $ENAMETOOLONG,%eax 284 jmp copystr_return 285 286ENTRY(copystr_fault) 287DECLARE_ONFAULT(copystr_fault) 288 movl $EFAULT,%eax 289copystr_return: 290 SMAP_CLAC 291 /* Set *lencopied and return %eax. */ 292 movq CPUVAR(CURPCB),%rcx 293 movq $0,PCB_ONFAULT(%rcx) 294 testq %r9,%r9 295 jz 8f 296 subq %rdx,%r8 297 movq %r8,(%r9) 2988: 299 RETGUARD_CHECK(copyoutstr, r10) 300 ret 301 lfence 302 303CODEPATCH_CODE(_stac, stac) 304CODEPATCH_CODE(_clac, clac) 305