1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2018 The FreeBSD Foundation 5 * 6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 * $FreeBSD$ 31 */ 32 33#include <machine/asmacros.h> 34#include <machine/cputypes.h> 35#include <machine/pmap.h> 36#include <machine/specialreg.h> 37 38#include "assym.inc" 39 40/* 41 * Fast path for copyout code. We switch to user space %cr3 and perform 42 * move operation between user memory and copyout buffer, located in the 43 * trampoline area. We must switch to trampoline stack, because both 44 * user and kernel buffer accesses might cause page fault. 45 * 46 * Page fault handler expects %edx to point to the onfault routine. 47 * Handler switches to idlePTD and calls the routine. 48 * The routine must restore the stack, enable interrupts, and 49 * return to the caller, informing it about failure. 50 */ 51 .text 52 53ENTRY(copyout_fast) 54 pushl %ebp 55 movl %esp, %ebp 56 pushl %esi 57 pushl %edi 58 pushl %ebx 59 60 movl 20(%ebp),%ebx /* KCR3 */ 61 /* bcopy(%esi = kaddr, %edi = PCPU(copyout_buf), %ecx = len) */ 62 movl 16(%ebp),%ecx 63 movl 8(%ebp),%esi 64 movl %esp,%eax 65 movl $copyout_fault,%edx 66 67 cli 68 movl PCPU(COPYOUT_BUF),%edi 69pf_y1: rep; movsb 70 71 movl 16(%ebp),%ecx /* len */ 72 movl PCPU(COPYOUT_BUF),%esi /* kaddr */ 73 movl 12(%ebp),%edi /* uaddr */ 74 movl PCPU(TRAMPSTK),%esp 75 movl PCPU(CURPCB),%edx 76 movl PCB_CR3(%edx),%edx /* UCR3 */ 77 movl %edx,%cr3 78 movl $copyout_fault,%edx 79 /* bcopy(%esi = PCPU(copyout_buf), %edi = udaddr, %ecx = len) */ 80pf_x1: rep; movsb 81 82 movl %ebx,%cr3 83 movl %eax,%esp 84 sti 85 xorl %eax,%eax 86 popl %ebx 87 popl %edi 88 popl %esi 89 leave 90 ret 91END(copyout_fast) 92 93ENTRY(copyin_fast) 94 pushl %ebp 95 movl %esp, %ebp 96 pushl %esi 97 pushl %edi 98 pushl %ebx 99 100 movl 20(%ebp),%ebx /* KCR3 */ 101 movl PCPU(CURPCB),%eax 102 movl PCB_CR3(%eax),%edx /* UCR3 */ 103 movl 16(%ebp),%ecx /* len */ 104 movl 8(%ebp),%esi /* udaddr */ 105 movl %esp,%eax 106 107 cli 108 movl PCPU(COPYOUT_BUF),%edi /* kaddr */ 109 movl PCPU(TRAMPSTK),%esp 110 movl %edx,%cr3 111 movl $copyout_fault,%edx 112 /* bcopy(%esi = udaddr, %edi = PCPU(copyout_buf), %ecx = len) */ 113pf_x2: rep; movsb 114 115 movl %ebx,%cr3 116 movl %eax,%esp 117 118 /* bcopy(%esi = PCPU(copyout_buf), %edi = kaddr, %ecx = len) */ 119 movl 16(%ebp),%ecx 120 movl 12(%ebp),%edi 121 movl PCPU(COPYOUT_BUF),%esi 122pf_y2: rep; movsb 123 124 sti 125 xorl %eax,%eax 126 popl %ebx 127 popl %edi 128 popl %esi 129 leave 130 ret 131END(copyin_fast) 132 133 ALIGN_TEXT 134copyout_fault: 135 movl %eax,%esp 136 sti 137 movl $EFAULT,%eax 138 popl %ebx 139 popl %edi 140 popl %esi 141 leave 142 ret 143 144ENTRY(fueword_fast) 145 pushl %ebp 146 movl %esp,%ebp 147 pushl %ebx 148 pushl %esi 149 pushl %edi 150 movl 8(%ebp),%ecx /* from */ 151 movl PCPU(CURPCB),%eax 152 movl PCB_CR3(%eax),%eax 153 movl $fusufault,%edx 154 movl 16(%ebp),%ebx 155 movl %esp,%esi 156 cli 157 movl PCPU(TRAMPSTK),%esp 158 movl %eax,%cr3 159pf_x3: movl (%ecx),%eax 160 movl %ebx,%cr3 161 movl %esi,%esp 162 sti 163 movl 12(%ebp),%edx 164 movl %eax,(%edx) 165 xorl %eax,%eax 166 popl %edi 167 popl %esi 168 popl %ebx 169 leave 170 ret 171END(fueword_fast) 172 173ENTRY(fuword16_fast) 174 pushl %ebp 175 movl %esp,%ebp 176 pushl %ebx 177 pushl %esi 178 pushl %edi 179 movl 8(%ebp),%ecx /* from */ 180 movl PCPU(CURPCB),%eax 181 movl PCB_CR3(%eax),%eax 182 movl $fusufault,%edx 183 movl 12(%ebp),%ebx 184 movl %esp,%esi 185 cli 186 movl PCPU(TRAMPSTK),%esp 187 movl %eax,%cr3 188pf_x4: movzwl (%ecx),%eax 189 movl %ebx,%cr3 190 movl %esi,%esp 191 sti 192 popl %edi 193 popl %esi 194 popl %ebx 195 leave 196 ret 197END(fuword16_fast) 198 199ENTRY(fubyte_fast) 200 pushl %ebp 201 movl %esp,%ebp 202 pushl %ebx 203 pushl %esi 204 pushl %edi 205 movl 8(%ebp),%ecx /* from */ 206 movl PCPU(CURPCB),%eax 207 movl PCB_CR3(%eax),%eax 208 movl $fusufault,%edx 209 movl 12(%ebp),%ebx 210 movl %esp,%esi 211 cli 212 movl PCPU(TRAMPSTK),%esp 213 movl %eax,%cr3 214pf_x5: movzbl (%ecx),%eax 215 movl %ebx,%cr3 216 movl %esi,%esp 217 sti 218 popl %edi 219 popl %esi 220 popl %ebx 221 leave 222 ret 223END(fubyte_fast) 224 225 ALIGN_TEXT 226fusufault: 227 movl %esi,%esp 228 sti 229 xorl %eax,%eax 230 decl %eax 231 popl %edi 232 popl %esi 233 popl %ebx 234 leave 235 ret 236 237ENTRY(suword_fast) 238 pushl %ebp 239 movl %esp,%ebp 240 pushl %ebx 241 pushl %esi 242 pushl %edi 243 movl PCPU(CURPCB),%eax 244 movl PCB_CR3(%eax),%eax 245 movl $fusufault,%edx 246 movl 8(%ebp),%ecx /* to */ 247 movl 12(%ebp),%edi /* val */ 248 movl 16(%ebp),%ebx 249 movl %esp,%esi 250 cli 251 movl PCPU(TRAMPSTK),%esp 252 movl %eax,%cr3 253pf_x6: movl %edi,(%ecx) 254 movl %ebx,%cr3 255 movl %esi,%esp 256 sti 257 xorl %eax,%eax 258 popl %edi 259 popl %esi 260 popl %ebx 261 leave 262 ret 263END(suword_fast) 264 265ENTRY(suword16_fast) 266 pushl %ebp 267 movl %esp,%ebp 268 pushl %ebx 269 pushl %esi 270 pushl %edi 271 movl PCPU(CURPCB),%eax 272 movl PCB_CR3(%eax),%eax 273 movl $fusufault,%edx 274 movl 8(%ebp),%ecx /* to */ 275 movl 12(%ebp),%edi /* val */ 276 movl 16(%ebp),%ebx 277 movl %esp,%esi 278 cli 279 movl PCPU(TRAMPSTK),%esp 280 movl %eax,%cr3 281pf_x7: movw %di,(%ecx) 282 movl %ebx,%cr3 283 movl %esi,%esp 284 sti 285 xorl %eax,%eax 286 popl %edi 287 popl %esi 288 popl %ebx 289 leave 290 ret 291END(suword16_fast) 292 293ENTRY(subyte_fast) 294 pushl %ebp 295 movl %esp,%ebp 296 pushl %ebx 297 pushl %esi 298 pushl %edi 299 movl PCPU(CURPCB),%eax 300 movl PCB_CR3(%eax),%eax 301 movl $fusufault,%edx 302 movl 8(%ebp),%ecx /* to */ 303 movl 12(%ebp),%edi /* val */ 304 movl 16(%ebp),%ebx 305 movl %esp,%esi 306 cli 307 movl PCPU(TRAMPSTK),%esp 308 movl %eax,%cr3 309 movl %edi,%eax 310pf_x8: movb %al,(%ecx) 311 movl %ebx,%cr3 312 movl %esi,%esp 313 sti 314 xorl %eax,%eax 315 popl %edi 316 popl %esi 317 popl %ebx 318 leave 319 ret 320END(subyte_fast) 321