1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2018 The FreeBSD Foundation 5 * 6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include <sys/param.h> 35 #include <sys/lock.h> 36 #include <sys/mutex.h> 37 #include <sys/pcpu.h> 38 #include <sys/proc.h> 39 #include <sys/sched.h> 40 #include <sys/sysctl.h> 41 #include <sys/systm.h> 42 #include <vm/vm.h> 43 #include <vm/vm_param.h> 44 #include <vm/vm_extern.h> 45 #include <vm/pmap.h> 46 #include <vm/vm_map.h> 47 #include <vm/vm_page.h> 48 49 int copyin_fast(const void *udaddr, void *kaddr, size_t len, u_int); 50 static int (*copyin_fast_tramp)(const void *, void *, size_t, u_int); 51 int copyout_fast(const void *kaddr, void *udaddr, size_t len, u_int); 52 static int (*copyout_fast_tramp)(const void *, void *, size_t, u_int); 53 int fubyte_fast(volatile const void *base, u_int kcr3); 54 static int (*fubyte_fast_tramp)(volatile const void *, u_int); 55 int fuword16_fast(volatile const void *base, u_int kcr3); 56 static int (*fuword16_fast_tramp)(volatile const void *, u_int); 57 int fueword_fast(volatile const void *base, long *val, u_int kcr3); 58 static int (*fueword_fast_tramp)(volatile const void *, long *, u_int); 59 int subyte_fast(volatile void *base, int val, u_int kcr3); 60 static int (*subyte_fast_tramp)(volatile void *, int, u_int); 61 int suword16_fast(volatile void *base, int val, u_int kcr3); 62 static int (*suword16_fast_tramp)(volatile void *, int, u_int); 63 int suword_fast(volatile void *base, long val, u_int kcr3); 64 static int (*suword_fast_tramp)(volatile void *, long, u_int); 65 66 static int fast_copyout = 1; 67 SYSCTL_INT(_machdep, OID_AUTO, fast_copyout, CTLFLAG_RWTUN, 68 &fast_copyout, 0, 69 ""); 70 71 void 72 copyout_init_tramp(void) 73 { 74 75 copyin_fast_tramp = (int (*)(const void *, void *, size_t, u_int))( 76 (uintptr_t)copyin_fast + setidt_disp); 77 copyout_fast_tramp = (int (*)(const void *, void *, size_t, u_int))( 78 (uintptr_t)copyout_fast + setidt_disp); 79 fubyte_fast_tramp = (int (*)(volatile const void *, u_int))( 80 (uintptr_t)fubyte_fast + setidt_disp); 81 fuword16_fast_tramp = (int (*)(volatile const void *, u_int))( 82 (uintptr_t)fuword16_fast + setidt_disp); 83 fueword_fast_tramp = (int (*)(volatile const void *, long *, u_int))( 84 (uintptr_t)fueword_fast + setidt_disp); 85 subyte_fast_tramp = (int (*)(volatile void *, int, u_int))( 86 (uintptr_t)subyte_fast + setidt_disp); 87 suword16_fast_tramp = (int (*)(volatile void *, int, u_int))( 88 (uintptr_t)suword16_fast + setidt_disp); 89 suword_fast_tramp = (int (*)(volatile void *, long, u_int))( 90 (uintptr_t)suword_fast + setidt_disp); 91 } 92 93 int 94 cp_slow0(vm_offset_t uva, size_t len, bool write, 95 void (*f)(vm_offset_t, void *), void *arg) 96 { 97 struct pcpu *pc; 98 vm_page_t m[2]; 99 vm_offset_t kaddr; 100 int error, i, plen; 101 bool sleepable; 102 103 plen = howmany(uva - trunc_page(uva) + len, PAGE_SIZE); 104 MPASS(plen <= nitems(m)); 105 error = 0; 106 i = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, uva, len, 107 (write ? VM_PROT_WRITE : VM_PROT_READ) | VM_PROT_QUICK_NOFAULT, 108 m, nitems(m)); 109 if (i != plen) 110 return (EFAULT); 111 sched_pin(); 112 pc = get_pcpu(); 113 if (!THREAD_CAN_SLEEP() || curthread->td_vslock_sz > 0 || 114 (curthread->td_pflags & TDP_NOFAULTING) != 0) { 115 sleepable = false; 116 mtx_lock(&pc->pc_copyout_mlock); 117 kaddr = pc->pc_copyout_maddr; 118 } else { 119 sleepable = true; 120 sx_xlock(&pc->pc_copyout_slock); 121 kaddr = pc->pc_copyout_saddr; 122 } 123 pmap_cp_slow0_map(kaddr, plen, m); 124 kaddr += uva - trunc_page(uva); 125 f(kaddr, arg); 126 sched_unpin(); 127 if (sleepable) 128 sx_xunlock(&pc->pc_copyout_slock); 129 else 130 mtx_unlock(&pc->pc_copyout_mlock); 131 vm_page_unhold_pages(m, plen); 132 return (error); 133 } 134 135 struct copyinstr_arg0 { 136 vm_offset_t kc; 137 size_t len; 138 size_t alen; 139 bool end; 140 }; 141 142 static void 143 copyinstr_slow0(vm_offset_t kva, void *arg) 144 { 145 struct copyinstr_arg0 *ca; 146 char c; 147 148 ca = arg; 149 MPASS(ca->alen == 0 && ca->len > 0 && !ca->end); 150 while (ca->alen < ca->len && !ca->end) { 151 c = *(char *)(kva + ca->alen); 152 *(char *)ca->kc = c; 153 ca->alen++; 154 ca->kc++; 155 if (c == '\0') 156 ca->end = true; 157 } 158 } 159 160 int 161 copyinstr(const void *udaddr, void *kaddr, size_t maxlen, size_t *lencopied) 162 { 163 struct copyinstr_arg0 ca; 164 vm_offset_t uc; 165 size_t plen; 166 int error; 167 168 error = 0; 169 ca.end = false; 170 for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr; 171 plen < maxlen && !ca.end; uc += ca.alen, plen += ca.alen) { 172 ca.len = round_page(uc) - uc; 173 if (ca.len == 0) 174 ca.len = PAGE_SIZE; 175 if (plen + ca.len > maxlen) 176 ca.len = maxlen - plen; 177 ca.alen = 0; 178 if (cp_slow0(uc, ca.len, false, copyinstr_slow0, &ca) != 0) { 179 error = EFAULT; 180 break; 181 } 182 } 183 if (!ca.end && plen == maxlen && error == 0) 184 error = ENAMETOOLONG; 185 if (lencopied != NULL) 186 *lencopied = plen; 187 return (error); 188 } 189 190 struct copyin_arg0 { 191 vm_offset_t kc; 192 size_t len; 193 }; 194 195 static void 196 copyin_slow0(vm_offset_t kva, void *arg) 197 { 198 struct copyin_arg0 *ca; 199 200 ca = arg; 201 bcopy((void *)kva, (void *)ca->kc, ca->len); 202 } 203 204 int 205 copyin(const void *udaddr, void *kaddr, size_t len) 206 { 207 struct copyin_arg0 ca; 208 vm_offset_t uc; 209 size_t plen; 210 211 if ((uintptr_t)udaddr + len < (uintptr_t)udaddr || 212 (uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS) 213 return (EFAULT); 214 if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ && 215 copyin_fast_tramp(udaddr, kaddr, len, pmap_get_kcr3()) == 0)) 216 return (0); 217 for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr; 218 plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) { 219 ca.len = round_page(uc) - uc; 220 if (ca.len == 0) 221 ca.len = PAGE_SIZE; 222 if (plen + ca.len > len) 223 ca.len = len - plen; 224 if (cp_slow0(uc, ca.len, false, copyin_slow0, &ca) != 0) 225 return (EFAULT); 226 } 227 return (0); 228 } 229 230 static void 231 copyout_slow0(vm_offset_t kva, void *arg) 232 { 233 struct copyin_arg0 *ca; 234 235 ca = arg; 236 bcopy((void *)ca->kc, (void *)kva, ca->len); 237 } 238 239 int 240 copyout(const void *kaddr, void *udaddr, size_t len) 241 { 242 struct copyin_arg0 ca; 243 vm_offset_t uc; 244 size_t plen; 245 246 if ((uintptr_t)udaddr + len < (uintptr_t)udaddr || 247 (uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS) 248 return (EFAULT); 249 if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ && 250 copyout_fast_tramp(kaddr, udaddr, len, pmap_get_kcr3()) == 0)) 251 return (0); 252 for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr; 253 plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) { 254 ca.len = round_page(uc) - uc; 255 if (ca.len == 0) 256 ca.len = PAGE_SIZE; 257 if (plen + ca.len > len) 258 ca.len = len - plen; 259 if (cp_slow0(uc, ca.len, true, copyout_slow0, &ca) != 0) 260 return (EFAULT); 261 } 262 return (0); 263 } 264 265 /* 266 * Fetch (load) a 32-bit word, a 16-bit word, or an 8-bit byte from user 267 * memory. 268 */ 269 270 static void 271 fubyte_slow0(vm_offset_t kva, void *arg) 272 { 273 274 *(int *)arg = *(u_char *)kva; 275 } 276 277 int 278 fubyte(volatile const void *base) 279 { 280 int res; 281 282 if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base || 283 (uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS) 284 return (-1); 285 if (fast_copyout) { 286 res = fubyte_fast_tramp(base, pmap_get_kcr3()); 287 if (res != -1) 288 return (res); 289 } 290 if (cp_slow0((vm_offset_t)base, sizeof(char), false, fubyte_slow0, 291 &res) != 0) 292 return (-1); 293 return (res); 294 } 295 296 static void 297 fuword16_slow0(vm_offset_t kva, void *arg) 298 { 299 300 *(int *)arg = *(uint16_t *)kva; 301 } 302 303 int 304 fuword16(volatile const void *base) 305 { 306 int res; 307 308 if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base || 309 (uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS) 310 return (-1); 311 if (fast_copyout) { 312 res = fuword16_fast_tramp(base, pmap_get_kcr3()); 313 if (res != -1) 314 return (res); 315 } 316 if (cp_slow0((vm_offset_t)base, sizeof(uint16_t), false, 317 fuword16_slow0, &res) != 0) 318 return (-1); 319 return (res); 320 } 321 322 static void 323 fueword_slow0(vm_offset_t kva, void *arg) 324 { 325 326 *(uint32_t *)arg = *(uint32_t *)kva; 327 } 328 329 int 330 fueword(volatile const void *base, long *val) 331 { 332 uint32_t res; 333 334 if ((uintptr_t)base + sizeof(*val) < (uintptr_t)base || 335 (uintptr_t)base + sizeof(*val) > VM_MAXUSER_ADDRESS) 336 return (-1); 337 if (fast_copyout) { 338 if (fueword_fast_tramp(base, val, pmap_get_kcr3()) == 0) 339 return (0); 340 } 341 if (cp_slow0((vm_offset_t)base, sizeof(long), false, fueword_slow0, 342 &res) != 0) 343 return (-1); 344 *val = res; 345 return (0); 346 } 347 348 int 349 fueword32(volatile const void *base, int32_t *val) 350 { 351 352 return (fueword(base, (long *)val)); 353 } 354 355 /* 356 * Store a 32-bit word, a 16-bit word, or an 8-bit byte to user memory. 357 */ 358 359 static void 360 subyte_slow0(vm_offset_t kva, void *arg) 361 { 362 363 *(u_char *)kva = *(int *)arg; 364 } 365 366 int 367 subyte(volatile void *base, int byte) 368 { 369 370 if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base || 371 (uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS) 372 return (-1); 373 if (fast_copyout && subyte_fast_tramp(base, byte, pmap_get_kcr3()) == 0) 374 return (0); 375 return (cp_slow0((vm_offset_t)base, sizeof(u_char), true, subyte_slow0, 376 &byte) != 0 ? -1 : 0); 377 } 378 379 static void 380 suword16_slow0(vm_offset_t kva, void *arg) 381 { 382 383 *(int *)kva = *(uint16_t *)arg; 384 } 385 386 int 387 suword16(volatile void *base, int word) 388 { 389 390 if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base || 391 (uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS) 392 return (-1); 393 if (fast_copyout && suword16_fast_tramp(base, word, pmap_get_kcr3()) 394 == 0) 395 return (0); 396 return (cp_slow0((vm_offset_t)base, sizeof(int16_t), true, 397 suword16_slow0, &word) != 0 ? -1 : 0); 398 } 399 400 static void 401 suword_slow0(vm_offset_t kva, void *arg) 402 { 403 404 *(int *)kva = *(uint32_t *)arg; 405 } 406 407 int 408 suword(volatile void *base, long word) 409 { 410 411 if ((uintptr_t)base + sizeof(word) < (uintptr_t)base || 412 (uintptr_t)base + sizeof(word) > VM_MAXUSER_ADDRESS) 413 return (-1); 414 if (fast_copyout && suword_fast_tramp(base, word, pmap_get_kcr3()) == 0) 415 return (0); 416 return (cp_slow0((vm_offset_t)base, sizeof(long), true, 417 suword_slow0, &word) != 0 ? -1 : 0); 418 } 419 420 int 421 suword32(volatile void *base, int32_t word) 422 { 423 424 return (suword(base, word)); 425 } 426 427 struct casueword_arg0 { 428 uint32_t oldval; 429 uint32_t newval; 430 int res; 431 }; 432 433 static void 434 casueword_slow0(vm_offset_t kva, void *arg) 435 { 436 struct casueword_arg0 *ca; 437 438 ca = arg; 439 ca->res = 1 - atomic_fcmpset_int((u_int *)kva, &ca->oldval, 440 ca->newval); 441 } 442 443 int 444 casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, 445 uint32_t newval) 446 { 447 struct casueword_arg0 ca; 448 int res; 449 450 ca.oldval = oldval; 451 ca.newval = newval; 452 res = cp_slow0((vm_offset_t)base, sizeof(int32_t), true, 453 casueword_slow0, &ca); 454 if (res == 0) { 455 *oldvalp = ca.oldval; 456 return (ca.res); 457 } 458 return (-1); 459 } 460 461 int 462 casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, u_long newval) 463 { 464 struct casueword_arg0 ca; 465 int res; 466 467 ca.oldval = oldval; 468 ca.newval = newval; 469 res = cp_slow0((vm_offset_t)base, sizeof(int32_t), true, 470 casueword_slow0, &ca); 471 if (res == 0) { 472 *oldvalp = ca.oldval; 473 return (ca.res); 474 } 475 return (-1); 476 } 477