1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause AND BSD-4-Clause 3 * 4 * Copyright (C) 2002 Benno Rice 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 /*- 28 * Copyright (C) 1993 Wolfgang Solfrank. 29 * Copyright (C) 1993 TooLs GmbH. 30 * All rights reserved. 31 * 32 * Redistribution and use in source and binary forms, with or without 33 * modification, are permitted provided that the following conditions 34 * are met: 35 * 1. Redistributions of source code must retain the above copyright 36 * notice, this list of conditions and the following disclaimer. 37 * 2. Redistributions in binary form must reproduce the above copyright 38 * notice, this list of conditions and the following disclaimer in the 39 * documentation and/or other materials provided with the distribution. 40 * 3. All advertising materials mentioning features or use of this software 41 * must display the following acknowledgement: 42 * This product includes software developed by TooLs GmbH. 43 * 4. The name of TooLs GmbH may not be used to endorse or promote products 44 * derived from this software without specific prior written permission. 45 * 46 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 47 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 48 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 49 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 50 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 51 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 52 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 53 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 54 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 55 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 56 */ 57 58 #include <sys/cdefs.h> 59 __FBSDID("$FreeBSD$"); 60 61 #include <sys/param.h> 62 #include <sys/lock.h> 63 #include <sys/mutex.h> 64 #include <sys/systm.h> 65 #include <sys/proc.h> 66 67 #include <vm/vm.h> 68 #include <vm/pmap.h> 69 #include <vm/vm_extern.h> 70 #include <vm/vm_map.h> 71 72 #include <machine/mmuvar.h> 73 #include <machine/pcb.h> 74 #include <machine/vmparam.h> 75 #include <machine/ifunc.h> 76 77 /* 78 * On powerpc64 (AIM only) the copy functions are IFUNCs, selecting the best 79 * option based on the PMAP in use. 80 * 81 * There are two options for copy functions on powerpc64: 82 * - 'remap' copies, which remap userspace segments into kernel space for 83 * copying. This is used by the 'oea64' pmap. 84 * - 'direct' copies, which copy directly from userspace. This does not require 85 * remapping user segments into kernel. This is used by the 'radix' pmap for 86 * performance. 87 * 88 * Book-E does not use the C 'remap' functions, opting instead to use the 89 * 'direct' copies, directly, avoiding the IFUNC overhead. 90 * 91 * On 32-bit AIM these functions bypass the IFUNC machinery for performance. 92 */ 93 #ifdef __powerpc64__ 94 int subyte_remap(volatile void *addr, int byte); 95 int subyte_direct(volatile void *addr, int byte); 96 int copyinstr_remap(const void *udaddr, void *kaddr, size_t len, size_t *done); 97 int copyinstr_direct(const void *udaddr, void *kaddr, size_t len, size_t *done); 98 int copyout_remap(const void *kaddr, void *udaddr, size_t len); 99 int copyout_direct(const void *kaddr, void *udaddr, size_t len); 100 int copyin_remap(const void *uaddr, void *kaddr, size_t len); 101 int copyin_direct(const void *uaddr, void *kaddr, size_t len); 102 int suword16_remap(volatile void *addr, int word); 103 int suword16_direct(volatile void *addr, int word); 104 int suword32_remap(volatile void *addr, int word); 105 int suword32_direct(volatile void *addr, int word); 106 int suword_remap(volatile void *addr, long word); 107 int suword_direct(volatile void *addr, long word); 108 int suword64_remap(volatile void *addr, int64_t word); 109 int suword64_direct(volatile void *addr, int64_t word); 110 int fubyte_remap(volatile const void *addr); 111 int fubyte_direct(volatile const void *addr); 112 int fuword16_remap(volatile const void *addr); 113 int fuword16_direct(volatile const void *addr); 114 int fueword32_remap(volatile const void *addr, int32_t *val); 115 int fueword32_direct(volatile const void *addr, int32_t *val); 116 int fueword64_remap(volatile const void *addr, int64_t *val); 117 int fueword64_direct(volatile const void *addr, int64_t *val); 118 int fueword_remap(volatile const void *addr, long *val); 119 int fueword_direct(volatile const void *addr, long *val); 120 int casueword32_remap(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp, 121 uint32_t new); 122 int casueword32_direct(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp, 123 uint32_t new); 124 int casueword_remap(volatile u_long *addr, u_long old, u_long *oldvalp, 125 u_long new); 126 int casueword_direct(volatile u_long *addr, u_long old, u_long *oldvalp, 127 u_long new); 128 129 /* 130 * The IFUNC resolver determines the copy based on whether the PMAP 131 * implementation includes a pmap_map_user_ptr function. 132 */ 133 #define DEFINE_COPY_FUNC(ret, func, args) \ 134 DEFINE_IFUNC(, ret, func, args) \ 135 { \ 136 return (PMAP_RESOLVE_FUNC(map_user_ptr) ? \ 137 func##_remap : func##_direct); \ 138 } 139 DEFINE_COPY_FUNC(int, subyte, (volatile void *, int)) 140 DEFINE_COPY_FUNC(int, copyinstr, (const void *, void *, size_t, size_t *)) 141 DEFINE_COPY_FUNC(int, copyin, (const void *, void *, size_t)) 142 DEFINE_COPY_FUNC(int, copyout, (const void *, void *, size_t)) 143 DEFINE_COPY_FUNC(int, suword, (volatile void *, long)) 144 DEFINE_COPY_FUNC(int, suword16, (volatile void *, int)) 145 DEFINE_COPY_FUNC(int, suword32, (volatile void *, int)) 146 DEFINE_COPY_FUNC(int, suword64, (volatile void *, int64_t)) 147 DEFINE_COPY_FUNC(int, fubyte, (volatile const void *)) 148 DEFINE_COPY_FUNC(int, fuword16, (volatile const void *)) 149 DEFINE_COPY_FUNC(int, fueword32, (volatile const void *, int32_t *)) 150 DEFINE_COPY_FUNC(int, fueword64, (volatile const void *, int64_t *)) 151 DEFINE_COPY_FUNC(int, fueword, (volatile const void *, long *)) 152 DEFINE_COPY_FUNC(int, casueword32, 153 (volatile uint32_t *, uint32_t, uint32_t *, uint32_t)) 154 DEFINE_COPY_FUNC(int, casueword, (volatile u_long *, u_long, u_long *, u_long)) 155 156 #define REMAP(x) x##_remap 157 #else 158 #define REMAP(x) x 159 #endif 160 161 int 162 REMAP(copyout)(const void *kaddr, void *udaddr, size_t len) 163 { 164 struct thread *td; 165 pmap_t pm; 166 jmp_buf env; 167 const char *kp; 168 char *up, *p; 169 size_t l; 170 171 td = curthread; 172 pm = &td->td_proc->p_vmspace->vm_pmap; 173 174 td->td_pcb->pcb_onfault = &env; 175 if (setjmp(env)) { 176 td->td_pcb->pcb_onfault = NULL; 177 return (EFAULT); 178 } 179 180 kp = kaddr; 181 up = udaddr; 182 183 while (len > 0) { 184 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) { 185 td->td_pcb->pcb_onfault = NULL; 186 return (EFAULT); 187 } 188 189 bcopy(kp, p, l); 190 191 up += l; 192 kp += l; 193 len -= l; 194 } 195 196 td->td_pcb->pcb_onfault = NULL; 197 return (0); 198 } 199 200 int 201 REMAP(copyin)(const void *udaddr, void *kaddr, size_t len) 202 { 203 struct thread *td; 204 pmap_t pm; 205 jmp_buf env; 206 const char *up; 207 char *kp, *p; 208 size_t l; 209 210 td = curthread; 211 pm = &td->td_proc->p_vmspace->vm_pmap; 212 213 td->td_pcb->pcb_onfault = &env; 214 if (setjmp(env)) { 215 td->td_pcb->pcb_onfault = NULL; 216 return (EFAULT); 217 } 218 219 kp = kaddr; 220 up = udaddr; 221 222 while (len > 0) { 223 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) { 224 td->td_pcb->pcb_onfault = NULL; 225 return (EFAULT); 226 } 227 228 bcopy(p, kp, l); 229 230 up += l; 231 kp += l; 232 len -= l; 233 } 234 235 td->td_pcb->pcb_onfault = NULL; 236 return (0); 237 } 238 239 int 240 REMAP(copyinstr)(const void *udaddr, void *kaddr, size_t len, size_t *done) 241 { 242 struct thread *td; 243 pmap_t pm; 244 jmp_buf env; 245 const char *up; 246 char *kp, *p; 247 size_t i, l, t; 248 int rv; 249 250 td = curthread; 251 pm = &td->td_proc->p_vmspace->vm_pmap; 252 253 t = 0; 254 rv = ENAMETOOLONG; 255 256 td->td_pcb->pcb_onfault = &env; 257 if (setjmp(env)) { 258 rv = EFAULT; 259 goto done; 260 } 261 262 kp = kaddr; 263 up = udaddr; 264 265 while (len > 0) { 266 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) { 267 rv = EFAULT; 268 goto done; 269 } 270 271 for (i = 0; len > 0 && i < l; i++, t++, len--) { 272 if ((*kp++ = *p++) == 0) { 273 i++, t++; 274 rv = 0; 275 goto done; 276 } 277 } 278 279 up += l; 280 } 281 282 done: 283 td->td_pcb->pcb_onfault = NULL; 284 285 if (done != NULL) { 286 *done = t; 287 } 288 289 return (rv); 290 } 291 292 int 293 REMAP(subyte)(volatile void *addr, int byte) 294 { 295 struct thread *td; 296 pmap_t pm; 297 jmp_buf env; 298 char *p; 299 300 td = curthread; 301 pm = &td->td_proc->p_vmspace->vm_pmap; 302 303 td->td_pcb->pcb_onfault = &env; 304 if (setjmp(env)) { 305 td->td_pcb->pcb_onfault = NULL; 306 return (-1); 307 } 308 309 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 310 td->td_pcb->pcb_onfault = NULL; 311 return (-1); 312 } 313 314 *p = (char)byte; 315 316 td->td_pcb->pcb_onfault = NULL; 317 return (0); 318 } 319 320 int 321 REMAP(suword16)(volatile void *addr, int word) 322 { 323 struct thread *td; 324 pmap_t pm; 325 jmp_buf env; 326 int16_t *p; 327 328 td = curthread; 329 pm = &td->td_proc->p_vmspace->vm_pmap; 330 331 td->td_pcb->pcb_onfault = &env; 332 if (setjmp(env)) { 333 td->td_pcb->pcb_onfault = NULL; 334 return (-1); 335 } 336 337 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 338 td->td_pcb->pcb_onfault = NULL; 339 return (-1); 340 } 341 342 *p = (int16_t)word; 343 344 td->td_pcb->pcb_onfault = NULL; 345 return (0); 346 } 347 348 #ifdef __powerpc64__ 349 int 350 REMAP(suword32)(volatile void *addr, int word) 351 { 352 struct thread *td; 353 pmap_t pm; 354 jmp_buf env; 355 int *p; 356 357 td = curthread; 358 pm = &td->td_proc->p_vmspace->vm_pmap; 359 360 td->td_pcb->pcb_onfault = &env; 361 if (setjmp(env)) { 362 td->td_pcb->pcb_onfault = NULL; 363 return (-1); 364 } 365 366 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 367 td->td_pcb->pcb_onfault = NULL; 368 return (-1); 369 } 370 371 *p = word; 372 373 td->td_pcb->pcb_onfault = NULL; 374 return (0); 375 } 376 #else 377 int 378 REMAP(suword32)(volatile void *addr, int32_t word) 379 { 380 REMAP( return (suword)(addr, (long)word)); 381 } 382 #endif 383 384 int 385 REMAP(suword)(volatile void *addr, long word) 386 { 387 struct thread *td; 388 pmap_t pm; 389 jmp_buf env; 390 long *p; 391 392 td = curthread; 393 pm = &td->td_proc->p_vmspace->vm_pmap; 394 395 td->td_pcb->pcb_onfault = &env; 396 if (setjmp(env)) { 397 td->td_pcb->pcb_onfault = NULL; 398 return (-1); 399 } 400 401 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 402 td->td_pcb->pcb_onfault = NULL; 403 return (-1); 404 } 405 406 *p = word; 407 408 td->td_pcb->pcb_onfault = NULL; 409 return (0); 410 } 411 412 #ifdef __powerpc64__ 413 int 414 REMAP(suword64)(volatile void *addr, int64_t word) 415 { 416 return (REMAP(suword)(addr, (long)word)); 417 } 418 #endif 419 420 int 421 REMAP(fubyte)(volatile const void *addr) 422 { 423 struct thread *td; 424 pmap_t pm; 425 jmp_buf env; 426 u_char *p; 427 int val; 428 429 td = curthread; 430 pm = &td->td_proc->p_vmspace->vm_pmap; 431 432 td->td_pcb->pcb_onfault = &env; 433 if (setjmp(env)) { 434 td->td_pcb->pcb_onfault = NULL; 435 return (-1); 436 } 437 438 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 439 td->td_pcb->pcb_onfault = NULL; 440 return (-1); 441 } 442 443 val = *p; 444 445 td->td_pcb->pcb_onfault = NULL; 446 return (val); 447 } 448 449 int 450 REMAP(fuword16)(volatile const void *addr) 451 { 452 struct thread *td; 453 pmap_t pm; 454 jmp_buf env; 455 uint16_t *p, val; 456 457 td = curthread; 458 pm = &td->td_proc->p_vmspace->vm_pmap; 459 460 td->td_pcb->pcb_onfault = &env; 461 if (setjmp(env)) { 462 td->td_pcb->pcb_onfault = NULL; 463 return (-1); 464 } 465 466 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 467 td->td_pcb->pcb_onfault = NULL; 468 return (-1); 469 } 470 471 val = *p; 472 473 td->td_pcb->pcb_onfault = NULL; 474 return (val); 475 } 476 477 int 478 REMAP(fueword32)(volatile const void *addr, int32_t *val) 479 { 480 struct thread *td; 481 pmap_t pm; 482 jmp_buf env; 483 int32_t *p; 484 485 td = curthread; 486 pm = &td->td_proc->p_vmspace->vm_pmap; 487 488 td->td_pcb->pcb_onfault = &env; 489 if (setjmp(env)) { 490 td->td_pcb->pcb_onfault = NULL; 491 return (-1); 492 } 493 494 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 495 td->td_pcb->pcb_onfault = NULL; 496 return (-1); 497 } 498 499 *val = *p; 500 501 td->td_pcb->pcb_onfault = NULL; 502 return (0); 503 } 504 505 #ifdef __powerpc64__ 506 int 507 REMAP(fueword64)(volatile const void *addr, int64_t *val) 508 { 509 struct thread *td; 510 pmap_t pm; 511 jmp_buf env; 512 int64_t *p; 513 514 td = curthread; 515 pm = &td->td_proc->p_vmspace->vm_pmap; 516 517 td->td_pcb->pcb_onfault = &env; 518 if (setjmp(env)) { 519 td->td_pcb->pcb_onfault = NULL; 520 return (-1); 521 } 522 523 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 524 td->td_pcb->pcb_onfault = NULL; 525 return (-1); 526 } 527 528 *val = *p; 529 530 td->td_pcb->pcb_onfault = NULL; 531 return (0); 532 } 533 #endif 534 535 int 536 REMAP(fueword)(volatile const void *addr, long *val) 537 { 538 struct thread *td; 539 pmap_t pm; 540 jmp_buf env; 541 long *p; 542 543 td = curthread; 544 pm = &td->td_proc->p_vmspace->vm_pmap; 545 546 td->td_pcb->pcb_onfault = &env; 547 if (setjmp(env)) { 548 td->td_pcb->pcb_onfault = NULL; 549 return (-1); 550 } 551 552 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 553 td->td_pcb->pcb_onfault = NULL; 554 return (-1); 555 } 556 557 *val = *p; 558 559 td->td_pcb->pcb_onfault = NULL; 560 return (0); 561 } 562 563 int 564 REMAP(casueword32)(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp, 565 uint32_t new) 566 { 567 struct thread *td; 568 pmap_t pm; 569 jmp_buf env; 570 uint32_t *p, val; 571 int res; 572 573 td = curthread; 574 pm = &td->td_proc->p_vmspace->vm_pmap; 575 576 td->td_pcb->pcb_onfault = &env; 577 if (setjmp(env)) { 578 td->td_pcb->pcb_onfault = NULL; 579 return (-1); 580 } 581 582 if (pmap_map_user_ptr(pm, (void *)(uintptr_t)addr, (void **)&p, 583 sizeof(*p), NULL)) { 584 td->td_pcb->pcb_onfault = NULL; 585 return (-1); 586 } 587 588 res = 0; 589 __asm __volatile ( 590 "lwarx %0, 0, %3\n\t" /* load old value */ 591 "cmplw %4, %0\n\t" /* compare */ 592 "bne 1f\n\t" /* exit if not equal */ 593 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 594 "bne- 2f\n\t" /* if failed */ 595 "b 3f\n\t" /* we've succeeded */ 596 "1:\n\t" 597 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 598 "2:li %2, 1\n\t" 599 "3:\n\t" 600 : "=&r" (val), "=m" (*p), "+&r" (res) 601 : "r" (p), "r" (old), "r" (new), "m" (*p) 602 : "cr0", "memory"); 603 604 td->td_pcb->pcb_onfault = NULL; 605 606 *oldvalp = val; 607 return (res); 608 } 609 610 #ifndef __powerpc64__ 611 int 612 REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new) 613 { 614 615 return (casueword32((volatile uint32_t *)addr, old, 616 (uint32_t *)oldvalp, new)); 617 } 618 #else 619 int 620 REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new) 621 { 622 struct thread *td; 623 pmap_t pm; 624 jmp_buf env; 625 u_long *p, val; 626 int res; 627 628 td = curthread; 629 pm = &td->td_proc->p_vmspace->vm_pmap; 630 631 td->td_pcb->pcb_onfault = &env; 632 if (setjmp(env)) { 633 td->td_pcb->pcb_onfault = NULL; 634 return (-1); 635 } 636 637 if (pmap_map_user_ptr(pm, (void *)(uintptr_t)addr, (void **)&p, 638 sizeof(*p), NULL)) { 639 td->td_pcb->pcb_onfault = NULL; 640 return (-1); 641 } 642 643 res = 0; 644 __asm __volatile ( 645 "ldarx %0, 0, %3\n\t" /* load old value */ 646 "cmpld %4, %0\n\t" /* compare */ 647 "bne 1f\n\t" /* exit if not equal */ 648 "stdcx. %5, 0, %3\n\t" /* attempt to store */ 649 "bne- 2f\n\t" /* if failed */ 650 "b 3f\n\t" /* we've succeeded */ 651 "1:\n\t" 652 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 653 "2:li %2, 1\n\t" 654 "3:\n\t" 655 : "=&r" (val), "=m" (*p), "+&r" (res) 656 : "r" (p), "r" (old), "r" (new), "m" (*p) 657 : "cr0", "memory"); 658 659 td->td_pcb->pcb_onfault = NULL; 660 661 *oldvalp = val; 662 return (res); 663 } 664 #endif 665