1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause AND BSD-4-Clause 3 * 4 * Copyright (C) 2002 Benno Rice 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 /*- 28 * Copyright (C) 1993 Wolfgang Solfrank. 29 * Copyright (C) 1993 TooLs GmbH. 30 * All rights reserved. 31 * 32 * Redistribution and use in source and binary forms, with or without 33 * modification, are permitted provided that the following conditions 34 * are met: 35 * 1. Redistributions of source code must retain the above copyright 36 * notice, this list of conditions and the following disclaimer. 37 * 2. Redistributions in binary form must reproduce the above copyright 38 * notice, this list of conditions and the following disclaimer in the 39 * documentation and/or other materials provided with the distribution. 40 * 3. All advertising materials mentioning features or use of this software 41 * must display the following acknowledgement: 42 * This product includes software developed by TooLs GmbH. 43 * 4. The name of TooLs GmbH may not be used to endorse or promote products 44 * derived from this software without specific prior written permission. 45 * 46 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 47 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 48 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 49 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 50 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 51 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 52 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 53 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 54 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 55 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 56 */ 57 58 #include <sys/cdefs.h> 59 #include <sys/param.h> 60 #include <sys/lock.h> 61 #include <sys/mutex.h> 62 #include <sys/systm.h> 63 #include <sys/proc.h> 64 65 #include <vm/vm.h> 66 #include <vm/pmap.h> 67 #include <vm/vm_extern.h> 68 #include <vm/vm_map.h> 69 70 #include <machine/mmuvar.h> 71 #include <machine/pcb.h> 72 #include <machine/vmparam.h> 73 #include <machine/ifunc.h> 74 75 /* 76 * On powerpc64 (AIM only) the copy functions are IFUNCs, selecting the best 77 * option based on the PMAP in use. 78 * 79 * There are two options for copy functions on powerpc64: 80 * - 'remap' copies, which remap userspace segments into kernel space for 81 * copying. This is used by the 'oea64' pmap. 82 * - 'direct' copies, which copy directly from userspace. This does not require 83 * remapping user segments into kernel. This is used by the 'radix' pmap for 84 * performance. 85 * 86 * Book-E does not use the C 'remap' functions, opting instead to use the 87 * 'direct' copies, directly, avoiding the IFUNC overhead. 88 * 89 * On 32-bit AIM these functions bypass the IFUNC machinery for performance. 90 */ 91 #ifdef __powerpc64__ 92 int subyte_remap(volatile void *addr, int byte); 93 int subyte_direct(volatile void *addr, int byte); 94 int copyinstr_remap(const void *udaddr, void *kaddr, size_t len, size_t *done); 95 int copyinstr_direct(const void *udaddr, void *kaddr, size_t len, size_t *done); 96 int copyout_remap(const void *kaddr, void *udaddr, size_t len); 97 int copyout_direct(const void *kaddr, void *udaddr, size_t len); 98 int copyin_remap(const void *uaddr, void *kaddr, size_t len); 99 int copyin_direct(const void *uaddr, void *kaddr, size_t len); 100 int suword16_remap(volatile void *addr, int word); 101 int suword16_direct(volatile void *addr, int word); 102 int suword32_remap(volatile void *addr, int word); 103 int suword32_direct(volatile void *addr, int word); 104 int suword_remap(volatile void *addr, long word); 105 int suword_direct(volatile void *addr, long word); 106 int suword64_remap(volatile void *addr, int64_t word); 107 int suword64_direct(volatile void *addr, int64_t word); 108 int fubyte_remap(volatile const void *addr); 109 int fubyte_direct(volatile const void *addr); 110 int fuword16_remap(volatile const void *addr); 111 int fuword16_direct(volatile const void *addr); 112 int fueword32_remap(volatile const void *addr, int32_t *val); 113 int fueword32_direct(volatile const void *addr, int32_t *val); 114 int fueword64_remap(volatile const void *addr, int64_t *val); 115 int fueword64_direct(volatile const void *addr, int64_t *val); 116 int fueword_remap(volatile const void *addr, long *val); 117 int fueword_direct(volatile const void *addr, long *val); 118 int casueword32_remap(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp, 119 uint32_t new); 120 int casueword32_direct(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp, 121 uint32_t new); 122 int casueword_remap(volatile u_long *addr, u_long old, u_long *oldvalp, 123 u_long new); 124 int casueword_direct(volatile u_long *addr, u_long old, u_long *oldvalp, 125 u_long new); 126 127 /* 128 * The IFUNC resolver determines the copy based on whether the PMAP 129 * implementation includes a pmap_map_user_ptr function. 130 */ 131 #define DEFINE_COPY_FUNC(ret, func, args) \ 132 DEFINE_IFUNC(, ret, func, args) \ 133 { \ 134 return (PMAP_RESOLVE_FUNC(map_user_ptr) ? \ 135 func##_remap : func##_direct); \ 136 } 137 DEFINE_COPY_FUNC(int, subyte, (volatile void *, int)) 138 DEFINE_COPY_FUNC(int, copyinstr, (const void *, void *, size_t, size_t *)) 139 DEFINE_COPY_FUNC(int, copyin, (const void *, void *, size_t)) 140 DEFINE_COPY_FUNC(int, copyout, (const void *, void *, size_t)) 141 DEFINE_COPY_FUNC(int, suword, (volatile void *, long)) 142 DEFINE_COPY_FUNC(int, suword16, (volatile void *, int)) 143 DEFINE_COPY_FUNC(int, suword32, (volatile void *, int)) 144 DEFINE_COPY_FUNC(int, suword64, (volatile void *, int64_t)) 145 DEFINE_COPY_FUNC(int, fubyte, (volatile const void *)) 146 DEFINE_COPY_FUNC(int, fuword16, (volatile const void *)) 147 DEFINE_COPY_FUNC(int, fueword32, (volatile const void *, int32_t *)) 148 DEFINE_COPY_FUNC(int, fueword64, (volatile const void *, int64_t *)) 149 DEFINE_COPY_FUNC(int, fueword, (volatile const void *, long *)) 150 DEFINE_COPY_FUNC(int, casueword32, 151 (volatile uint32_t *, uint32_t, uint32_t *, uint32_t)) 152 DEFINE_COPY_FUNC(int, casueword, (volatile u_long *, u_long, u_long *, u_long)) 153 154 #define REMAP(x) x##_remap 155 #else 156 #define REMAP(x) x 157 #endif 158 159 int 160 REMAP(copyout)(const void *kaddr, void *udaddr, size_t len) 161 { 162 struct thread *td; 163 pmap_t pm; 164 jmp_buf env; 165 const char *kp; 166 char *up, *p; 167 size_t l; 168 169 td = curthread; 170 pm = &td->td_proc->p_vmspace->vm_pmap; 171 172 td->td_pcb->pcb_onfault = &env; 173 if (setjmp(env)) { 174 td->td_pcb->pcb_onfault = NULL; 175 return (EFAULT); 176 } 177 178 kp = kaddr; 179 up = udaddr; 180 181 while (len > 0) { 182 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) { 183 td->td_pcb->pcb_onfault = NULL; 184 return (EFAULT); 185 } 186 187 bcopy(kp, p, l); 188 189 up += l; 190 kp += l; 191 len -= l; 192 } 193 194 td->td_pcb->pcb_onfault = NULL; 195 return (0); 196 } 197 198 int 199 REMAP(copyin)(const void *udaddr, void *kaddr, size_t len) 200 { 201 struct thread *td; 202 pmap_t pm; 203 jmp_buf env; 204 const char *up; 205 char *kp, *p; 206 size_t l; 207 208 td = curthread; 209 pm = &td->td_proc->p_vmspace->vm_pmap; 210 211 td->td_pcb->pcb_onfault = &env; 212 if (setjmp(env)) { 213 td->td_pcb->pcb_onfault = NULL; 214 return (EFAULT); 215 } 216 217 kp = kaddr; 218 up = udaddr; 219 220 while (len > 0) { 221 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) { 222 td->td_pcb->pcb_onfault = NULL; 223 return (EFAULT); 224 } 225 226 bcopy(p, kp, l); 227 228 up += l; 229 kp += l; 230 len -= l; 231 } 232 233 td->td_pcb->pcb_onfault = NULL; 234 return (0); 235 } 236 237 int 238 REMAP(copyinstr)(const void *udaddr, void *kaddr, size_t len, size_t *done) 239 { 240 struct thread *td; 241 pmap_t pm; 242 jmp_buf env; 243 const char *up; 244 char *kp, *p; 245 size_t i, l, t; 246 int rv; 247 248 td = curthread; 249 pm = &td->td_proc->p_vmspace->vm_pmap; 250 251 t = 0; 252 rv = ENAMETOOLONG; 253 254 td->td_pcb->pcb_onfault = &env; 255 if (setjmp(env)) { 256 rv = EFAULT; 257 goto done; 258 } 259 260 kp = kaddr; 261 up = udaddr; 262 263 while (len > 0) { 264 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) { 265 rv = EFAULT; 266 goto done; 267 } 268 269 for (i = 0; len > 0 && i < l; i++, t++, len--) { 270 if ((*kp++ = *p++) == 0) { 271 i++, t++; 272 rv = 0; 273 goto done; 274 } 275 } 276 277 up += l; 278 } 279 280 done: 281 td->td_pcb->pcb_onfault = NULL; 282 283 if (done != NULL) { 284 *done = t; 285 } 286 287 return (rv); 288 } 289 290 int 291 REMAP(subyte)(volatile void *addr, int byte) 292 { 293 struct thread *td; 294 pmap_t pm; 295 jmp_buf env; 296 char *p; 297 298 td = curthread; 299 pm = &td->td_proc->p_vmspace->vm_pmap; 300 301 td->td_pcb->pcb_onfault = &env; 302 if (setjmp(env)) { 303 td->td_pcb->pcb_onfault = NULL; 304 return (-1); 305 } 306 307 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 308 td->td_pcb->pcb_onfault = NULL; 309 return (-1); 310 } 311 312 *p = (char)byte; 313 314 td->td_pcb->pcb_onfault = NULL; 315 return (0); 316 } 317 318 int 319 REMAP(suword16)(volatile void *addr, int word) 320 { 321 struct thread *td; 322 pmap_t pm; 323 jmp_buf env; 324 int16_t *p; 325 326 td = curthread; 327 pm = &td->td_proc->p_vmspace->vm_pmap; 328 329 td->td_pcb->pcb_onfault = &env; 330 if (setjmp(env)) { 331 td->td_pcb->pcb_onfault = NULL; 332 return (-1); 333 } 334 335 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 336 td->td_pcb->pcb_onfault = NULL; 337 return (-1); 338 } 339 340 *p = (int16_t)word; 341 342 td->td_pcb->pcb_onfault = NULL; 343 return (0); 344 } 345 346 #ifdef __powerpc64__ 347 int 348 REMAP(suword32)(volatile void *addr, int word) 349 { 350 struct thread *td; 351 pmap_t pm; 352 jmp_buf env; 353 int *p; 354 355 td = curthread; 356 pm = &td->td_proc->p_vmspace->vm_pmap; 357 358 td->td_pcb->pcb_onfault = &env; 359 if (setjmp(env)) { 360 td->td_pcb->pcb_onfault = NULL; 361 return (-1); 362 } 363 364 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 365 td->td_pcb->pcb_onfault = NULL; 366 return (-1); 367 } 368 369 *p = word; 370 371 td->td_pcb->pcb_onfault = NULL; 372 return (0); 373 } 374 #else 375 int 376 REMAP(suword32)(volatile void *addr, int32_t word) 377 { 378 REMAP( return (suword)(addr, (long)word)); 379 } 380 #endif 381 382 int 383 REMAP(suword)(volatile void *addr, long word) 384 { 385 struct thread *td; 386 pmap_t pm; 387 jmp_buf env; 388 long *p; 389 390 td = curthread; 391 pm = &td->td_proc->p_vmspace->vm_pmap; 392 393 td->td_pcb->pcb_onfault = &env; 394 if (setjmp(env)) { 395 td->td_pcb->pcb_onfault = NULL; 396 return (-1); 397 } 398 399 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 400 td->td_pcb->pcb_onfault = NULL; 401 return (-1); 402 } 403 404 *p = word; 405 406 td->td_pcb->pcb_onfault = NULL; 407 return (0); 408 } 409 410 #ifdef __powerpc64__ 411 int 412 REMAP(suword64)(volatile void *addr, int64_t word) 413 { 414 return (REMAP(suword)(addr, (long)word)); 415 } 416 #endif 417 418 int 419 REMAP(fubyte)(volatile const void *addr) 420 { 421 struct thread *td; 422 pmap_t pm; 423 jmp_buf env; 424 u_char *p; 425 int val; 426 427 td = curthread; 428 pm = &td->td_proc->p_vmspace->vm_pmap; 429 430 td->td_pcb->pcb_onfault = &env; 431 if (setjmp(env)) { 432 td->td_pcb->pcb_onfault = NULL; 433 return (-1); 434 } 435 436 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 437 td->td_pcb->pcb_onfault = NULL; 438 return (-1); 439 } 440 441 val = *p; 442 443 td->td_pcb->pcb_onfault = NULL; 444 return (val); 445 } 446 447 int 448 REMAP(fuword16)(volatile const void *addr) 449 { 450 struct thread *td; 451 pmap_t pm; 452 jmp_buf env; 453 uint16_t *p, val; 454 455 td = curthread; 456 pm = &td->td_proc->p_vmspace->vm_pmap; 457 458 td->td_pcb->pcb_onfault = &env; 459 if (setjmp(env)) { 460 td->td_pcb->pcb_onfault = NULL; 461 return (-1); 462 } 463 464 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 465 td->td_pcb->pcb_onfault = NULL; 466 return (-1); 467 } 468 469 val = *p; 470 471 td->td_pcb->pcb_onfault = NULL; 472 return (val); 473 } 474 475 int 476 REMAP(fueword32)(volatile const void *addr, int32_t *val) 477 { 478 struct thread *td; 479 pmap_t pm; 480 jmp_buf env; 481 int32_t *p; 482 483 td = curthread; 484 pm = &td->td_proc->p_vmspace->vm_pmap; 485 486 td->td_pcb->pcb_onfault = &env; 487 if (setjmp(env)) { 488 td->td_pcb->pcb_onfault = NULL; 489 return (-1); 490 } 491 492 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 493 td->td_pcb->pcb_onfault = NULL; 494 return (-1); 495 } 496 497 *val = *p; 498 499 td->td_pcb->pcb_onfault = NULL; 500 return (0); 501 } 502 503 #ifdef __powerpc64__ 504 int 505 REMAP(fueword64)(volatile const void *addr, int64_t *val) 506 { 507 struct thread *td; 508 pmap_t pm; 509 jmp_buf env; 510 int64_t *p; 511 512 td = curthread; 513 pm = &td->td_proc->p_vmspace->vm_pmap; 514 515 td->td_pcb->pcb_onfault = &env; 516 if (setjmp(env)) { 517 td->td_pcb->pcb_onfault = NULL; 518 return (-1); 519 } 520 521 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 522 td->td_pcb->pcb_onfault = NULL; 523 return (-1); 524 } 525 526 *val = *p; 527 528 td->td_pcb->pcb_onfault = NULL; 529 return (0); 530 } 531 #endif 532 533 int 534 REMAP(fueword)(volatile const void *addr, long *val) 535 { 536 struct thread *td; 537 pmap_t pm; 538 jmp_buf env; 539 long *p; 540 541 td = curthread; 542 pm = &td->td_proc->p_vmspace->vm_pmap; 543 544 td->td_pcb->pcb_onfault = &env; 545 if (setjmp(env)) { 546 td->td_pcb->pcb_onfault = NULL; 547 return (-1); 548 } 549 550 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 551 td->td_pcb->pcb_onfault = NULL; 552 return (-1); 553 } 554 555 *val = *p; 556 557 td->td_pcb->pcb_onfault = NULL; 558 return (0); 559 } 560 561 int 562 REMAP(casueword32)(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp, 563 uint32_t new) 564 { 565 struct thread *td; 566 pmap_t pm; 567 jmp_buf env; 568 uint32_t *p, val; 569 int res; 570 571 td = curthread; 572 pm = &td->td_proc->p_vmspace->vm_pmap; 573 574 td->td_pcb->pcb_onfault = &env; 575 if (setjmp(env)) { 576 td->td_pcb->pcb_onfault = NULL; 577 return (-1); 578 } 579 580 if (pmap_map_user_ptr(pm, (void *)(uintptr_t)addr, (void **)&p, 581 sizeof(*p), NULL)) { 582 td->td_pcb->pcb_onfault = NULL; 583 return (-1); 584 } 585 586 res = 0; 587 __asm __volatile ( 588 "lwarx %0, 0, %3\n\t" /* load old value */ 589 "cmplw %4, %0\n\t" /* compare */ 590 "bne 1f\n\t" /* exit if not equal */ 591 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 592 "bne- 2f\n\t" /* if failed */ 593 "b 3f\n\t" /* we've succeeded */ 594 "1:\n\t" 595 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 596 "2:li %2, 1\n\t" 597 "3:\n\t" 598 : "=&r" (val), "=m" (*p), "+&r" (res) 599 : "r" (p), "r" (old), "r" (new), "m" (*p) 600 : "cr0", "memory"); 601 602 td->td_pcb->pcb_onfault = NULL; 603 604 *oldvalp = val; 605 return (res); 606 } 607 608 #ifndef __powerpc64__ 609 int 610 REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new) 611 { 612 613 return (casueword32((volatile uint32_t *)addr, old, 614 (uint32_t *)oldvalp, new)); 615 } 616 #else 617 int 618 REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new) 619 { 620 struct thread *td; 621 pmap_t pm; 622 jmp_buf env; 623 u_long *p, val; 624 int res; 625 626 td = curthread; 627 pm = &td->td_proc->p_vmspace->vm_pmap; 628 629 td->td_pcb->pcb_onfault = &env; 630 if (setjmp(env)) { 631 td->td_pcb->pcb_onfault = NULL; 632 return (-1); 633 } 634 635 if (pmap_map_user_ptr(pm, (void *)(uintptr_t)addr, (void **)&p, 636 sizeof(*p), NULL)) { 637 td->td_pcb->pcb_onfault = NULL; 638 return (-1); 639 } 640 641 res = 0; 642 __asm __volatile ( 643 "ldarx %0, 0, %3\n\t" /* load old value */ 644 "cmpld %4, %0\n\t" /* compare */ 645 "bne 1f\n\t" /* exit if not equal */ 646 "stdcx. %5, 0, %3\n\t" /* attempt to store */ 647 "bne- 2f\n\t" /* if failed */ 648 "b 3f\n\t" /* we've succeeded */ 649 "1:\n\t" 650 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 651 "2:li %2, 1\n\t" 652 "3:\n\t" 653 : "=&r" (val), "=m" (*p), "+&r" (res) 654 : "r" (p), "r" (old), "r" (new), "m" (*p) 655 : "cr0", "memory"); 656 657 td->td_pcb->pcb_onfault = NULL; 658 659 *oldvalp = val; 660 return (res); 661 } 662 #endif 663