1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause AND BSD-4-Clause 3 * 4 * Copyright (C) 2002 Benno Rice 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 /*- 28 * Copyright (C) 1993 Wolfgang Solfrank. 29 * Copyright (C) 1993 TooLs GmbH. 30 * All rights reserved. 31 * 32 * Redistribution and use in source and binary forms, with or without 33 * modification, are permitted provided that the following conditions 34 * are met: 35 * 1. Redistributions of source code must retain the above copyright 36 * notice, this list of conditions and the following disclaimer. 37 * 2. Redistributions in binary form must reproduce the above copyright 38 * notice, this list of conditions and the following disclaimer in the 39 * documentation and/or other materials provided with the distribution. 40 * 3. All advertising materials mentioning features or use of this software 41 * must display the following acknowledgement: 42 * This product includes software developed by TooLs GmbH. 43 * 4. The name of TooLs GmbH may not be used to endorse or promote products 44 * derived from this software without specific prior written permission. 45 * 46 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 47 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 48 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 49 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 50 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 51 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 52 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 53 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 54 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 55 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 56 */ 57 58 #include <sys/param.h> 59 #include <sys/lock.h> 60 #include <sys/mutex.h> 61 #include <sys/systm.h> 62 #include <sys/proc.h> 63 64 #include <vm/vm.h> 65 #include <vm/pmap.h> 66 #include <vm/vm_extern.h> 67 #include <vm/vm_map.h> 68 69 #include <machine/mmuvar.h> 70 #include <machine/pcb.h> 71 #include <machine/vmparam.h> 72 #include <machine/ifunc.h> 73 74 /* 75 * On powerpc64 (AIM only) the copy functions are IFUNCs, selecting the best 76 * option based on the PMAP in use. 77 * 78 * There are two options for copy functions on powerpc64: 79 * - 'remap' copies, which remap userspace segments into kernel space for 80 * copying. This is used by the 'oea64' pmap. 81 * - 'direct' copies, which copy directly from userspace. This does not require 82 * remapping user segments into kernel. This is used by the 'radix' pmap for 83 * performance. 84 * 85 * Book-E does not use the C 'remap' functions, opting instead to use the 86 * 'direct' copies, directly, avoiding the IFUNC overhead. 87 * 88 * On 32-bit AIM these functions bypass the IFUNC machinery for performance. 89 */ 90 #ifdef __powerpc64__ 91 int subyte_remap(volatile void *addr, int byte); 92 int subyte_direct(volatile void *addr, int byte); 93 int copyinstr_remap(const void *udaddr, void *kaddr, size_t len, size_t *done); 94 int copyinstr_direct(const void *udaddr, void *kaddr, size_t len, size_t *done); 95 int copyout_remap(const void *kaddr, void *udaddr, size_t len); 96 int copyout_direct(const void *kaddr, void *udaddr, size_t len); 97 int copyin_remap(const void *uaddr, void *kaddr, size_t len); 98 int copyin_direct(const void *uaddr, void *kaddr, size_t len); 99 int suword16_remap(volatile void *addr, int word); 100 int suword16_direct(volatile void *addr, int word); 101 int suword32_remap(volatile void *addr, int word); 102 int suword32_direct(volatile void *addr, int word); 103 int suword_remap(volatile void *addr, long word); 104 int suword_direct(volatile void *addr, long word); 105 int suword64_remap(volatile void *addr, int64_t word); 106 int suword64_direct(volatile void *addr, int64_t word); 107 int fubyte_remap(volatile const void *addr); 108 int fubyte_direct(volatile const void *addr); 109 int fuword16_remap(volatile const void *addr); 110 int fuword16_direct(volatile const void *addr); 111 int fueword32_remap(volatile const void *addr, int32_t *val); 112 int fueword32_direct(volatile const void *addr, int32_t *val); 113 int fueword64_remap(volatile const void *addr, int64_t *val); 114 int fueword64_direct(volatile const void *addr, int64_t *val); 115 int fueword_remap(volatile const void *addr, long *val); 116 int fueword_direct(volatile const void *addr, long *val); 117 int casueword32_remap(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp, 118 uint32_t new); 119 int casueword32_direct(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp, 120 uint32_t new); 121 int casueword_remap(volatile u_long *addr, u_long old, u_long *oldvalp, 122 u_long new); 123 int casueword_direct(volatile u_long *addr, u_long old, u_long *oldvalp, 124 u_long new); 125 126 /* 127 * The IFUNC resolver determines the copy based on whether the PMAP 128 * implementation includes a pmap_map_user_ptr function. 129 */ 130 #define DEFINE_COPY_FUNC(ret, func, args) \ 131 DEFINE_IFUNC(, ret, func, args) \ 132 { \ 133 return (PMAP_RESOLVE_FUNC(map_user_ptr) ? \ 134 func##_remap : func##_direct); \ 135 } 136 DEFINE_COPY_FUNC(int, subyte, (volatile void *, int)) 137 DEFINE_COPY_FUNC(int, copyinstr, (const void *, void *, size_t, size_t *)) 138 DEFINE_COPY_FUNC(int, copyin, (const void *, void *, size_t)) 139 DEFINE_COPY_FUNC(int, copyout, (const void *, void *, size_t)) 140 DEFINE_COPY_FUNC(int, suword, (volatile void *, long)) 141 DEFINE_COPY_FUNC(int, suword16, (volatile void *, int)) 142 DEFINE_COPY_FUNC(int, suword32, (volatile void *, int)) 143 DEFINE_COPY_FUNC(int, suword64, (volatile void *, int64_t)) 144 DEFINE_COPY_FUNC(int, fubyte, (volatile const void *)) 145 DEFINE_COPY_FUNC(int, fuword16, (volatile const void *)) 146 DEFINE_COPY_FUNC(int, fueword32, (volatile const void *, int32_t *)) 147 DEFINE_COPY_FUNC(int, fueword64, (volatile const void *, int64_t *)) 148 DEFINE_COPY_FUNC(int, fueword, (volatile const void *, long *)) 149 DEFINE_COPY_FUNC(int, casueword32, 150 (volatile uint32_t *, uint32_t, uint32_t *, uint32_t)) 151 DEFINE_COPY_FUNC(int, casueword, (volatile u_long *, u_long, u_long *, u_long)) 152 153 #define REMAP(x) x##_remap 154 #else 155 #define REMAP(x) x 156 #endif 157 158 int 159 REMAP(copyout)(const void *kaddr, void *udaddr, size_t len) 160 { 161 struct thread *td; 162 pmap_t pm; 163 jmp_buf env; 164 const char *kp; 165 char *up, *p; 166 size_t l; 167 168 td = curthread; 169 pm = &td->td_proc->p_vmspace->vm_pmap; 170 171 td->td_pcb->pcb_onfault = &env; 172 if (setjmp(env)) { 173 td->td_pcb->pcb_onfault = NULL; 174 return (EFAULT); 175 } 176 177 kp = kaddr; 178 up = udaddr; 179 180 while (len > 0) { 181 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) { 182 td->td_pcb->pcb_onfault = NULL; 183 return (EFAULT); 184 } 185 186 bcopy(kp, p, l); 187 188 up += l; 189 kp += l; 190 len -= l; 191 } 192 193 td->td_pcb->pcb_onfault = NULL; 194 return (0); 195 } 196 197 int 198 REMAP(copyin)(const void *udaddr, void *kaddr, size_t len) 199 { 200 struct thread *td; 201 pmap_t pm; 202 jmp_buf env; 203 const char *up; 204 char *kp, *p; 205 size_t l; 206 207 td = curthread; 208 pm = &td->td_proc->p_vmspace->vm_pmap; 209 210 td->td_pcb->pcb_onfault = &env; 211 if (setjmp(env)) { 212 td->td_pcb->pcb_onfault = NULL; 213 return (EFAULT); 214 } 215 216 kp = kaddr; 217 up = udaddr; 218 219 while (len > 0) { 220 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) { 221 td->td_pcb->pcb_onfault = NULL; 222 return (EFAULT); 223 } 224 225 bcopy(p, kp, l); 226 227 up += l; 228 kp += l; 229 len -= l; 230 } 231 232 td->td_pcb->pcb_onfault = NULL; 233 return (0); 234 } 235 236 int 237 REMAP(copyinstr)(const void *udaddr, void *kaddr, size_t len, size_t *done) 238 { 239 struct thread *td; 240 pmap_t pm; 241 jmp_buf env; 242 const char *up; 243 char *kp, *p; 244 size_t i, l, t; 245 int rv; 246 247 td = curthread; 248 pm = &td->td_proc->p_vmspace->vm_pmap; 249 250 t = 0; 251 rv = ENAMETOOLONG; 252 253 td->td_pcb->pcb_onfault = &env; 254 if (setjmp(env)) { 255 rv = EFAULT; 256 goto done; 257 } 258 259 kp = kaddr; 260 up = udaddr; 261 262 while (len > 0) { 263 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) { 264 rv = EFAULT; 265 goto done; 266 } 267 268 for (i = 0; len > 0 && i < l; i++, t++, len--) { 269 if ((*kp++ = *p++) == 0) { 270 i++, t++; 271 rv = 0; 272 goto done; 273 } 274 } 275 276 up += l; 277 } 278 279 done: 280 td->td_pcb->pcb_onfault = NULL; 281 282 if (done != NULL) { 283 *done = t; 284 } 285 286 return (rv); 287 } 288 289 int 290 REMAP(subyte)(volatile void *addr, int byte) 291 { 292 struct thread *td; 293 pmap_t pm; 294 jmp_buf env; 295 char *p; 296 297 td = curthread; 298 pm = &td->td_proc->p_vmspace->vm_pmap; 299 300 td->td_pcb->pcb_onfault = &env; 301 if (setjmp(env)) { 302 td->td_pcb->pcb_onfault = NULL; 303 return (-1); 304 } 305 306 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 307 td->td_pcb->pcb_onfault = NULL; 308 return (-1); 309 } 310 311 *p = (char)byte; 312 313 td->td_pcb->pcb_onfault = NULL; 314 return (0); 315 } 316 317 int 318 REMAP(suword16)(volatile void *addr, int word) 319 { 320 struct thread *td; 321 pmap_t pm; 322 jmp_buf env; 323 int16_t *p; 324 325 td = curthread; 326 pm = &td->td_proc->p_vmspace->vm_pmap; 327 328 td->td_pcb->pcb_onfault = &env; 329 if (setjmp(env)) { 330 td->td_pcb->pcb_onfault = NULL; 331 return (-1); 332 } 333 334 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 335 td->td_pcb->pcb_onfault = NULL; 336 return (-1); 337 } 338 339 *p = (int16_t)word; 340 341 td->td_pcb->pcb_onfault = NULL; 342 return (0); 343 } 344 345 #ifdef __powerpc64__ 346 int 347 REMAP(suword32)(volatile void *addr, int word) 348 { 349 struct thread *td; 350 pmap_t pm; 351 jmp_buf env; 352 int *p; 353 354 td = curthread; 355 pm = &td->td_proc->p_vmspace->vm_pmap; 356 357 td->td_pcb->pcb_onfault = &env; 358 if (setjmp(env)) { 359 td->td_pcb->pcb_onfault = NULL; 360 return (-1); 361 } 362 363 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 364 td->td_pcb->pcb_onfault = NULL; 365 return (-1); 366 } 367 368 *p = word; 369 370 td->td_pcb->pcb_onfault = NULL; 371 return (0); 372 } 373 #else 374 int 375 REMAP(suword32)(volatile void *addr, int32_t word) 376 { 377 REMAP( return (suword)(addr, (long)word)); 378 } 379 #endif 380 381 int 382 REMAP(suword)(volatile void *addr, long word) 383 { 384 struct thread *td; 385 pmap_t pm; 386 jmp_buf env; 387 long *p; 388 389 td = curthread; 390 pm = &td->td_proc->p_vmspace->vm_pmap; 391 392 td->td_pcb->pcb_onfault = &env; 393 if (setjmp(env)) { 394 td->td_pcb->pcb_onfault = NULL; 395 return (-1); 396 } 397 398 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 399 td->td_pcb->pcb_onfault = NULL; 400 return (-1); 401 } 402 403 *p = word; 404 405 td->td_pcb->pcb_onfault = NULL; 406 return (0); 407 } 408 409 #ifdef __powerpc64__ 410 int 411 REMAP(suword64)(volatile void *addr, int64_t word) 412 { 413 return (REMAP(suword)(addr, (long)word)); 414 } 415 #endif 416 417 int 418 REMAP(fubyte)(volatile const void *addr) 419 { 420 struct thread *td; 421 pmap_t pm; 422 jmp_buf env; 423 u_char *p; 424 int val; 425 426 td = curthread; 427 pm = &td->td_proc->p_vmspace->vm_pmap; 428 429 td->td_pcb->pcb_onfault = &env; 430 if (setjmp(env)) { 431 td->td_pcb->pcb_onfault = NULL; 432 return (-1); 433 } 434 435 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 436 td->td_pcb->pcb_onfault = NULL; 437 return (-1); 438 } 439 440 val = *p; 441 442 td->td_pcb->pcb_onfault = NULL; 443 return (val); 444 } 445 446 int 447 REMAP(fuword16)(volatile const void *addr) 448 { 449 struct thread *td; 450 pmap_t pm; 451 jmp_buf env; 452 uint16_t *p, val; 453 454 td = curthread; 455 pm = &td->td_proc->p_vmspace->vm_pmap; 456 457 td->td_pcb->pcb_onfault = &env; 458 if (setjmp(env)) { 459 td->td_pcb->pcb_onfault = NULL; 460 return (-1); 461 } 462 463 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 464 td->td_pcb->pcb_onfault = NULL; 465 return (-1); 466 } 467 468 val = *p; 469 470 td->td_pcb->pcb_onfault = NULL; 471 return (val); 472 } 473 474 int 475 REMAP(fueword32)(volatile const void *addr, int32_t *val) 476 { 477 struct thread *td; 478 pmap_t pm; 479 jmp_buf env; 480 int32_t *p; 481 482 td = curthread; 483 pm = &td->td_proc->p_vmspace->vm_pmap; 484 485 td->td_pcb->pcb_onfault = &env; 486 if (setjmp(env)) { 487 td->td_pcb->pcb_onfault = NULL; 488 return (-1); 489 } 490 491 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 492 td->td_pcb->pcb_onfault = NULL; 493 return (-1); 494 } 495 496 *val = *p; 497 498 td->td_pcb->pcb_onfault = NULL; 499 return (0); 500 } 501 502 #ifdef __powerpc64__ 503 int 504 REMAP(fueword64)(volatile const void *addr, int64_t *val) 505 { 506 struct thread *td; 507 pmap_t pm; 508 jmp_buf env; 509 int64_t *p; 510 511 td = curthread; 512 pm = &td->td_proc->p_vmspace->vm_pmap; 513 514 td->td_pcb->pcb_onfault = &env; 515 if (setjmp(env)) { 516 td->td_pcb->pcb_onfault = NULL; 517 return (-1); 518 } 519 520 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 521 td->td_pcb->pcb_onfault = NULL; 522 return (-1); 523 } 524 525 *val = *p; 526 527 td->td_pcb->pcb_onfault = NULL; 528 return (0); 529 } 530 #endif 531 532 int 533 REMAP(fueword)(volatile const void *addr, long *val) 534 { 535 struct thread *td; 536 pmap_t pm; 537 jmp_buf env; 538 long *p; 539 540 td = curthread; 541 pm = &td->td_proc->p_vmspace->vm_pmap; 542 543 td->td_pcb->pcb_onfault = &env; 544 if (setjmp(env)) { 545 td->td_pcb->pcb_onfault = NULL; 546 return (-1); 547 } 548 549 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 550 td->td_pcb->pcb_onfault = NULL; 551 return (-1); 552 } 553 554 *val = *p; 555 556 td->td_pcb->pcb_onfault = NULL; 557 return (0); 558 } 559 560 int 561 REMAP(casueword32)(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp, 562 uint32_t new) 563 { 564 struct thread *td; 565 pmap_t pm; 566 jmp_buf env; 567 uint32_t *p, val; 568 int res; 569 570 td = curthread; 571 pm = &td->td_proc->p_vmspace->vm_pmap; 572 573 td->td_pcb->pcb_onfault = &env; 574 if (setjmp(env)) { 575 td->td_pcb->pcb_onfault = NULL; 576 return (-1); 577 } 578 579 if (pmap_map_user_ptr(pm, (void *)(uintptr_t)addr, (void **)&p, 580 sizeof(*p), NULL)) { 581 td->td_pcb->pcb_onfault = NULL; 582 return (-1); 583 } 584 585 res = 0; 586 __asm __volatile ( 587 "lwarx %0, 0, %3\n\t" /* load old value */ 588 "cmplw %4, %0\n\t" /* compare */ 589 "bne 1f\n\t" /* exit if not equal */ 590 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 591 "bne- 2f\n\t" /* if failed */ 592 "b 3f\n\t" /* we've succeeded */ 593 "1:\n\t" 594 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 595 "2:li %2, 1\n\t" 596 "3:\n\t" 597 : "=&r" (val), "=m" (*p), "+&r" (res) 598 : "r" (p), "r" (old), "r" (new), "m" (*p) 599 : "cr0", "memory"); 600 601 td->td_pcb->pcb_onfault = NULL; 602 603 *oldvalp = val; 604 return (res); 605 } 606 607 #ifndef __powerpc64__ 608 int 609 REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new) 610 { 611 612 return (casueword32((volatile uint32_t *)addr, old, 613 (uint32_t *)oldvalp, new)); 614 } 615 #else 616 int 617 REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new) 618 { 619 struct thread *td; 620 pmap_t pm; 621 jmp_buf env; 622 u_long *p, val; 623 int res; 624 625 td = curthread; 626 pm = &td->td_proc->p_vmspace->vm_pmap; 627 628 td->td_pcb->pcb_onfault = &env; 629 if (setjmp(env)) { 630 td->td_pcb->pcb_onfault = NULL; 631 return (-1); 632 } 633 634 if (pmap_map_user_ptr(pm, (void *)(uintptr_t)addr, (void **)&p, 635 sizeof(*p), NULL)) { 636 td->td_pcb->pcb_onfault = NULL; 637 return (-1); 638 } 639 640 res = 0; 641 __asm __volatile ( 642 "ldarx %0, 0, %3\n\t" /* load old value */ 643 "cmpld %4, %0\n\t" /* compare */ 644 "bne 1f\n\t" /* exit if not equal */ 645 "stdcx. %5, 0, %3\n\t" /* attempt to store */ 646 "bne- 2f\n\t" /* if failed */ 647 "b 3f\n\t" /* we've succeeded */ 648 "1:\n\t" 649 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 650 "2:li %2, 1\n\t" 651 "3:\n\t" 652 : "=&r" (val), "=m" (*p), "+&r" (res) 653 : "r" (p), "r" (old), "r" (new), "m" (*p) 654 : "cr0", "memory"); 655 656 td->td_pcb->pcb_onfault = NULL; 657 658 *oldvalp = val; 659 return (res); 660 } 661 #endif 662