1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD AND BSD-4-Clause 3 * 4 * Copyright (C) 2002 Benno Rice 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 /*- 28 * Copyright (C) 1993 Wolfgang Solfrank. 29 * Copyright (C) 1993 TooLs GmbH. 30 * All rights reserved. 31 * 32 * Redistribution and use in source and binary forms, with or without 33 * modification, are permitted provided that the following conditions 34 * are met: 35 * 1. Redistributions of source code must retain the above copyright 36 * notice, this list of conditions and the following disclaimer. 37 * 2. Redistributions in binary form must reproduce the above copyright 38 * notice, this list of conditions and the following disclaimer in the 39 * documentation and/or other materials provided with the distribution. 40 * 3. All advertising materials mentioning features or use of this software 41 * must display the following acknowledgement: 42 * This product includes software developed by TooLs GmbH. 43 * 4. The name of TooLs GmbH may not be used to endorse or promote products 44 * derived from this software without specific prior written permission. 45 * 46 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 47 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 48 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 49 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 50 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 51 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 52 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 53 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 54 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 55 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 56 */ 57 58 #include <sys/cdefs.h> 59 __FBSDID("$FreeBSD$"); 60 61 #include <sys/param.h> 62 #include <sys/lock.h> 63 #include <sys/mutex.h> 64 #include <sys/systm.h> 65 #include <sys/proc.h> 66 67 #include <vm/vm.h> 68 #include <vm/pmap.h> 69 #include <vm/vm_extern.h> 70 #include <vm/vm_map.h> 71 72 #include <machine/mmuvar.h> 73 #include <machine/pcb.h> 74 #include <machine/vmparam.h> 75 #include <machine/ifunc.h> 76 77 /* 78 * On powerpc64 (AIM only) the copy functions are IFUNCs, selecting the best 79 * option based on the PMAP in use. 80 * 81 * There are two options for copy functions on powerpc64: 82 * - 'remap' copies, which remap userspace segments into kernel space for 83 * copying. This is used by the 'oea64' pmap. 84 * - 'direct' copies, which copy directly from userspace. This does not require 85 * remapping user segments into kernel. This is used by the 'radix' pmap for 86 * performance. 87 * 88 * Book-E does not use the C 'remap' functions, opting instead to use the 89 * 'direct' copies, directly, avoiding the IFUNC overhead. 90 * 91 * On 32-bit AIM these functions bypass the IFUNC machinery for performance. 92 */ 93 #ifdef __powerpc64__ 94 int subyte_remap(volatile void *addr, int byte); 95 int subyte_direct(volatile void *addr, int byte); 96 int copyinstr_remap(const void *udaddr, void *kaddr, size_t len, size_t *done); 97 int copyinstr_direct(const void *udaddr, void *kaddr, size_t len, size_t *done); 98 int copyout_remap(const void *kaddr, void *udaddr, size_t len); 99 int copyout_direct(const void *kaddr, void *udaddr, size_t len); 100 int copyin_remap(const void *uaddr, void *kaddr, size_t len); 101 int copyin_direct(const void *uaddr, void *kaddr, size_t len); 102 int suword32_remap(volatile void *addr, int word); 103 int suword32_direct(volatile void *addr, int word); 104 int suword_remap(volatile void *addr, long word); 105 int suword_direct(volatile void *addr, long word); 106 int suword64_remap(volatile void *addr, int64_t word); 107 int suword64_direct(volatile void *addr, int64_t word); 108 int fubyte_remap(volatile const void *addr); 109 int fubyte_direct(volatile const void *addr); 110 int fuword16_remap(volatile const void *addr); 111 int fuword16_direct(volatile const void *addr); 112 int fueword32_remap(volatile const void *addr, int32_t *val); 113 int fueword32_direct(volatile const void *addr, int32_t *val); 114 int fueword64_remap(volatile const void *addr, int64_t *val); 115 int fueword64_direct(volatile const void *addr, int64_t *val); 116 int fueword_remap(volatile const void *addr, long *val); 117 int fueword_direct(volatile const void *addr, long *val); 118 int casueword32_remap(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp, 119 uint32_t new); 120 int casueword32_direct(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp, 121 uint32_t new); 122 int casueword_remap(volatile u_long *addr, u_long old, u_long *oldvalp, 123 u_long new); 124 int casueword_direct(volatile u_long *addr, u_long old, u_long *oldvalp, 125 u_long new); 126 127 /* 128 * The IFUNC resolver determines the copy based on whether the PMAP 129 * implementation includes a pmap_map_user_ptr function. 130 */ 131 #define DEFINE_COPY_FUNC(ret, func, args) \ 132 DEFINE_IFUNC(, ret, func, args) \ 133 { \ 134 return (PMAP_RESOLVE_FUNC(map_user_ptr) ? \ 135 func##_remap : func##_direct); \ 136 } 137 DEFINE_COPY_FUNC(int, subyte, (volatile void *, int)) 138 DEFINE_COPY_FUNC(int, copyinstr, (const void *, void *, size_t, size_t *)) 139 DEFINE_COPY_FUNC(int, copyin, (const void *, void *, size_t)) 140 DEFINE_COPY_FUNC(int, copyout, (const void *, void *, size_t)) 141 DEFINE_COPY_FUNC(int, suword, (volatile void *, long)) 142 DEFINE_COPY_FUNC(int, suword32, (volatile void *, int)) 143 DEFINE_COPY_FUNC(int, suword64, (volatile void *, int64_t)) 144 DEFINE_COPY_FUNC(int, fubyte, (volatile const void *)) 145 DEFINE_COPY_FUNC(int, fuword16, (volatile const void *)) 146 DEFINE_COPY_FUNC(int, fueword32, (volatile const void *, int32_t *)) 147 DEFINE_COPY_FUNC(int, fueword64, (volatile const void *, int64_t *)) 148 DEFINE_COPY_FUNC(int, fueword, (volatile const void *, long *)) 149 DEFINE_COPY_FUNC(int, casueword32, 150 (volatile uint32_t *, uint32_t, uint32_t *, uint32_t)) 151 DEFINE_COPY_FUNC(int, casueword, (volatile u_long *, u_long, u_long *, u_long)) 152 153 #define REMAP(x) x##_remap 154 #else 155 #define REMAP(x) x 156 #endif 157 158 159 int 160 REMAP(copyout)(const void *kaddr, void *udaddr, size_t len) 161 { 162 struct thread *td; 163 pmap_t pm; 164 jmp_buf env; 165 const char *kp; 166 char *up, *p; 167 size_t l; 168 169 td = curthread; 170 pm = &td->td_proc->p_vmspace->vm_pmap; 171 172 td->td_pcb->pcb_onfault = &env; 173 if (setjmp(env)) { 174 td->td_pcb->pcb_onfault = NULL; 175 return (EFAULT); 176 } 177 178 kp = kaddr; 179 up = udaddr; 180 181 while (len > 0) { 182 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) { 183 td->td_pcb->pcb_onfault = NULL; 184 return (EFAULT); 185 } 186 187 bcopy(kp, p, l); 188 189 up += l; 190 kp += l; 191 len -= l; 192 } 193 194 td->td_pcb->pcb_onfault = NULL; 195 return (0); 196 } 197 198 int 199 REMAP(copyin)(const void *udaddr, void *kaddr, size_t len) 200 { 201 struct thread *td; 202 pmap_t pm; 203 jmp_buf env; 204 const char *up; 205 char *kp, *p; 206 size_t l; 207 208 td = curthread; 209 pm = &td->td_proc->p_vmspace->vm_pmap; 210 211 td->td_pcb->pcb_onfault = &env; 212 if (setjmp(env)) { 213 td->td_pcb->pcb_onfault = NULL; 214 return (EFAULT); 215 } 216 217 kp = kaddr; 218 up = udaddr; 219 220 while (len > 0) { 221 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) { 222 td->td_pcb->pcb_onfault = NULL; 223 return (EFAULT); 224 } 225 226 bcopy(p, kp, l); 227 228 up += l; 229 kp += l; 230 len -= l; 231 } 232 233 td->td_pcb->pcb_onfault = NULL; 234 return (0); 235 } 236 237 int 238 REMAP(copyinstr)(const void *udaddr, void *kaddr, size_t len, size_t *done) 239 { 240 const char *up; 241 char *kp; 242 size_t l; 243 int rv, c; 244 245 kp = kaddr; 246 up = udaddr; 247 248 rv = ENAMETOOLONG; 249 250 for (l = 0; len-- > 0; l++) { 251 if ((c = fubyte(up++)) < 0) { 252 rv = EFAULT; 253 break; 254 } 255 256 if (!(*kp++ = c)) { 257 l++; 258 rv = 0; 259 break; 260 } 261 } 262 263 if (done != NULL) { 264 *done = l; 265 } 266 267 return (rv); 268 } 269 270 int 271 REMAP(subyte)(volatile void *addr, int byte) 272 { 273 struct thread *td; 274 pmap_t pm; 275 jmp_buf env; 276 char *p; 277 278 td = curthread; 279 pm = &td->td_proc->p_vmspace->vm_pmap; 280 281 td->td_pcb->pcb_onfault = &env; 282 if (setjmp(env)) { 283 td->td_pcb->pcb_onfault = NULL; 284 return (-1); 285 } 286 287 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 288 td->td_pcb->pcb_onfault = NULL; 289 return (-1); 290 } 291 292 *p = (char)byte; 293 294 td->td_pcb->pcb_onfault = NULL; 295 return (0); 296 } 297 298 #ifdef __powerpc64__ 299 int 300 REMAP(suword32)(volatile void *addr, int word) 301 { 302 struct thread *td; 303 pmap_t pm; 304 jmp_buf env; 305 int *p; 306 307 td = curthread; 308 pm = &td->td_proc->p_vmspace->vm_pmap; 309 310 td->td_pcb->pcb_onfault = &env; 311 if (setjmp(env)) { 312 td->td_pcb->pcb_onfault = NULL; 313 return (-1); 314 } 315 316 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 317 td->td_pcb->pcb_onfault = NULL; 318 return (-1); 319 } 320 321 *p = word; 322 323 td->td_pcb->pcb_onfault = NULL; 324 return (0); 325 } 326 #else 327 int 328 REMAP(suword32)(volatile void *addr, int32_t word) 329 { 330 REMAP( return (suword)(addr, (long)word)); 331 } 332 #endif 333 334 int 335 REMAP(suword)(volatile void *addr, long word) 336 { 337 struct thread *td; 338 pmap_t pm; 339 jmp_buf env; 340 long *p; 341 342 td = curthread; 343 pm = &td->td_proc->p_vmspace->vm_pmap; 344 345 td->td_pcb->pcb_onfault = &env; 346 if (setjmp(env)) { 347 td->td_pcb->pcb_onfault = NULL; 348 return (-1); 349 } 350 351 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 352 td->td_pcb->pcb_onfault = NULL; 353 return (-1); 354 } 355 356 *p = word; 357 358 td->td_pcb->pcb_onfault = NULL; 359 return (0); 360 } 361 362 #ifdef __powerpc64__ 363 int 364 REMAP(suword64)(volatile void *addr, int64_t word) 365 { 366 return (REMAP(suword)(addr, (long)word)); 367 } 368 #endif 369 370 int 371 REMAP(fubyte)(volatile const void *addr) 372 { 373 struct thread *td; 374 pmap_t pm; 375 jmp_buf env; 376 u_char *p; 377 int val; 378 379 td = curthread; 380 pm = &td->td_proc->p_vmspace->vm_pmap; 381 382 td->td_pcb->pcb_onfault = &env; 383 if (setjmp(env)) { 384 td->td_pcb->pcb_onfault = NULL; 385 return (-1); 386 } 387 388 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 389 td->td_pcb->pcb_onfault = NULL; 390 return (-1); 391 } 392 393 val = *p; 394 395 td->td_pcb->pcb_onfault = NULL; 396 return (val); 397 } 398 399 int 400 REMAP(fuword16)(volatile const void *addr) 401 { 402 struct thread *td; 403 pmap_t pm; 404 jmp_buf env; 405 uint16_t *p, val; 406 407 td = curthread; 408 pm = &td->td_proc->p_vmspace->vm_pmap; 409 410 td->td_pcb->pcb_onfault = &env; 411 if (setjmp(env)) { 412 td->td_pcb->pcb_onfault = NULL; 413 return (-1); 414 } 415 416 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 417 td->td_pcb->pcb_onfault = NULL; 418 return (-1); 419 } 420 421 val = *p; 422 423 td->td_pcb->pcb_onfault = NULL; 424 return (val); 425 } 426 427 int 428 REMAP(fueword32)(volatile const void *addr, int32_t *val) 429 { 430 struct thread *td; 431 pmap_t pm; 432 jmp_buf env; 433 int32_t *p; 434 435 td = curthread; 436 pm = &td->td_proc->p_vmspace->vm_pmap; 437 438 td->td_pcb->pcb_onfault = &env; 439 if (setjmp(env)) { 440 td->td_pcb->pcb_onfault = NULL; 441 return (-1); 442 } 443 444 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 445 td->td_pcb->pcb_onfault = NULL; 446 return (-1); 447 } 448 449 *val = *p; 450 451 td->td_pcb->pcb_onfault = NULL; 452 return (0); 453 } 454 455 #ifdef __powerpc64__ 456 int 457 REMAP(fueword64)(volatile const void *addr, int64_t *val) 458 { 459 struct thread *td; 460 pmap_t pm; 461 jmp_buf env; 462 int64_t *p; 463 464 td = curthread; 465 pm = &td->td_proc->p_vmspace->vm_pmap; 466 467 td->td_pcb->pcb_onfault = &env; 468 if (setjmp(env)) { 469 td->td_pcb->pcb_onfault = NULL; 470 return (-1); 471 } 472 473 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 474 td->td_pcb->pcb_onfault = NULL; 475 return (-1); 476 } 477 478 *val = *p; 479 480 td->td_pcb->pcb_onfault = NULL; 481 return (0); 482 } 483 #endif 484 485 int 486 REMAP(fueword)(volatile const void *addr, long *val) 487 { 488 struct thread *td; 489 pmap_t pm; 490 jmp_buf env; 491 long *p; 492 493 td = curthread; 494 pm = &td->td_proc->p_vmspace->vm_pmap; 495 496 td->td_pcb->pcb_onfault = &env; 497 if (setjmp(env)) { 498 td->td_pcb->pcb_onfault = NULL; 499 return (-1); 500 } 501 502 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 503 td->td_pcb->pcb_onfault = NULL; 504 return (-1); 505 } 506 507 *val = *p; 508 509 td->td_pcb->pcb_onfault = NULL; 510 return (0); 511 } 512 513 int 514 REMAP(casueword32)(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp, 515 uint32_t new) 516 { 517 struct thread *td; 518 pmap_t pm; 519 jmp_buf env; 520 uint32_t *p, val; 521 int res; 522 523 td = curthread; 524 pm = &td->td_proc->p_vmspace->vm_pmap; 525 526 td->td_pcb->pcb_onfault = &env; 527 if (setjmp(env)) { 528 td->td_pcb->pcb_onfault = NULL; 529 return (-1); 530 } 531 532 if (pmap_map_user_ptr(pm, (void *)(uintptr_t)addr, (void **)&p, 533 sizeof(*p), NULL)) { 534 td->td_pcb->pcb_onfault = NULL; 535 return (-1); 536 } 537 538 res = 0; 539 __asm __volatile ( 540 "lwarx %0, 0, %3\n\t" /* load old value */ 541 "cmplw %4, %0\n\t" /* compare */ 542 "bne 1f\n\t" /* exit if not equal */ 543 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 544 "bne- 2f\n\t" /* if failed */ 545 "b 3f\n\t" /* we've succeeded */ 546 "1:\n\t" 547 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 548 "2:li %2, 1\n\t" 549 "3:\n\t" 550 : "=&r" (val), "=m" (*p), "+&r" (res) 551 : "r" (p), "r" (old), "r" (new), "m" (*p) 552 : "cr0", "memory"); 553 554 td->td_pcb->pcb_onfault = NULL; 555 556 *oldvalp = val; 557 return (res); 558 } 559 560 #ifndef __powerpc64__ 561 int 562 REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new) 563 { 564 565 return (casueword32((volatile uint32_t *)addr, old, 566 (uint32_t *)oldvalp, new)); 567 } 568 #else 569 int 570 REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new) 571 { 572 struct thread *td; 573 pmap_t pm; 574 jmp_buf env; 575 u_long *p, val; 576 int res; 577 578 td = curthread; 579 pm = &td->td_proc->p_vmspace->vm_pmap; 580 581 td->td_pcb->pcb_onfault = &env; 582 if (setjmp(env)) { 583 td->td_pcb->pcb_onfault = NULL; 584 return (-1); 585 } 586 587 if (pmap_map_user_ptr(pm, (void *)(uintptr_t)addr, (void **)&p, 588 sizeof(*p), NULL)) { 589 td->td_pcb->pcb_onfault = NULL; 590 return (-1); 591 } 592 593 res = 0; 594 __asm __volatile ( 595 "ldarx %0, 0, %3\n\t" /* load old value */ 596 "cmpld %4, %0\n\t" /* compare */ 597 "bne 1f\n\t" /* exit if not equal */ 598 "stdcx. %5, 0, %3\n\t" /* attempt to store */ 599 "bne- 2f\n\t" /* if failed */ 600 "b 3f\n\t" /* we've succeeded */ 601 "1:\n\t" 602 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 603 "2:li %2, 1\n\t" 604 "3:\n\t" 605 : "=&r" (val), "=m" (*p), "+&r" (res) 606 : "r" (p), "r" (old), "r" (new), "m" (*p) 607 : "cr0", "memory"); 608 609 td->td_pcb->pcb_onfault = NULL; 610 611 *oldvalp = val; 612 return (res); 613 } 614 #endif 615