1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2018, Matthew Macy <mmacy@freebsd.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * $FreeBSD$ 28 */ 29 30/* 31 * Assembly variants of various functions, for those that don't need generic C 32 * implementations. Currently this includes: 33 * 34 * - Direct-access versions of copyin/copyout methods. 35 * - These are used by Radix AIM pmap (ISA 3.0), and Book-E, to avoid 36 * unnecessary pmap_map_usr_ptr() calls. 37 */ 38 39#include "assym.inc" 40#include "opt_sched.h" 41 42#include <sys/syscall.h> 43#include <sys/errno.h> 44 45#include <machine/param.h> 46#include <machine/asm.h> 47#include <machine/spr.h> 48#include <machine/trap.h> 49#include <machine/vmparam.h> 50 51#ifdef _CALL_ELF 52.abiversion _CALL_ELF 53#endif 54 55#ifdef __powerpc64__ 56#define LOAD ld 57#define STORE std 58#define WORD 8 59#define CMPI cmpdi 60#define CMPLI cmpldi 61/* log_2(8 * WORD) */ 62#define LOOP_LOG 6 63#define LOG_WORD 3 64#define CURTHREAD %r13 65#else 66#define LOAD lwz 67#define STORE stw 68#define WORD 4 69#define CMPI cmpwi 70#define CMPLI cmplwi 71/* log_2(8 * WORD) */ 72#define LOOP_LOG 5 73#define LOG_WORD 2 74#define CURTHREAD %r2 75#endif 76 77#ifdef AIM 78#define ENTRY_DIRECT(x) ENTRY(x ## _direct) 79#define END_DIRECT(x) END(x ## _direct) 80#else 81#define ENTRY_DIRECT(x) ENTRY(x) 82#define END_DIRECT(x) END(x) 83#endif 84 85#ifdef __powerpc64__ 86#define PROLOGUE ;\ 87 mflr %r0 ;\ 88 std %r0, 16(%r1) ;\ 89 90#define EPILOGUE ;\ 91 ld %r0, 16(%r1) ;\ 92 mtlr %r0 ;\ 93 blr ;\ 94 nop 95 96#define VALIDATE_TRUNCATE_ADDR_COPY VALIDATE_ADDR_COPY 97#define VALIDATE_ADDR_COPY(raddr, len) \ 98 srdi %r0, raddr, 52 ;\ 99 cmpwi %r0, 1 ;\ 100 bge- copy_fault ;\ 101 nop 102 103#define VALIDATE_ADDR_FUSU(raddr) ;\ 104 srdi %r0, raddr, 52 ;\ 105 cmpwi %r0, 1 ;\ 106 bge- fusufault ;\ 107 nop 108 109#else 110#define PROLOGUE ;\ 111 mflr %r0 ;\ 112 stw %r0, 4(%r1) ;\ 113 114#define EPILOGUE ;\ 115 lwz %r0, 4(%r1) ;\ 116 mtlr %r0 ;\ 117 blr ;\ 118 nop 119 120/* %r0 is temporary */ 121/* 122 * Validate address and length are valid. 123 * For VALIDATE_ADDR_COPY() have to account for wraparound. 124 */ 125#define VALIDATE_ADDR_COPY(raddr, len) \ 126 lis %r0, VM_MAXUSER_ADDRESS@h ;\ 127 ori %r0, %r0, VM_MAXUSER_ADDRESS@l ;\ 128 cmplw %r0, raddr ;\ 129 blt- copy_fault ;\ 130 add %r0, raddr, len ;\ 131 cmplw 7, %r0, raddr ;\ 132 blt- 7, copy_fault ;\ 133 mtcrf 0x80, %r0 ;\ 134 bt- 0, copy_fault ;\ 135 nop 136 137#define VALIDATE_TRUNCATE_ADDR_COPY(raddr, len) \ 138 lis %r0, VM_MAXUSER_ADDRESS@h ;\ 139 ori %r0, %r0, VM_MAXUSER_ADDRESS@l ;\ 140 cmplw %r0, raddr ;\ 141 blt- copy_fault ;\ 142 sub %r0, %r0, raddr ;\ 143 cmplw len, %r0 ;\ 144 isel len, len, %r0, 0 ;\ 145 146#define VALIDATE_ADDR_FUSU(raddr) \ 147 lis %r0, VM_MAXUSER_ADDRESS@h ;\ 148 ori %r0, %r0, VM_MAXUSER_ADDRESS@l ;\ 149 cmplw %r0, raddr ;\ 150 ble- fusufault 151 152#endif 153 154#define SET_COPYFAULT(raddr, rpcb, len) \ 155 VALIDATE_ADDR_COPY(raddr, len) ;\ 156 li %r0, COPYFAULT ;\ 157 LOAD rpcb, TD_PCB(CURTHREAD) ;\ 158 STORE %r0, PCB_ONFAULT(rpcb) ;\ 159 160#define SET_COPYFAULT_TRUNCATE(raddr, rpcb, len)\ 161 VALIDATE_TRUNCATE_ADDR_COPY(raddr, len) ;\ 162 li %r0, COPYFAULT ;\ 163 LOAD rpcb, TD_PCB(CURTHREAD) ;\ 164 STORE %r0, PCB_ONFAULT(rpcb) 165 166#define SET_FUSUFAULT(raddr, rpcb) \ 167 VALIDATE_ADDR_FUSU(raddr) ;\ 168 li %r0, FUSUFAULT ;\ 169 LOAD rpcb, TD_PCB(CURTHREAD) ;\ 170 STORE %r0, PCB_ONFAULT(rpcb) 171 172#define CLEAR_FAULT_NO_CLOBBER(rpcb) \ 173 LOAD rpcb, TD_PCB(CURTHREAD) ;\ 174 li %r0, 0 ;\ 175 STORE %r0, PCB_ONFAULT(rpcb) 176 177#define CLEAR_FAULT(rpcb) \ 178 CLEAR_FAULT_NO_CLOBBER(rpcb) ;\ 179 li %r3, 0 180 181/* 182 * bcopy(src, dst, len) 183 * %r3 %r4 %r5 184 * 185 * %r7 is the pcb pointer 186 * 187 * %r0 and %r8-%r10 are volatile 188 * %r11 and %r12 are generally volatile, used in linking and exception 189 * handling. Can be clobbered here. 190 * 191 * Does not allocate or use stack space, but clobbers all volatile registers. 192 */ 193 194#define rs %r3 195#define rd %r4 196#define rl %r5 197 198#define t1 %r6 199#define t2 %r7 200#define t3 %r8 201#define t4 %r9 202#define t5 %r10 203#define t6 %r11 204#define t7 %r12 205#define t8 %r0 206 207#define Thresh WORD * 8 208#define W4 3 209#define W2 2 210#define W1 1 211#define WORDS(n) (32 - LOG_WORD - W##n) 212.text 213ENTRY(bcopy_generic) 214 CMPLI 0, %r5, 0 215 beq .Lend 216 dcbtst 0, rd 217 dcbt 0, rs 218 CMPLI rl, Thresh 219 blt .Lsmall 220 b .Llarge 221/* memcpy */ 222/* ... */ 223.Lsmall: /* < 8 words remaining */ 224 mtcrf 0x3, rl 225.Lsmall_start: 226 bf WORDS(4), 0f 227 LOAD t1, 0(rs) 228 LOAD t2, WORD*1(rs) 229 LOAD t3, WORD*2(rs) 230 LOAD t4, WORD*3(rs) 231 addi rs, rs, WORD*4 232 STORE t1, 0(rd) 233 STORE t2, WORD*1(rd) 234 STORE t3, WORD*2(rd) 235 STORE t4, WORD*3(rd) 236 addi rd, rd, WORD*4 2370: /* < 4 words remaining */ 238 bf WORDS(2), 1f 239 LOAD t1, 0(rs) 240 LOAD t2, WORD*1(rs) 241 addi rs, rs, WORD*2 242 STORE t1, 0(rd) 243 STORE t2, WORD*1(rd) 244 addi rd, rd, WORD*2 2451: /* < 2 words remaining */ 246 bf WORDS(1), 2f 247 LOAD t1, 0(rs) 248 addi rs, rs, WORD 249 STORE t1, 0(rd) 250 addi rd, rd, WORD 2512: /* < 1 word remaining */ 252#ifdef __powerpc64__ 253 bf 29, 3f 254 lwz t1, 0(rs) 255 addi rs, rs, 4 256 stw t1, 0(rd) 257 addi rd, rd, 4 2583: /* < 4 bytes remaining */ 259#endif 260 bf 30, 4f 261 lhz t1, 0(rs) 262 addi rs, rs, 2 263 sth t1, 0(rd) 264 addi rd, rd, 2 2654: /* < 2 bytes remaining */ 266 bf 31, .Lout 267 lbz t1, 0(rs) 268 addi rs, rs, 1 269 stb t1, 0(rd) 270 addi rd, rd, 1 271 b .Lout 272 273 .align 4 274.Llarge: 275 neg t3, rd 276 andi. t6, t3, WORD-1 /* Align rd to word size */ 277 mtctr t6 278 sub rl, rl, t6 279 beq+ .Llargealigned 2801: 281 lbz t1, 0(rs) 282 addi rs, rs, 1 283 stb t1, 0(rd) 284 addi rd, rd, 1 285 bdnz 1b 286 287.Llargealigned: 288 srwi. t2, rl, LOOP_LOG /* length >> log_2(loop_size) => 8W iterations */ 289 mtcrf 0x3, rl 290 beq .Lsmall_start 291 mtctr t2 292 b 1f 293 294 .align 5 2951: 296 LOAD t1, 0(rs) 297 LOAD t2, WORD(rs) 298 LOAD t3, WORD*2(rs) 299 LOAD t4, WORD*3(rs) 300 LOAD t5, WORD*4(rs) 301 LOAD t6, WORD*5(rs) 302 LOAD t7, WORD*6(rs) 303 LOAD t8, WORD*7(rs) 304 addi rs, rs, WORD*8 305 STORE t1, 0(rd) 306 STORE t2, WORD*1(rd) 307 STORE t3, WORD*2(rd) 308 STORE t4, WORD*3(rd) 309 STORE t5, WORD*4(rd) 310 STORE t6, WORD*5(rd) 311 STORE t7, WORD*6(rd) 312 STORE t8, WORD*7(rd) 313 addi rd, rd, WORD*8 314 bdnz 1b 315 316 b .Lsmall_start 317.Lout: 318/* done */ 319.Lend: 320 blr 321END(bcopy_generic) 322 323/* 324 * copyout(from_kernel, to_user, len) 325 * %r3, %r4, %r5 326 */ 327ENTRY_DIRECT(copyout) 328 PROLOGUE 329 SET_COPYFAULT(%r4, %r7, %r5) 330 bl bcopy_generic 331 nop 332 CLEAR_FAULT(%r7) 333 EPILOGUE 334END_DIRECT(copyout) 335 336/* 337 * copyin(from_user, to_kernel, len) 338 * %r3, %r4, %r5 339 */ 340ENTRY_DIRECT(copyin) 341 PROLOGUE 342 SET_COPYFAULT(%r3, %r7, %r5) 343 bl bcopy_generic 344 nop 345 CLEAR_FAULT(%r7) 346 EPILOGUE 347END_DIRECT(copyin) 348 349/* 350 * copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done) 351 * %r3 %r4 %r5 %r6 352 */ 353 354ENTRY_DIRECT(copyinstr) 355 PROLOGUE 356 SET_COPYFAULT_TRUNCATE(%r3, %r7, %r5) 357 addi %r9, %r5, 1 358 mtctr %r9 359 mr %r8, %r3 360 addi %r8, %r8, -1 361 addi %r4, %r4, -1 362 li %r3, ENAMETOOLONG 3630: 364 bdz- 2f 365 lbzu %r0, 1(%r8) 366 stbu %r0, 1(%r4) 367 368 // NULL byte reached ? 369 CMPI %r0, 0 370 beq- 1f 371 b 0b 3721: 373 li %r3, 0 3742: 375 /* skip storing length if done is NULL */ 376 CMPI %r6, 0 377 beq- 3f 378 mfctr %r0 379 sub %r0, %r9, %r0 380 STORE %r0, 0(%r6) 3813: 382 CLEAR_FAULT_NO_CLOBBER(%r7) 383 EPILOGUE 384END_DIRECT(copyinstr) 385 386ENTRY_DIRECT(subyte) 387 PROLOGUE 388 SET_FUSUFAULT(%r3, %r7) 389 stb %r4, 0(%r3) 390 CLEAR_FAULT(%r7) 391 EPILOGUE 392END_DIRECT(subyte) 393 394#ifndef __powerpc64__ 395ENTRY_DIRECT(suword) 396 PROLOGUE 397 SET_FUSUFAULT(%r3, %r7) 398 stw %r4, 0(%r3) 399 CLEAR_FAULT(%r7) 400 EPILOGUE 401END_DIRECT(suword) 402#endif 403 404ENTRY_DIRECT(suword32) 405 PROLOGUE 406 SET_FUSUFAULT(%r3, %r7) 407 stw %r4, 0(%r3) 408 CLEAR_FAULT(%r7) 409 EPILOGUE 410END_DIRECT(suword32) 411 412#ifdef __powerpc64__ 413ENTRY_DIRECT(suword64) 414 PROLOGUE 415 SET_FUSUFAULT(%r3, %r7) 416 std %r4, 0(%r3) 417 CLEAR_FAULT(%r7) 418 EPILOGUE 419END_DIRECT(suword64) 420 421ENTRY_DIRECT(suword) 422 PROLOGUE 423 SET_FUSUFAULT(%r3, %r7) 424 std %r4, 0(%r3) 425 CLEAR_FAULT(%r7) 426 EPILOGUE 427END_DIRECT(suword) 428#endif 429 430ENTRY_DIRECT(fubyte) 431 PROLOGUE 432 SET_FUSUFAULT(%r3, %r7) 433 lbz %r3, 0(%r3) 434 CLEAR_FAULT_NO_CLOBBER(%r7) 435 EPILOGUE 436END_DIRECT(fubyte) 437 438ENTRY_DIRECT(fuword16) 439 PROLOGUE 440 SET_FUSUFAULT(%r3, %r7) 441 lhz %r3, 0(%r3) 442 CLEAR_FAULT_NO_CLOBBER(%r7) 443 EPILOGUE 444END_DIRECT(fuword16) 445 446#ifndef __powerpc64__ 447ENTRY_DIRECT(fueword) 448 PROLOGUE 449 SET_FUSUFAULT(%r3, %r7) 450 lwz %r0, 0(%r3) 451 stw %r0, 0(%r4) 452 CLEAR_FAULT(%r7) 453 EPILOGUE 454END_DIRECT(fueword) 455#endif 456ENTRY_DIRECT(fueword32) 457 PROLOGUE 458 SET_FUSUFAULT(%r3, %r7) 459 lwz %r0, 0(%r3) 460 stw %r0, 0(%r4) 461 CLEAR_FAULT(%r7) 462 EPILOGUE 463END_DIRECT(fueword32) 464 465#ifdef __powerpc64__ 466ENTRY_DIRECT(fueword) 467 PROLOGUE 468 SET_FUSUFAULT(%r3, %r7) 469 ld %r0, 0(%r3) 470 std %r0, 0(%r4) 471 CLEAR_FAULT(%r7) 472 EPILOGUE 473END_DIRECT(fueword) 474 475ENTRY_DIRECT(fueword64) 476 PROLOGUE 477 SET_FUSUFAULT(%r3, %r7) 478 ld %r0, 0(%r3) 479 std %r0, 0(%r4) 480 CLEAR_FAULT(%r7) 481 EPILOGUE 482END_DIRECT(fueword64) 483#endif 484 485/* 486 * casueword(volatile u_long *base, u_long old, u_long *oldp, u_long new) 487 * %r3 %r4 %r5 %r6 488 */ 489 490#define CASUEWORD32(raddr, rpcb) ;\ 491 PROLOGUE ;\ 492 SET_FUSUFAULT(raddr, rpcb) ;\ 493 li %r8, 0 ;\ 4941: ;\ 495 lwarx %r0, 0, %r3 ;\ 496 cmplw %r4, %r0 ;\ 497 bne 2f ;\ 498 stwcx. %r6, 0, %r3 ;\ 499 bne- 3f ;\ 500 b 4f ;\ 5012: ;\ 502 stwcx. %r0, 0, %r3 /* clear reservation (74xx) */ ;\ 5033: ;\ 504 li %r8, 1 ;\ 5054: ;\ 506 stw %r0, 0(%r5) ;\ 507 CLEAR_FAULT_NO_CLOBBER(rpcb) ;\ 508 mr %r3, %r8 ;\ 509 EPILOGUE 510 511ENTRY_DIRECT(casueword32) 512 CASUEWORD32(%r3, %r7) 513END_DIRECT(casueword32) 514 515#ifdef __powerpc64__ 516#define CASUEWORD64(raddr, rpcb) ;\ 517 PROLOGUE ;\ 518 SET_FUSUFAULT(raddr, rpcb) ;\ 519 li %r8, 0 ;\ 5201: ;\ 521 ldarx %r0, 0, %r3 ;\ 522 cmpld %r4, %r0 ;\ 523 bne 2f ;\ 524 stdcx. %r6, 0, %r3 ;\ 525 bne- 3f ;\ 526 b 4f ;\ 5272: ;\ 528 stdcx. %r0, 0, %r3 /* clear reservation (74xx) */ ;\ 5293: ;\ 530 li %r8, 1 ;\ 5314: ;\ 532 std %r0, 0(%r5) ;\ 533 CLEAR_FAULT_NO_CLOBBER(rpcb) ;\ 534 mr %r3, %r8 ;\ 535 EPILOGUE 536 537ENTRY_DIRECT(casueword) 538 CASUEWORD64(%r3, %r7) 539END_DIRECT(casueword) 540 541ENTRY_DIRECT(casueword64) 542 CASUEWORD64(%r3, %r7) 543END_DIRECT(casueword64) 544#else 545ENTRY_DIRECT(casueword) 546 CASUEWORD32(%r3, %r7) 547END_DIRECT(casueword) 548#endif 549 550_NAKED_ENTRY(fusufault) 551 CLEAR_FAULT_NO_CLOBBER(%r7) 552 li %r3, -1 553 EPILOGUE 554_END(fusufault) 555 556_NAKED_ENTRY(copy_fault) 557 CLEAR_FAULT_NO_CLOBBER(%r7) 558 li %r3, EFAULT 559 EPILOGUE 560_END(copy_fault) 561