1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2018, Matthew Macy <mmacy@freebsd.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * $FreeBSD$ 28 */ 29 30/* 31 * Assembly variants of various functions, for those that don't need generic C 32 * implementations. Currently this includes: 33 * 34 * - Direct-access versions of copyin/copyout methods. 35 * - These are used by Radix AIM pmap (ISA 3.0), and Book-E, to avoid 36 * unnecessary pmap_map_usr_ptr() calls. 37 */ 38 39#include "assym.inc" 40#include "opt_sched.h" 41 42#include <sys/syscall.h> 43#include <sys/errno.h> 44 45#include <machine/param.h> 46#include <machine/asm.h> 47#include <machine/spr.h> 48#include <machine/trap.h> 49#include <machine/vmparam.h> 50 51#ifdef _CALL_ELF 52.abiversion _CALL_ELF 53#endif 54 55#ifdef __powerpc64__ 56#define LOAD ld 57#define STORE std 58#define WORD 8 59#define CMPI cmpdi 60#define CMPLI cmpldi 61/* log_2(8 * WORD) */ 62#define LOOP_LOG 6 63#define LOG_WORD 3 64#else 65#define LOAD lwz 66#define STORE stw 67#define WORD 4 68#define CMPI cmpwi 69#define CMPLI cmplwi 70/* log_2(8 * WORD) */ 71#define LOOP_LOG 5 72#define LOG_WORD 2 73#endif 74 75#ifdef AIM 76#define ENTRY_DIRECT(x) ENTRY(x ## _direct) 77#else 78#define ENTRY_DIRECT(x) ENTRY(x) 79#endif 80 81#ifdef __powerpc64__ 82#define PROLOGUE ;\ 83 mflr %r0 ;\ 84 std %r0, 16(%r1) ;\ 85 86#define EPILOGUE ;\ 87 ld %r0, 16(%r1) ;\ 88 mtlr %r0 ;\ 89 blr ;\ 90 nop 91 92#define VALIDATE_TRUNCATE_ADDR_COPY VALIDATE_ADDR_COPY 93#define VALIDATE_ADDR_COPY(raddr, len) \ 94 srdi %r0, raddr, 52 ;\ 95 cmpwi %r0, 1 ;\ 96 bge- copy_fault ;\ 97 nop 98 99#define VALIDATE_ADDR_FUSU(raddr) ;\ 100 srdi %r0, raddr, 52 ;\ 101 cmpwi %r0, 1 ;\ 102 bge- fusufault ;\ 103 nop 104 105#else 106#define PROLOGUE ;\ 107 mflr %r0 ;\ 108 stw %r0, 4(%r1) ;\ 109 110#define EPILOGUE ;\ 111 lwz %r0, 4(%r1) ;\ 112 mtlr %r0 ;\ 113 blr ;\ 114 nop 115 116/* %r0 is temporary */ 117/* 118 * Validate address and length are valid. 119 * For VALIDATE_ADDR_COPY() have to account for wraparound. 120 */ 121#define VALIDATE_ADDR_COPY(raddr, len) \ 122 lis %r0, VM_MAXUSER_ADDRESS@h ;\ 123 ori %r0, %r0, VM_MAXUSER_ADDRESS@l ;\ 124 cmplw %r0, raddr ;\ 125 blt- copy_fault ;\ 126 add %r0, raddr, len ;\ 127 cmplw 7, %r0, raddr ;\ 128 blt- 7, copy_fault ;\ 129 mtcrf 0x80, %r0 ;\ 130 bt- 0, copy_fault ;\ 131 nop 132 133#define VALIDATE_TRUNCATE_ADDR_COPY(raddr, len) \ 134 lis %r0, VM_MAXUSER_ADDRESS@h ;\ 135 ori %r0, %r0, VM_MAXUSER_ADDRESS@l ;\ 136 cmplw %r0, raddr ;\ 137 blt- copy_fault ;\ 138 sub %r0, %r0, raddr ;\ 139 cmplw len, %r0 ;\ 140 isel len, len, %r0, 0 ;\ 141 142#define VALIDATE_ADDR_FUSU(raddr) \ 143 lis %r0, VM_MAXUSER_ADDRESS@h ;\ 144 ori %r0, %r0, VM_MAXUSER_ADDRESS@l ;\ 145 cmplw %r0, raddr ;\ 146 ble- fusufault 147 148#endif 149 150#define PCPU(reg) mfsprg reg, 0 151 152#define SET_COPYFAULT(raddr, rpcb, len) \ 153 VALIDATE_ADDR_COPY(raddr, len) ;\ 154 PCPU(%r9) ;\ 155 li %r0, COPYFAULT ;\ 156 LOAD rpcb, PC_CURPCB(%r9) ;\ 157 STORE %r0, PCB_ONFAULT(rpcb) ;\ 158 159#define SET_COPYFAULT_TRUNCATE(raddr, rpcb, len)\ 160 VALIDATE_TRUNCATE_ADDR_COPY(raddr, len) ;\ 161 PCPU(%r9) ;\ 162 li %r0, COPYFAULT ;\ 163 LOAD rpcb, PC_CURPCB(%r9) ;\ 164 STORE %r0, PCB_ONFAULT(rpcb) 165 166#define SET_FUSUFAULT(raddr, rpcb) \ 167 VALIDATE_ADDR_FUSU(raddr) ;\ 168 PCPU(%r9) ;\ 169 li %r0, FUSUFAULT ;\ 170 LOAD rpcb, PC_CURPCB(%r9) ;\ 171 STORE %r0, PCB_ONFAULT(rpcb) 172 173#define CLEAR_FAULT_NO_CLOBBER(rpcb) \ 174 PCPU(%r9) ;\ 175 LOAD rpcb, PC_CURPCB(%r9) ;\ 176 li %r0, 0 ;\ 177 STORE %r0, PCB_ONFAULT(rpcb) 178 179#define CLEAR_FAULT(rpcb) \ 180 CLEAR_FAULT_NO_CLOBBER(rpcb) ;\ 181 li %r3, 0 182 183/* 184 * bcopy(src, dst, len) 185 * %r3 %r4 %r5 186 * 187 * %r7 is the pcb pointer 188 * 189 * %r0 and %r8-%r10 are volatile 190 * %r11 and %r12 are generally volatile, used in linking and exception 191 * handling. Can be clobbered here. 192 * 193 * Does not allocate or use stack space, but clobbers all volatile registers. 194 */ 195 196#define rs %r3 197#define rd %r4 198#define rl %r5 199 200#define t1 %r6 201#define t2 %r7 202#define t3 %r8 203#define t4 %r9 204#define t5 %r10 205#define t6 %r11 206#define t7 %r12 207#define t8 %r0 208 209#define Thresh WORD * 8 210#define W4 3 211#define W2 2 212#define W1 1 213#define WORDS(n) (32 - LOG_WORD - W##n) 214.text 215ENTRY(bcopy_generic) 216 CMPLI 0, %r5, 0 217 beq .Lend 218 dcbtst 0, rd 219 dcbt 0, rs 220 CMPLI rl, Thresh 221 blt .Lsmall 222 b .Llarge 223/* memcpy */ 224/* ... */ 225.Lsmall: /* < 8 words remaining */ 226 mtcrf 0x3, rl 227.Lsmall_start: 228 bf WORDS(4), 0f 229 LOAD t1, 0(rs) 230 LOAD t2, WORD*1(rs) 231 LOAD t3, WORD*2(rs) 232 LOAD t4, WORD*3(rs) 233 addi rs, rs, WORD*4 234 STORE t1, 0(rd) 235 STORE t2, WORD*1(rd) 236 STORE t3, WORD*2(rd) 237 STORE t4, WORD*3(rd) 238 addi rd, rd, WORD*4 2390: /* < 4 words remaining */ 240 bf WORDS(2), 1f 241 LOAD t1, 0(rs) 242 LOAD t2, WORD*1(rs) 243 addi rs, rs, WORD*2 244 STORE t1, 0(rd) 245 STORE t2, WORD*1(rd) 246 addi rd, rd, WORD*2 2471: /* < 2 words remaining */ 248 bf WORDS(1), 2f 249 LOAD t1, 0(rs) 250 addi rs, rs, WORD 251 STORE t1, 0(rd) 252 addi rd, rd, WORD 2532: /* < 1 word remaining */ 254#ifdef __powerpc64__ 255 bf 29, 3f 256 lwz t1, 0(rs) 257 addi rs, rs, 4 258 stw t1, 0(rd) 259 addi rd, rd, 4 2603: /* < 4 bytes remaining */ 261#endif 262 bf 30, 4f 263 lhz t1, 0(rs) 264 addi rs, rs, 2 265 sth t1, 0(rd) 266 addi rd, rd, 2 2674: /* < 2 bytes remaining */ 268 bf 31, .Lout 269 lbz t1, 0(rs) 270 addi rs, rs, 1 271 stb t1, 0(rd) 272 addi rd, rd, 1 273 b .Lout 274 275 .align 4 276.Llarge: 277 neg t3, rd 278 andi. t6, t3, WORD-1 /* Align rd to word size */ 279 mtctr t6 280 sub rl, rl, t6 281 beq+ .Llargealigned 2821: 283 lbz t1, 0(rs) 284 addi rs, rs, 1 285 stb t1, 0(rd) 286 addi rd, rd, 1 287 bdnz 1b 288 289.Llargealigned: 290 srwi. t2, rl, LOOP_LOG /* length >> log_2(loop_size) => 8W iterations */ 291 mtcrf 0x3, rl 292 beq .Lsmall_start 293 mtctr t2 294 b 1f 295 296 .align 5 2971: 298 LOAD t1, 0(rs) 299 LOAD t2, WORD(rs) 300 LOAD t3, WORD*2(rs) 301 LOAD t4, WORD*3(rs) 302 LOAD t5, WORD*4(rs) 303 LOAD t6, WORD*5(rs) 304 LOAD t7, WORD*6(rs) 305 LOAD t8, WORD*7(rs) 306 addi rs, rs, WORD*8 307 STORE t1, 0(rd) 308 STORE t2, WORD*1(rd) 309 STORE t3, WORD*2(rd) 310 STORE t4, WORD*3(rd) 311 STORE t5, WORD*4(rd) 312 STORE t6, WORD*5(rd) 313 STORE t7, WORD*6(rd) 314 STORE t8, WORD*7(rd) 315 addi rd, rd, WORD*8 316 bdnz 1b 317 318 b .Lsmall_start 319.Lout: 320/* done */ 321.Lend: 322 blr 323 324/* 325 * copyout(from_kernel, to_user, len) 326 * %r3, %r4, %r5 327 */ 328ENTRY_DIRECT(copyout) 329 PROLOGUE 330 SET_COPYFAULT(%r4, %r7, %r5) 331 bl bcopy_generic 332 nop 333 CLEAR_FAULT(%r7) 334 EPILOGUE 335 336/* 337 * copyin(from_user, to_kernel, len) 338 * %r3, %r4, %r5 339 */ 340ENTRY_DIRECT(copyin) 341 PROLOGUE 342 SET_COPYFAULT(%r3, %r7, %r5) 343 bl bcopy_generic 344 nop 345 CLEAR_FAULT(%r7) 346 EPILOGUE 347/* 348 * copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done) 349 * %r3 %r4 %r5 %r6 350 */ 351 352ENTRY_DIRECT(copyinstr) 353 PROLOGUE 354 SET_COPYFAULT_TRUNCATE(%r3, %r7, %r5) 355 addi %r9, %r5, 1 356 mtctr %r9 357 mr %r8, %r3 358 addi %r8, %r8, -1 359 addi %r4, %r4, -1 360 li %r3, ENAMETOOLONG 3610: 362 bdz- 2f 363 lbzu %r0, 1(%r8) 364 stbu %r0, 1(%r4) 365 366 // NULL byte reached ? 367 CMPI %r0, 0 368 beq- 1f 369 b 0b 3701: 371 li %r3, 0 3722: 373 /* skip storing length if done is NULL */ 374 CMPI %r6, 0 375 beq- 3f 376 mfctr %r0 377 sub %r0, %r9, %r0 378 STORE %r0, 0(%r6) 3793: 380 CLEAR_FAULT_NO_CLOBBER(%r7) 381 EPILOGUE 382 383ENTRY_DIRECT(subyte) 384 PROLOGUE 385 SET_FUSUFAULT(%r3, %r7) 386 stb %r4, 0(%r3) 387 CLEAR_FAULT(%r7) 388 EPILOGUE 389 390#ifndef __powerpc64__ 391ENTRY_DIRECT(suword) 392 PROLOGUE 393 SET_FUSUFAULT(%r3, %r7) 394 stw %r4, 0(%r3) 395 CLEAR_FAULT(%r7) 396 EPILOGUE 397#endif 398 399ENTRY_DIRECT(suword32) 400 PROLOGUE 401 SET_FUSUFAULT(%r3, %r7) 402 stw %r4, 0(%r3) 403 CLEAR_FAULT(%r7) 404 EPILOGUE 405 406#ifdef __powerpc64__ 407ENTRY_DIRECT(suword64) 408 PROLOGUE 409 SET_FUSUFAULT(%r3, %r7) 410 std %r4, 0(%r3) 411 CLEAR_FAULT(%r7) 412 EPILOGUE 413ENTRY_DIRECT(suword) 414 PROLOGUE 415 SET_FUSUFAULT(%r3, %r7) 416 std %r4, 0(%r3) 417 CLEAR_FAULT(%r7) 418 EPILOGUE 419#endif 420 421ENTRY_DIRECT(fubyte) 422 PROLOGUE 423 SET_FUSUFAULT(%r3, %r7) 424 lbz %r3, 0(%r3) 425 CLEAR_FAULT_NO_CLOBBER(%r7) 426 EPILOGUE 427 428ENTRY_DIRECT(fuword16) 429 PROLOGUE 430 SET_FUSUFAULT(%r3, %r7) 431 lhz %r3, 0(%r3) 432 CLEAR_FAULT_NO_CLOBBER(%r7) 433 EPILOGUE 434 435#ifndef __powerpc64__ 436ENTRY_DIRECT(fueword) 437 PROLOGUE 438 SET_FUSUFAULT(%r3, %r7) 439 lwz %r0, 0(%r3) 440 stw %r0, 0(%r4) 441 CLEAR_FAULT(%r7) 442 EPILOGUE 443#endif 444ENTRY_DIRECT(fueword32) 445 PROLOGUE 446 SET_FUSUFAULT(%r3, %r7) 447 lwz %r0, 0(%r3) 448 stw %r0, 0(%r4) 449 CLEAR_FAULT(%r7) 450 EPILOGUE 451 452#ifdef __powerpc64__ 453ENTRY_DIRECT(fueword) 454 PROLOGUE 455 SET_FUSUFAULT(%r3, %r7) 456 ld %r0, 0(%r3) 457 std %r0, 0(%r4) 458 CLEAR_FAULT(%r7) 459 EPILOGUE 460 461ENTRY_DIRECT(fueword64) 462 PROLOGUE 463 SET_FUSUFAULT(%r3, %r7) 464 ld %r0, 0(%r3) 465 std %r0, 0(%r4) 466 CLEAR_FAULT(%r7) 467 EPILOGUE 468#endif 469 470/* 471 * casueword(volatile u_long *base, u_long old, u_long *oldp, u_long new) 472 * %r3 %r4 %r5 %r6 473 */ 474 475#define CASUEWORD32(raddr, rpcb) ;\ 476 PROLOGUE ;\ 477 SET_FUSUFAULT(raddr, rpcb) ;\ 478 li %r8, 0 ;\ 4791: ;\ 480 lwarx %r0, 0, %r3 ;\ 481 cmplw %r4, %r0 ;\ 482 bne 2f ;\ 483 stwcx. %r6, 0, %r3 ;\ 484 bne- 3f ;\ 485 b 4f ;\ 4862: ;\ 487 stwcx. %r0, 0, %r3 /* clear reservation (74xx) */ ;\ 4883: ;\ 489 li %r8, 1 ;\ 4904: ;\ 491 stw %r0, 0(%r5) ;\ 492 CLEAR_FAULT_NO_CLOBBER(rpcb) ;\ 493 mr %r3, %r8 ;\ 494 EPILOGUE 495 496ENTRY_DIRECT(casueword32) 497 CASUEWORD32(%r3, %r7) 498 499#ifdef __powerpc64__ 500#define CASUEWORD64(raddr, rpcb) ;\ 501 PROLOGUE ;\ 502 SET_FUSUFAULT(raddr, rpcb) ;\ 503 li %r8, 0 ;\ 5041: ;\ 505 ldarx %r0, 0, %r3 ;\ 506 cmpld %r4, %r0 ;\ 507 bne 2f ;\ 508 stdcx. %r6, 0, %r3 ;\ 509 bne- 3f ;\ 510 b 4f ;\ 5112: ;\ 512 stdcx. %r0, 0, %r3 /* clear reservation (74xx) */ ;\ 5133: ;\ 514 li %r8, 1 ;\ 5154: ;\ 516 std %r0, 0(%r5) ;\ 517 CLEAR_FAULT_NO_CLOBBER(rpcb) ;\ 518 mr %r3, %r8 ;\ 519 EPILOGUE 520 521ENTRY_DIRECT(casueword) 522 CASUEWORD64(%r3, %r7) 523 524ENTRY_DIRECT(casueword64) 525 CASUEWORD64(%r3, %r7) 526#else 527ENTRY_DIRECT(casueword) 528 CASUEWORD32(%r3, %r7) 529#endif 530 531_NAKED_ENTRY(fusufault) 532 CLEAR_FAULT_NO_CLOBBER(%r7) 533 li %r3, -1 534 EPILOGUE 535 536_NAKED_ENTRY(copy_fault) 537 CLEAR_FAULT_NO_CLOBBER(%r7) 538 li %r3, EFAULT 539 EPILOGUE 540