1/* $NetBSD: copy.S,v 1.6 2015/11/22 10:18:14 martin Exp $ */ 2 3/* 4 * Copyright (c) 2006-2010 Matthew R. Green 5 * Copyright (c) 1996-2002 Eduardo Horvath 6 * Copyright (c) 1996 Paul Kranenburg 7 * Copyright (c) 1996 8 * The President and Fellows of Harvard College. 9 * All rights reserved. 10 * Copyright (c) 1992, 1993 11 * The Regents of the University of California. 12 * All rights reserved. 13 * 14 * This software was developed by the Computer Systems Engineering group 15 * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and 16 * contributed to Berkeley. 17 * 18 * All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the University of 21 * California, Lawrence Berkeley Laboratory. 22 * This product includes software developed by Harvard University. 23 * 24 * Redistribution and use in source and binary forms, with or without 25 * modification, are permitted provided that the following conditions 26 * are met: 27 28 * 1. Redistributions of source code must retain the above copyright 29 * notice, this list of conditions and the following disclaimer. 30 * 2. Redistributions in binary form must reproduce the above copyright 31 * notice, this list of conditions and the following disclaimer in the 32 * documentation and/or other materials provided with the 33 * distribution. 34 * 3. All advertising materials mentioning features or use of this 35 * software must display the following acknowledgement: 36 * This product includes software developed by the University of 37 * California, Berkeley and its contributors. 38 * This product includes software developed by Harvard University. 39 * This product includes software developed by Paul Kranenburg. 40 * 4. Neither the name of the University nor the names of its 41 * contributors may be used to endorse or promote products derived 42 * from this software without specific prior written permission. 43 * 44 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' 45 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 46 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 47 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR 48 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 49 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 50 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 51 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 52 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 53 * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF 54 * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 55 * DAMAGE. 56 * 57 * @(#)locore.s 8.4 (Berkeley) 12/10/93 58 */ 59 60 61#include "opt_ddb.h" 62#include "opt_kgdb.h" 63#include "opt_multiprocessor.h" 64#include "opt_compat_netbsd.h" 65#include "opt_compat_netbsd32.h" 66#include "opt_lockdebug.h" 67 68#include "assym.h" 69#include <machine/param.h> 70#include <machine/ctlreg.h> 71#include <machine/asm.h> 72#include <machine/locore.h> 73#include <machine/psl.h> 74 75#include "ksyms.h" 76 77 .register %g2,#scratch 78 .register %g3,#scratch 79 80/* 81 * copyinstr(fromaddr, toaddr, maxlength, &lencopied) 82 * 83 * Copy a null terminated string from the user address space into 84 * the kernel address space. 85 */ 86ENTRY(copyinstr) 87 ! %o0 = fromaddr, %o1 = toaddr, %o2 = maxlen, %o3 = &lencopied 88#ifdef NOTDEF_DEBUG 89 save %sp, -CC64FSZ, %sp 90 set 8f, %o0 91 mov %i0, %o1 92 mov %i1, %o2 93 mov %i2, %o3 94 call printf 95 mov %i3, %o4 96 restore 97 .data 988: .asciz "copyinstr: from=%x to=%x max=%x &len=%x\n" 99 _ALIGN 100 .text 101#endif 102 brgz,pt %o2, 1f ! Make sure len is valid 103 sethi %hi(CPCB), %o4 ! (first instr of copy) 104 retl 105 mov ENAMETOOLONG, %o0 1061: 107 LDPTR [%o4 + %lo(CPCB)], %o4 ! catch faults 108 set Lcsdone, %o5 109 membar #Sync 110 STPTR %o5, [%o4 + PCB_ONFAULT] 111 112 mov %o1, %o5 ! save = toaddr; 113! XXX should do this in bigger chunks when possible 1140: ! loop: 115 ldsba [%o0] ASI_AIUS, %g1 ! c = *fromaddr; 116 stb %g1, [%o1] ! *toaddr++ = c; 117 inc %o1 118 brz,a,pn %g1, Lcsdone ! if (c == NULL) 119 clr %o0 ! { error = 0; done; } 120 deccc %o2 ! if (--len > 0) { 121 bg,pt %icc, 0b ! fromaddr++; 122 inc %o0 ! goto loop; 123 ba,pt %xcc, Lcsdone ! } 124 mov ENAMETOOLONG, %o0 ! error = ENAMETOOLONG; 125 NOTREACHED 126 127/* 128 * copyoutstr(fromaddr, toaddr, maxlength, &lencopied) 129 * 130 * Copy a null terminated string from the kernel 131 * address space to the user address space. 132 */ 133ENTRY(copyoutstr) 134 ! %o0 = fromaddr, %o1 = toaddr, %o2 = maxlen, %o3 = &lencopied 135#ifdef NOTDEF_DEBUG 136 save %sp, -CC64FSZ, %sp 137 set 8f, %o0 138 mov %i0, %o1 139 mov %i1, %o2 140 mov %i2, %o3 141 call printf 142 mov %i3, %o4 143 restore 144 .data 1458: .asciz "copyoutstr: from=%x to=%x max=%x &len=%x\n" 146 _ALIGN 147 .text 148#endif 149 brgz,pt %o2, 1f ! Make sure len is valid 150 sethi %hi(CPCB), %o4 ! (first instr of copy) 151 retl 152 mov ENAMETOOLONG, %o0 1531: 154 LDPTR [%o4 + %lo(CPCB)], %o4 ! catch faults 155 set Lcsdone, %o5 156 membar #Sync 157 STPTR %o5, [%o4 + PCB_ONFAULT] 158 159 mov %o1, %o5 ! save = toaddr; 160! XXX should do this in bigger chunks when possible 1610: ! loop: 162 ldsb [%o0], %g1 ! c = *fromaddr; 163 stba %g1, [%o1] ASI_AIUS ! *toaddr++ = c; 164 inc %o1 165 brz,a,pn %g1, Lcsdone ! if (c == NULL) 166 clr %o0 ! { error = 0; done; } 167 deccc %o2 ! if (--len > 0) { 168 bg,pt %icc, 0b ! fromaddr++; 169 inc %o0 ! goto loop; 170 ! } 171 mov ENAMETOOLONG, %o0 ! error = ENAMETOOLONG; 172Lcsdone: ! done: 173 sub %o1, %o5, %o1 ! len = to - save; 174 brnz,a %o3, 1f ! if (lencopied) 175 STPTR %o1, [%o3] ! *lencopied = len; 1761: 177 retl ! cpcb->pcb_onfault = 0; 178 STPTR %g0, [%o4 + PCB_ONFAULT]! return (error); 179 180 181/* 182 * copyin(src, dst, len) 183 * 184 * Copy specified amount of data from user space into the kernel. 185 * 186 * This is a modified version of memcpy that uses ASI_AIUS. When 187 * memcpy is optimized to use block copy ASIs, this should be also. 188 */ 189 190ENTRY(copyin) 191! flushw ! Make sure we don't have stack probs & lose hibits of %o 192#ifdef NOTDEF_DEBUG 193 save %sp, -CC64FSZ, %sp 194 set 1f, %o0 195 mov %i0, %o1 196 mov %i1, %o2 197 call printf 198 mov %i2, %o3 199 restore 200 .data 2011: .asciz "copyin: src=%x dest=%x len=%x\n" 202 _ALIGN 203 .text 204#endif 205 sethi %hi(CPCB), %o3 206 wr %g0, ASI_AIUS, %asi 207 LDPTR [%o3 + %lo(CPCB)], %o3 208 set Lcopyfault, %o4 209! mov %o7, %g7 ! save return address 210 membar #Sync 211 STPTR %o4, [%o3 + PCB_ONFAULT] 212 cmp %o2, BCOPY_SMALL 213Lcopyin_start: 214 bge,a Lcopyin_fancy ! if >= this many, go be fancy. 215 btst 7, %o0 ! (part of being fancy) 216 217 /* 218 * Not much to copy, just do it a byte at a time. 219 */ 220 deccc %o2 ! while (--len >= 0) 221 bl 1f 2220: 223 inc %o0 224 ldsba [%o0 - 1] %asi, %o4! *dst++ = (++src)[-1]; 225 stb %o4, [%o1] 226 deccc %o2 227 bge 0b 228 inc %o1 2291: 230 ba Lcopyin_done 231 clr %o0 232 NOTREACHED 233 234 /* 235 * Plenty of data to copy, so try to do it optimally. 236 */ 237Lcopyin_fancy: 238 ! check for common case first: everything lines up. 239! btst 7, %o0 ! done already 240 bne 1f 241 .empty 242 btst 7, %o1 243 be,a Lcopyin_doubles 244 dec 8, %o2 ! if all lined up, len -= 8, goto copyin_doubes 245 246 ! If the low bits match, we can make these line up. 2471: 248 xor %o0, %o1, %o3 ! t = src ^ dst; 249 btst 1, %o3 ! if (t & 1) { 250 be,a 1f 251 btst 1, %o0 ! [delay slot: if (src & 1)] 252 253 ! low bits do not match, must copy by bytes. 2540: 255 ldsba [%o0] %asi, %o4 ! do { 256 inc %o0 ! (++dst)[-1] = *src++; 257 inc %o1 258 deccc %o2 259 bnz 0b ! } while (--len != 0); 260 stb %o4, [%o1 - 1] 261 ba Lcopyin_done 262 clr %o0 263 NOTREACHED 264 265 ! lowest bit matches, so we can copy by words, if nothing else 2661: 267 be,a 1f ! if (src & 1) { 268 btst 2, %o3 ! [delay slot: if (t & 2)] 269 270 ! although low bits match, both are 1: must copy 1 byte to align 271 ldsba [%o0] %asi, %o4 ! *dst++ = *src++; 272 stb %o4, [%o1] 273 inc %o0 274 inc %o1 275 dec %o2 ! len--; 276 btst 2, %o3 ! } [if (t & 2)] 2771: 278 be,a 1f ! if (t & 2) { 279 btst 2, %o0 ! [delay slot: if (src & 2)] 280 dec 2, %o2 ! len -= 2; 2810: 282 ldsha [%o0] %asi, %o4 ! do { 283 sth %o4, [%o1] ! *(short *)dst = *(short *)src; 284 inc 2, %o0 ! dst += 2, src += 2; 285 deccc 2, %o2 ! } while ((len -= 2) >= 0); 286 bge 0b 287 inc 2, %o1 288 b Lcopyin_mopb ! goto mop_up_byte; 289 btst 1, %o2 ! } [delay slot: if (len & 1)] 290 NOTREACHED 291 292 ! low two bits match, so we can copy by longwords 2931: 294 be,a 1f ! if (src & 2) { 295 btst 4, %o3 ! [delay slot: if (t & 4)] 296 297 ! although low 2 bits match, they are 10: must copy one short to align 298 ldsha [%o0] %asi, %o4 ! (*short *)dst = *(short *)src; 299 sth %o4, [%o1] 300 inc 2, %o0 ! dst += 2; 301 inc 2, %o1 ! src += 2; 302 dec 2, %o2 ! len -= 2; 303 btst 4, %o3 ! } [if (t & 4)] 3041: 305 be,a 1f ! if (t & 4) { 306 btst 4, %o0 ! [delay slot: if (src & 4)] 307 dec 4, %o2 ! len -= 4; 3080: 309 lduwa [%o0] %asi, %o4 ! do { 310 st %o4, [%o1] ! *(int *)dst = *(int *)src; 311 inc 4, %o0 ! dst += 4, src += 4; 312 deccc 4, %o2 ! } while ((len -= 4) >= 0); 313 bge 0b 314 inc 4, %o1 315 b Lcopyin_mopw ! goto mop_up_word_and_byte; 316 btst 2, %o2 ! } [delay slot: if (len & 2)] 317 NOTREACHED 318 319 ! low three bits match, so we can copy by doublewords 3201: 321 be 1f ! if (src & 4) { 322 dec 8, %o2 ! [delay slot: len -= 8] 323 lduwa [%o0] %asi, %o4 ! *(int *)dst = *(int *)src; 324 st %o4, [%o1] 325 inc 4, %o0 ! dst += 4, src += 4, len -= 4; 326 inc 4, %o1 327 dec 4, %o2 ! } 3281: 329Lcopyin_doubles: 330 ldxa [%o0] %asi, %g1 ! do { 331 stx %g1, [%o1] ! *(double *)dst = *(double *)src; 332 inc 8, %o0 ! dst += 8, src += 8; 333 deccc 8, %o2 ! } while ((len -= 8) >= 0); 334 bge Lcopyin_doubles 335 inc 8, %o1 336 337 ! check for a usual case again (save work) 338 btst 7, %o2 ! if ((len & 7) == 0) 339 be Lcopyin_done ! goto copyin_done; 340 341 btst 4, %o2 ! if ((len & 4)) == 0) 342 be,a Lcopyin_mopw ! goto mop_up_word_and_byte; 343 btst 2, %o2 ! [delay slot: if (len & 2)] 344 lduwa [%o0] %asi, %o4 ! *(int *)dst = *(int *)src; 345 st %o4, [%o1] 346 inc 4, %o0 ! dst += 4; 347 inc 4, %o1 ! src += 4; 348 btst 2, %o2 ! } [if (len & 2)] 349 3501: 351 ! mop up trailing word (if present) and byte (if present). 352Lcopyin_mopw: 353 be Lcopyin_mopb ! no word, go mop up byte 354 btst 1, %o2 ! [delay slot: if (len & 1)] 355 ldsha [%o0] %asi, %o4 ! *(short *)dst = *(short *)src; 356 be Lcopyin_done ! if ((len & 1) == 0) goto done; 357 sth %o4, [%o1] 358 ldsba [%o0 + 2] %asi, %o4 ! dst[2] = src[2]; 359 stb %o4, [%o1 + 2] 360 ba Lcopyin_done 361 clr %o0 362 NOTREACHED 363 364 ! mop up trailing byte (if present). 365Lcopyin_mopb: 366 be,a Lcopyin_done 367 nop 368 ldsba [%o0] %asi, %o4 369 stb %o4, [%o1] 370 371Lcopyin_done: 372 sethi %hi(CPCB), %o3 373! stb %o4,[%o1] ! Store last byte -- should not be needed 374 LDPTR [%o3 + %lo(CPCB)], %o3 375 membar #Sync 376 STPTR %g0, [%o3 + PCB_ONFAULT] 377 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore ASI 378 retl 379 clr %o0 ! return 0 380 381/* 382 * copyout(src, dst, len) 383 * 384 * Copy specified amount of data from kernel to user space. 385 * Just like copyin, except that the `dst' addresses are user space 386 * rather than the `src' addresses. 387 * 388 * This is a modified version of memcpy that uses ASI_AIUS. When 389 * memcpy is optimized to use block copy ASIs, this should be also. 390 */ 391 /* 392 * This needs to be reimplemented to really do the copy. 393 */ 394ENTRY(copyout) 395 /* 396 * ******NOTE****** this depends on memcpy() not using %g7 397 */ 398#ifdef NOTDEF_DEBUG 399 save %sp, -CC64FSZ, %sp 400 set 1f, %o0 401 mov %i0, %o1 402 set CTX_SECONDARY, %o4 403 mov %i1, %o2 404 ldxa [%o4] ASI_DMMU, %o4 405 call printf 406 mov %i2, %o3 407 restore 408 .data 4091: .asciz "copyout: src=%x dest=%x len=%x ctx=%d\n" 410 _ALIGN 411 .text 412#endif 413Ldocopy: 414 sethi %hi(CPCB), %o3 415 wr %g0, ASI_AIUS, %asi 416 LDPTR [%o3 + %lo(CPCB)], %o3 417 set Lcopyfault, %o4 418! mov %o7, %g7 ! save return address 419 membar #Sync 420 STPTR %o4, [%o3 + PCB_ONFAULT] 421 cmp %o2, BCOPY_SMALL 422Lcopyout_start: 423 membar #StoreStore 424 bge,a Lcopyout_fancy ! if >= this many, go be fancy. 425 btst 7, %o0 ! (part of being fancy) 426 427 /* 428 * Not much to copy, just do it a byte at a time. 429 */ 430 deccc %o2 ! while (--len >= 0) 431 bl 1f 432 .empty 4330: 434 inc %o0 435 ldsb [%o0 - 1], %o4! (++dst)[-1] = *src++; 436 stba %o4, [%o1] %asi 437 deccc %o2 438 bge 0b 439 inc %o1 4401: 441 ba Lcopyout_done 442 clr %o0 443 NOTREACHED 444 445 /* 446 * Plenty of data to copy, so try to do it optimally. 447 */ 448Lcopyout_fancy: 449 ! check for common case first: everything lines up. 450! btst 7, %o0 ! done already 451 bne 1f 452 .empty 453 btst 7, %o1 454 be,a Lcopyout_doubles 455 dec 8, %o2 ! if all lined up, len -= 8, goto copyout_doubes 456 457 ! If the low bits match, we can make these line up. 4581: 459 xor %o0, %o1, %o3 ! t = src ^ dst; 460 btst 1, %o3 ! if (t & 1) { 461 be,a 1f 462 btst 1, %o0 ! [delay slot: if (src & 1)] 463 464 ! low bits do not match, must copy by bytes. 4650: 466 ldsb [%o0], %o4 ! do { 467 inc %o0 ! (++dst)[-1] = *src++; 468 inc %o1 469 deccc %o2 470 bnz 0b ! } while (--len != 0); 471 stba %o4, [%o1 - 1] %asi 472 ba Lcopyout_done 473 clr %o0 474 NOTREACHED 475 476 ! lowest bit matches, so we can copy by words, if nothing else 4771: 478 be,a 1f ! if (src & 1) { 479 btst 2, %o3 ! [delay slot: if (t & 2)] 480 481 ! although low bits match, both are 1: must copy 1 byte to align 482 ldsb [%o0], %o4 ! *dst++ = *src++; 483 stba %o4, [%o1] %asi 484 inc %o0 485 inc %o1 486 dec %o2 ! len--; 487 btst 2, %o3 ! } [if (t & 2)] 4881: 489 be,a 1f ! if (t & 2) { 490 btst 2, %o0 ! [delay slot: if (src & 2)] 491 dec 2, %o2 ! len -= 2; 4920: 493 ldsh [%o0], %o4 ! do { 494 stha %o4, [%o1] %asi ! *(short *)dst = *(short *)src; 495 inc 2, %o0 ! dst += 2, src += 2; 496 deccc 2, %o2 ! } while ((len -= 2) >= 0); 497 bge 0b 498 inc 2, %o1 499 b Lcopyout_mopb ! goto mop_up_byte; 500 btst 1, %o2 ! } [delay slot: if (len & 1)] 501 NOTREACHED 502 503 ! low two bits match, so we can copy by longwords 5041: 505 be,a 1f ! if (src & 2) { 506 btst 4, %o3 ! [delay slot: if (t & 4)] 507 508 ! although low 2 bits match, they are 10: must copy one short to align 509 ldsh [%o0], %o4 ! (*short *)dst = *(short *)src; 510 stha %o4, [%o1] %asi 511 inc 2, %o0 ! dst += 2; 512 inc 2, %o1 ! src += 2; 513 dec 2, %o2 ! len -= 2; 514 btst 4, %o3 ! } [if (t & 4)] 5151: 516 be,a 1f ! if (t & 4) { 517 btst 4, %o0 ! [delay slot: if (src & 4)] 518 dec 4, %o2 ! len -= 4; 5190: 520 lduw [%o0], %o4 ! do { 521 sta %o4, [%o1] %asi ! *(int *)dst = *(int *)src; 522 inc 4, %o0 ! dst += 4, src += 4; 523 deccc 4, %o2 ! } while ((len -= 4) >= 0); 524 bge 0b 525 inc 4, %o1 526 b Lcopyout_mopw ! goto mop_up_word_and_byte; 527 btst 2, %o2 ! } [delay slot: if (len & 2)] 528 NOTREACHED 529 530 ! low three bits match, so we can copy by doublewords 5311: 532 be 1f ! if (src & 4) { 533 dec 8, %o2 ! [delay slot: len -= 8] 534 lduw [%o0], %o4 ! *(int *)dst = *(int *)src; 535 sta %o4, [%o1] %asi 536 inc 4, %o0 ! dst += 4, src += 4, len -= 4; 537 inc 4, %o1 538 dec 4, %o2 ! } 5391: 540Lcopyout_doubles: 541 ldx [%o0], %g1 ! do { 542 stxa %g1, [%o1] %asi ! *(double *)dst = *(double *)src; 543 inc 8, %o0 ! dst += 8, src += 8; 544 deccc 8, %o2 ! } while ((len -= 8) >= 0); 545 bge Lcopyout_doubles 546 inc 8, %o1 547 548 ! check for a usual case again (save work) 549 btst 7, %o2 ! if ((len & 7) == 0) 550 be Lcopyout_done ! goto copyout_done; 551 552 btst 4, %o2 ! if ((len & 4)) == 0) 553 be,a Lcopyout_mopw ! goto mop_up_word_and_byte; 554 btst 2, %o2 ! [delay slot: if (len & 2)] 555 lduw [%o0], %o4 ! *(int *)dst = *(int *)src; 556 sta %o4, [%o1] %asi 557 inc 4, %o0 ! dst += 4; 558 inc 4, %o1 ! src += 4; 559 btst 2, %o2 ! } [if (len & 2)] 560 5611: 562 ! mop up trailing word (if present) and byte (if present). 563Lcopyout_mopw: 564 be Lcopyout_mopb ! no word, go mop up byte 565 btst 1, %o2 ! [delay slot: if (len & 1)] 566 ldsh [%o0], %o4 ! *(short *)dst = *(short *)src; 567 be Lcopyout_done ! if ((len & 1) == 0) goto done; 568 stha %o4, [%o1] %asi 569 ldsb [%o0 + 2], %o4 ! dst[2] = src[2]; 570 stba %o4, [%o1 + 2] %asi 571 ba Lcopyout_done 572 clr %o0 573 NOTREACHED 574 575 ! mop up trailing byte (if present). 576Lcopyout_mopb: 577 be,a Lcopyout_done 578 nop 579 ldsb [%o0], %o4 580 stba %o4, [%o1] %asi 581 582Lcopyout_done: 583 sethi %hi(CPCB), %o3 584 LDPTR [%o3 + %lo(CPCB)], %o3 585 membar #Sync 586 STPTR %g0, [%o3 + PCB_ONFAULT] 587! jmp %g7 + 8 ! Original instr 588 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore ASI 589 membar #StoreStore|#StoreLoad 590 retl ! New instr 591 clr %o0 ! return 0 592 593! Copyin or copyout fault. Clear cpcb->pcb_onfault and return error. 594! Note that although we were in memcpy, there is no state to clean up; 595! the only special thing is that we have to return to [g7 + 8] rather than 596! [o7 + 8]. 597Lcopyfault: 598 sethi %hi(CPCB), %o3 599 LDPTR [%o3 + %lo(CPCB)], %o3 600 STPTR %g0, [%o3 + PCB_ONFAULT] 601 membar #StoreStore|#StoreLoad 602#ifdef NOTDEF_DEBUG 603 save %sp, -CC64FSZ, %sp 604 set 1f, %o0 605 call printf 606 nop 607 restore 608 .data 6091: .asciz "copyfault: fault occurred\n" 610 _ALIGN 611 .text 612#endif 613 retl 614 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore ASI 615 616 617STRONG_ALIAS(ucas_int,ucas_32) 618#ifdef _LP64 619STRONG_ALIAS(ucas_ptr,ucas_64) 620#else 621STRONG_ALIAS(ucas_ptr,ucas_32) 622#endif 623 624/* 625 * Compare-and-swap the pointer in the user-space. 626 * 627 * int ucas_32(volatile int32_t *uptr, int32_t old, int32_t new, int32_t *ret); 628 */ 629ENTRY(ucas_32) 630 sethi %hi(CPCB), %o4 631 wr %g0, ASI_AIUS, %asi 632 LDPTR [%o4 + %lo(CPCB)], %o4 633 set Lcopyfault, %o5 ! reusing copyin/copyout 634 membar #Sync ! fault handler 635 STPTR %o5, [%o4 + PCB_ONFAULT] 636 637 casa [%o0] %asi, %o1, %o2 ! cas, result in %o2 638 639 sethi %hi(CPCB), %o4 640 LDPTR [%o4 + %lo(CPCB)], %o4 641 membar #Sync 642 STPTR %g0, [%o4 + PCB_ONFAULT] 643 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore ASI 644 membar #StoreStore|#StoreLoad 645 stw %o2, [%o3] ! store the cas result in *result 646 retl 647 clr %o0 ! return 0 648 649/* 650 * Compare-and-swap the 64-bit integer in the user-space. 651 * 652 * int ucas_64(volatile int64_t *uptr, int64_t old, int64_t new, int64_t *ret); 653 */ 654ENTRY(ucas_64) 655#ifndef _LP64 656 COMBINE(%o1, %o2, %o1) 657 COMBINE(%o3, %o4, %o2) 658 mov %o5, %o3 659#endif 660 sethi %hi(CPCB), %o4 661 wr %g0, ASI_AIUS, %asi 662 LDPTR [%o4 + %lo(CPCB)], %o4 663 set Lcopyfault, %o5 ! reusing copyin/copyout 664 membar #Sync ! fault handler 665 STPTR %o5, [%o4 + PCB_ONFAULT] 666 667 casxa [%o0] %asi, %o1, %o2 ! cas, result in %o2 668 669 sethi %hi(CPCB), %o4 670 LDPTR [%o4 + %lo(CPCB)], %o4 671 membar #Sync 672 STPTR %g0, [%o4 + PCB_ONFAULT] 673 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore ASI 674 membar #StoreStore|#StoreLoad 675 stx %o2, [%o3] ! store the cas result in *result 676 retl 677 clr %o0 ! return 0 678 679/* 680 * {fu,su}{,i}{byte,word} 681 */ 682Lfserr: 683 STPTR %g0, [%o2 + PCB_ONFAULT]! error in r/w, clear pcb_onfault 684 membar #StoreStore|#StoreLoad 685Lfsbadaddr: 686#ifndef _LP64 687 mov -1, %o1 688#endif 689 retl ! and return error indicator 690 mov -1, %o0 691 692 /* 693 * This is just like Lfserr, but it's a global label that allows 694 * mem_access_fault() to check to see that we don't want to try to 695 * page in the fault. It's used by fuswintr() etc. 696 */ 697 .globl _C_LABEL(Lfsbail) 698_C_LABEL(Lfsbail): 699 STPTR %g0, [%o2 + PCB_ONFAULT]! error in r/w, clear pcb_onfault 700 membar #StoreStore|#StoreLoad 701 retl ! and return error indicator 702 mov -1, %o0 703 704 /* 705 * Like fusword but callable from interrupt context. 706 * Fails if data isn't resident. 707 */ 708ENTRY(fuswintr) 709 sethi %hi(CPCB), %o2 ! cpcb->pcb_onfault = _Lfsbail; 710 LDPTR [%o2 + %lo(CPCB)], %o2 711 set _C_LABEL(Lfsbail), %o3 712 STPTR %o3, [%o2 + PCB_ONFAULT] 713 membar #Sync 714 lduha [%o0] ASI_AIUS, %o0 ! fetch the halfword 715 membar #Sync 716 STPTR %g0, [%o2 + PCB_ONFAULT]! but first clear onfault 717 retl ! made it 718 membar #StoreStore|#StoreLoad 719 720ENTRY(fusword) 721 sethi %hi(CPCB), %o2 ! cpcb->pcb_onfault = Lfserr; 722 LDPTR [%o2 + %lo(CPCB)], %o2 723 set Lfserr, %o3 724 STPTR %o3, [%o2 + PCB_ONFAULT] 725 membar #Sync 726 lduha [%o0] ASI_AIUS, %o0 ! fetch the halfword 727 membar #Sync 728 STPTR %g0, [%o2 + PCB_ONFAULT]! but first clear onfault 729 retl ! made it 730 membar #StoreStore|#StoreLoad 731 732ALTENTRY(fuibyte) 733ENTRY(fubyte) 734 sethi %hi(CPCB), %o2 ! cpcb->pcb_onfault = Lfserr; 735 LDPTR [%o2 + %lo(CPCB)], %o2 736 set Lfserr, %o3 737 STPTR %o3, [%o2 + PCB_ONFAULT] 738 membar #Sync 739 lduba [%o0] ASI_AIUS, %o0 ! fetch the byte 740 membar #Sync 741 STPTR %g0, [%o2 + PCB_ONFAULT]! but first clear onfault 742 retl ! made it 743 membar #StoreStore|#StoreLoad 744 745ENTRY(suswintr) 746 sethi %hi(CPCB), %o2 ! cpcb->pcb_onfault = _Lfsbail; 747 LDPTR [%o2 + %lo(CPCB)], %o2 748 set _C_LABEL(Lfsbail), %o3 749 STPTR %o3, [%o2 + PCB_ONFAULT] 750 membar #Sync 751 stha %o1, [%o0] ASI_AIUS ! store the halfword 752 membar #Sync 753 STPTR %g0, [%o2 + PCB_ONFAULT]! made it, clear onfault 754 membar #StoreStore|#StoreLoad 755 retl ! and return 0 756 clr %o0 757 758ENTRY(susword) 759 sethi %hi(CPCB), %o2 ! cpcb->pcb_onfault = Lfserr; 760 LDPTR [%o2 + %lo(CPCB)], %o2 761 set Lfserr, %o3 762 STPTR %o3, [%o2 + PCB_ONFAULT] 763 membar #Sync 764 stha %o1, [%o0] ASI_AIUS ! store the halfword 765 membar #Sync 766 STPTR %g0, [%o2 + PCB_ONFAULT]! made it, clear onfault 767 membar #StoreStore|#StoreLoad 768 retl ! and return 0 769 clr %o0 770 771ENTRY(subyte) 772 sethi %hi(CPCB), %o2 ! cpcb->pcb_onfault = Lfserr; 773 LDPTR [%o2 + %lo(CPCB)], %o2 774 set Lfserr, %o3 775 STPTR %o3, [%o2 + PCB_ONFAULT] 776 membar #Sync 777 stba %o1, [%o0] ASI_AIUS ! store the byte 778 membar #Sync 779 STPTR %g0, [%o2 + PCB_ONFAULT]! made it, clear onfault 780 membar #StoreStore|#StoreLoad 781 retl ! and return 0 782 clr %o0 783 784/* probeget and probeset are meant to be used during autoconfiguration */ 785/* 786 * The following probably need to be changed, but to what I don't know. 787 */ 788 789/* 790 * uint64_t 791 * probeget(addr, asi, size) 792 * paddr_t addr; 793 * int asi; 794 * int size; 795 * 796 * Read or write a (byte,word,longword) from the given address. 797 * Like {fu,su}{byte,halfword,word} but our caller is supposed 798 * to know what he is doing... the address can be anywhere. 799 * 800 * We optimize for space, rather than time, here. 801 */ 802ENTRY(probeget) 803#ifndef _LP64 804 !! Shuffle the args around into LP64 format 805 COMBINE(%o0, %o1, %o0) 806 mov %o2, %o1 807 mov %o3, %o2 808#endif 809 mov %o2, %o4 810 ! %o0 = addr, %o1 = asi, %o4 = (1,2,4) 811 sethi %hi(CPCB), %o2 812 LDPTR [%o2 + %lo(CPCB)], %o2 ! cpcb->pcb_onfault = Lfserr; 813#ifdef _LP64 814 set _C_LABEL(Lfsbail), %o5 815#else 816 set _C_LABEL(Lfsprobe), %o5 817#endif 818 STPTR %o5, [%o2 + PCB_ONFAULT] 819 or %o0, 0x9, %o3 ! if (PHYS_ASI(asi)) { 820 sub %o3, 0x1d, %o3 821 brz,a %o3, 0f 822 mov %g0, %o5 823 DLFLUSH(%o0,%o5) ! flush cache line 824 ! } 8250: 826#ifndef _LP64 827 rdpr %pstate, %g1 828 wrpr %g1, PSTATE_AM, %pstate 829#endif 830 btst 1, %o4 831 wr %o1, 0, %asi 832 membar #Sync 833 bz 0f ! if (len & 1) 834 btst 2, %o4 835 ba,pt %icc, 1f 836 lduba [%o0] %asi, %o0 ! value = *(char *)addr; 8370: 838 bz 0f ! if (len & 2) 839 btst 4, %o4 840 ba,pt %icc, 1f 841 lduha [%o0] %asi, %o0 ! value = *(short *)addr; 8420: 843 bz 0f ! if (len & 4) 844 btst 8, %o4 845 ba,pt %icc, 1f 846 lda [%o0] %asi, %o0 ! value = *(int *)addr; 8470: 848 ldxa [%o0] %asi, %o0 ! value = *(long *)addr; 8491: 850#ifndef _LP64 851 SPLIT(%o0, %o0, %o1) 852#endif 853 membar #Sync 854#ifndef _LP64 855 wrpr %g1, 0, %pstate 856#endif 857 brz %o5, 1f ! if (cache flush addr != 0) 858 nop 859 DLFLUSH2(%o5) ! flush cache line again 8601: 861 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 862 STPTR %g0, [%o2 + PCB_ONFAULT] 863 retl ! made it, clear onfault and return 864 membar #StoreStore|#StoreLoad 865 866 /* 867 * Fault handler for probeget 868 */ 869_C_LABEL(Lfsprobe): 870#ifndef _LP64 871 wrpr %g1, 0, %pstate 872#endif 873 STPTR %g0, [%o2 + PCB_ONFAULT]! error in r/w, clear pcb_onfault 874 mov -1, %o1 875 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 876 membar #StoreStore|#StoreLoad 877 retl ! and return error indicator 878 mov -1, %o0 879 880/* 881 * probeset(addr, asi, size, val) 882 * paddr_t addr; 883 * int asi; 884 * int size; 885 * long val; 886 * 887 * As above, but we return 0 on success. 888 */ 889ENTRY(probeset) 890#ifndef _LP64 891 !! Shuffle the args around into LP64 format 892 COMBINE(%o0, %o1, %o0) 893 mov %o2, %o1 894 mov %o3, %o2 895 COMBINE(%o4, %o5, %o3) 896#endif 897 mov %o2, %o4 898 ! %o0 = addr, %o1 = asi, %o4 = (1,2,4), %o3 = val 899 sethi %hi(CPCB), %o2 ! Lfserr requires CPCB in %o2 900 LDPTR [%o2 + %lo(CPCB)], %o2 ! cpcb->pcb_onfault = Lfserr; 901 set _C_LABEL(Lfsbail), %o5 902 STPTR %o5, [%o2 + PCB_ONFAULT] 903 btst 1, %o4 904 wr %o1, 0, %asi 905 membar #Sync 906 bz 0f ! if (len & 1) 907 btst 2, %o4 908 ba,pt %icc, 1f 909 stba %o3, [%o0] %asi ! *(char *)addr = value; 9100: 911 bz 0f ! if (len & 2) 912 btst 4, %o4 913 ba,pt %icc, 1f 914 stha %o3, [%o0] %asi ! *(short *)addr = value; 9150: 916 bz 0f ! if (len & 4) 917 btst 8, %o4 918 ba,pt %icc, 1f 919 sta %o3, [%o0] %asi ! *(int *)addr = value; 9200: 921 bz Lfserr ! if (len & 8) 922 ba,pt %icc, 1f 923 sta %o3, [%o0] %asi ! *(int *)addr = value; 9241: membar #Sync 925 clr %o0 ! made it, clear onfault and return 0 926 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 927 STPTR %g0, [%o2 + PCB_ONFAULT] 928 retl 929 membar #StoreStore|#StoreLoad 930 931 932/* 933 * kcopy() is exactly like bcopy except that it set pcb_onfault such that 934 * when a fault occurs, it is able to return -1 to indicate this to the 935 * caller. 936 */ 937ENTRY(kcopy) 938#ifdef DEBUG 939 set pmapdebug, %o4 940 ld [%o4], %o4 941 btst 0x80, %o4 ! PDB_COPY 942 bz,pt %icc, 3f 943 nop 944 save %sp, -CC64FSZ, %sp 945 mov %i0, %o1 946 set 2f, %o0 947 mov %i1, %o2 948 call printf 949 mov %i2, %o3 950! ta 1; nop 951 restore 952 .data 9532: .asciz "kcopy(%p->%p,%x)\n" 954 _ALIGN 955 .text 9563: 957#endif 958 sethi %hi(CPCB), %o5 ! cpcb->pcb_onfault = Lkcerr; 959 LDPTR [%o5 + %lo(CPCB)], %o5 960 set Lkcerr, %o3 961 LDPTR [%o5 + PCB_ONFAULT], %g1! save current onfault handler 962 membar #LoadStore 963 STPTR %o3, [%o5 + PCB_ONFAULT] 964 membar #StoreStore|#StoreLoad 965 966 cmp %o2, BCOPY_SMALL 967Lkcopy_start: 968 bge,a Lkcopy_fancy ! if >= this many, go be fancy. 969 btst 7, %o0 ! (part of being fancy) 970 971 /* 972 * Not much to copy, just do it a byte at a time. 973 */ 974 deccc %o2 ! while (--len >= 0) 975 bl 1f 976 .empty 9770: 978 ldsb [%o0], %o4 ! *dst++ = *src++; 979 inc %o0 980 stb %o4, [%o1] 981 deccc %o2 982 bge 0b 983 inc %o1 9841: 985 membar #Sync ! Make sure all fauls are processed 986 STPTR %g1, [%o5 + PCB_ONFAULT]! restore fault handler 987 membar #StoreStore|#StoreLoad 988 retl 989 clr %o0 990 NOTREACHED 991 992 /* 993 * Plenty of data to copy, so try to do it optimally. 994 */ 995Lkcopy_fancy: 996 ! check for common case first: everything lines up. 997! btst 7, %o0 ! done already 998 bne 1f 999 .empty 1000 btst 7, %o1 1001 be,a Lkcopy_doubles 1002 dec 8, %o2 ! if all lined up, len -= 8, goto kcopy_doubes 1003 1004 ! If the low bits match, we can make these line up. 10051: 1006 xor %o0, %o1, %o3 ! t = src ^ dst; 1007 btst 1, %o3 ! if (t & 1) { 1008 be,a 1f 1009 btst 1, %o0 ! [delay slot: if (src & 1)] 1010 1011 ! low bits do not match, must copy by bytes. 10120: 1013 ldsb [%o0], %o4 ! do { 1014 inc %o0 ! *dst++ = *src++; 1015 stb %o4, [%o1] 1016 deccc %o2 1017 bnz 0b ! } while (--len != 0); 1018 inc %o1 1019 membar #Sync ! Make sure all traps are taken 1020 STPTR %g1, [%o5 + PCB_ONFAULT]! restore fault handler 1021 membar #StoreStore|#StoreLoad 1022 retl 1023 clr %o0 1024 NOTREACHED 1025 1026 ! lowest bit matches, so we can copy by words, if nothing else 10271: 1028 be,a 1f ! if (src & 1) { 1029 btst 2, %o3 ! [delay slot: if (t & 2)] 1030 1031 ! although low bits match, both are 1: must copy 1 byte to align 1032 ldsb [%o0], %o4 ! *dst++ = *src++; 1033 inc %o0 1034 stb %o4, [%o1] 1035 dec %o2 ! len--; 1036 inc %o1 1037 btst 2, %o3 ! } [if (t & 2)] 10381: 1039 be,a 1f ! if (t & 2) { 1040 btst 2, %o0 ! [delay slot: if (src & 2)] 1041 dec 2, %o2 ! len -= 2; 10420: 1043 ldsh [%o0], %o4 ! do { 1044 inc 2, %o0 ! dst += 2, src += 2; 1045 sth %o4, [%o1] ! *(short *)dst = *(short *)src; 1046 deccc 2, %o2 ! } while ((len -= 2) >= 0); 1047 bge 0b 1048 inc 2, %o1 1049 b Lkcopy_mopb ! goto mop_up_byte; 1050 btst 1, %o2 ! } [delay slot: if (len & 1)] 1051 NOTREACHED 1052 1053 ! low two bits match, so we can copy by longwords 10541: 1055 be,a 1f ! if (src & 2) { 1056 btst 4, %o3 ! [delay slot: if (t & 4)] 1057 1058 ! although low 2 bits match, they are 10: must copy one short to align 1059 ldsh [%o0], %o4 ! (*short *)dst = *(short *)src; 1060 inc 2, %o0 ! dst += 2; 1061 sth %o4, [%o1] 1062 dec 2, %o2 ! len -= 2; 1063 inc 2, %o1 ! src += 2; 1064 btst 4, %o3 ! } [if (t & 4)] 10651: 1066 be,a 1f ! if (t & 4) { 1067 btst 4, %o0 ! [delay slot: if (src & 4)] 1068 dec 4, %o2 ! len -= 4; 10690: 1070 ld [%o0], %o4 ! do { 1071 inc 4, %o0 ! dst += 4, src += 4; 1072 st %o4, [%o1] ! *(int *)dst = *(int *)src; 1073 deccc 4, %o2 ! } while ((len -= 4) >= 0); 1074 bge 0b 1075 inc 4, %o1 1076 b Lkcopy_mopw ! goto mop_up_word_and_byte; 1077 btst 2, %o2 ! } [delay slot: if (len & 2)] 1078 NOTREACHED 1079 1080 ! low three bits match, so we can copy by doublewords 10811: 1082 be 1f ! if (src & 4) { 1083 dec 8, %o2 ! [delay slot: len -= 8] 1084 ld [%o0], %o4 ! *(int *)dst = *(int *)src; 1085 inc 4, %o0 ! dst += 4, src += 4, len -= 4; 1086 st %o4, [%o1] 1087 dec 4, %o2 ! } 1088 inc 4, %o1 10891: 1090Lkcopy_doubles: 1091 ldx [%o0], %g5 ! do { 1092 inc 8, %o0 ! dst += 8, src += 8; 1093 stx %g5, [%o1] ! *(double *)dst = *(double *)src; 1094 deccc 8, %o2 ! } while ((len -= 8) >= 0); 1095 bge Lkcopy_doubles 1096 inc 8, %o1 1097 1098 ! check for a usual case again (save work) 1099 btst 7, %o2 ! if ((len & 7) == 0) 1100 be Lkcopy_done ! goto kcopy_done; 1101 1102 btst 4, %o2 ! if ((len & 4)) == 0) 1103 be,a Lkcopy_mopw ! goto mop_up_word_and_byte; 1104 btst 2, %o2 ! [delay slot: if (len & 2)] 1105 ld [%o0], %o4 ! *(int *)dst = *(int *)src; 1106 inc 4, %o0 ! dst += 4; 1107 st %o4, [%o1] 1108 inc 4, %o1 ! src += 4; 1109 btst 2, %o2 ! } [if (len & 2)] 1110 11111: 1112 ! mop up trailing word (if present) and byte (if present). 1113Lkcopy_mopw: 1114 be Lkcopy_mopb ! no word, go mop up byte 1115 btst 1, %o2 ! [delay slot: if (len & 1)] 1116 ldsh [%o0], %o4 ! *(short *)dst = *(short *)src; 1117 be Lkcopy_done ! if ((len & 1) == 0) goto done; 1118 sth %o4, [%o1] 1119 ldsb [%o0 + 2], %o4 ! dst[2] = src[2]; 1120 stb %o4, [%o1 + 2] 1121 membar #Sync ! Make sure all traps are taken 1122 STPTR %g1, [%o5 + PCB_ONFAULT]! restore fault handler 1123 membar #StoreStore|#StoreLoad 1124 retl 1125 clr %o0 1126 NOTREACHED 1127 1128 ! mop up trailing byte (if present). 1129Lkcopy_mopb: 1130 bne,a 1f 1131 ldsb [%o0], %o4 1132 1133Lkcopy_done: 1134 membar #Sync ! Make sure all traps are taken 1135 STPTR %g1, [%o5 + PCB_ONFAULT]! restore fault handler 1136 membar #StoreStore|#StoreLoad 1137 retl 1138 clr %o0 1139 NOTREACHED 1140 11411: 1142 stb %o4, [%o1] 1143 membar #Sync ! Make sure all traps are taken 1144 STPTR %g1, [%o5 + PCB_ONFAULT]! restore fault handler 1145 membar #StoreStore|#StoreLoad 1146 retl 1147 clr %o0 1148 NOTREACHED 1149 1150Lkcerr: 1151#ifdef DEBUG 1152 set pmapdebug, %o4 1153 ld [%o4], %o4 1154 btst 0x80, %o4 ! PDB_COPY 1155 bz,pt %icc, 3f 1156 nop 1157 save %sp, -CC64FSZ, %sp 1158 set 2f, %o0 1159 call printf 1160 nop 1161! ta 1; nop 1162 restore 1163 .data 11642: .asciz "kcopy error\n" 1165 _ALIGN 1166 .text 11673: 1168#endif 1169 STPTR %g1, [%o5 + PCB_ONFAULT]! restore fault handler 1170 retl ! and return error indicator 1171 membar #StoreStore|#StoreLoad 1172 NOTREACHED 1173 1174/* 1175 * copystr(fromaddr, toaddr, maxlength, &lencopied) 1176 * 1177 * Copy a null terminated string from one point to another in 1178 * the kernel address space. (This is a leaf procedure, but 1179 * it does not seem that way to the C compiler.) 1180 */ 1181ENTRY(copystr) 1182 brgz,pt %o2, 0f ! Make sure len is valid 1183 mov %o1, %o5 ! to0 = to; 1184 retl 1185 mov ENAMETOOLONG, %o0 11860: ! loop: 1187 ldsb [%o0], %o4 ! c = *from; 1188 tst %o4 1189 stb %o4, [%o1] ! *to++ = c; 1190 be 1f ! if (c == 0) 1191 inc %o1 ! goto ok; 1192 deccc %o2 ! if (--len > 0) { 1193 bg,a 0b ! from++; 1194 inc %o0 ! goto loop; 1195 b 2f ! } 1196 mov ENAMETOOLONG, %o0 ! ret = ENAMETOOLONG; goto done; 11971: ! ok: 1198 clr %o0 ! ret = 0; 11992: 1200 sub %o1, %o5, %o1 ! len = to - to0; 1201 tst %o3 ! if (lencopied) 1202 bnz,a 3f 1203 STPTR %o1, [%o3] ! *lencopied = len; 12043: 1205 retl 1206 nop 1207#ifdef DIAGNOSTIC 12084: 1209 sethi %hi(5f), %o0 1210 call _C_LABEL(panic) 1211 or %lo(5f), %o0, %o0 1212 .data 12135: 1214 .asciz "copystr" 1215 _ALIGN 1216 .text 1217#endif 1218 1219