1/* $OpenBSD: copy_subr.S,v 1.6 2013/06/15 18:38:18 miod Exp $ */ 2/* 3 * Mach Operating System 4 * Copyright (c) 1993-1992 Carnegie Mellon University 5 * Copyright (c) 1991 OMRON Corporation 6 * Copyright (c) 1996 Nivas Madhur 7 * Copyright (c) 1998 Steve Murphree, Jr. 8 * All Rights Reserved. 9 * 10 * Permission to use, copy, modify and distribute this software and its 11 * documentation is hereby granted, provided that both the copyright 12 * notice and this permission notice appear in all copies of the 13 * software, derivative works or modified versions, and any portions 14 * thereof, and that both notices appear in supporting documentation. 15 * 16 * CARNEGIE MELLON AND OMRON ALLOW FREE USE OF THIS SOFTWARE IN ITS "AS IS" 17 * CONDITION. CARNEGIE MELLON AND OMRON DISCLAIM ANY LIABILITY OF ANY KIND 18 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 19 * 20 * Carnegie Mellon requests users of this software to return to 21 * 22 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 23 * School of Computer Science 24 * Carnegie Mellon University 25 * Pittsburgh PA 15213-3890 26 * 27 * any improvements or extensions that they make and grant Carnegie the 28 * rights to redistribute these changes. 29 */ 30 31#include <machine/asm.h> 32 33/* 34 * copy count bytes of data from source to destination 35 * Don Harper (don@omron.co.jp), Omron Corporation. 36 */ 37 38#if defined(MEMCPY) || defined(MEMMOVE) 39#define SRC %r3 40#define DEST %r2 41#define SAVE %r5 42#else 43#define SRC %r2 44#define DEST %r3 45#endif 46#define LEN %r4 47 48#ifdef MEMCPY 49ENTRY(memcpy) 50#endif 51#ifdef MEMMOVE 52ENTRY(memmove) 53#endif 54#ifdef BCOPY 55#define OVBCOPY 56ENTRY(bcopy) 57#endif 58 59#if defined(MEMCPY) || defined(MEMMOVE) 60 or SAVE, DEST, %r0 61#endif 62 63 bcnd eq0,LEN,_ASM_LABEL(bcopy_out) /* nothing to do if == 0 */ 64 65/* 66 * check position of source and destination data 67 */ 68 cmp %r9,SRC,DEST /* compare source address to destination */ 69 bb1 eq,%r9,_ASM_LABEL(bcopy_out) /* nothing to do if equal */ 70#if defined(MEMMOVE) || defined(OVBCOPY) 71 bb1 lo,%r9,_ASM_LABEL(bcopy_reverse) /* reverse copy if src < dest */ 72#endif 73 74/* 75 * source address is greater than destination address, or we do 76 * not have to care about overlapping areas: copy forward 77 */ 78 cmp %r9,LEN,16 /* see if we have at least 16 bytes */ 79 bb1 lt,%r9,_ASM_LABEL(f_byte_copy) /* copy bytes for small data length */ 80/* 81 * determine copy strategy based on alignment of source and destination 82 */ 83 mask %r6,SRC,3 /* get 2 low order bits of source address */ 84 mask %r7,DEST,3 /* get 2 low order bits of destination addr */ 85 mak %r6,%r6,0<4> /* convert source bits to table offset */ 86 mak %r7,%r7,0<2> /* convert destination bits to table offset */ 87 or.u %r12,%r0,%hi16(_ASM_LABEL(f_strat)) 88 or %r12,%r12,%lo16(_ASM_LABEL(f_strat)) 89 addu %r6,%r6,%r7 /* compute final table offset for strategy */ 90 ld %r12,%r12,%r6 /* load the strategy routine */ 91 jmp %r12 /* branch to strategy routine */ 92 93/* 94 * Copy three bytes from src to destination then copy words 95 */ 96ASLOCAL(f_3byte_word_copy) 97 ld.bu %r6,SRC,0 /* load byte from source */ 98 ld.bu %r7,SRC,1 /* load byte from source */ 99 ld.bu %r8,SRC,2 /* load byte from source */ 100 st.b %r6,DEST,0 /* store byte to destination */ 101 st.b %r7,DEST,1 /* store byte to destination */ 102 st.b %r8,DEST,2 /* store byte to destination */ 103 addu SRC,SRC,3 /* increment source pointer */ 104 addu DEST,DEST,3 /* increment destination pointer */ 105 br.n _ASM_LABEL(f_word_copy) /* copy full words */ 106 subu LEN,LEN,3 /* decrement length */ 107 108/* 109 * Copy 1 halfword from src to destination then copy words 110 */ 111ASLOCAL(f_1half_word_copy) 112 ld.hu %r6,SRC,0 /* load half-word from source */ 113 st.h %r6,DEST,0 /* store half-word to destination */ 114 addu SRC,SRC,2 /* increment source pointer */ 115 addu DEST,DEST,2 /* increment destination pointer */ 116 br.n _ASM_LABEL(f_word_copy) /* copy full words */ 117 subu LEN,LEN,2 /* decrement remaining length */ 118 119/* 120 * Copy 1 byte from src to destination then copy words 121 */ 122ASLOCAL(f_1byte_word_copy) 123 ld.bu %r6,SRC,0 /* load 1 byte from source */ 124 st.b %r6,DEST,0 /* store 1 byte to destination */ 125 addu SRC,SRC,1 /* increment source pointer */ 126 addu DEST,DEST,1 /* increment destination pointer */ 127 subu LEN,LEN,1 /* decrement remaining length */ 128 /* FALLTHROUGH */ 129/* 130 * Copy as many full words as possible, 4 words per loop 131 */ 132ASLOCAL(f_word_copy) 133 cmp %r10,LEN,16 /* see if we have 16 bytes remaining */ 134 bb1 lo,%r10,_ASM_LABEL(f_byte_copy) /* not enough left, copy bytes */ 135 ld %r6,SRC,0 /* load first word */ 136 ld %r7,SRC,4 /* load second word */ 137 ld %r8,SRC,8 /* load third word */ 138 ld %r9,SRC,12 /* load fourth word */ 139 st %r6,DEST,0 /* store first word */ 140 st %r7,DEST,4 /* store second word */ 141 st %r8,DEST,8 /* store third word */ 142 st %r9,DEST,12 /* store fourth word */ 143 addu SRC,SRC,16 /* increment source pointer */ 144 addu DEST,DEST,16 /* increment destination pointer */ 145 br.n _ASM_LABEL(f_word_copy) /* branch to copy another block */ 146 subu LEN,LEN,16 /* decrement remaining length */ 147 148ASLOCAL(f_1byte_half_copy) 149 ld.bu %r6,SRC,0 /* load 1 byte from source */ 150 st.b %r6,DEST,0 /* store 1 byte to destination */ 151 addu SRC,SRC,1 /* increment source pointer */ 152 addu DEST,DEST,1 /* increment destination pointer */ 153 subu LEN,LEN,1 /* decrement remaining length */ 154 /* FALLTHROUGH */ 155 156ASLOCAL(f_half_copy) 157 cmp %r10,LEN,16 /* see if we have 16 bytes remaining */ 158 bb1 lo,%r10,_ASM_LABEL(f_byte_copy) /* not enough left, copy bytes */ 159 ld.hu %r6,SRC,0 /* load first half-word */ 160 ld.hu %r7,SRC,2 /* load second half-word */ 161 ld.hu %r8,SRC,4 /* load third half-word */ 162 ld.hu %r9,SRC,6 /* load fourth half-word */ 163 ld.hu %r10,SRC,8 /* load fifth half-word */ 164 ld.hu %r11,SRC,10 /* load sixth half-word */ 165 ld.hu %r12,SRC,12 /* load seventh half-word */ 166 ld.hu %r13,SRC,14 /* load eighth half-word */ 167 st.h %r6,DEST,0 /* store first half-word */ 168 st.h %r7,DEST,2 /* store second half-word */ 169 st.h %r8,DEST,4 /* store third half-word */ 170 st.h %r9,DEST,6 /* store fourth half-word */ 171 st.h %r10,DEST,8 /* store fifth half-word */ 172 st.h %r11,DEST,10 /* store sixth half-word */ 173 st.h %r12,DEST,12 /* store seventh half-word */ 174 st.h %r13,DEST,14 /* store eighth half-word */ 175 addu SRC,SRC,16 /* increment source pointer */ 176 addu DEST,DEST,16 /* increment destination pointer */ 177 br.n _ASM_LABEL(f_half_copy) /* branch to copy another block */ 178 subu LEN,LEN,16 /* decrement remaining length */ 179 180ASLOCAL(f_byte_copy) 181 bcnd eq0,LEN,_ASM_LABEL(bcopy_out) /* branch if nothing left to copy */ 182 ld.bu %r6,SRC,0 /* load byte from source */ 183 st.b %r6,DEST,0 /* store byte in destination */ 184 addu SRC,SRC,1 /* increment source pointer */ 185 addu DEST,DEST,1 /* increment destination pointer */ 186 br.n _ASM_LABEL(f_byte_copy) /* branch for next byte */ 187 subu LEN,LEN,1 /* decrement remaining length */ 188 189#if defined(MEMMOVE) || defined(OVBCOPY) 190/* 191 * source address is less than destination address, copy in reverse 192 */ 193ASLOCAL(bcopy_reverse) 194/* 195 * start copy pointers at end of data 196 */ 197 addu SRC,SRC,LEN /* start source at end of data */ 198 addu DEST,DEST,LEN /* start destination at end of data */ 199/* 200 * check for short data 201 */ 202 cmp %r9,LEN,16 /* see if we have at least 16 bytes */ 203 bb1 lt,%r9,_ASM_LABEL(r_byte_copy) /* copy bytes for small data length */ 204/* 205 * determine copy strategy based on alignment of source and destination 206 */ 207 mask %r6,SRC,3 /* get 2 low order bits of source address */ 208 mask %r7,DEST,3 /* get 2 low order bits of destination addr */ 209 mak %r6,%r6,0<4> /* convert source bits to table offset */ 210 mak %r7,%r7,0<2> /* convert destination bits to table offset */ 211 or.u %r12,%r0,%hi16(_ASM_LABEL(r_strat)) 212 or %r12,%r12,%lo16(_ASM_LABEL(r_strat)) 213 addu %r6,%r6,%r7 /* compute final table offset for strategy */ 214 ld %r12,%r12,%r6 /* load the strategy routine */ 215 jmp %r12 /* branch to strategy routine */ 216 217/* 218 * Copy three bytes from src to destination then copy words 219 */ 220ASLOCAL(r_3byte_word_copy) 221 subu SRC,SRC,3 /* decrement source pointer */ 222 subu DEST,DEST,3 /* decrement destination pointer */ 223 ld.bu %r6,SRC,0 /* load byte from source */ 224 ld.bu %r7,SRC,1 /* load byte from source */ 225 ld.bu %r8,SRC,2 /* load byte from source */ 226 st.b %r6,DEST,0 /* store byte to destination */ 227 st.b %r7,DEST,1 /* store byte to destination */ 228 st.b %r8,DEST,2 /* store byte to destination */ 229 br.n _ASM_LABEL(r_word_copy) /* copy full words */ 230 subu LEN,LEN,3 /* decrement length */ 231 232/* 233 * Copy 1 halfword from src to destination then copy words 234 */ 235ASLOCAL(r_1half_word_copy) 236 subu SRC,SRC,2 /* decrement source pointer */ 237 subu DEST,DEST,2 /* decrement destination pointer */ 238 ld.hu %r6,SRC,0 /* load half-word from source */ 239 st.h %r6,DEST,0 /* store half-word to destination */ 240 br.n _ASM_LABEL(r_word_copy) /* copy full words */ 241 subu LEN,LEN,2 /* decrement remaining length */ 242 243/* 244 * Copy 1 byte from src to destination then copy words 245 */ 246ASLOCAL(r_1byte_word_copy) 247 subu SRC,SRC,1 /* decrement source pointer */ 248 subu DEST,DEST,1 /* decrement destination pointer */ 249 ld.bu %r6,SRC,0 /* load 1 byte from source */ 250 st.b %r6,DEST,0 /* store 1 byte to destination */ 251 subu LEN,LEN,1 /* decrement remaining length */ 252 /* FALLTHROUGH */ 253/* 254 * Copy as many full words as possible, 4 words per loop 255 */ 256ASLOCAL(r_word_copy) 257 cmp %r10,LEN,16 /* see if we have 16 bytes remaining */ 258 bb1 lo,%r10,_ASM_LABEL(r_byte_copy) /* not enough left, copy bytes */ 259 subu SRC,SRC,16 /* decrement source pointer */ 260 subu DEST,DEST,16 /* decrement destination pointer */ 261 ld %r6,SRC,0 /* load first word */ 262 ld %r7,SRC,4 /* load second word */ 263 ld %r8,SRC,8 /* load third word */ 264 ld %r9,SRC,12 /* load fourth word */ 265 st %r6,DEST,0 /* store first word */ 266 st %r7,DEST,4 /* store second word */ 267 st %r8,DEST,8 /* store third word */ 268 st %r9,DEST,12 /* store fourth word */ 269 br.n _ASM_LABEL(r_word_copy) /* branch to copy another block */ 270 subu LEN,LEN,16 /* decrement remaining length */ 271 272ASLOCAL(r_1byte_half_copy) 273 subu SRC,SRC,1 /* decrement source pointer */ 274 subu DEST,DEST,1 /* decrement destination pointer */ 275 ld.bu %r6,SRC,0 /* load 1 byte from source */ 276 st.b %r6,DEST,0 /* store 1 byte to destination */ 277 subu LEN,LEN,1 /* decrement remaining length */ 278 /* FALLTHROUGH */ 279 280ASLOCAL(r_half_copy) 281 cmp %r10,LEN,16 /* see if we have 16 bytes remaining */ 282 bb1 lo,%r10,_ASM_LABEL(r_byte_copy) /* not enough left, copy bytes */ 283 subu SRC,SRC,16 /* decrement source pointer */ 284 subu DEST,DEST,16 /* decrement destination pointer */ 285 ld.hu %r6,SRC,0 /* load first half-word */ 286 ld.hu %r7,SRC,2 /* load second half-word */ 287 ld.hu %r8,SRC,4 /* load third half-word */ 288 ld.hu %r9,SRC,6 /* load fourth half-word */ 289 ld.hu %r10,SRC,8 /* load fifth half-word */ 290 ld.hu %r11,SRC,10 /* load sixth half-word */ 291 ld.hu %r12,SRC,12 /* load seventh half-word */ 292 ld.hu %r13,SRC,14 /* load eighth half-word */ 293 st.h %r6,DEST,0 /* store first half-word */ 294 st.h %r7,DEST,2 /* store second half-word */ 295 st.h %r8,DEST,4 /* store third half-word */ 296 st.h %r9,DEST,6 /* store fourth half-word */ 297 st.h %r10,DEST,8 /* store fifth half-word */ 298 st.h %r11,DEST,10 /* store sixth half-word */ 299 st.h %r12,DEST,12 /* store seventh half-word */ 300 st.h %r13,DEST,14 /* store eighth half-word */ 301 br.n _ASM_LABEL(r_half_copy) /* branch to copy another block */ 302 subu LEN,LEN,16 /* decrement remaining length */ 303 304ASLOCAL(r_byte_copy) 305 bcnd eq0,LEN,_ASM_LABEL(bcopy_out) /* branch if nothing left to copy */ 306 subu SRC,SRC,1 /* decrement source pointer */ 307 subu DEST,DEST,1 /* decrement destination pointer */ 308 ld.bu %r6,SRC,0 /* load byte from source */ 309 st.b %r6,DEST,0 /* store byte in destination */ 310 br.n _ASM_LABEL(r_byte_copy) /* branch for next byte */ 311 subu LEN,LEN,1 /* decrement remaining length */ 312#endif /* MEMMOVE || OVBCOPY */ 313 314ASLOCAL(bcopy_out) 315#if defined(MEMCPY) || defined(MEMMOVE) 316 jmp.n %r1 /* all done, return to caller */ 317 or %r2, SAVE, %r0 318#else 319 jmp %r1 /* all done, return to caller */ 320#endif 321 322 .data 323 .align 2 324ASLOCAL(f_strat) 325 .word _ASM_LABEL(f_word_copy) 326 .word _ASM_LABEL(f_byte_copy) 327 .word _ASM_LABEL(f_half_copy) 328 .word _ASM_LABEL(f_byte_copy) 329 .word _ASM_LABEL(f_byte_copy) 330 .word _ASM_LABEL(f_3byte_word_copy) 331 .word _ASM_LABEL(f_byte_copy) 332 .word _ASM_LABEL(f_1byte_half_copy) 333 .word _ASM_LABEL(f_half_copy) 334 .word _ASM_LABEL(f_byte_copy) 335 .word _ASM_LABEL(f_1half_word_copy) 336 .word _ASM_LABEL(f_byte_copy) 337 .word _ASM_LABEL(f_byte_copy) 338 .word _ASM_LABEL(f_1byte_half_copy) 339 .word _ASM_LABEL(f_byte_copy) 340 .word _ASM_LABEL(f_1byte_word_copy) 341 342#if defined(MEMMOVE) || defined(OVBCOPY) 343ASLOCAL(r_strat) 344 .word _ASM_LABEL(r_word_copy) 345 .word _ASM_LABEL(r_byte_copy) 346 .word _ASM_LABEL(r_half_copy) 347 .word _ASM_LABEL(r_byte_copy) 348 .word _ASM_LABEL(r_byte_copy) 349 .word _ASM_LABEL(r_1byte_word_copy) 350 .word _ASM_LABEL(r_byte_copy) 351 .word _ASM_LABEL(r_1byte_half_copy) 352 .word _ASM_LABEL(r_half_copy) 353 .word _ASM_LABEL(r_byte_copy) 354 .word _ASM_LABEL(r_1half_word_copy) 355 .word _ASM_LABEL(r_byte_copy) 356 .word _ASM_LABEL(r_byte_copy) 357 .word _ASM_LABEL(r_1byte_half_copy) 358 .word _ASM_LABEL(r_byte_copy) 359 .word _ASM_LABEL(r_3byte_word_copy) 360#endif 361