xref: /freebsd/lib/libc/amd64/string/memmove.S (revision 38a52bd3) 
1/*-
2 * Copyright (c) 2018 The FreeBSD Foundation
3 *
4 * This software was developed by Mateusz Guzik <mjg@FreeBSD.org>
5 * under sponsorship from the FreeBSD Foundation.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29#include <machine/asm.h>
30__FBSDID("$FreeBSD$");
31
32/*
33 * Note: this routine was written with kernel use in mind (read: no simd),
34 * it is only present in userspace as a temporary measure until something
35 * better gets imported.
36 */
37
38#define	ALIGN_TEXT	.p2align 4,0x90 /* 16-byte alignment, nop filled */
39
40/*
41 * memmove(dst, src, cnt)
42 *         rdi, rsi, rdx
43 */
44
45/*
46 * Register state at entry is supposed to be as follows:
47 * rdi - destination
48 * rsi - source
49 * rdx - count
50 *
51 * The macro possibly clobbers the above and: rcx, r8, r9, 10
52 * It does not clobber rax nor r11.
53 */
54.macro MEMMOVE erms overlap begin end
55	\begin
56
57	/*
58	 * For sizes 0..32 all data is read before it is written, so there
59	 * is no correctness issue with direction of copying.
60	 */
61	cmpq	$32,%rcx
62	jbe	101632f
63
64.if \overlap == 1
65	movq	%rdi,%r8
66	subq	%rsi,%r8
67	cmpq	%rcx,%r8	/* overlapping && src < dst? */
68	jb	2f
69.endif
70
71	cmpq	$256,%rcx
72	ja	1256f
73
74	ALIGN_TEXT
75103200:
76	movq	(%rsi),%rdx
77	movq	%rdx,(%rdi)
78	movq	8(%rsi),%rdx
79	movq	%rdx,8(%rdi)
80	movq	16(%rsi),%rdx
81	movq	%rdx,16(%rdi)
82	movq	24(%rsi),%rdx
83	movq	%rdx,24(%rdi)
84	leaq	32(%rsi),%rsi
85	leaq	32(%rdi),%rdi
86	subq	$32,%rcx
87	cmpq	$32,%rcx
88	jae	103200b
89	cmpb	$0,%cl
90	jne	101632f
91	\end
92	ret
93	ALIGN_TEXT
94101632:
95	cmpb	$16,%cl
96	jl	100816f
97	movq	(%rsi),%rdx
98	movq	8(%rsi),%r8
99	movq	-16(%rsi,%rcx),%r9
100	movq	-8(%rsi,%rcx),%r10
101	movq	%rdx,(%rdi)
102	movq	%r8,8(%rdi)
103	movq	%r9,-16(%rdi,%rcx)
104	movq	%r10,-8(%rdi,%rcx)
105	\end
106	ret
107	ALIGN_TEXT
108100816:
109	cmpb	$8,%cl
110	jl	100408f
111	movq	(%rsi),%rdx
112	movq	-8(%rsi,%rcx),%r8
113	movq	%rdx,(%rdi)
114	movq	%r8,-8(%rdi,%rcx,)
115	\end
116	ret
117	ALIGN_TEXT
118100408:
119	cmpb	$4,%cl
120	jl	100204f
121	movl	(%rsi),%edx
122	movl	-4(%rsi,%rcx),%r8d
123	movl	%edx,(%rdi)
124	movl	%r8d,-4(%rdi,%rcx)
125	\end
126	ret
127	ALIGN_TEXT
128100204:
129	cmpb	$2,%cl
130	jl	100001f
131	movzwl	(%rsi),%edx
132	movzwl	-2(%rsi,%rcx),%r8d
133	movw	%dx,(%rdi)
134	movw	%r8w,-2(%rdi,%rcx)
135	\end
136	ret
137	ALIGN_TEXT
138100001:
139	cmpb	$1,%cl
140	jl	100000f
141	movb	(%rsi),%dl
142	movb	%dl,(%rdi)
143100000:
144	\end
145	ret
146
147	ALIGN_TEXT
1481256:
149	testb	$15,%dil
150	jnz	100f
151.if \erms == 1
152	rep
153	movsb
154.else
155	shrq	$3,%rcx                         /* copy by 64-bit words */
156	rep
157	movsq
158	movq	%rdx,%rcx
159	andl	$7,%ecx                         /* any bytes left? */
160	jne	100408b
161.endif
162	\end
163	ret
164100:
165	movq	(%rsi),%r8
166	movq	8(%rsi),%r9
167	movq	%rdi,%r10
168	movq	%rdi,%rcx
169	andq	$15,%rcx
170	leaq	-16(%rdx,%rcx),%rdx
171	neg	%rcx
172	leaq	16(%rdi,%rcx),%rdi
173	leaq	16(%rsi,%rcx),%rsi
174	movq	%rdx,%rcx
175.if \erms == 1
176	rep
177	movsb
178	movq	%r8,(%r10)
179	movq	%r9,8(%r10)
180.else
181	shrq	$3,%rcx                         /* copy by 64-bit words */
182	rep
183	movsq
184	movq	%r8,(%r10)
185	movq	%r9,8(%r10)
186	movq	%rdx,%rcx
187	andl	$7,%ecx                         /* any bytes left? */
188	jne	100408b
189.endif
190	\end
191	ret
192
193.if \overlap == 1
194	/*
195	 * Copy backwards.
196	 */
197        ALIGN_TEXT
1982:
199	cmpq	$256,%rcx
200	ja	2256f
201
202	leaq	-8(%rdi,%rcx),%rdi
203	leaq	-8(%rsi,%rcx),%rsi
204
205	cmpq	$32,%rcx
206	jb	2016f
207
208	ALIGN_TEXT
2092032:
210	movq	(%rsi),%rdx
211	movq	%rdx,(%rdi)
212	movq	-8(%rsi),%rdx
213	movq	%rdx,-8(%rdi)
214	movq	-16(%rsi),%rdx
215	movq	%rdx,-16(%rdi)
216	movq	-24(%rsi),%rdx
217	movq	%rdx,-24(%rdi)
218	leaq	-32(%rsi),%rsi
219	leaq	-32(%rdi),%rdi
220	subq	$32,%rcx
221	cmpq	$32,%rcx
222	jae	2032b
223	cmpb	$0,%cl
224	jne	2016f
225	\end
226	ret
227	ALIGN_TEXT
2282016:
229	cmpb	$16,%cl
230	jl	2008f
231	movq	(%rsi),%rdx
232	movq	%rdx,(%rdi)
233	movq	-8(%rsi),%rdx
234	movq	%rdx,-8(%rdi)
235	subb	$16,%cl
236	jz	2000f
237	leaq	-16(%rsi),%rsi
238	leaq	-16(%rdi),%rdi
2392008:
240	cmpb	$8,%cl
241	jl	2004f
242	movq	(%rsi),%rdx
243	movq	%rdx,(%rdi)
244	subb	$8,%cl
245	jz	2000f
246	leaq	-8(%rsi),%rsi
247	leaq	-8(%rdi),%rdi
2482004:
249	cmpb	$4,%cl
250	jl	2002f
251	movl	4(%rsi),%edx
252	movl	%edx,4(%rdi)
253	subb	$4,%cl
254	jz	2000f
255	leaq	-4(%rsi),%rsi
256	leaq	-4(%rdi),%rdi
2572002:
258	cmpb	$2,%cl
259	jl	2001f
260	movw	6(%rsi),%dx
261	movw	%dx,6(%rdi)
262	subb	$2,%cl
263	jz	2000f
264	leaq	-2(%rsi),%rsi
265	leaq	-2(%rdi),%rdi
2662001:
267	cmpb	$1,%cl
268	jl	2000f
269	movb	7(%rsi),%dl
270	movb	%dl,7(%rdi)
2712000:
272	\end
273	ret
274	ALIGN_TEXT
2752256:
276	std
277	leaq	-8(%rdi,%rcx),%rdi
278	leaq	-8(%rsi,%rcx),%rsi
279	shrq	$3,%rcx
280	rep
281	movsq
282	cld
283	movq	%rdx,%rcx
284	andb	$7,%cl
285	jne	2004b
286	\end
287	ret
288.endif
289.endm
290
291
292.macro MEMMOVE_BEGIN
293	movq	%rdi,%rax
294	movq	%rdx,%rcx
295.endm
296
297.macro MEMMOVE_END
298.endm
299
300#ifndef MEMCPY
301ENTRY(memmove)
302	MEMMOVE erms=0 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END
303END(memmove)
304#else
305ENTRY(memcpy)
306	MEMMOVE erms=0 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END
307END(memcpy)
308#endif
309