1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * String handling functions.
4 *
5 * Copyright IBM Corp. 2012
6 */
7
8#include <linux/linkage.h>
9#include <asm/export.h>
10#include <asm/nospec-insn.h>
11
12	GEN_BR_THUNK %r14
13
14/*
15 * void *memmove(void *dest, const void *src, size_t n)
16 */
17WEAK(memmove)
18ENTRY(__memmove)
19	ltgr	%r4,%r4
20	lgr	%r1,%r2
21	jz	.Lmemmove_exit
22	aghi	%r4,-1
23	clgr	%r2,%r3
24	jnh	.Lmemmove_forward
25	la	%r5,1(%r4,%r3)
26	clgr	%r2,%r5
27	jl	.Lmemmove_reverse
28.Lmemmove_forward:
29	srlg	%r0,%r4,8
30	ltgr	%r0,%r0
31	jz	.Lmemmove_forward_remainder
32.Lmemmove_forward_loop:
33	mvc	0(256,%r1),0(%r3)
34	la	%r1,256(%r1)
35	la	%r3,256(%r3)
36	brctg	%r0,.Lmemmove_forward_loop
37.Lmemmove_forward_remainder:
38	larl	%r5,.Lmemmove_mvc
39	ex	%r4,0(%r5)
40.Lmemmove_exit:
41	BR_EX	%r14
42.Lmemmove_reverse:
43	ic	%r0,0(%r4,%r3)
44	stc	%r0,0(%r4,%r1)
45	brctg	%r4,.Lmemmove_reverse
46	ic	%r0,0(%r4,%r3)
47	stc	%r0,0(%r4,%r1)
48	BR_EX	%r14
49.Lmemmove_mvc:
50	mvc	0(1,%r1),0(%r3)
51ENDPROC(__memmove)
52EXPORT_SYMBOL(memmove)
53
54/*
55 * memset implementation
56 *
57 * This code corresponds to the C construct below. We do distinguish
58 * between clearing (c == 0) and setting a memory array (c != 0) simply
59 * because nearly all memset invocations in the kernel clear memory and
60 * the xc instruction is preferred in such cases.
61 *
62 * void *memset(void *s, int c, size_t n)
63 * {
64 *	if (likely(c == 0))
65 *		return __builtin_memset(s, 0, n);
66 *	return __builtin_memset(s, c, n);
67 * }
68 */
69WEAK(memset)
70ENTRY(__memset)
71	ltgr	%r4,%r4
72	jz	.Lmemset_exit
73	ltgr	%r3,%r3
74	jnz	.Lmemset_fill
75	aghi	%r4,-1
76	srlg	%r3,%r4,8
77	ltgr	%r3,%r3
78	lgr	%r1,%r2
79	jz	.Lmemset_clear_remainder
80.Lmemset_clear_loop:
81	xc	0(256,%r1),0(%r1)
82	la	%r1,256(%r1)
83	brctg	%r3,.Lmemset_clear_loop
84.Lmemset_clear_remainder:
85	larl	%r3,.Lmemset_xc
86	ex	%r4,0(%r3)
87.Lmemset_exit:
88	BR_EX	%r14
89.Lmemset_fill:
90	cghi	%r4,1
91	lgr	%r1,%r2
92	je	.Lmemset_fill_exit
93	aghi	%r4,-2
94	srlg	%r5,%r4,8
95	ltgr	%r5,%r5
96	jz	.Lmemset_fill_remainder
97.Lmemset_fill_loop:
98	stc	%r3,0(%r1)
99	mvc	1(255,%r1),0(%r1)
100	la	%r1,256(%r1)
101	brctg	%r5,.Lmemset_fill_loop
102.Lmemset_fill_remainder:
103	stc	%r3,0(%r1)
104	larl	%r5,.Lmemset_mvc
105	ex	%r4,0(%r5)
106	BR_EX	%r14
107.Lmemset_fill_exit:
108	stc	%r3,0(%r1)
109	BR_EX	%r14
110.Lmemset_xc:
111	xc	0(1,%r1),0(%r1)
112.Lmemset_mvc:
113	mvc	1(1,%r1),0(%r1)
114ENDPROC(__memset)
115EXPORT_SYMBOL(memset)
116
117/*
118 * memcpy implementation
119 *
120 * void *memcpy(void *dest, const void *src, size_t n)
121 */
122WEAK(memcpy)
123ENTRY(__memcpy)
124	ltgr	%r4,%r4
125	jz	.Lmemcpy_exit
126	aghi	%r4,-1
127	srlg	%r5,%r4,8
128	ltgr	%r5,%r5
129	lgr	%r1,%r2
130	jnz	.Lmemcpy_loop
131.Lmemcpy_remainder:
132	larl	%r5,.Lmemcpy_mvc
133	ex	%r4,0(%r5)
134.Lmemcpy_exit:
135	BR_EX	%r14
136.Lmemcpy_loop:
137	mvc	0(256,%r1),0(%r3)
138	la	%r1,256(%r1)
139	la	%r3,256(%r3)
140	brctg	%r5,.Lmemcpy_loop
141	j	.Lmemcpy_remainder
142.Lmemcpy_mvc:
143	mvc	0(1,%r1),0(%r3)
144ENDPROC(__memcpy)
145EXPORT_SYMBOL(memcpy)
146
147/*
148 * __memset16/32/64
149 *
150 * void *__memset16(uint16_t *s, uint16_t v, size_t count)
151 * void *__memset32(uint32_t *s, uint32_t v, size_t count)
152 * void *__memset64(uint64_t *s, uint64_t v, size_t count)
153 */
154.macro __MEMSET bits,bytes,insn
155ENTRY(__memset\bits)
156	ltgr	%r4,%r4
157	jz	.L__memset_exit\bits
158	cghi	%r4,\bytes
159	je	.L__memset_store\bits
160	aghi	%r4,-(\bytes+1)
161	srlg	%r5,%r4,8
162	ltgr	%r5,%r5
163	lgr	%r1,%r2
164	jz	.L__memset_remainder\bits
165.L__memset_loop\bits:
166	\insn	%r3,0(%r1)
167	mvc	\bytes(256-\bytes,%r1),0(%r1)
168	la	%r1,256(%r1)
169	brctg	%r5,.L__memset_loop\bits
170.L__memset_remainder\bits:
171	\insn	%r3,0(%r1)
172	larl	%r5,.L__memset_mvc\bits
173	ex	%r4,0(%r5)
174	BR_EX	%r14
175.L__memset_store\bits:
176	\insn	%r3,0(%r2)
177.L__memset_exit\bits:
178	BR_EX	%r14
179.L__memset_mvc\bits:
180	mvc	\bytes(1,%r1),0(%r1)
181ENDPROC(__memset\bits)
182.endm
183
184__MEMSET 16,2,sth
185EXPORT_SYMBOL(__memset16)
186
187__MEMSET 32,4,st
188EXPORT_SYMBOL(__memset32)
189
190__MEMSET 64,8,stg
191EXPORT_SYMBOL(__memset64)
192