1 /* $NetBSD: copyin.c,v 1.9 2020/07/06 09:34:16 rin Exp $ */
2
3 /*-
4 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects
9 * Agency and which was developed by Matt Thomas of 3am Software Foundry.
10 *
11 * This material is based upon work supported by the Defense Advanced Research
12 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under
13 * Contract No. N66001-09-C-2073.
14 * Approved for Public Release, Distribution Unlimited
15 *
16 * Redistribution and use in source and binary forms, with or without
17 * modification, are permitted provided that the following conditions
18 * are met:
19 * 1. Redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer.
21 * 2. Redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
26 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 * POSSIBILITY OF SUCH DAMAGE.
36 */
37
38 #define __UFETCHSTORE_PRIVATE
39
40 #include <sys/cdefs.h>
41 __KERNEL_RCSID(0, "$NetBSD: copyin.c,v 1.9 2020/07/06 09:34:16 rin Exp $");
42
43 #include <sys/param.h>
44 #include <sys/lwp.h>
45 #include <sys/systm.h>
46
47 #include <powerpc/pcb.h>
48
49 #include <powerpc/booke/cpuvar.h>
50
51 static inline uint8_t
copyin_byte(const uint8_t * const usaddr8,register_t ds_msr)52 copyin_byte(const uint8_t * const usaddr8, register_t ds_msr)
53 {
54 register_t msr;
55 uint8_t data;
56 __asm volatile(
57 "mfmsr %[msr]; " /* Save MSR */
58 "mtmsr %[ds_msr]; sync; isync; " /* DS on */
59 "lbz %[data],0(%[usaddr8]); " /* fetch user byte */
60 "mtmsr %[msr]; sync; isync; " /* DS off */
61 : [msr] "=&r" (msr), [data] "=r" (data)
62 : [ds_msr] "r" (ds_msr), [usaddr8] "b" (usaddr8));
63 return data;
64 }
65
66 static inline uint16_t
copyin_halfword(const uint16_t * const usaddr16,register_t ds_msr)67 copyin_halfword(const uint16_t * const usaddr16, register_t ds_msr)
68 {
69 register_t msr;
70 uint16_t data;
71 __asm volatile(
72 "mfmsr %[msr]; " /* Save MSR */
73 "mtmsr %[ds_msr]; sync; isync; " /* DS on */
74 "lhz %[data],0(%[usaddr16]); " /* fetch user byte */
75 "mtmsr %[msr]; sync; isync; " /* DS off */
76 : [msr] "=&r" (msr), [data] "=r" (data)
77 : [ds_msr] "r" (ds_msr), [usaddr16] "b" (usaddr16));
78 return data;
79 }
80
81 static inline uint32_t
copyin_word(const uint32_t * const usaddr32,register_t ds_msr)82 copyin_word(const uint32_t * const usaddr32, register_t ds_msr)
83 {
84 register_t msr;
85 uint32_t data;
86 __asm volatile(
87 "mfmsr %[msr]; " /* Save MSR */
88 "mtmsr %[ds_msr]; sync; isync; " /* DS on */
89 "lwz %[data],0(%[usaddr32]); " /* load user byte */
90 "mtmsr %[msr]; sync; isync; " /* DS off */
91 : [msr] "=&r" (msr), [data] "=r" (data)
92 : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32));
93 return data;
94 }
95
96 static inline uint32_t
copyin_word_bswap(const uint32_t * const usaddr32,register_t ds_msr)97 copyin_word_bswap(const uint32_t * const usaddr32, register_t ds_msr)
98 {
99 register_t msr;
100 uint32_t data;
101 __asm volatile(
102 "mfmsr %[msr]; " /* Save MSR */
103 "mtmsr %[ds_msr]; sync; isync; " /* DS on */
104 "lwbrx %[data],0,%[usaddr32]; " /* load user LE word */
105 "mtmsr %[msr]; sync; isync; " /* DS off */
106 : [msr] "=&r" (msr), [data] "=r" (data)
107 : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32));
108 return data;
109 }
110
111 static inline void
copyin_8words(const uint32_t * usaddr32,uint32_t * kdaddr32,register_t ds_msr)112 copyin_8words(const uint32_t *usaddr32, uint32_t *kdaddr32, register_t ds_msr)
113 {
114 register_t msr;
115 //uint32_t data[8];
116 __asm volatile(
117 "mfmsr %[msr]" /* Save MSR */
118 "\n\t" "mtmsr %[ds_msr]; sync; isync" /* DS on */
119 "\n\t" "lwz %[data0],0(%[usaddr32])" /* fetch user data */
120 "\n\t" "lwz %[data1],4(%[usaddr32])" /* fetch user data */
121 "\n\t" "lwz %[data2],8(%[usaddr32])" /* fetch user data */
122 "\n\t" "lwz %[data3],12(%[usaddr32])" /* fetch user data */
123 "\n\t" "lwz %[data4],16(%[usaddr32])" /* fetch user data */
124 "\n\t" "lwz %[data5],20(%[usaddr32])" /* fetch user data */
125 "\n\t" "lwz %[data6],24(%[usaddr32])" /* fetch user data */
126 "\n\t" "lwz %[data7],28(%[usaddr32])" /* fetch user data */
127 "\n\t" "mtmsr %[msr]; sync; isync" /* DS off */
128 : [msr] "=&r" (msr),
129 [data0] "=&r" (kdaddr32[0]), [data1] "=&r" (kdaddr32[1]),
130 [data2] "=&r" (kdaddr32[2]), [data3] "=&r" (kdaddr32[3]),
131 [data4] "=&r" (kdaddr32[4]), [data5] "=&r" (kdaddr32[5]),
132 [data6] "=&r" (kdaddr32[6]), [data7] "=&r" (kdaddr32[7])
133 : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32));
134 }
135
136 static inline void
copyin_16words(const uint32_t * usaddr32,uint32_t * kdaddr32,register_t ds_msr)137 copyin_16words(const uint32_t *usaddr32, uint32_t *kdaddr32, register_t ds_msr)
138 {
139 register_t msr;
140 __asm volatile(
141 "mfmsr %[msr]" /* Save MSR */
142 "\n\t" "mtmsr %[ds_msr]; sync; isync" /* DS on */
143 "\n\t" "lwz %[data0],0(%[usaddr32])" /* fetch user data */
144 "\n\t" "lwz %[data1],4(%[usaddr32])" /* fetch user data */
145 "\n\t" "lwz %[data2],8(%[usaddr32])" /* fetch user data */
146 "\n\t" "lwz %[data3],12(%[usaddr32])" /* fetch user data */
147 "\n\t" "lwz %[data4],16(%[usaddr32])" /* fetch user data */
148 "\n\t" "lwz %[data5],20(%[usaddr32])" /* fetch user data */
149 "\n\t" "lwz %[data6],24(%[usaddr32])" /* fetch user data */
150 "\n\t" "lwz %[data7],28(%[usaddr32])" /* fetch user data */
151 "\n\t" "lwz %[data8],32(%[usaddr32])" /* fetch user data */
152 "\n\t" "lwz %[data9],36(%[usaddr32])" /* fetch user data */
153 "\n\t" "lwz %[data10],40(%[usaddr32])" /* fetch user data */
154 "\n\t" "lwz %[data11],44(%[usaddr32])" /* fetch user data */
155 "\n\t" "lwz %[data12],48(%[usaddr32])" /* fetch user data */
156 "\n\t" "lwz %[data13],52(%[usaddr32])" /* fetch user data */
157 "\n\t" "lwz %[data14],56(%[usaddr32])" /* fetch user data */
158 "\n\t" "lwz %[data15],60(%[usaddr32])" /* fetch user data */
159 "\n\t" "mtmsr %[msr]; sync; isync" /* DS off */
160 : [msr] "=&r" (msr),
161 [data0] "=&r" (kdaddr32[0]), [data1] "=&r" (kdaddr32[1]),
162 [data2] "=&r" (kdaddr32[2]), [data3] "=&r" (kdaddr32[3]),
163 [data4] "=&r" (kdaddr32[4]), [data5] "=&r" (kdaddr32[5]),
164 [data6] "=&r" (kdaddr32[6]), [data7] "=&r" (kdaddr32[7]),
165 [data8] "=&r" (kdaddr32[8]), [data9] "=&r" (kdaddr32[9]),
166 [data10] "=&r" (kdaddr32[10]), [data11] "=&r" (kdaddr32[11]),
167 [data12] "=&r" (kdaddr32[12]), [data13] "=&r" (kdaddr32[13]),
168 [data14] "=&r" (kdaddr32[14]), [data15] "=&r" (kdaddr32[15])
169 : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32));
170 }
171 static inline void
copyin_bytes(vaddr_t usaddr,vaddr_t kdaddr,size_t len,register_t ds_msr)172 copyin_bytes(vaddr_t usaddr, vaddr_t kdaddr, size_t len, register_t ds_msr)
173 {
174 const uint8_t *usaddr8 = (void *)usaddr;
175 uint8_t *kdaddr8 = (void *)kdaddr;
176 while (len-- > 0) {
177 *kdaddr8++ = copyin_byte(usaddr8++, ds_msr);
178 }
179 }
180
181 static inline void
copyin_words(vaddr_t usaddr,vaddr_t kdaddr,size_t len,register_t ds_msr)182 copyin_words(vaddr_t usaddr, vaddr_t kdaddr, size_t len, register_t ds_msr)
183 {
184 KASSERT((kdaddr & 3) == 0);
185 KASSERT((usaddr & 3) == 0);
186 const uint32_t *usaddr32 = (void *)usaddr;
187 uint32_t *kdaddr32 = (void *)kdaddr;
188 len >>= 2;
189 while (len >= 16) {
190 copyin_16words(usaddr32, kdaddr32, ds_msr);
191 usaddr32 += 16, kdaddr32 += 16, len -= 16;
192 }
193 KASSERT(len < 16);
194 if (len >= 8) {
195 copyin_8words(usaddr32, kdaddr32, ds_msr);
196 usaddr32 += 8, kdaddr32 += 8, len -= 8;
197 }
198 while (len-- > 0) {
199 *kdaddr32++ = copyin_word(usaddr32++, ds_msr);
200 }
201 }
202
203 int
_ufetch_8(const uint8_t * vusaddr,uint8_t * valp)204 _ufetch_8(const uint8_t *vusaddr, uint8_t *valp)
205 {
206 struct pcb * const pcb = lwp_getpcb(curlwp);
207 struct faultbuf env;
208
209 if (setfault(&env) != 0) {
210 pcb->pcb_onfault = NULL;
211 return EFAULT;
212 }
213
214 *valp = copyin_byte(vusaddr, mfmsr() | PSL_DS);
215
216 pcb->pcb_onfault = NULL;
217
218 return 0;
219 }
220
221 int
_ufetch_16(const uint16_t * vusaddr,uint16_t * valp)222 _ufetch_16(const uint16_t *vusaddr, uint16_t *valp)
223 {
224 struct pcb * const pcb = lwp_getpcb(curlwp);
225 struct faultbuf env;
226
227 if (setfault(&env) != 0) {
228 pcb->pcb_onfault = NULL;
229 return EFAULT;
230 }
231
232 *valp = copyin_halfword(vusaddr, mfmsr() | PSL_DS);
233
234 pcb->pcb_onfault = NULL;
235
236 return 0;
237 }
238
239 int
_ufetch_32(const uint32_t * vusaddr,uint32_t * valp)240 _ufetch_32(const uint32_t *vusaddr, uint32_t *valp)
241 {
242 struct pcb * const pcb = lwp_getpcb(curlwp);
243 struct faultbuf env;
244
245 if (setfault(&env) != 0) {
246 pcb->pcb_onfault = NULL;
247 return EFAULT;
248 }
249
250 *valp = copyin_word(vusaddr, mfmsr() | PSL_DS);
251
252 pcb->pcb_onfault = NULL;
253
254 return 0;
255 }
256
257 int
copyin(const void * vusaddr,void * vkdaddr,size_t len)258 copyin(const void *vusaddr, void *vkdaddr, size_t len)
259 {
260 struct pcb * const pcb = lwp_getpcb(curlwp);
261 struct faultbuf env;
262 vaddr_t usaddr = (vaddr_t) vusaddr;
263 vaddr_t kdaddr = (vaddr_t) vkdaddr;
264
265 if (__predict_false(len == 0)) {
266 return 0;
267 }
268
269 const register_t ds_msr = mfmsr() | PSL_DS;
270
271 int rv = setfault(&env);
272 if (rv != 0) {
273 pcb->pcb_onfault = NULL;
274 return rv;
275 }
276
277 if (__predict_false(len < 4)) {
278 copyin_bytes(usaddr, kdaddr, len, ds_msr);
279 pcb->pcb_onfault = NULL;
280 return 0;
281 }
282
283 const size_t alignment = (usaddr ^ kdaddr) & 3;
284 if (__predict_true(alignment == 0)) {
285 size_t slen;
286 if (__predict_false(kdaddr & 3)) {
287 slen = 4 - (kdaddr & 3);
288 copyin_bytes(usaddr, kdaddr, slen, ds_msr);
289 usaddr += slen, kdaddr += slen, len -= slen;
290 }
291 slen = len & ~3;
292 if (__predict_true(slen >= 4)) {
293 copyin_words(usaddr, kdaddr, slen, ds_msr);
294 usaddr += slen, kdaddr += slen, len -= slen;
295 }
296 }
297 if (len > 0) {
298 copyin_bytes(usaddr, kdaddr, len, ds_msr);
299 }
300 pcb->pcb_onfault = NULL;
301 return 0;
302 }
303
304 int
copyinstr(const void * usaddr,void * kdaddr,size_t len,size_t * done)305 copyinstr(const void *usaddr, void *kdaddr, size_t len, size_t *done)
306 {
307 struct pcb * const pcb = lwp_getpcb(curlwp);
308 struct faultbuf env;
309 int rv;
310
311 if (__predict_false(len == 0)) {
312 if (done)
313 *done = 0;
314 return 0;
315 }
316
317 rv = setfault(&env);
318 if (rv != 0) {
319 pcb->pcb_onfault = NULL;
320 if (done)
321 *done = 0;
322 return rv;
323 }
324
325 const register_t ds_msr = mfmsr() | PSL_DS;
326 const uint32_t *usaddr32 = (const void *)((uintptr_t)usaddr & ~3);
327 uint8_t *kdaddr8 = kdaddr;
328 size_t copylen, wlen;
329 uint32_t data;
330 size_t uoff = (uintptr_t)usaddr & 3;
331 wlen = 4 - uoff;
332 /*
333 * We need discard any leading bytes if the address was
334 * unaligned. We read the words byteswapped so that the LSB
335 * contains the lowest address byte.
336 */
337 data = copyin_word_bswap(usaddr32++, ds_msr) >> (8 * uoff);
338 for (copylen = 0; copylen < len; copylen++, wlen--, data >>= 8) {
339 if (wlen == 0) {
340 /*
341 * If we've depleted the data in the word, fetch the
342 * next one.
343 */
344 data = copyin_word_bswap(usaddr32++, ds_msr);
345 wlen = 4;
346 }
347 *kdaddr8++ = data;
348 if ((uint8_t) data == 0) {
349 copylen++;
350 goto out;
351 }
352 }
353 rv = ENAMETOOLONG;
354
355 out:
356 pcb->pcb_onfault = NULL;
357 if (done)
358 *done = copylen;
359 return rv;
360 }
361