1 /* $NetBSD: copyin.c,v 1.5 2012/03/16 07:23:38 matt Exp $ */ 2 3 /*- 4 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects 9 * Agency and which was developed by Matt Thomas of 3am Software Foundry. 10 * 11 * This material is based upon work supported by the Defense Advanced Research 12 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under 13 * Contract No. N66001-09-C-2073. 14 * Approved for Public Release, Distribution Unlimited 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 1. Redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer. 21 * 2. Redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 26 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 #include <sys/cdefs.h> 39 __KERNEL_RCSID(0, "$NetBSD: copyin.c,v 1.5 2012/03/16 07:23:38 matt Exp $"); 40 41 #include <sys/param.h> 42 #include <sys/lwp.h> 43 44 #include <powerpc/pcb.h> 45 46 #include <powerpc/booke/cpuvar.h> 47 48 static inline uint8_t 49 copyin_byte(const uint8_t * const usaddr8, register_t ds_msr) 50 { 51 register_t msr; 52 uint8_t data; 53 __asm volatile( 54 "mfmsr %[msr]; " /* Save MSR */ 55 "mtmsr %[ds_msr]; sync; isync; " /* DS on */ 56 "lbz %[data],0(%[usaddr8]); " /* fetch user byte */ 57 "mtmsr %[msr]; sync; isync; " /* DS off */ 58 : [msr] "=&r" (msr), [data] "=r" (data) 59 : [ds_msr] "r" (ds_msr), [usaddr8] "b" (usaddr8)); 60 return data; 61 } 62 63 static inline uint16_t 64 copyin_halfword(const uint16_t * const usaddr16, register_t ds_msr) 65 { 66 register_t msr; 67 uint16_t data; 68 __asm volatile( 69 "mfmsr %[msr]; " /* Save MSR */ 70 "mtmsr %[ds_msr]; sync; isync; " /* DS on */ 71 "lhz %[data],0(%[usaddr16]); " /* fetch user byte */ 72 "mtmsr %[msr]; sync; isync; " /* DS off */ 73 : [msr] "=&r" (msr), [data] "=r" (data) 74 : [ds_msr] "r" (ds_msr), [usaddr16] "b" (usaddr16)); 75 return data; 76 } 77 78 static inline uint32_t 79 copyin_word(const uint32_t * const usaddr32, register_t ds_msr) 80 { 81 register_t msr; 82 uint32_t data; 83 __asm volatile( 84 "mfmsr %[msr]; " /* Save MSR */ 85 "mtmsr %[ds_msr]; sync; isync; " /* DS on */ 86 "lwz %[data],0(%[usaddr32]); " /* load user byte */ 87 "mtmsr %[msr]; sync; isync; " /* DS off */ 88 : [msr] "=&r" (msr), [data] "=r" (data) 89 : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32)); 90 return data; 91 } 92 93 static inline uint32_t 94 copyin_word_bswap(const uint32_t * const usaddr32, register_t ds_msr) 95 { 96 register_t msr; 97 uint32_t data; 98 __asm volatile( 99 "mfmsr %[msr]; " /* Save MSR */ 100 "mtmsr %[ds_msr]; sync; isync; " /* DS on */ 101 "lwbrx %[data],0,%[usaddr32]; " /* load user LE word */ 102 "mtmsr %[msr]; sync; isync; " /* DS off */ 103 : [msr] "=&r" (msr), [data] "=r" (data) 104 : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32)); 105 return data; 106 } 107 108 static inline void 109 copyin_8words(const uint32_t *usaddr32, uint32_t *kdaddr32, register_t ds_msr) 110 { 111 register_t msr; 112 //uint32_t data[8]; 113 __asm volatile( 114 "mfmsr %[msr]" /* Save MSR */ 115 "\n\t" "mtmsr %[ds_msr]; sync; isync" /* DS on */ 116 "\n\t" "lwz %[data0],0(%[usaddr32])" /* fetch user data */ 117 "\n\t" "lwz %[data1],4(%[usaddr32])" /* fetch user data */ 118 "\n\t" "lwz %[data2],8(%[usaddr32])" /* fetch user data */ 119 "\n\t" "lwz %[data3],12(%[usaddr32])" /* fetch user data */ 120 "\n\t" "lwz %[data4],16(%[usaddr32])" /* fetch user data */ 121 "\n\t" "lwz %[data5],20(%[usaddr32])" /* fetch user data */ 122 "\n\t" "lwz %[data6],24(%[usaddr32])" /* fetch user data */ 123 "\n\t" "lwz %[data7],28(%[usaddr32])" /* fetch user data */ 124 "\n\t" "mtmsr %[msr]; sync; isync" /* DS off */ 125 : [msr] "=&r" (msr), 126 [data0] "=&r" (kdaddr32[0]), [data1] "=&r" (kdaddr32[1]), 127 [data2] "=&r" (kdaddr32[2]), [data3] "=&r" (kdaddr32[3]), 128 [data4] "=&r" (kdaddr32[4]), [data5] "=&r" (kdaddr32[5]), 129 [data6] "=&r" (kdaddr32[6]), [data7] "=&r" (kdaddr32[7]) 130 : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32)); 131 } 132 133 static inline void 134 copyin_16words(const uint32_t *usaddr32, uint32_t *kdaddr32, register_t ds_msr) 135 { 136 register_t msr; 137 __asm volatile( 138 "mfmsr %[msr]" /* Save MSR */ 139 "\n\t" "mtmsr %[ds_msr]; sync; isync" /* DS on */ 140 "\n\t" "lwz %[data0],0(%[usaddr32])" /* fetch user data */ 141 "\n\t" "lwz %[data1],4(%[usaddr32])" /* fetch user data */ 142 "\n\t" "lwz %[data2],8(%[usaddr32])" /* fetch user data */ 143 "\n\t" "lwz %[data3],12(%[usaddr32])" /* fetch user data */ 144 "\n\t" "lwz %[data4],16(%[usaddr32])" /* fetch user data */ 145 "\n\t" "lwz %[data5],20(%[usaddr32])" /* fetch user data */ 146 "\n\t" "lwz %[data6],24(%[usaddr32])" /* fetch user data */ 147 "\n\t" "lwz %[data7],28(%[usaddr32])" /* fetch user data */ 148 "\n\t" "lwz %[data8],32(%[usaddr32])" /* fetch user data */ 149 "\n\t" "lwz %[data9],36(%[usaddr32])" /* fetch user data */ 150 "\n\t" "lwz %[data10],40(%[usaddr32])" /* fetch user data */ 151 "\n\t" "lwz %[data11],44(%[usaddr32])" /* fetch user data */ 152 "\n\t" "lwz %[data12],48(%[usaddr32])" /* fetch user data */ 153 "\n\t" "lwz %[data13],52(%[usaddr32])" /* fetch user data */ 154 "\n\t" "lwz %[data14],56(%[usaddr32])" /* fetch user data */ 155 "\n\t" "lwz %[data15],60(%[usaddr32])" /* fetch user data */ 156 "\n\t" "mtmsr %[msr]; sync; isync" /* DS off */ 157 : [msr] "=&r" (msr), 158 [data0] "=&r" (kdaddr32[0]), [data1] "=&r" (kdaddr32[1]), 159 [data2] "=&r" (kdaddr32[2]), [data3] "=&r" (kdaddr32[3]), 160 [data4] "=&r" (kdaddr32[4]), [data5] "=&r" (kdaddr32[5]), 161 [data6] "=&r" (kdaddr32[6]), [data7] "=&r" (kdaddr32[7]), 162 [data8] "=&r" (kdaddr32[8]), [data9] "=&r" (kdaddr32[9]), 163 [data10] "=&r" (kdaddr32[10]), [data11] "=&r" (kdaddr32[11]), 164 [data12] "=&r" (kdaddr32[12]), [data13] "=&r" (kdaddr32[13]), 165 [data14] "=&r" (kdaddr32[14]), [data15] "=&r" (kdaddr32[15]) 166 : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32)); 167 } 168 static inline void 169 copyin_bytes(vaddr_t usaddr, vaddr_t kdaddr, size_t len, register_t ds_msr) 170 { 171 const uint8_t *usaddr8 = (void *)usaddr; 172 uint8_t *kdaddr8 = (void *)kdaddr; 173 while (len-- > 0) { 174 *kdaddr8++ = copyin_byte(usaddr8++, ds_msr); 175 } 176 } 177 178 static inline void 179 copyin_words(vaddr_t usaddr, vaddr_t kdaddr, size_t len, register_t ds_msr) 180 { 181 KASSERT((kdaddr & 3) == 0); 182 KASSERT((usaddr & 3) == 0); 183 const uint32_t *usaddr32 = (void *)usaddr; 184 uint32_t *kdaddr32 = (void *)kdaddr; 185 len >>= 2; 186 while (len >= 16) { 187 copyin_16words(usaddr32, kdaddr32, ds_msr); 188 usaddr32 += 16, kdaddr32 += 16, len -= 16; 189 } 190 KASSERT(len < 16); 191 if (len >= 8) { 192 copyin_8words(usaddr32, kdaddr32, ds_msr); 193 usaddr32 += 8, kdaddr32 += 8, len -= 8; 194 } 195 while (len-- > 0) { 196 *kdaddr32++ = copyin_word(usaddr32++, ds_msr); 197 } 198 } 199 200 uint32_t 201 ufetch_32(const void *vusaddr) 202 { 203 struct pcb * const pcb = lwp_getpcb(curlwp); 204 struct faultbuf env; 205 206 if (setfault(&env) != 0) { 207 pcb->pcb_onfault = NULL; 208 return -1; 209 } 210 211 uint32_t rv = copyin_word(vusaddr, mfmsr() | PSL_DS); 212 213 pcb->pcb_onfault = NULL; 214 215 return rv; 216 } 217 218 int 219 copyin(const void *vusaddr, void *vkdaddr, size_t len) 220 { 221 struct pcb * const pcb = lwp_getpcb(curlwp); 222 struct faultbuf env; 223 vaddr_t usaddr = (vaddr_t) vusaddr; 224 vaddr_t kdaddr = (vaddr_t) vkdaddr; 225 226 if (__predict_false(len == 0)) { 227 return 0; 228 } 229 230 const register_t ds_msr = mfmsr() | PSL_DS; 231 232 int rv = setfault(&env); 233 if (rv != 0) { 234 pcb->pcb_onfault = NULL; 235 return rv; 236 } 237 238 if (__predict_false(len < 4)) { 239 copyin_bytes(usaddr, kdaddr, len, ds_msr); 240 pcb->pcb_onfault = NULL; 241 return 0; 242 } 243 244 const size_t alignment = (usaddr ^ kdaddr) & 3; 245 if (__predict_true(alignment == 0)) { 246 size_t slen; 247 if (__predict_false(kdaddr & 3)) { 248 slen = 4 - (kdaddr & 3); 249 copyin_bytes(usaddr, kdaddr, slen, ds_msr); 250 usaddr += slen, kdaddr += slen, len -= slen; 251 } 252 slen = len & ~3; 253 if (__predict_true(slen >= 4)) { 254 copyin_words(usaddr, kdaddr, slen, ds_msr); 255 usaddr += slen, kdaddr += slen, len -= slen; 256 } 257 } 258 if (len > 0) { 259 copyin_bytes(usaddr, kdaddr, len, ds_msr); 260 } 261 pcb->pcb_onfault = NULL; 262 return 0; 263 } 264 265 int 266 copyinstr(const void *usaddr, void *kdaddr, size_t len, size_t *done) 267 { 268 struct pcb * const pcb = lwp_getpcb(curlwp); 269 struct faultbuf env; 270 271 if (__predict_false(len == 0)) { 272 if (done) 273 *done = 0; 274 return 0; 275 } 276 277 int rv = setfault(&env); 278 if (rv != 0) { 279 pcb->pcb_onfault = NULL; 280 if (done) 281 *done = 0; 282 return rv; 283 } 284 285 const register_t ds_msr = mfmsr() | PSL_DS; 286 const uint32_t *usaddr32 = (const void *)((uintptr_t)usaddr & ~3); 287 uint8_t *kdaddr8 = kdaddr; 288 size_t copylen, wlen; 289 uint32_t data; 290 size_t uoff = (uintptr_t)usaddr & 3; 291 wlen = 4 - uoff; 292 /* 293 * We need discard any leading bytes if the address was 294 * unaligned. We read the words byteswapped so that the LSB 295 * contains the lowest address byte. 296 */ 297 data = copyin_word_bswap(usaddr32++, ds_msr) >> (8 * uoff); 298 for (copylen = 0; copylen < len; copylen++, wlen--, data >>= 8) { 299 if (wlen == 0) { 300 /* 301 * If we've depleted the data in the word, fetch the 302 * next one. 303 */ 304 data = copyin_word_bswap(usaddr32++, ds_msr); 305 wlen = 4; 306 } 307 *kdaddr8++ = data; 308 if ((uint8_t) data == 0) { 309 copylen++; 310 break; 311 } 312 } 313 314 pcb->pcb_onfault = NULL; 315 if (done) 316 *done = copylen; 317 /* 318 * If the last byte is not NUL (0), then the name is too long. 319 */ 320 return (uint8_t)data ? ENAMETOOLONG : 0; 321 } 322