1 /* $NetBSD: altivec.c,v 1.20 2011/01/23 17:36:09 matt Exp $ */ 2 3 /* 4 * Copyright (C) 1996 Wolfgang Solfrank. 5 * Copyright (C) 1996 TooLs GmbH. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by TooLs GmbH. 19 * 4. The name of TooLs GmbH may not be used to endorse or promote products 20 * derived from this software without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 25 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 27 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 28 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 29 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 30 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 31 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: altivec.c,v 1.20 2011/01/23 17:36:09 matt Exp $"); 36 37 #include "opt_multiprocessor.h" 38 39 #include <sys/param.h> 40 #include <sys/proc.h> 41 #include <sys/systm.h> 42 #include <sys/atomic.h> 43 44 #include <uvm/uvm_extern.h> /* for vcopypage/vzeropage */ 45 46 #include <powerpc/pcb.h> 47 #include <powerpc/altivec.h> 48 #include <powerpc/spr.h> 49 #include <powerpc/oea/spr.h> 50 #include <powerpc/psl.h> 51 52 #ifdef MULTIPROCESSOR 53 #include <arch/powerpc/pic/picvar.h> 54 #include <arch/powerpc/pic/ipivar.h> 55 static void vec_mp_save_lwp(struct lwp *); 56 #endif 57 58 void 59 vec_enable(void) 60 { 61 struct cpu_info *ci = curcpu(); 62 struct lwp *l = curlwp; 63 register_t msr; 64 65 KASSERT(l->l_md.md_veccpu != NULL); 66 67 l->l_md.md_flags |= MDLWP_USEDVEC; 68 69 /* 70 * Enable AltiVec temporarily (and disable interrupts). 71 */ 72 msr = mfmsr(); 73 mtmsr((msr & ~PSL_EE) | PSL_VEC); 74 __asm volatile ("isync"); 75 76 if (ci->ci_veclwp != l) { 77 struct pcb * const pcb = lwp_getpcb(l); 78 struct trapframe * const tf = l->l_md.md_utf; 79 80 vec_save_cpu(VEC_SAVE_AND_RELEASE); 81 82 /* 83 * Load the vector unit from vreg which is best done in 84 * assembly. 85 */ 86 vec_load_from_vreg(&pcb->pcb_vr); 87 88 /* 89 * VRSAVE will be restored when trap frame returns 90 */ 91 tf->tf_vrsave = pcb->pcb_vr.vrsave; 92 93 /* 94 * Enable AltiVec when we return to user-mode. 95 * Record the new ownership of the AltiVec unit. 96 */ 97 ci->ci_veclwp = l; 98 l->l_md.md_veccpu = ci; 99 __asm volatile ("sync"); 100 } 101 l->l_md.md_flags |= MDLWP_OWNVEC; 102 103 /* 104 * Restore MSR (turn off AltiVec) 105 */ 106 mtmsr(msr); 107 } 108 109 void 110 vec_save_cpu(enum vec_op op) 111 { 112 /* 113 * Turn on AltiVEC, turn off interrupts. 114 */ 115 const register_t msr = mfmsr(); 116 mtmsr((msr & ~PSL_EE) | PSL_VEC); 117 __asm volatile ("isync"); 118 119 struct cpu_info * const ci = curcpu(); 120 lwp_t * const l = ci->ci_veclwp; 121 122 if (l->l_md.md_flags & MDLWP_OWNVEC) { 123 struct pcb * const pcb = lwp_getpcb(l); 124 struct trapframe * const tf = l->l_md.md_utf; 125 126 /* 127 * Grab contents of vector unit. 128 */ 129 vec_unload_to_vreg(&pcb->pcb_vr); 130 131 /* 132 * Save VRSAVE 133 */ 134 pcb->pcb_vr.vrsave = tf->tf_vrsave; 135 136 /* 137 * Note that we aren't using any CPU resources and stop any 138 * data streams. 139 */ 140 __asm volatile ("dssall; sync"); 141 142 /* 143 * Disclaim ownership. 144 */ 145 l->l_md.md_flags &= ~MDLWP_OWNVEC; 146 147 /* 148 * Give up the VEC unit if are releasing it too. 149 */ 150 if (op == VEC_SAVE_AND_RELEASE) 151 ci->ci_veclwp = ci->ci_data.cpu_idlelwp; 152 } 153 154 /* 155 * Restore MSR (turn off AltiVec) 156 */ 157 mtmsr(msr); 158 } 159 160 #ifdef MULTIPROCESSOR 161 /* 162 * Save a process's AltiVEC state to its PCB. The state may be in any CPU. 163 * The process must either be curproc or traced by curproc (and stopped). 164 * (The point being that the process must not run on another CPU during 165 * this function). 166 */ 167 static void 168 vec_mp_save_lwp(struct lwp *l) 169 { 170 /* 171 * Send an IPI to the other CPU with the data and wait for that CPU 172 * to flush the data. Note that the other CPU might have switched 173 * to a different proc's AltiVEC state by the time it receives the IPI, 174 * but that will only result in an unnecessary reload. 175 */ 176 177 if ((l->l_md.md_flags & MDLWP_OWNVEC) == 0) 178 return; 179 180 ppc_send_ipi(l->l_md.md_veccpu->ci_index, PPC_IPI_FLUSH_VEC); 181 182 /* Wait for flush. */ 183 for (u_int i = 0; i < 0x3fffffff; i++) { 184 if ((l->l_md.md_flags & MDLWP_OWNVEC) == 0) 185 return; 186 } 187 188 panic("%s/%d timed out: pid = %d.%d, veccpu->ci_cpuid = %d\n", 189 __func__, cpu_number(), l->l_proc->p_pid, l->l_lid, 190 l->l_md.md_veccpu->ci_cpuid); 191 } 192 #endif /*MULTIPROCESSOR*/ 193 194 /* 195 * Save a process's AltiVEC state to its PCB. The state may be in any CPU. 196 * The process must either be curproc or traced by curproc (and stopped). 197 * (The point being that the process must not run on another CPU during 198 * this function). 199 */ 200 void 201 vec_save_lwp(struct lwp *l, enum vec_op op) 202 { 203 struct cpu_info * const ci = curcpu(); 204 205 KASSERT(l->l_md.md_veccpu != NULL); 206 207 /* 208 * If it's already in the PCB, there's nothing to do. 209 */ 210 if ((l->l_md.md_flags & MDLWP_OWNVEC) == 0) 211 return; 212 213 /* 214 * If we simply need to discard the information, then don't 215 * to save anything. 216 */ 217 if (op == VEC_DISCARD) { 218 #ifndef MULTIPROCESSOR 219 KASSERT(ci == l->l_md.md_veccpu); 220 #endif 221 KASSERT(l == l->l_md.md_veccpu->ci_veclwp); 222 KASSERT(l == curlwp || ci == l->l_md.md_veccpu); 223 ci->ci_veclwp = ci->ci_data.cpu_idlelwp; 224 atomic_and_uint(&l->l_md.md_flags, ~MDLWP_OWNVEC); 225 return; 226 } 227 228 /* 229 * If the state is in the current CPU, just flush the current CPU's 230 * state. 231 */ 232 if (l == ci->ci_veclwp) { 233 vec_save_cpu(op); 234 return; 235 } 236 237 238 #ifdef MULTIPROCESSOR 239 /* 240 * It must be on another CPU, flush it from there. 241 */ 242 vec_mp_save_lwp(l); 243 #endif 244 } 245 246 void 247 vec_restore_from_mcontext(struct lwp *l, const mcontext_t *mcp) 248 { 249 struct pcb * const pcb = lwp_getpcb(l); 250 251 /* we don't need to save the state, just drop it */ 252 vec_save_lwp(l, VEC_DISCARD); 253 memcpy(pcb->pcb_vr.vreg, &mcp->__vrf.__vrs, sizeof (pcb->pcb_vr.vreg)); 254 pcb->pcb_vr.vscr = mcp->__vrf.__vscr; 255 pcb->pcb_vr.vrsave = mcp->__vrf.__vrsave; 256 l->l_md.md_utf->tf_vrsave = pcb->pcb_vr.vrsave; 257 } 258 259 bool 260 vec_save_to_mcontext(struct lwp *l, mcontext_t *mcp, unsigned int *flagp) 261 { 262 /* Save AltiVec context, if any. */ 263 if ((l->l_md.md_flags & MDLWP_USEDVEC) == 0) 264 return false; 265 266 struct pcb * const pcb = lwp_getpcb(l); 267 268 /* 269 * If we're the AltiVec owner, dump its context to the PCB first. 270 */ 271 vec_save_lwp(l, VEC_SAVE); 272 273 mcp->__gregs[_REG_MSR] |= PSL_VEC; 274 mcp->__vrf.__vscr = pcb->pcb_vr.vscr; 275 mcp->__vrf.__vrsave = l->l_md.md_utf->tf_vrsave; 276 memcpy(mcp->__vrf.__vrs, pcb->pcb_vr.vreg, sizeof (mcp->__vrf.__vrs)); 277 *flagp |= _UC_POWERPC_VEC; 278 return true; 279 } 280 281 #define ZERO_VEC 19 282 283 void 284 vzeropage(paddr_t pa) 285 { 286 const paddr_t ea = pa + PAGE_SIZE; 287 uint32_t vec[7], *vp = (void *) roundup((uintptr_t) vec, 16); 288 register_t omsr, msr; 289 290 __asm volatile("mfmsr %0" : "=r"(omsr) :); 291 292 /* 293 * Turn on AltiVec, turn off interrupts. 294 */ 295 msr = (omsr & ~PSL_EE) | PSL_VEC; 296 __asm volatile("sync; mtmsr %0; isync" :: "r"(msr)); 297 298 /* 299 * Save the VEC register we are going to use before we disable 300 * relocation. 301 */ 302 __asm("stvx %1,0,%0" :: "r"(vp), "n"(ZERO_VEC)); 303 __asm("vxor %0,%0,%0" :: "n"(ZERO_VEC)); 304 305 /* 306 * Zero the page using a single cache line. 307 */ 308 __asm volatile( 309 " sync ;" 310 " mfmsr %[msr];" 311 " rlwinm %[msr],%[msr],0,28,26;" /* Clear PSL_DR */ 312 " mtmsr %[msr];" /* Turn off DMMU */ 313 " isync;" 314 "1: stvx %[zv], %[pa], %[off0];" 315 " stvxl %[zv], %[pa], %[off16];" 316 " stvx %[zv], %[pa], %[off32];" 317 " stvxl %[zv], %[pa], %[off48];" 318 " addi %[pa], %[pa], 64;" 319 " cmplw %[pa], %[ea];" 320 " blt+ 1b;" 321 " ori %[msr], %[msr], 0x10;" /* Set PSL_DR */ 322 " sync;" 323 " mtmsr %[msr];" /* Turn on DMMU */ 324 " isync;" 325 :: [msr] "r"(msr), [pa] "b"(pa), [ea] "b"(ea), 326 [off0] "r"(0), [off16] "r"(16), [off32] "r"(32), [off48] "r"(48), 327 [zv] "n"(ZERO_VEC)); 328 329 /* 330 * Restore VEC register (now that we can access the stack again). 331 */ 332 __asm("lvx %1,0,%0" :: "r"(vp), "n"(ZERO_VEC)); 333 334 /* 335 * Restore old MSR (AltiVec OFF). 336 */ 337 __asm volatile("sync; mtmsr %0; isync" :: "r"(omsr)); 338 } 339 340 #define LO_VEC 16 341 #define HI_VEC 17 342 343 void 344 vcopypage(paddr_t dst, paddr_t src) 345 { 346 const paddr_t edst = dst + PAGE_SIZE; 347 uint32_t vec[11], *vp = (void *) roundup((uintptr_t) vec, 16); 348 register_t omsr, msr; 349 350 __asm volatile("mfmsr %0" : "=r"(omsr) :); 351 352 /* 353 * Turn on AltiVec, turn off interrupts. 354 */ 355 msr = (omsr & ~PSL_EE) | PSL_VEC; 356 __asm volatile("sync; mtmsr %0; isync" :: "r"(msr)); 357 358 /* 359 * Save the VEC registers we will be using before we disable 360 * relocation. 361 */ 362 __asm("stvx %2,%1,%0" :: "b"(vp), "r"( 0), "n"(LO_VEC)); 363 __asm("stvx %2,%1,%0" :: "b"(vp), "r"(16), "n"(HI_VEC)); 364 365 /* 366 * Copy the page using a single cache line, with DMMU 367 * disabled. On most PPCs, two vector registers occupy one 368 * cache line. 369 */ 370 __asm volatile( 371 " sync ;" 372 " mfmsr %[msr];" 373 " rlwinm %[msr],%[msr],0,28,26;" /* Clear PSL_DR */ 374 " mtmsr %[msr];" /* Turn off DMMU */ 375 " isync;" 376 "1: lvx %[lv], %[src], %[off0];" 377 " stvx %[lv], %[dst], %[off0];" 378 " lvxl %[hv], %[src], %[off16];" 379 " stvxl %[hv], %[dst], %[off16];" 380 " addi %[src], %[src], 32;" 381 " addi %[dst], %[dst], 32;" 382 " cmplw %[dst], %[edst];" 383 " blt+ 1b;" 384 " ori %[msr], %[msr], 0x10;" /* Set PSL_DR */ 385 " sync;" 386 " mtmsr %[msr];" /* Turn on DMMU */ 387 " isync;" 388 :: [msr] "r"(msr), [src] "b"(src), [dst] "b"(dst), 389 [edst] "b"(edst), [off0] "r"(0), [off16] "r"(16), 390 [lv] "n"(LO_VEC), [hv] "n"(HI_VEC)); 391 392 /* 393 * Restore VEC registers (now that we can access the stack again). 394 */ 395 __asm("lvx %2,%1,%0" :: "b"(vp), "r"( 0), "n"(LO_VEC)); 396 __asm("lvx %2,%1,%0" :: "b"(vp), "r"(16), "n"(HI_VEC)); 397 398 /* 399 * Restore old MSR (AltiVec OFF). 400 */ 401 __asm volatile("sync; mtmsr %0; isync" :: "r"(omsr)); 402 } 403