1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2014 Ian Lepore <ian@freebsd.org> 5 * Copyright (c) 2012 Mark Tinguely 6 * 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #ifdef VFP 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/limits.h> 38 #include <sys/proc.h> 39 #include <sys/imgact_elf.h> 40 #include <sys/kernel.h> 41 42 #include <machine/armreg.h> 43 #include <machine/elf.h> 44 #include <machine/frame.h> 45 #include <machine/md_var.h> 46 #include <machine/pcb.h> 47 #include <machine/undefined.h> 48 #include <machine/vfp.h> 49 50 /* function prototypes */ 51 static int vfp_bounce(u_int, u_int, struct trapframe *, int); 52 static void vfp_restore(struct vfp_state *); 53 54 extern int vfp_exists; 55 static struct undefined_handler vfp10_uh, vfp11_uh; 56 /* If true the VFP unit has 32 double registers, otherwise it has 16 */ 57 static int is_d32; 58 59 struct fpu_kern_ctx { 60 struct vfp_state *prev; 61 #define FPU_KERN_CTX_DUMMY 0x01 /* avoided save for the kern thread */ 62 #define FPU_KERN_CTX_INUSE 0x02 63 uint32_t flags; 64 struct vfp_state state; 65 }; 66 67 /* 68 * About .fpu directives in this file... 69 * 70 * We should need simply .fpu vfpv3, but clang 3.5 has a quirk where setting 71 * vfpv3 doesn't imply that vfp2 features are also available -- both have to be 72 * explicitly set to get all the features of both. This is probably a bug in 73 * clang, so it may get fixed and require changes here some day. Other changes 74 * are probably coming in clang too, because there is email and open PRs 75 * indicating they want to completely disable the ability to use .fpu and 76 * similar directives in inline asm. That would be catastrophic for us, 77 * hopefully they come to their senses. There was also some discusion of a new 78 * syntax such as .push fpu=vfpv3; ...; .pop fpu; and that would be ideal for 79 * us, better than what we have now really. 80 * 81 * For gcc, each .fpu directive completely overrides the prior directive, unlike 82 * with clang, but luckily on gcc saying v3 implies all the v2 features as well. 83 */ 84 85 #define fmxr(reg, val) \ 86 __asm __volatile(" .fpu vfpv2\n .fpu vfpv3\n" \ 87 " vmsr " __STRING(reg) ", %0" :: "r"(val)); 88 89 #define fmrx(reg) \ 90 ({ u_int val = 0;\ 91 __asm __volatile(" .fpu vfpv2\n .fpu vfpv3\n" \ 92 " vmrs %0, " __STRING(reg) : "=r"(val)); \ 93 val; \ 94 }) 95 96 static u_int 97 get_coprocessorACR(void) 98 { 99 u_int val; 100 __asm __volatile("mrc p15, 0, %0, c1, c0, 2" : "=r" (val) : : "cc"); 101 return val; 102 } 103 104 static void 105 set_coprocessorACR(u_int val) 106 { 107 __asm __volatile("mcr p15, 0, %0, c1, c0, 2\n\t" 108 : : "r" (val) : "cc"); 109 isb(); 110 } 111 112 static void 113 vfp_enable(void) 114 { 115 uint32_t fpexc; 116 117 fpexc = fmrx(fpexc); 118 fmxr(fpexc, fpexc | VFPEXC_EN); 119 isb(); 120 } 121 122 static void 123 vfp_disable(void) 124 { 125 uint32_t fpexc; 126 127 fpexc = fmrx(fpexc); 128 fmxr(fpexc, fpexc & ~VFPEXC_EN); 129 isb(); 130 } 131 132 /* called for each cpu */ 133 void 134 vfp_init(void) 135 { 136 u_int fpsid, tmp; 137 u_int coproc, vfp_arch; 138 139 coproc = get_coprocessorACR(); 140 coproc |= COPROC10 | COPROC11; 141 set_coprocessorACR(coproc); 142 143 fpsid = fmrx(fpsid); /* read the vfp system id */ 144 145 if (!(fpsid & VFPSID_HARDSOFT_IMP)) { 146 vfp_exists = 1; 147 is_d32 = 0; 148 PCPU_SET(vfpsid, fpsid); /* save the fpsid */ 149 elf_hwcap |= HWCAP_VFP; 150 151 vfp_arch = 152 (fpsid & VFPSID_SUBVERSION2_MASK) >> VFPSID_SUBVERSION_OFF; 153 154 if (vfp_arch >= VFP_ARCH3) { 155 tmp = fmrx(mvfr0); 156 PCPU_SET(vfpmvfr0, tmp); 157 elf_hwcap |= HWCAP_VFPv3; 158 159 if ((tmp & VMVFR0_RB_MASK) == 2) { 160 elf_hwcap |= HWCAP_VFPD32; 161 is_d32 = 1; 162 } else 163 elf_hwcap |= HWCAP_VFPv3D16; 164 165 tmp = fmrx(mvfr1); 166 PCPU_SET(vfpmvfr1, tmp); 167 168 if (PCPU_GET(cpuid) == 0) { 169 if ((tmp & VMVFR1_FZ_MASK) == 0x1) { 170 /* Denormals arithmetic support */ 171 initial_fpscr &= ~VFPSCR_FZ; 172 thread0.td_pcb->pcb_vfpstate.fpscr = 173 initial_fpscr; 174 } 175 } 176 177 if ((tmp & VMVFR1_LS_MASK) >> VMVFR1_LS_OFF == 1 && 178 (tmp & VMVFR1_I_MASK) >> VMVFR1_I_OFF == 1 && 179 (tmp & VMVFR1_SP_MASK) >> VMVFR1_SP_OFF == 1) 180 elf_hwcap |= HWCAP_NEON; 181 if ((tmp & VMVFR1_FMAC_MASK) >> VMVFR1_FMAC_OFF == 1) 182 elf_hwcap |= HWCAP_VFPv4; 183 } 184 185 /* initialize the coprocess 10 and 11 calls 186 * These are called to restore the registers and enable 187 * the VFP hardware. 188 */ 189 if (vfp10_uh.uh_handler == NULL) { 190 vfp10_uh.uh_handler = vfp_bounce; 191 vfp11_uh.uh_handler = vfp_bounce; 192 install_coproc_handler_static(10, &vfp10_uh); 193 install_coproc_handler_static(11, &vfp11_uh); 194 } 195 } 196 } 197 198 SYSINIT(vfp, SI_SUB_CPU, SI_ORDER_ANY, vfp_init, NULL); 199 200 /* start VFP unit, restore the vfp registers from the PCB and retry 201 * the instruction 202 */ 203 static int 204 vfp_bounce(u_int addr, u_int insn, struct trapframe *frame, int code) 205 { 206 u_int cpu, fpexc; 207 struct pcb *curpcb; 208 ksiginfo_t ksi; 209 210 if ((code & FAULT_USER) == 0) 211 panic("undefined floating point instruction in supervisor mode"); 212 213 critical_enter(); 214 215 /* 216 * If the VFP is already on and we got an undefined instruction, then 217 * something tried to executate a truly invalid instruction that maps to 218 * the VFP. 219 */ 220 fpexc = fmrx(fpexc); 221 if (fpexc & VFPEXC_EN) { 222 /* Clear any exceptions */ 223 fmxr(fpexc, fpexc & ~(VFPEXC_EX | VFPEXC_FP2V)); 224 225 /* kill the process - we do not handle emulation */ 226 critical_exit(); 227 228 if (fpexc & VFPEXC_EX) { 229 /* We have an exception, signal a SIGFPE */ 230 ksiginfo_init_trap(&ksi); 231 ksi.ksi_signo = SIGFPE; 232 if (fpexc & VFPEXC_UFC) 233 ksi.ksi_code = FPE_FLTUND; 234 else if (fpexc & VFPEXC_OFC) 235 ksi.ksi_code = FPE_FLTOVF; 236 else if (fpexc & VFPEXC_IOC) 237 ksi.ksi_code = FPE_FLTINV; 238 ksi.ksi_addr = (void *)addr; 239 trapsignal(curthread, &ksi); 240 return 0; 241 } 242 243 return 1; 244 } 245 246 /* 247 * If the last time this thread used the VFP it was on this core, and 248 * the last thread to use the VFP on this core was this thread, then the 249 * VFP state is valid, otherwise restore this thread's state to the VFP. 250 */ 251 fmxr(fpexc, fpexc | VFPEXC_EN); 252 curpcb = curthread->td_pcb; 253 cpu = PCPU_GET(cpuid); 254 if (curpcb->pcb_vfpcpu != cpu || curthread != PCPU_GET(fpcurthread)) { 255 vfp_restore(curpcb->pcb_vfpsaved); 256 curpcb->pcb_vfpcpu = cpu; 257 PCPU_SET(fpcurthread, curthread); 258 } 259 260 critical_exit(); 261 262 KASSERT(curpcb->pcb_vfpsaved == &curpcb->pcb_vfpstate, 263 ("Kernel VFP state in use when entering userspace")); 264 265 return (0); 266 } 267 268 /* 269 * Update the VFP state for a forked process or new thread. The PCB will 270 * have been copied from the old thread. 271 * The code is heavily based on arm64 logic. 272 */ 273 void 274 vfp_new_thread(struct thread *newtd, struct thread *oldtd, bool fork) 275 { 276 struct pcb *newpcb; 277 278 newpcb = newtd->td_pcb; 279 280 /* Kernel threads start with clean VFP */ 281 if ((oldtd->td_pflags & TDP_KTHREAD) != 0) { 282 newpcb->pcb_fpflags &= 283 ~(PCB_FP_STARTED | PCB_FP_KERN | PCB_FP_NOSAVE); 284 } else { 285 MPASS((newpcb->pcb_fpflags & (PCB_FP_KERN|PCB_FP_NOSAVE)) == 0); 286 if (!fork) { 287 newpcb->pcb_fpflags &= ~PCB_FP_STARTED; 288 } 289 } 290 291 newpcb->pcb_vfpsaved = &newpcb->pcb_vfpstate; 292 newpcb->pcb_vfpcpu = UINT_MAX; 293 } 294 /* 295 * Restore the given state to the VFP hardware. 296 */ 297 static void 298 vfp_restore(struct vfp_state *vfpsave) 299 { 300 uint32_t fpexc; 301 302 /* On vfpv3 we may need to restore FPINST and FPINST2 */ 303 fpexc = vfpsave->fpexec; 304 if (fpexc & VFPEXC_EX) { 305 fmxr(fpinst, vfpsave->fpinst); 306 if (fpexc & VFPEXC_FP2V) 307 fmxr(fpinst2, vfpsave->fpinst2); 308 } 309 fmxr(fpscr, vfpsave->fpscr); 310 311 __asm __volatile( 312 " .fpu vfpv2\n" 313 " .fpu vfpv3\n" 314 " vldmia %0!, {d0-d15}\n" /* d0-d15 */ 315 " cmp %1, #0\n" /* -D16 or -D32? */ 316 " vldmiane %0!, {d16-d31}\n" /* d16-d31 */ 317 " addeq %0, %0, #128\n" /* skip missing regs */ 318 : "+&r" (vfpsave) : "r" (is_d32) : "cc" 319 ); 320 321 fmxr(fpexc, fpexc); 322 } 323 324 /* 325 * If the VFP is on, save its current state and turn it off if requested to do 326 * so. If the VFP is not on, does not change the values at *vfpsave. Caller is 327 * responsible for preventing a context switch while this is running. 328 */ 329 void 330 vfp_store(struct vfp_state *vfpsave, boolean_t disable_vfp) 331 { 332 uint32_t fpexc; 333 334 fpexc = fmrx(fpexc); /* Is the vfp enabled? */ 335 if (fpexc & VFPEXC_EN) { 336 vfpsave->fpexec = fpexc; 337 vfpsave->fpscr = fmrx(fpscr); 338 339 /* On vfpv3 we may need to save FPINST and FPINST2 */ 340 if (fpexc & VFPEXC_EX) { 341 vfpsave->fpinst = fmrx(fpinst); 342 if (fpexc & VFPEXC_FP2V) 343 vfpsave->fpinst2 = fmrx(fpinst2); 344 fpexc &= ~VFPEXC_EX; 345 } 346 347 __asm __volatile( 348 " .fpu vfpv2\n" 349 " .fpu vfpv3\n" 350 " vstmia %0!, {d0-d15}\n" /* d0-d15 */ 351 " cmp %1, #0\n" /* -D16 or -D32? */ 352 " vstmiane %0!, {d16-d31}\n" /* d16-d31 */ 353 " addeq %0, %0, #128\n" /* skip missing regs */ 354 : "+&r" (vfpsave) : "r" (is_d32) : "cc" 355 ); 356 357 if (disable_vfp) 358 fmxr(fpexc , fpexc & ~VFPEXC_EN); 359 } 360 } 361 362 /* 363 * The current thread is dying. If the state currently in the hardware belongs 364 * to the current thread, set fpcurthread to NULL to indicate that the VFP 365 * hardware state does not belong to any thread. If the VFP is on, turn it off. 366 */ 367 void 368 vfp_discard(struct thread *td) 369 { 370 u_int tmp; 371 372 if (PCPU_GET(fpcurthread) == td) 373 PCPU_SET(fpcurthread, NULL); 374 375 tmp = fmrx(fpexc); 376 if (tmp & VFPEXC_EN) 377 fmxr(fpexc, tmp & ~VFPEXC_EN); 378 } 379 380 void 381 vfp_save_state(struct thread *td, struct pcb *pcb) 382 { 383 int32_t fpexc; 384 385 KASSERT(pcb != NULL, ("NULL vfp pcb")); 386 KASSERT(td == NULL || td->td_pcb == pcb, ("Invalid vfp pcb")); 387 388 /* 389 * savectx() will be called on panic with dumppcb as an argument, 390 * dumppcb doesn't have pcb_vfpsaved set, so set it to save 391 * the VFP registers. 392 */ 393 if (pcb->pcb_vfpsaved == NULL) 394 pcb->pcb_vfpsaved = &pcb->pcb_vfpstate; 395 396 if (td == NULL) 397 td = curthread; 398 399 critical_enter(); 400 /* 401 * Only store the registers if the VFP is enabled, 402 * i.e. return if we are trapping on FP access. 403 */ 404 fpexc = fmrx(fpexc); 405 if (fpexc & VFPEXC_EN) { 406 KASSERT(PCPU_GET(fpcurthread) == td, 407 ("Storing an invalid VFP state")); 408 409 vfp_store(pcb->pcb_vfpsaved, true); 410 } 411 critical_exit(); 412 } 413 414 void 415 fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags) 416 { 417 struct pcb *pcb; 418 419 pcb = td->td_pcb; 420 KASSERT((flags & FPU_KERN_NOCTX) != 0 || ctx != NULL, 421 ("ctx is required when !FPU_KERN_NOCTX")); 422 KASSERT(ctx == NULL || (ctx->flags & FPU_KERN_CTX_INUSE) == 0, 423 ("using inuse ctx")); 424 KASSERT((pcb->pcb_fpflags & PCB_FP_NOSAVE) == 0, 425 ("recursive fpu_kern_enter while in PCB_FP_NOSAVE state")); 426 427 if ((flags & FPU_KERN_NOCTX) != 0) { 428 critical_enter(); 429 if (curthread == PCPU_GET(fpcurthread)) { 430 vfp_save_state(curthread, pcb); 431 } 432 PCPU_SET(fpcurthread, NULL); 433 434 vfp_enable(); 435 pcb->pcb_fpflags |= PCB_FP_KERN | PCB_FP_NOSAVE | 436 PCB_FP_STARTED; 437 return; 438 } 439 440 if ((flags & FPU_KERN_KTHR) != 0 && is_fpu_kern_thread(0)) { 441 ctx->flags = FPU_KERN_CTX_DUMMY | FPU_KERN_CTX_INUSE; 442 return; 443 } 444 /* 445 * Check either we are already using the VFP in the kernel, or 446 * the the saved state points to the default user space. 447 */ 448 KASSERT((pcb->pcb_fpflags & PCB_FP_KERN) != 0 || 449 pcb->pcb_vfpsaved == &pcb->pcb_vfpstate, 450 ("Mangled pcb_vfpsaved %x %p %p", pcb->pcb_fpflags, pcb->pcb_vfpsaved, 451 &pcb->pcb_vfpstate)); 452 ctx->flags = FPU_KERN_CTX_INUSE; 453 vfp_save_state(curthread, pcb); 454 ctx->prev = pcb->pcb_vfpsaved; 455 pcb->pcb_vfpsaved = &ctx->state; 456 pcb->pcb_fpflags |= PCB_FP_KERN; 457 pcb->pcb_fpflags &= ~PCB_FP_STARTED; 458 459 return; 460 } 461 462 int 463 fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx) 464 { 465 struct pcb *pcb; 466 467 pcb = td->td_pcb; 468 469 if ((pcb->pcb_fpflags & PCB_FP_NOSAVE) != 0) { 470 KASSERT(ctx == NULL, ("non-null ctx after FPU_KERN_NOCTX")); 471 KASSERT(PCPU_GET(fpcurthread) == NULL, 472 ("non-NULL fpcurthread for PCB_FP_NOSAVE")); 473 CRITICAL_ASSERT(td); 474 475 vfp_disable(); 476 pcb->pcb_fpflags &= ~(PCB_FP_NOSAVE | PCB_FP_STARTED); 477 critical_exit(); 478 } else { 479 KASSERT((ctx->flags & FPU_KERN_CTX_INUSE) != 0, 480 ("FPU context not inuse")); 481 ctx->flags &= ~FPU_KERN_CTX_INUSE; 482 483 if (is_fpu_kern_thread(0) && 484 (ctx->flags & FPU_KERN_CTX_DUMMY) != 0) 485 return (0); 486 KASSERT((ctx->flags & FPU_KERN_CTX_DUMMY) == 0, ("dummy ctx")); 487 critical_enter(); 488 vfp_discard(td); 489 critical_exit(); 490 pcb->pcb_fpflags &= ~PCB_FP_STARTED; 491 pcb->pcb_vfpsaved = ctx->prev; 492 } 493 494 if (pcb->pcb_vfpsaved == &pcb->pcb_vfpstate) { 495 pcb->pcb_fpflags &= ~PCB_FP_KERN; 496 } else { 497 KASSERT((pcb->pcb_fpflags & PCB_FP_KERN) != 0, 498 ("unpaired fpu_kern_leave")); 499 } 500 501 return (0); 502 } 503 504 int 505 fpu_kern_thread(u_int flags __unused) 506 { 507 struct pcb *pcb = curthread->td_pcb; 508 509 KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0, 510 ("Only kthread may use fpu_kern_thread")); 511 KASSERT(pcb->pcb_vfpsaved == &pcb->pcb_vfpstate, 512 ("Mangled pcb_vfpsaved")); 513 KASSERT((pcb->pcb_fpflags & PCB_FP_KERN) == 0, 514 ("Thread already setup for the VFP")); 515 pcb->pcb_fpflags |= PCB_FP_KERN; 516 return (0); 517 } 518 519 int 520 is_fpu_kern_thread(u_int flags __unused) 521 { 522 struct pcb *curpcb; 523 524 if ((curthread->td_pflags & TDP_KTHREAD) == 0) 525 return (0); 526 curpcb = curthread->td_pcb; 527 return ((curpcb->pcb_fpflags & PCB_FP_KERN) != 0); 528 } 529 530 #endif 531