1 /* $OpenBSD: npx.c,v 1.76 2024/05/13 01:15:50 jsg Exp $ */
2 /* $NetBSD: npx.c,v 1.57 1996/05/12 23:12:24 mycroft Exp $ */
3
4 #if 0
5 #define IPRINTF(x) printf x
6 #else
7 #define IPRINTF(x)
8 #endif
9
10 /*-
11 * Copyright (c) 1994, 1995 Charles M. Hannum. All rights reserved.
12 * Copyright (c) 1990 William Jolitz.
13 * Copyright (c) 1991 The Regents of the University of California.
14 * All rights reserved.
15 *
16 * Redistribution and use in source and binary forms, with or without
17 * modification, are permitted provided that the following conditions
18 * are met:
19 * 1. Redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer.
21 * 2. Redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution.
24 * 3. Neither the name of the University nor the names of its contributors
25 * may be used to endorse or promote products derived from this software
26 * without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 * SUCH DAMAGE.
39 *
40 * @(#)npx.c 7.2 (Berkeley) 5/12/91
41 */
42
43 #include <sys/param.h>
44 #include <sys/systm.h>
45 #include <sys/proc.h>
46 #include <sys/signalvar.h>
47 #include <sys/user.h>
48 #include <sys/device.h>
49
50 #include <uvm/uvm_extern.h>
51
52 #include <machine/intr.h>
53 #include <machine/npx.h>
54 #include <machine/pio.h>
55 #include <machine/cpufunc.h>
56 #include <machine/pcb.h>
57 #include <machine/trap.h>
58 #include <machine/specialreg.h>
59 #include <machine/i8259.h>
60
61 #include <dev/isa/isavar.h>
62
63 /*
64 * 387 and 287 Numeric Coprocessor Extension (NPX) Driver.
65 *
66 * We do lazy initialization and switching using the TS bit in cr0 and the
67 * MDP_USEDFPU bit in mdproc.
68 *
69 * DNA exceptions are handled like this:
70 *
71 * 1) If there is no NPX, return and go to the emulator.
72 * 2) If someone else has used the NPX, save its state into that process's PCB.
73 * 3a) If MDP_USEDFPU is not set, set it and initialize the NPX.
74 * 3b) Otherwise, reload the process's previous NPX state.
75 *
76 * When a process is created or exec()s, its saved cr0 image has the TS bit
77 * set and the MDP_USEDFPU bit clear. The MDP_USEDFPU bit is set when the
78 * process first gets a DNA and the NPX is initialized. The TS bit is turned
79 * off when the NPX is used, and turned on again later when the process's NPX
80 * state is saved.
81 */
82
83 #define fldcw(addr) __asm("fldcw %0" : : "m" (*addr))
84 #define fnclex() __asm("fnclex")
85 #define fninit() __asm("fninit")
86 #define fnsave(addr) __asm("fnsave %0" : "=m" (*addr))
87 #define fnstcw(addr) __asm("fnstcw %0" : "=m" (*addr))
88 #define fnstsw(addr) __asm("fnstsw %0" : "=m" (*addr))
89 #define fp_divide_by_0() __asm("fldz; fld1; fdiv %st,%st(1); fwait")
90 #define frstor(addr) __asm("frstor %0" : : "m" (*addr))
91 #define fwait() __asm("fwait")
92 #define clts() __asm("clts")
93 #define stts() lcr0(rcr0() | CR0_TS)
94
95 /*
96 * The mxcsr_mask for this host, taken from fxsave() on the primary CPU
97 */
98 uint32_t fpu_mxcsr_mask;
99
100 int npxintr(void *);
101 static int npxprobe1(struct isa_attach_args *);
102 static int x86fpflags_to_siginfo(u_int32_t);
103
104
105 struct npx_softc {
106 struct device sc_dev;
107 void *sc_ih;
108 };
109
110 int npxprobe(struct device *, void *, void *);
111 void npxattach(struct device *, struct device *, void *);
112
113 const struct cfattach npx_ca = {
114 sizeof(struct npx_softc), npxprobe, npxattach
115 };
116
117 struct cfdriver npx_cd = {
118 NULL, "npx", DV_DULL
119 };
120
121 enum npx_type {
122 NPX_NONE = 0,
123 NPX_INTERRUPT,
124 NPX_EXCEPTION,
125 NPX_BROKEN,
126 NPX_CPUID,
127 };
128
129 static enum npx_type npx_type;
130 static volatile u_int npx_intrs_while_probing
131 __attribute__((section(".kudata")));
132 static volatile u_int npx_traps_while_probing
133 __attribute__((section(".kudata")));
134
135 #define fxsave(addr) __asm("fxsave %0" : "=m" (*addr))
136 #define fxrstor(addr) __asm("fxrstor %0" : : "m" (*addr))
137 #define ldmxcsr(addr) __asm("ldmxcsr %0" : : "m" (*addr))
138
139 static __inline void
fpu_save(union savefpu * addr)140 fpu_save(union savefpu *addr)
141 {
142
143 if (i386_use_fxsave) {
144 fxsave(&addr->sv_xmm);
145 /* FXSAVE doesn't FNINIT like FNSAVE does -- so do it here. */
146 fninit();
147 } else
148 fnsave(&addr->sv_87);
149 }
150
151 static int
npxdna_notset(struct cpu_info * ci)152 npxdna_notset(struct cpu_info *ci)
153 {
154 panic("npxdna vector not initialized");
155 }
156
157 int (*npxdna_func)(struct cpu_info *) = npxdna_notset;
158 int npxdna_s87(struct cpu_info *);
159 int npxdna_xmm(struct cpu_info *);
160
161 /*
162 * Special interrupt handlers. Someday intr0-intr15 will be used to count
163 * interrupts. We'll still need a special exception 16 handler. The busy
164 * latch stuff in probintr() can be moved to npxprobe().
165 */
166 void probeintr(void);
167 asm (".text\n\t"
168 "probeintr:\n\t"
169 "ss\n\t"
170 "incl npx_intrs_while_probing\n\t"
171 "pushl %eax\n\t"
172 "movb $0x20,%al # EOI (asm in strings loses cpp features)\n\t"
173 "outb %al,$0xa0 # IO_ICU2\n\t"
174 "outb %al,$0x20 # IO_ICU1\n\t"
175 "movb $0,%al\n\t"
176 "outb %al,$0xf0 # clear BUSY# latch\n\t"
177 "popl %eax\n\t"
178 "iret\n\t");
179
180 void probetrap(void);
181 asm (".text\n\t"
182 "probetrap:\n\t"
183 "ss\n\t"
184 "incl npx_traps_while_probing\n\t"
185 "fnclex\n\t"
186 "iret\n\t");
187
188 static inline int
npxprobe1(struct isa_attach_args * ia)189 npxprobe1(struct isa_attach_args *ia)
190 {
191 int control;
192 int status;
193
194 ia->ia_iosize = 16;
195 ia->ia_msize = 0;
196
197 /*
198 * Finish resetting the coprocessor, if any. If there is an error
199 * pending, then we may get a bogus IRQ13, but probeintr() will handle
200 * it OK. Bogus halts have never been observed, but we enabled
201 * IRQ13 and cleared the BUSY# latch early to handle them anyway.
202 */
203 fninit();
204 delay(1000); /* wait for any IRQ13 (fwait might hang) */
205
206 /*
207 * Check for a status of mostly zero.
208 */
209 status = 0x5a5a;
210 fnstsw(&status);
211 if ((status & 0xb8ff) == 0) {
212 /*
213 * Good, now check for a proper control word.
214 */
215 control = 0x5a5a;
216 fnstcw(&control);
217 if ((control & 0x1f3f) == 0x033f) {
218 /*
219 * We have an npx, now divide by 0 to see if exception
220 * 16 works.
221 */
222 control &= ~(1 << 2); /* enable divide by 0 trap */
223 fldcw(&control);
224 npx_traps_while_probing = npx_intrs_while_probing = 0;
225 fp_divide_by_0();
226 delay(1);
227 if (npx_traps_while_probing != 0) {
228 /*
229 * Good, exception 16 works.
230 */
231 npx_type = NPX_EXCEPTION;
232 ia->ia_irq = IRQUNK; /* zap the interrupt */
233 } else if (npx_intrs_while_probing != 0) {
234 /*
235 * Bad, we are stuck with IRQ13.
236 */
237 npx_type = NPX_INTERRUPT;
238 } else {
239 /*
240 * Worse, even IRQ13 is broken.
241 */
242 npx_type = NPX_BROKEN;
243 ia->ia_irq = IRQUNK;
244 }
245 return 1;
246 }
247 }
248
249 /*
250 * Probe failed. There is no usable FPU.
251 */
252 npx_type = NPX_NONE;
253 return 0;
254 }
255
256 /*
257 * Probe routine. Initialize cr0 to give correct behaviour for [f]wait
258 * whether the device exists or not (XXX should be elsewhere). Set flags
259 * to tell npxattach() what to do. Modify device struct if npx doesn't
260 * need to use interrupts. Return 1 if device exists.
261 */
262 int
npxprobe(struct device * parent,void * match,void * aux)263 npxprobe(struct device *parent, void *match, void *aux)
264 {
265 struct isa_attach_args *ia = aux;
266 int irq;
267 int result;
268 u_long s;
269 unsigned save_imen;
270 struct gate_descriptor save_idt_npxintr;
271 struct gate_descriptor save_idt_npxtrap;
272
273 if (cpu_feature & CPUID_FPU) {
274 npx_type = NPX_CPUID;
275 ia->ia_irq = IRQUNK; /* Don't want the interrupt vector */
276 ia->ia_iosize = 16;
277 ia->ia_msize = 0;
278 return 1;
279 }
280
281 /*
282 * This routine is now just a wrapper for npxprobe1(), to install
283 * special npx interrupt and trap handlers, to enable npx interrupts
284 * and to disable other interrupts. Someday isa_configure() will
285 * install suitable handlers and run with interrupts enabled so we
286 * won't need to do so much here.
287 */
288 irq = NRSVIDT + ia->ia_irq;
289 s = intr_disable();
290 save_idt_npxintr = idt[irq];
291 save_idt_npxtrap = idt[16];
292 setgate(&idt[irq], probeintr, 0, SDT_SYS386IGT, SEL_KPL, GICODE_SEL);
293 setgate(&idt[16], probetrap, 0, SDT_SYS386TGT, SEL_KPL, GCODE_SEL);
294 save_imen = imen;
295 imen = ~((1 << IRQ_SLAVE) | (1 << ia->ia_irq));
296 SET_ICUS();
297
298 /*
299 * Partially reset the coprocessor, if any. Some BIOS's don't reset
300 * it after a warm boot.
301 */
302 outb(0xf1, 0); /* full reset on some systems, NOP on others */
303 delay(1000);
304 outb(0xf0, 0); /* clear BUSY# latch */
305
306 /*
307 * We set CR0 in locore to trap all ESC and WAIT instructions.
308 * We have to turn off the CR0_EM bit temporarily while probing.
309 */
310 lcr0(rcr0() & ~(CR0_EM|CR0_TS));
311 intr_restore(s);
312 result = npxprobe1(ia);
313 s = intr_disable();
314 lcr0(rcr0() | (CR0_EM|CR0_TS));
315
316 imen = save_imen;
317 SET_ICUS();
318 idt[irq] = save_idt_npxintr;
319 idt[16] = save_idt_npxtrap;
320 intr_restore(s);
321 return (result);
322 }
323
324 int npx586bug1(int, int);
325 asm (".text\n\t"
326 "npx586bug1:\n\t"
327 "fildl 4(%esp) # x\n\t"
328 "fildl 8(%esp) # y\n\t"
329 "fld %st(1)\n\t"
330 "fdiv %st(1),%st # x/y\n\t"
331 "fmulp %st,%st(1) # (x/y)*y\n\t"
332 "fsubrp %st,%st(1) # x-(x/y)*y\n\t"
333 "pushl $0\n\t"
334 "fistpl (%esp)\n\t"
335 "popl %eax\n\t"
336 "ret\n\t");
337
338 void
npxinit(struct cpu_info * ci)339 npxinit(struct cpu_info *ci)
340 {
341 lcr0(rcr0() & ~(CR0_EM|CR0_TS));
342 fninit();
343 if (npx586bug1(4195835, 3145727) != 0) {
344 printf("%s: WARNING: Pentium FDIV bug detected!\n",
345 ci->ci_dev->dv_xname);
346 }
347 if (fpu_mxcsr_mask == 0 && i386_use_fxsave) {
348 struct savexmm xm __attribute__((aligned(16)));
349
350 bzero(&xm, sizeof(xm));
351 fxsave(&xm);
352 if (xm.sv_env.en_mxcsr_mask)
353 fpu_mxcsr_mask = xm.sv_env.en_mxcsr_mask;
354 else
355 fpu_mxcsr_mask = __INITIAL_MXCSR_MASK__;
356 }
357 lcr0(rcr0() | (CR0_TS));
358 }
359
360 /*
361 * Attach routine - announce which it is, and wire into system
362 */
363 void
npxattach(struct device * parent,struct device * self,void * aux)364 npxattach(struct device *parent, struct device *self, void *aux)
365 {
366 struct npx_softc *sc = (void *)self;
367 struct isa_attach_args *ia = aux;
368
369 switch (npx_type) {
370 case NPX_INTERRUPT:
371 printf("\n");
372 lcr0(rcr0() & ~CR0_NE);
373 sc->sc_ih = isa_intr_establish(ia->ia_ic, ia->ia_irq,
374 IST_EDGE, IPL_NONE, npxintr, 0, sc->sc_dev.dv_xname);
375 break;
376 case NPX_EXCEPTION:
377 printf(": using exception 16\n");
378 break;
379 case NPX_CPUID:
380 printf(": reported by CPUID; using exception 16\n");
381 npx_type = NPX_EXCEPTION;
382 break;
383 case NPX_BROKEN:
384 printf(": error reporting broken; not using\n");
385 npx_type = NPX_NONE;
386 return;
387 case NPX_NONE:
388 return;
389 }
390
391 npxinit(&cpu_info_primary);
392
393 if (i386_use_fxsave)
394 npxdna_func = npxdna_xmm;
395 else
396 npxdna_func = npxdna_s87;
397 }
398
399 /*
400 * Record the FPU state and reinitialize it all except for the control word.
401 * Then generate a SIGFPE.
402 *
403 * Reinitializing the state allows naive SIGFPE handlers to longjmp without
404 * doing any fixups.
405 *
406 * XXX there is currently no way to pass the full error state to signal
407 * handlers, and if this is a nested interrupt there is no way to pass even
408 * a status code! So there is no way to have a non-naive SIGFPE handler. At
409 * best a handler could do an fninit followed by an fldcw of a static value.
410 * fnclex would be of little use because it would leave junk on the FPU stack.
411 * Returning from the handler would be even less safe than usual because
412 * IRQ13 exception handling makes exceptions even less precise than usual.
413 */
414 int
npxintr(void * arg)415 npxintr(void *arg)
416 {
417 struct cpu_info *ci = curcpu();
418 struct proc *p = ci->ci_fpcurproc;
419 union savefpu *addr;
420 struct intrframe *frame = arg;
421 int code;
422 union sigval sv;
423
424 uvmexp.traps++;
425 IPRINTF(("%s: fp intr\n", ci->ci_dev->dv_xname));
426
427 if (p == NULL || npx_type == NPX_NONE) {
428 /* XXX no %p in stand/printf.c. Cast to quiet gcc -Wall. */
429 printf("npxintr: p = %lx, curproc = %lx, npx_type = %d\n",
430 (u_long) p, (u_long) curproc, npx_type);
431 panic("npxintr from nowhere");
432 }
433 /*
434 * Clear the interrupt latch.
435 */
436 outb(0xf0, 0);
437 /*
438 * If we're saving, ignore the interrupt. The FPU will happily
439 * generate another one when we restore the state later.
440 */
441 if (ci->ci_fpsaving)
442 return (1);
443
444 #ifdef DIAGNOSTIC
445 /*
446 * At this point, fpcurproc should be curproc. If it wasn't, the TS
447 * bit should be set, and we should have gotten a DNA exception.
448 */
449 if (p != curproc)
450 panic("npxintr: wrong process");
451 #endif
452
453 /*
454 * Find the address of fpcurproc's saved FPU state. (Given the
455 * invariant above, this is always the one in curpcb.)
456 */
457 addr = &p->p_addr->u_pcb.pcb_savefpu;
458 /*
459 * Save state. This does an implied fninit. It had better not halt
460 * the cpu or we'll hang.
461 */
462 fpu_save(addr);
463 fwait();
464 /*
465 * Restore control word (was clobbered by fpu_save).
466 */
467 if (i386_use_fxsave) {
468 fldcw(&addr->sv_xmm.sv_env.en_cw);
469 /*
470 * FNINIT doesn't affect MXCSR or the XMM registers;
471 * no need to re-load MXCSR here.
472 */
473 } else
474 fldcw(&addr->sv_87.sv_env.en_cw);
475 fwait();
476 /*
477 * Remember the exception status word and tag word. The current
478 * (almost fninit'ed) fpu state is in the fpu and the exception
479 * state just saved will soon be junk. However, the implied fninit
480 * doesn't change the error pointers or register contents, and we
481 * preserved the control word and will copy the status and tag
482 * words, so the complete exception state can be recovered.
483 */
484 if (i386_use_fxsave) {
485 addr->sv_xmm.sv_ex_sw = addr->sv_xmm.sv_env.en_sw;
486 addr->sv_xmm.sv_ex_tw = addr->sv_xmm.sv_env.en_tw;
487 } else {
488 addr->sv_87.sv_ex_sw = addr->sv_87.sv_env.en_sw;
489 addr->sv_87.sv_ex_tw = addr->sv_87.sv_env.en_tw;
490 }
491
492 /*
493 * Pass exception to process. If it's the current process, try to do
494 * it immediately.
495 */
496 if (p == curproc && USERMODE(frame->if_cs, frame->if_eflags)) {
497 /*
498 * Interrupt is essentially a trap, so we can afford to call
499 * the SIGFPE handler (if any) as soon as the interrupt
500 * returns.
501 *
502 * XXX little or nothing is gained from this, and plenty is
503 * lost - the interrupt frame has to contain the trap frame
504 * (this is otherwise only necessary for the rescheduling trap
505 * in doreti, and the frame for that could easily be set up
506 * just before it is used).
507 */
508 p->p_md.md_regs = (struct trapframe *)&frame->if_fs;
509
510 /*
511 * Encode the appropriate code for detailed information on
512 * this exception.
513 */
514 if (i386_use_fxsave)
515 code = x86fpflags_to_siginfo(addr->sv_xmm.sv_ex_sw);
516 else
517 code = x86fpflags_to_siginfo(addr->sv_87.sv_ex_sw);
518 sv.sival_int = frame->if_eip;
519 trapsignal(p, SIGFPE, T_ARITHTRAP, code, sv);
520 } else {
521 /*
522 * Nested interrupt. These losers occur when:
523 * o an IRQ13 is bogusly generated at a bogus time, e.g.:
524 * o immediately after an fnsave or frstor of an
525 * error state.
526 * o a couple of 386 instructions after
527 * "fstpl _memvar" causes a stack overflow.
528 * These are especially nasty when combined with a
529 * trace trap.
530 * o an IRQ13 occurs at the same time as another higher-
531 * priority interrupt.
532 *
533 * Treat them like a true async interrupt.
534 */
535 KERNEL_LOCK();
536 psignal(p, SIGFPE);
537 KERNEL_UNLOCK();
538 }
539
540 return (1);
541 }
542
543 void
npxtrap(struct trapframe * frame)544 npxtrap(struct trapframe *frame)
545 {
546 struct proc *p = curcpu()->ci_fpcurproc;
547 union savefpu *addr = &p->p_addr->u_pcb.pcb_savefpu;
548 u_int32_t mxcsr, statbits;
549 int code;
550 union sigval sv;
551
552 #ifdef DIAGNOSTIC
553 /*
554 * At this point, fpcurproc should be curproc. If it wasn't, the TS
555 * bit should be set, and we should have gotten a DNA exception.
556 */
557 if (p != curproc)
558 panic("npxtrap: wrong process");
559 #endif
560
561 fxsave(&addr->sv_xmm);
562 mxcsr = addr->sv_xmm.sv_env.en_mxcsr;
563 statbits = mxcsr;
564 mxcsr &= ~0x3f;
565 ldmxcsr(&mxcsr);
566 addr->sv_xmm.sv_ex_sw = addr->sv_xmm.sv_env.en_sw;
567 addr->sv_xmm.sv_ex_tw = addr->sv_xmm.sv_env.en_tw;
568 code = x86fpflags_to_siginfo(statbits);
569 sv.sival_int = frame->tf_eip;
570 trapsignal(p, SIGFPE, frame->tf_err, code, sv);
571 }
572
573 static int
x86fpflags_to_siginfo(u_int32_t flags)574 x86fpflags_to_siginfo(u_int32_t flags)
575 {
576 int i;
577 static int x86fp_siginfo_table[] = {
578 FPE_FLTINV, /* bit 0 - invalid operation */
579 FPE_FLTRES, /* bit 1 - denormal operand */
580 FPE_FLTDIV, /* bit 2 - divide by zero */
581 FPE_FLTOVF, /* bit 3 - fp overflow */
582 FPE_FLTUND, /* bit 4 - fp underflow */
583 FPE_FLTRES, /* bit 5 - fp precision */
584 FPE_FLTINV, /* bit 6 - stack fault */
585 };
586
587 for (i=0;i < sizeof(x86fp_siginfo_table)/sizeof(int); i++) {
588 if (flags & (1 << i))
589 return (x86fp_siginfo_table[i]);
590 }
591 /* punt if flags not set */
592 return (FPE_FLTINV);
593 }
594
595 /*
596 * Implement device not available (DNA) exception
597 *
598 * If we were the last process to use the FPU, we can simply return.
599 * Otherwise, we save the previous state, if necessary, and restore our last
600 * saved state.
601 */
602 int
npxdna_xmm(struct cpu_info * ci)603 npxdna_xmm(struct cpu_info *ci)
604 {
605 union savefpu *sfp;
606 struct proc *p;
607 int s;
608
609 if (ci->ci_fpsaving) {
610 printf("recursive npx trap; cr0=%x\n", rcr0());
611 return (0);
612 }
613
614 s = splipi(); /* lock out IPI's while we clean house.. */
615
616 #ifdef MULTIPROCESSOR
617 p = ci->ci_curproc;
618 #else
619 p = curproc;
620 #endif
621
622 IPRINTF(("%s: dna for %lx%s\n", ci->ci_dev->dv_xname, (u_long)p,
623 (p->p_md.md_flags & MDP_USEDFPU) ? " (used fpu)" : ""));
624
625 /*
626 * XXX should have a fast-path here when no save/restore is necessary
627 */
628 /*
629 * Initialize the FPU state to clear any exceptions. If someone else
630 * was using the FPU, save their state (which does an implicit
631 * initialization).
632 */
633 if (ci->ci_fpcurproc != NULL) {
634 IPRINTF(("%s: fp save %lx\n", ci->ci_dev->dv_xname,
635 (u_long)ci->ci_fpcurproc));
636 npxsave_cpu(ci, ci->ci_fpcurproc != &proc0);
637 } else {
638 clts();
639 IPRINTF(("%s: fp init\n", ci->ci_dev->dv_xname));
640 fninit();
641 fwait();
642 stts();
643 }
644 splx(s);
645
646 IPRINTF(("%s: done saving\n", ci->ci_dev->dv_xname));
647 KDASSERT(ci->ci_fpcurproc == NULL);
648 #ifndef MULTIPROCESSOR
649 KDASSERT(p->p_addr->u_pcb.pcb_fpcpu == NULL);
650 #else
651 if (p->p_addr->u_pcb.pcb_fpcpu != NULL)
652 npxsave_proc(p, 1);
653 #endif
654 p->p_addr->u_pcb.pcb_cr0 &= ~CR0_TS;
655 clts();
656 s = splipi();
657 ci->ci_fpcurproc = p;
658 p->p_addr->u_pcb.pcb_fpcpu = ci;
659 splx(s);
660 uvmexp.fpswtch++;
661
662 sfp = &p->p_addr->u_pcb.pcb_savefpu;
663
664 if ((p->p_md.md_flags & MDP_USEDFPU) == 0) {
665 bzero(&sfp->sv_xmm, sizeof(sfp->sv_xmm));
666 sfp->sv_xmm.sv_env.en_cw = __INITIAL_NPXCW__;
667 sfp->sv_xmm.sv_env.en_mxcsr = __INITIAL_MXCSR__;
668 fxrstor(&sfp->sv_xmm);
669 p->p_md.md_flags |= MDP_USEDFPU;
670 } else {
671 static double zero = 0.0;
672
673 /*
674 * amd fpu does not restore fip, fdp, fop on fxrstor
675 * thus leaking other process's execution history.
676 */
677 fnclex();
678 __asm volatile("ffree %%st(7)\n\tfldl %0" : : "m" (zero));
679 fxrstor(&sfp->sv_xmm);
680 }
681
682 return (1);
683 }
684
685 int
npxdna_s87(struct cpu_info * ci)686 npxdna_s87(struct cpu_info *ci)
687 {
688 union savefpu *sfp;
689 struct proc *p;
690 int s;
691
692 KDASSERT(i386_use_fxsave == 0);
693
694 if (ci->ci_fpsaving) {
695 printf("recursive npx trap; cr0=%x\n", rcr0());
696 return (0);
697 }
698
699 s = splipi(); /* lock out IPI's while we clean house.. */
700 #ifdef MULTIPROCESSOR
701 p = ci->ci_curproc;
702 #else
703 p = curproc;
704 #endif
705
706 IPRINTF(("%s: dna for %lx%s\n", ci->ci_dev->dv_xname, (u_long)p,
707 (p->p_md.md_flags & MDP_USEDFPU) ? " (used fpu)" : ""));
708
709 /*
710 * If someone else was using our FPU, save their state (which does an
711 * implicit initialization); otherwise, initialize the FPU state to
712 * clear any exceptions.
713 */
714 if (ci->ci_fpcurproc != NULL) {
715 IPRINTF(("%s: fp save %lx\n", ci->ci_dev->dv_xname,
716 (u_long)ci->ci_fpcurproc));
717 npxsave_cpu(ci, ci->ci_fpcurproc != &proc0);
718 } else {
719 clts();
720 IPRINTF(("%s: fp init\n", ci->ci_dev->dv_xname));
721 fninit();
722 fwait();
723 stts();
724 }
725 splx(s);
726
727 IPRINTF(("%s: done saving\n", ci->ci_dev->dv_xname));
728 KDASSERT(ci->ci_fpcurproc == NULL);
729 #ifndef MULTIPROCESSOR
730 KDASSERT(p->p_addr->u_pcb.pcb_fpcpu == NULL);
731 #else
732 if (p->p_addr->u_pcb.pcb_fpcpu != NULL)
733 npxsave_proc(p, 1);
734 #endif
735 p->p_addr->u_pcb.pcb_cr0 &= ~CR0_TS;
736 clts();
737 s = splipi();
738 ci->ci_fpcurproc = p;
739 p->p_addr->u_pcb.pcb_fpcpu = ci;
740 splx(s);
741 uvmexp.fpswtch++;
742
743 sfp = &p->p_addr->u_pcb.pcb_savefpu;
744
745 if ((p->p_md.md_flags & MDP_USEDFPU) == 0) {
746 bzero(&sfp->sv_87, sizeof(sfp->sv_87));
747 sfp->sv_87.sv_env.en_cw = __INITIAL_NPXCW__;
748 sfp->sv_87.sv_env.en_tw = 0xffff;
749 frstor(&sfp->sv_87);
750 p->p_md.md_flags |= MDP_USEDFPU;
751 } else {
752 /*
753 * The following frstor may cause an IRQ13 when the state being
754 * restored has a pending error. The error will appear to have
755 * been triggered by the current (npx) user instruction even
756 * when that instruction is a no-wait instruction that should
757 * not trigger an error (e.g., fnclex). On at least one 486
758 * system all of the no-wait instructions are broken the same
759 * as frstor, so our treatment does not amplify the breakage.
760 * On at least one 386/Cyrix 387 system, fnclex works correctly
761 * while frstor and fnsave are broken, so our treatment breaks
762 * fnclex if it is the first FPU instruction after a context
763 * switch.
764 */
765 frstor(&sfp->sv_87);
766 }
767
768 return (1);
769 }
770
771 /*
772 * The FNSAVE instruction clears the FPU state. Rather than reloading the FPU
773 * immediately, we clear fpcurproc and turn on CR0_TS to force a DNA and a
774 * reload of the FPU state the next time we try to use it. This routine
775 * is only called when forking, core dumping, or debugging, or swapping,
776 * so the lazy reload at worst forces us to trap once per fork(), and at best
777 * saves us a reload once per fork().
778 */
779 void
npxsave_cpu(struct cpu_info * ci,int save)780 npxsave_cpu(struct cpu_info *ci, int save)
781 {
782 struct proc *p;
783 int s;
784
785 KDASSERT(ci == curcpu());
786
787 p = ci->ci_fpcurproc;
788 if (p == NULL)
789 return;
790
791 IPRINTF(("%s: fp cpu %s %lx\n", ci->ci_dev->dv_xname,
792 save ? "save" : "flush", (u_long)p));
793
794 if (save) {
795 #ifdef DIAGNOSTIC
796 if (ci->ci_fpsaving != 0)
797 panic("npxsave_cpu: recursive save!");
798 #endif
799 /*
800 * Set ci->ci_fpsaving, so that any pending exception will be
801 * thrown away. (It will be caught again if/when the FPU
802 * state is restored.)
803 *
804 * XXX on i386 and earlier, this routine should always be
805 * called at spl0; if it might called with the NPX interrupt
806 * masked, it would be necessary to forcibly unmask the NPX
807 * interrupt so that it could succeed.
808 * XXX this is irrelevant on 486 and above (systems
809 * which report FP failures via traps rather than irq13).
810 * XXX punting for now..
811 */
812 clts();
813 ci->ci_fpsaving = 1;
814 fpu_save(&p->p_addr->u_pcb.pcb_savefpu);
815 ci->ci_fpsaving = 0;
816 /* It is unclear if this is needed. */
817 fwait();
818 }
819
820 /*
821 * We set the TS bit in the saved CR0 for this process, so that it
822 * will get a DNA exception on any FPU instruction and force a reload.
823 */
824 stts();
825 p->p_addr->u_pcb.pcb_cr0 |= CR0_TS;
826
827 s = splipi();
828 p->p_addr->u_pcb.pcb_fpcpu = NULL;
829 ci->ci_fpcurproc = NULL;
830 splx(s);
831 }
832
833 /*
834 * Save p's FPU state, which may be on this processor or another processor.
835 */
836 void
npxsave_proc(struct proc * p,int save)837 npxsave_proc(struct proc *p, int save)
838 {
839 struct cpu_info *ci = curcpu();
840 struct cpu_info *oci;
841
842 KDASSERT(p->p_addr != NULL);
843
844 oci = p->p_addr->u_pcb.pcb_fpcpu;
845 if (oci == NULL)
846 return;
847
848 IPRINTF(("%s: fp proc %s %lx\n", ci->ci_dev->dv_xname,
849 save ? "save" : "flush", (u_long)p));
850
851 #if defined(MULTIPROCESSOR)
852 if (oci == ci) {
853 int s = splipi();
854 npxsave_cpu(ci, save);
855 splx(s);
856 } else {
857 IPRINTF(("%s: fp ipi to %s %s %lx\n", ci->ci_dev->dv_xname,
858 oci->ci_dev->dv_xname, save ? "save" : "flush", (u_long)p));
859
860 oci->ci_fpsaveproc = p;
861 i386_send_ipi(oci,
862 save ? I386_IPI_SYNCH_FPU : I386_IPI_FLUSH_FPU);
863 while (p->p_addr->u_pcb.pcb_fpcpu != NULL)
864 CPU_BUSY_CYCLE();
865 }
866 #else
867 KASSERT(ci->ci_fpcurproc == p);
868 npxsave_cpu(ci, save);
869 #endif
870 }
871
872 void
fpu_kernel_enter(void)873 fpu_kernel_enter(void)
874 {
875 struct cpu_info *ci = curcpu();
876 uint32_t cw;
877 int s;
878
879 /*
880 * Fast path. If the kernel was using the FPU before, there
881 * is no work to do besides clearing TS.
882 */
883 if (ci->ci_fpcurproc == &proc0) {
884 clts();
885 return;
886 }
887
888 s = splipi();
889
890 if (ci->ci_fpcurproc != NULL) {
891 npxsave_cpu(ci, 1);
892 uvmexp.fpswtch++;
893 }
894
895 /* Claim the FPU */
896 ci->ci_fpcurproc = &proc0;
897
898 splx(s);
899
900 /* Disable DNA exceptions */
901 clts();
902
903 /* Initialize the FPU */
904 fninit();
905 cw = __INITIAL_NPXCW__;
906 fldcw(&cw);
907 if (i386_has_sse || i386_has_sse2) {
908 cw = __INITIAL_MXCSR__;
909 ldmxcsr(&cw);
910 }
911 }
912
913 void
fpu_kernel_exit(void)914 fpu_kernel_exit(void)
915 {
916 /* Enable DNA exceptions */
917 stts();
918 }
919