1/*	$NetBSD: cpufunc.S,v 1.25 2014/02/12 23:24:09 dsl Exp $	*/
2
3/*-
4 * Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Charles M. Hannum, and by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in the
17 *    documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * Functions to provide access to i386-specific instructions.
34 */
35
36#include <sys/errno.h>
37
38#include <machine/asm.h>
39#include <machine/frameasm.h>
40#include <machine/specialreg.h>
41#include <machine/segments.h>
42
43#include "opt_xen.h"
44
45#include "assym.h"
46
47/* Small and slow, so align less. */
48#undef _ALIGN_TEXT
49#define	_ALIGN_TEXT	.align 8
50
51ENTRY(x86_lfence)
52	lfence
53	ret
54
55ENTRY(x86_sfence)
56	sfence
57	ret
58
59ENTRY(x86_mfence)
60	mfence
61	ret
62
63#ifndef XEN
64ENTRY(invlpg)
65	invlpg	(%rdi)
66	ret
67
68ENTRY(lidt)
69	lidt	(%rdi)
70	ret
71
72ENTRY(lldt)
73	cmpl	%edi, CPUVAR(CURLDT)
74	jne	1f
75	ret
761:
77	movl	%edi, CPUVAR(CURLDT)
78	lldt	%di
79	ret
80
81ENTRY(ltr)
82	ltr	%di
83	ret
84
85ENTRY(lcr0)
86	movq	%rdi, %cr0
87	ret
88
89ENTRY(rcr0)
90	movq	%cr0, %rax
91	ret
92
93ENTRY(lcr2)
94	movq	%rdi, %cr2
95	ret
96
97ENTRY(rcr2)
98	movq	%cr2, %rax
99	ret
100
101ENTRY(lcr3)
102	movq	%rdi, %cr3
103	ret
104
105ENTRY(rcr3)
106	movq	%cr3, %rax
107	ret
108#endif
109
110ENTRY(lcr4)
111	movq	%rdi, %cr4
112	ret
113
114ENTRY(rcr4)
115	movq	%cr4, %rax
116	ret
117
118ENTRY(lcr8)
119	movq	%rdi, %cr8
120	ret
121
122ENTRY(rcr8)
123	movq	%cr8, %rax
124	ret
125
126/*
127 * Big hammer: flush all TLB entries, including ones from PTE's
128 * with the G bit set.  This should only be necessary if TLB
129 * shootdown falls far behind.
130 *
131 * Intel Architecture Software Developer's Manual, Volume 3,
132 *	System Programming, section 9.10, "Invalidating the
133 * Translation Lookaside Buffers (TLBS)":
134 * "The following operations invalidate all TLB entries, irrespective
135 * of the setting of the G flag:
136 * ...
137 * "(P6 family processors only): Writing to control register CR4 to
138 * modify the PSE, PGE, or PAE flag."
139 *
140 * (the alternatives not quoted above are not an option here.)
141 *
142 * If PGE is not in use, we reload CR3.
143 */
144#ifndef XEN
145ENTRY(tlbflushg)
146	movq	%cr4, %rax
147	testq	$CR4_PGE, %rax
148	jz	1f
149	movq	%rax, %rdx
150	andq	$~CR4_PGE, %rdx
151	movq	%rdx, %cr4
152	movq	%rax, %cr4
153	ret
154
155ENTRY(tlbflush)
1561:
157	movq	%cr3, %rax
158	movq	%rax, %cr3
159	ret
160
161ENTRY(ldr6)
162	movq	%rdi, %dr6
163	ret
164
165ENTRY(rdr6)
166	movq	%dr6, %rdi
167	ret
168
169ENTRY(x86_disable_intr)
170	cli
171	ret
172
173ENTRY(x86_enable_intr)
174	sti
175	ret
176
177ENTRY(x86_read_flags)
178	pushfq
179	popq	%rax
180	ret
181
182STRONG_ALIAS(x86_read_psl,x86_read_flags)
183
184ENTRY(x86_write_flags)
185	pushq	%rdi
186	popfq
187	ret
188
189STRONG_ALIAS(x86_write_psl,x86_write_flags)
190#endif /* XEN */
191
192ENTRY(rdmsr)
193	movq	%rdi, %rcx
194	xorq	%rax, %rax
195	rdmsr
196	shlq	$32, %rdx
197	orq	%rdx, %rax
198	ret
199
200ENTRY(wrmsr)
201	movq	%rdi, %rcx
202	movq	%rsi, %rax
203	movq	%rsi, %rdx
204	shrq	$32, %rdx
205	wrmsr
206	ret
207
208ENTRY(rdmsr_locked)
209	movq	%rdi, %rcx
210	xorq	%rax, %rax
211	movl	$OPTERON_MSR_PASSCODE, %edi
212	rdmsr
213	shlq	$32, %rdx
214	orq	%rdx, %rax
215	ret
216
217ENTRY(wrmsr_locked)
218	movq	%rdi, %rcx
219	movq	%rsi, %rax
220	movq	%rsi, %rdx
221	shrq	$32, %rdx
222	movl	$OPTERON_MSR_PASSCODE, %edi
223	wrmsr
224	ret
225
226/*
227 * Support for reading MSRs in the safe manner (returns EFAULT on fault)
228 */
229/* int rdmsr_safe(u_int msr, uint64_t *data) */
230ENTRY(rdmsr_safe)
231	movq	CPUVAR(CURLWP), %r8
232	movq	L_PCB(%r8), %r8
233	movq	$_C_LABEL(msr_onfault), PCB_ONFAULT(%r8)
234
235	movl	%edi, %ecx /* u_int msr */
236	rdmsr			/* Read MSR pointed by %ecx. Returns
237				   hi byte in edx, lo in %eax */
238	salq	$32, %rdx	/* sign-shift %rdx left */
239	movl	%eax, %eax	/* zero-extend %eax -> %rax */
240	orq	%rdx, %rax
241	movq	%rax, (%rsi)  /* *data */
242	xorq	%rax, %rax    /* "no error" */
243
244	movq	%rax, PCB_ONFAULT(%r8)
245	ret
246
247ENTRY(rdxcr)
248	movq	%rdi, %rcx
249	xgetbv
250	shlq	$32, %rdx
251	orq	%rdx, %rax
252	ret
253
254ENTRY(wrxcr)
255	movq	%rdi, %rcx
256	movq	%rsi, %rax
257	movq	%rsi, %rdx
258	shrq	$32, %rdx
259	xsetbv
260	ret
261
262/*
263 * MSR operations fault handler
264 */
265NENTRY(msr_onfault)
266	movq	CPUVAR(CURLWP), %r8
267	movq	L_PCB(%r8), %r8
268	movq	$0, PCB_ONFAULT(%r8)
269	movl	$EFAULT, %eax
270	ret
271
272#ifndef XEN
273ENTRY(wbinvd)
274	wbinvd
275	ret
276#endif
277
278ENTRY(cpu_counter)
279	xorq	%rax, %rax
280	rdtsc
281	shlq	$32, %rdx
282	orq	%rdx, %rax
283	addq	CPUVAR(CC_SKEW), %rax
284	ret
285
286ENTRY(cpu_counter32)
287	rdtsc
288	addl	CPUVAR(CC_SKEW), %eax
289	ret
290
291ENTRY(rdpmc)
292	movq	%rdi, %rcx
293	xorq	%rax, %rax
294	rdpmc
295	shlq	$32, %rdx
296	orq	%rdx, %rax
297	ret
298
299ENTRY(breakpoint)
300	pushq	%rbp
301	movq	%rsp, %rbp
302	int	$0x03		/* paranoid, not 'int3' */
303	leave
304	ret
305
306ENTRY(x86_curcpu)
307	movq	%gs:(CPU_INFO_SELF), %rax
308	ret
309
310ENTRY(x86_curlwp)
311	movq	%gs:(CPU_INFO_CURLWP), %rax
312	ret
313
314ENTRY(cpu_set_curpri)
315	movl	%edi, %gs:(CPU_INFO_CURPRIORITY)
316	ret
317
318ENTRY(__byte_swap_u32_variable)
319	movl	%edi, %eax
320	bswapl	%eax
321	ret
322
323ENTRY(__byte_swap_u16_variable)
324	movl	%edi, %eax
325	xchgb	%al, %ah
326	ret
327
328/*
329 * void lgdt(struct region_descriptor *rdp);
330 *
331 * Load a new GDT pointer (and do any necessary cleanup).
332 * XXX It's somewhat questionable whether reloading all the segment registers
333 * is necessary, since the actual descriptor data is not changed except by
334 * process creation and exit, both of which clean up via task switches.  OTOH,
335 * this only happens at run time when the GDT is resized.
336 */
337#ifndef XEN
338ENTRY(lgdt)
339	/* Reload the descriptor table. */
340	movq	%rdi,%rax
341	lgdt	(%rax)
342	/* Flush the prefetch q. */
343	jmp	1f
344	nop
3451:	/* Reload "stale" selectors. */
346#else /* XEN */
347/*
348 * void lgdt_finish(void);
349 * Reload segments after a GDT change
350 */
351ENTRY(lgdt_finish)
352#endif /* XEN */
353	movl	$GSEL(GDATA_SEL, SEL_KPL),%eax
354	movl	%eax,%ds
355	movl	%eax,%es
356	movl	%eax,%ss
357	/* FALLTHROUGH */
358
359/*
360 * void x86_flush()
361 *
362 * Flush instruction pipelines by doing an intersegment (far) return.
363 */
364ENTRY(x86_flush)
365	popq	%rax
366	pushq	$GSEL(GCODE_SEL, SEL_KPL)
367	pushq	%rax
368	lretq
369
370/* Waits - set up stack frame. */
371ENTRY(x86_hlt)
372	pushq	%rbp
373	movq	%rsp, %rbp
374	hlt
375	leave
376	ret
377
378/* Waits - set up stack frame. */
379ENTRY(x86_stihlt)
380	pushq	%rbp
381	movq	%rsp, %rbp
382	sti
383	hlt
384	leave
385	ret
386
387ENTRY(x86_monitor)
388	movq	%rdi, %rax
389	movq	%rsi, %rcx
390	monitor	%rax, %rcx, %rdx
391	ret
392
393/* Waits - set up stack frame. */
394ENTRY(x86_mwait)
395	pushq	%rbp
396	movq	%rsp, %rbp
397	movq	%rdi, %rax
398	movq	%rsi, %rcx
399	mwait	%rax, %rcx
400	leave
401	ret
402
403NENTRY(x86_pause)
404	pause
405	ret
406
407ENTRY(x86_cpuid2)
408	movq	%rbx, %r8
409	movq	%rdi, %rax
410	movq	%rsi, %rcx
411	movq	%rdx, %rsi
412	cpuid
413	movl	%eax, 0(%rsi)
414	movl	%ebx, 4(%rsi)
415	movl	%ecx, 8(%rsi)
416	movl	%edx, 12(%rsi)
417	movq	%r8, %rbx
418	ret
419
420ENTRY(x86_getss)
421	movl	%ss, %eax
422	ret
423
424ENTRY(fldcw)
425	fldcw	(%rdi)
426	ret
427
428ENTRY(fnclex)
429	fnclex
430	ret
431
432ENTRY(fninit)
433	fninit
434	ret
435
436ENTRY(fnsave)
437	fnsave	(%rdi)
438	ret
439
440ENTRY(fnstcw)
441	fnstcw	(%rdi)
442	ret
443
444ENTRY(fngetsw)
445	fnstsw	%ax
446	ret
447
448ENTRY(fnstsw)
449	fnstsw	(%rdi)
450	ret
451
452ENTRY(fp_divide_by_0)
453	fldz
454	fld1
455	fdiv	%st, %st(1)
456	fwait
457	ret
458
459ENTRY(frstor)
460	frstor	(%rdi)
461	ret
462
463ENTRY(fwait)
464	fwait
465	ret
466
467ENTRY(clts)
468	clts
469	ret
470
471ENTRY(stts)
472	movq	%cr0, %rax
473	orq	$CR0_TS, %rax
474	movq	%rax, %cr0
475	ret
476
477ENTRY(fxsave)
478	fxsave	(%rdi)
479	ret
480
481ENTRY(fxrstor)
482	fxrstor	(%rdi)
483	ret
484
485ENTRY(fldummy)
486	ffree	%st(7)
487	fldz
488	ret
489
490ENTRY(xsave)
491	movq	%rsi, %rax
492	movq	%rsi, %rdx
493	shrq	$32, %rdx
494	xsave	(%rdi)
495	ret
496
497ENTRY(xsaveopt)
498	movq	%rsi, %rax
499	movq	%rsi, %rdx
500	shrq	$32, %rdx
501	xsaveopt	(%rdi)
502	ret
503
504ENTRY(xrstor)
505	movq	%rsi, %rax
506	movq	%rsi, %rdx
507	shrq	$32, %rdx
508	xrstor	(%rdi)
509	ret
510
511ENTRY(x86_stmxcsr)
512	stmxcsr	(%rdi)
513	ret
514
515ENTRY(x86_ldmxcsr)
516	ldmxcsr	(%rdi)
517	ret
518
519ENTRY(inb)
520	movq	%rdi, %rdx
521	xorq	%rax, %rax
522	inb	%dx, %al
523	ret
524
525ENTRY(insb)
526	movl	%edx, %ecx
527	movl	%edi, %edx
528	movq	%rsi, %rdi
529	rep
530	insb
531	ret
532
533ENTRY(inw)
534	movq	%rdi, %rdx
535	xorq	%rax, %rax
536	inw	%dx, %ax
537	ret
538
539ENTRY(insw)
540	movl	%edx, %ecx
541	movl	%edi, %edx
542	movq	%rsi, %rdi
543	rep
544	insw
545	ret
546
547ENTRY(inl)
548	movq	%rdi, %rdx
549	xorq	%rax, %rax
550	inl	%dx, %eax
551	ret
552
553ENTRY(insl)
554	movl	%edx, %ecx
555	movl	%edi, %edx
556	movq	%rsi, %rdi
557	rep
558	insl
559	ret
560
561ENTRY(outb)
562	movq	%rdi, %rdx
563	movq	%rsi, %rax
564	outb	%al, %dx
565	ret
566
567ENTRY(outsb)
568	movl	%edx, %ecx
569	movl	%edi, %edx
570	rep
571	outsb
572	ret
573
574ENTRY(outw)
575	movq	%rdi, %rdx
576	movq	%rsi, %rax
577	outw	%ax, %dx
578	ret
579
580ENTRY(outsw)
581	movl	%edx, %ecx
582	movl	%edi, %edx
583	rep
584	outsw
585	ret
586
587ENTRY(outl)
588	movq	%rdi, %rdx
589	movq	%rsi, %rax
590	outl	%eax, %dx
591	ret
592
593ENTRY(outsl)
594	movl	%edx, %ecx
595	movl	%edi, %edx
596	rep
597	outsl
598	ret
599
600ENTRY(setfs)
601	movw	%di, %fs
602	ret
603
604ENTRY(setusergs)
605	CLI(ax)
606	swapgs
607	movw	%di, %gs
608	swapgs
609	STI(ax)
610	ret
611