xref: /freebsd/sys/i386/i386/sys_machdep.c (revision 315ee00f) 
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1990 The Regents of the University of California.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  * 3. Neither the name of the University nor the names of its contributors
16  *    may be used to endorse or promote products derived from this software
17  *    without specific prior written permission.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29  * SUCH DAMAGE.
30  *
31  *	from: @(#)sys_machdep.c	5.5 (Berkeley) 1/19/91
32  */
33 
34 #include <sys/cdefs.h>
35 #include "opt_capsicum.h"
36 #include "opt_kstack_pages.h"
37 #include "opt_ktrace.h"
38 
39 #include <sys/param.h>
40 #include <sys/capsicum.h>
41 #include <sys/systm.h>
42 #include <sys/ktrace.h>
43 #include <sys/lock.h>
44 #include <sys/malloc.h>
45 #include <sys/mutex.h>
46 #include <sys/priv.h>
47 #include <sys/proc.h>
48 #include <sys/smp.h>
49 #include <sys/sysproto.h>
50 
51 #include <vm/vm.h>
52 #include <vm/pmap.h>
53 #include <vm/vm_map.h>
54 #include <vm/vm_extern.h>
55 
56 #include <machine/atomic.h>
57 #include <machine/cpu.h>
58 #include <machine/pcb.h>
59 #include <machine/pcb_ext.h>
60 #include <machine/proc.h>
61 #include <machine/sysarch.h>
62 
63 #include <security/audit/audit.h>
64 
65 #include <vm/vm_kern.h>		/* for kernel_map */
66 
67 #define MAX_LD 8192
68 #define LD_PER_PAGE 512
69 #define	NEW_MAX_LD(num)  rounddown2(num + LD_PER_PAGE, LD_PER_PAGE)
70 #define SIZE_FROM_LARGEST_LD(num) (NEW_MAX_LD(num) << 3)
71 #define	NULL_LDT_BASE	((caddr_t)NULL)
72 
73 #ifdef SMP
74 static void set_user_ldt_rv(void *arg);
75 #endif
76 static int i386_set_ldt_data(struct thread *, int start, int num,
77     union descriptor *descs);
78 static int i386_ldt_grow(struct thread *td, int len);
79 
80 void
81 fill_based_sd(struct segment_descriptor *sdp, uint32_t base)
82 {
83 
84 	sdp->sd_lobase = base & 0xffffff;
85 	sdp->sd_hibase = (base >> 24) & 0xff;
86 	sdp->sd_lolimit = 0xffff;	/* 4GB limit, wraps around */
87 	sdp->sd_hilimit = 0xf;
88 	sdp->sd_type = SDT_MEMRWA;
89 	sdp->sd_dpl = SEL_UPL;
90 	sdp->sd_p = 1;
91 	sdp->sd_xx = 0;
92 	sdp->sd_def32 = 1;
93 	sdp->sd_gran = 1;
94 }
95 
96 /*
97  * Construct special descriptors for "base" selectors.  Store them in
98  * the PCB for later use by cpu_switch().  Store them in the GDT for
99  * more immediate use.  The GDT entries are part of the current
100  * context.  Callers must load related segment registers to complete
101  * setting up the current context.
102  */
103 void
104 set_fsbase(struct thread *td, uint32_t base)
105 {
106 	struct segment_descriptor sd;
107 
108 	fill_based_sd(&sd, base);
109 	critical_enter();
110 	td->td_pcb->pcb_fsd = sd;
111 	if (td == curthread)
112 		PCPU_GET(fsgs_gdt)[0] = sd;
113 	critical_exit();
114 }
115 
116 void
117 set_gsbase(struct thread *td, uint32_t base)
118 {
119 	struct segment_descriptor sd;
120 
121 	fill_based_sd(&sd, base);
122 	critical_enter();
123 	td->td_pcb->pcb_gsd = sd;
124 	if (td == curthread)
125 		PCPU_GET(fsgs_gdt)[1] = sd;
126 	critical_exit();
127 }
128 
129 #ifndef _SYS_SYSPROTO_H_
130 struct sysarch_args {
131 	int op;
132 	char *parms;
133 };
134 #endif
135 
136 int
137 sysarch(struct thread *td, struct sysarch_args *uap)
138 {
139 	int error;
140 	union descriptor *lp;
141 	union {
142 		struct i386_ldt_args largs;
143 		struct i386_ioperm_args iargs;
144 		struct i386_get_xfpustate xfpu;
145 	} kargs;
146 	uint32_t base;
147 	struct segment_descriptor *sdp;
148 
149 	AUDIT_ARG_CMD(uap->op);
150 
151 #ifdef CAPABILITY_MODE
152 	/*
153 	 * When adding new operations, add a new case statement here to
154 	 * explicitly indicate whether or not the operation is safe to
155 	 * perform in capability mode.
156 	 */
157 	if (IN_CAPABILITY_MODE(td)) {
158 		switch (uap->op) {
159 		case I386_GET_LDT:
160 		case I386_SET_LDT:
161 		case I386_GET_IOPERM:
162 		case I386_GET_FSBASE:
163 		case I386_SET_FSBASE:
164 		case I386_GET_GSBASE:
165 		case I386_SET_GSBASE:
166 		case I386_GET_XFPUSTATE:
167 			break;
168 
169 		case I386_SET_IOPERM:
170 		default:
171 #ifdef KTRACE
172 			if (KTRPOINT(td, KTR_CAPFAIL))
173 				ktrcapfail(CAPFAIL_SYSCALL, NULL, NULL);
174 #endif
175 			return (ECAPMODE);
176 		}
177 	}
178 #endif
179 
180 	switch (uap->op) {
181 	case I386_GET_IOPERM:
182 	case I386_SET_IOPERM:
183 		if ((error = copyin(uap->parms, &kargs.iargs,
184 		    sizeof(struct i386_ioperm_args))) != 0)
185 			return (error);
186 		break;
187 	case I386_GET_LDT:
188 	case I386_SET_LDT:
189 		if ((error = copyin(uap->parms, &kargs.largs,
190 		    sizeof(struct i386_ldt_args))) != 0)
191 			return (error);
192 		break;
193 	case I386_GET_XFPUSTATE:
194 		if ((error = copyin(uap->parms, &kargs.xfpu,
195 		    sizeof(struct i386_get_xfpustate))) != 0)
196 			return (error);
197 		break;
198 	default:
199 		break;
200 	}
201 
202 	switch (uap->op) {
203 	case I386_GET_LDT:
204 		error = i386_get_ldt(td, &kargs.largs);
205 		break;
206 	case I386_SET_LDT:
207 		if (kargs.largs.descs != NULL) {
208 			if (kargs.largs.num > MAX_LD)
209 				return (EINVAL);
210 			lp = malloc(kargs.largs.num * sizeof(union descriptor),
211 			    M_TEMP, M_WAITOK);
212 			error = copyin(kargs.largs.descs, lp,
213 			    kargs.largs.num * sizeof(union descriptor));
214 			if (error == 0)
215 				error = i386_set_ldt(td, &kargs.largs, lp);
216 			free(lp, M_TEMP);
217 		} else {
218 			error = i386_set_ldt(td, &kargs.largs, NULL);
219 		}
220 		break;
221 	case I386_GET_IOPERM:
222 		error = i386_get_ioperm(td, &kargs.iargs);
223 		if (error == 0)
224 			error = copyout(&kargs.iargs, uap->parms,
225 			    sizeof(struct i386_ioperm_args));
226 		break;
227 	case I386_SET_IOPERM:
228 		error = i386_set_ioperm(td, &kargs.iargs);
229 		break;
230 	case I386_VM86:
231 		error = vm86_sysarch(td, uap->parms);
232 		break;
233 	case I386_GET_FSBASE:
234 		sdp = &td->td_pcb->pcb_fsd;
235 		base = sdp->sd_hibase << 24 | sdp->sd_lobase;
236 		error = copyout(&base, uap->parms, sizeof(base));
237 		break;
238 	case I386_SET_FSBASE:
239 		error = copyin(uap->parms, &base, sizeof(base));
240 		if (error == 0) {
241 			/*
242 			 * Construct the special descriptor for fsbase
243 			 * and arrange for doreti to load its selector
244 			 * soon enough.
245 			 */
246 			set_fsbase(td, base);
247 			td->td_frame->tf_fs = GSEL(GUFS_SEL, SEL_UPL);
248 		}
249 		break;
250 	case I386_GET_GSBASE:
251 		sdp = &td->td_pcb->pcb_gsd;
252 		base = sdp->sd_hibase << 24 | sdp->sd_lobase;
253 		error = copyout(&base, uap->parms, sizeof(base));
254 		break;
255 	case I386_SET_GSBASE:
256 		error = copyin(uap->parms, &base, sizeof(base));
257 		if (error == 0) {
258 			/*
259 			 * Construct the special descriptor for gsbase.
260 			 * The selector is loaded immediately, since we
261 			 * normally only reload %gs on context switches.
262 			 */
263 			set_gsbase(td, base);
264 			load_gs(GSEL(GUGS_SEL, SEL_UPL));
265 		}
266 		break;
267 	case I386_GET_XFPUSTATE:
268 		if (kargs.xfpu.len > cpu_max_ext_state_size -
269 		    sizeof(union savefpu))
270 			return (EINVAL);
271 		npxgetregs(td);
272 		error = copyout((char *)(get_pcb_user_save_td(td) + 1),
273 		    kargs.xfpu.addr, kargs.xfpu.len);
274 		break;
275 	default:
276 		error = EINVAL;
277 		break;
278 	}
279 	return (error);
280 }
281 
282 int
283 i386_extend_pcb(struct thread *td)
284 {
285 	int i, offset;
286 	u_long *addr;
287 	struct pcb_ext *ext;
288 	struct soft_segment_descriptor ssd = {
289 		0,			/* segment base address (overwritten) */
290 		ctob(IOPAGES + 1) - 1,	/* length */
291 		SDT_SYS386TSS,		/* segment type */
292 		0,			/* priority level */
293 		1,			/* descriptor present */
294 		0, 0,
295 		0,			/* default 32 size */
296 		0			/* granularity */
297 	};
298 
299 	ext = pmap_trm_alloc(ctob(IOPAGES + 1), M_WAITOK | M_ZERO);
300 	/* -16 is so we can convert a trapframe into vm86trapframe inplace */
301 	ext->ext_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
302 	/*
303 	 * The last byte of the i/o map must be followed by an 0xff byte.
304 	 * We arbitrarily allocate 16 bytes here, to keep the starting
305 	 * address on a doubleword boundary.
306 	 */
307 	offset = PAGE_SIZE - 16;
308 	ext->ext_tss.tss_ioopt =
309 	    (offset - ((unsigned)&ext->ext_tss - (unsigned)ext)) << 16;
310 	ext->ext_iomap = (caddr_t)ext + offset;
311 	ext->ext_vm86.vm86_intmap = (caddr_t)ext + offset - 32;
312 
313 	addr = (u_long *)ext->ext_vm86.vm86_intmap;
314 	for (i = 0; i < (ctob(IOPAGES) + 32 + 16) / sizeof(u_long); i++)
315 		*addr++ = ~0;
316 
317 	ssd.ssd_base = (unsigned)&ext->ext_tss;
318 	ssd.ssd_limit -= ((unsigned)&ext->ext_tss - (unsigned)ext);
319 	ssdtosd(&ssd, &ext->ext_tssd);
320 
321 	KASSERT(td == curthread, ("giving TSS to !curthread"));
322 	KASSERT(td->td_pcb->pcb_ext == 0, ("already have a TSS!"));
323 
324 	/* Switch to the new TSS. */
325 	critical_enter();
326 	ext->ext_tss.tss_esp0 = PCPU_GET(trampstk);
327 	td->td_pcb->pcb_ext = ext;
328 	PCPU_SET(private_tss, 1);
329 	*PCPU_GET(tss_gdt) = ext->ext_tssd;
330 	ltr(GSEL(GPROC0_SEL, SEL_KPL));
331 	critical_exit();
332 
333 	return 0;
334 }
335 
336 int
337 i386_set_ioperm(struct thread *td, struct i386_ioperm_args *uap)
338 {
339 	char *iomap;
340 	u_int i;
341 	int error;
342 
343 	if ((error = priv_check(td, PRIV_IO)) != 0)
344 		return (error);
345 	if ((error = securelevel_gt(td->td_ucred, 0)) != 0)
346 		return (error);
347 	/*
348 	 * XXX
349 	 * While this is restricted to root, we should probably figure out
350 	 * whether any other driver is using this i/o address, as so not to
351 	 * cause confusion.  This probably requires a global 'usage registry'.
352 	 */
353 
354 	if (td->td_pcb->pcb_ext == 0)
355 		if ((error = i386_extend_pcb(td)) != 0)
356 			return (error);
357 	iomap = (char *)td->td_pcb->pcb_ext->ext_iomap;
358 
359 	if (uap->start > uap->start + uap->length ||
360 	    uap->start + uap->length > IOPAGES * PAGE_SIZE * NBBY)
361 		return (EINVAL);
362 
363 	for (i = uap->start; i < uap->start + uap->length; i++) {
364 		if (uap->enable)
365 			iomap[i >> 3] &= ~(1 << (i & 7));
366 		else
367 			iomap[i >> 3] |= (1 << (i & 7));
368 	}
369 	return (error);
370 }
371 
372 int
373 i386_get_ioperm(struct thread *td, struct i386_ioperm_args *uap)
374 {
375 	int i, state;
376 	char *iomap;
377 
378 	if (uap->start >= IOPAGES * PAGE_SIZE * NBBY)
379 		return (EINVAL);
380 
381 	if (td->td_pcb->pcb_ext == 0) {
382 		uap->length = 0;
383 		goto done;
384 	}
385 
386 	iomap = (char *)td->td_pcb->pcb_ext->ext_iomap;
387 
388 	i = uap->start;
389 	state = (iomap[i >> 3] >> (i & 7)) & 1;
390 	uap->enable = !state;
391 	uap->length = 1;
392 
393 	for (i = uap->start + 1; i < IOPAGES * PAGE_SIZE * NBBY; i++) {
394 		if (state != ((iomap[i >> 3] >> (i & 7)) & 1))
395 			break;
396 		uap->length++;
397 	}
398 
399 done:
400 	return (0);
401 }
402 
403 /*
404  * Update the GDT entry pointing to the LDT to point to the LDT of the
405  * current process. Manage dt_lock holding/unholding autonomously.
406  */
407 static void
408 set_user_ldt_locked(struct mdproc *mdp)
409 {
410 	struct proc_ldt *pldt;
411 	int gdt_idx;
412 
413 	mtx_assert(&dt_lock, MA_OWNED);
414 
415 	pldt = mdp->md_ldt;
416 	gdt_idx = GUSERLDT_SEL;
417 	gdt_idx += PCPU_GET(cpuid) * NGDT;	/* always 0 on UP */
418 	gdt[gdt_idx].sd = pldt->ldt_sd;
419 	lldt(GSEL(GUSERLDT_SEL, SEL_KPL));
420 	PCPU_SET(currentldt, GSEL(GUSERLDT_SEL, SEL_KPL));
421 }
422 
423 void
424 set_user_ldt(struct mdproc *mdp)
425 {
426 
427 	mtx_lock_spin(&dt_lock);
428 	set_user_ldt_locked(mdp);
429 	mtx_unlock_spin(&dt_lock);
430 }
431 
432 #ifdef SMP
433 static void
434 set_user_ldt_rv(void *arg)
435 {
436 	struct proc *p;
437 
438 	p = curproc;
439 	if (arg == p->p_vmspace)
440 		set_user_ldt(&p->p_md);
441 }
442 #endif
443 
444 /*
445  * dt_lock must be held. Returns with dt_lock held.
446  */
447 struct proc_ldt *
448 user_ldt_alloc(struct mdproc *mdp, int len)
449 {
450 	struct proc_ldt *pldt, *new_ldt;
451 
452 	mtx_assert(&dt_lock, MA_OWNED);
453 	mtx_unlock_spin(&dt_lock);
454 	new_ldt = malloc(sizeof(struct proc_ldt), M_SUBPROC, M_WAITOK);
455 
456 	new_ldt->ldt_len = len = NEW_MAX_LD(len);
457 	new_ldt->ldt_base = pmap_trm_alloc(len * sizeof(union descriptor),
458 	    M_WAITOK | M_ZERO);
459 	new_ldt->ldt_refcnt = 1;
460 	new_ldt->ldt_active = 0;
461 
462 	mtx_lock_spin(&dt_lock);
463 	gdt_segs[GUSERLDT_SEL].ssd_base = (unsigned)new_ldt->ldt_base;
464 	gdt_segs[GUSERLDT_SEL].ssd_limit = len * sizeof(union descriptor) - 1;
465 	ssdtosd(&gdt_segs[GUSERLDT_SEL], &new_ldt->ldt_sd);
466 
467 	if ((pldt = mdp->md_ldt) != NULL) {
468 		if (len > pldt->ldt_len)
469 			len = pldt->ldt_len;
470 		bcopy(pldt->ldt_base, new_ldt->ldt_base,
471 		    len * sizeof(union descriptor));
472 	} else
473 		bcopy(ldt, new_ldt->ldt_base, sizeof(union descriptor) * NLDT);
474 
475 	return (new_ldt);
476 }
477 
478 /*
479  * Must be called with dt_lock held.  Returns with dt_lock unheld.
480  */
481 void
482 user_ldt_free(struct thread *td)
483 {
484 	struct mdproc *mdp;
485 	struct proc_ldt *pldt;
486 
487 	mtx_assert(&dt_lock, MA_OWNED);
488 	mdp = &td->td_proc->p_md;
489 	if ((pldt = mdp->md_ldt) == NULL) {
490 		mtx_unlock_spin(&dt_lock);
491 		return;
492 	}
493 
494 	if (td == curthread) {
495 		lldt(_default_ldt);
496 		PCPU_SET(currentldt, _default_ldt);
497 	}
498 
499 	mdp->md_ldt = NULL;
500 	user_ldt_deref(pldt);
501 }
502 
503 void
504 user_ldt_deref(struct proc_ldt *pldt)
505 {
506 
507 	mtx_assert(&dt_lock, MA_OWNED);
508 	if (--pldt->ldt_refcnt == 0) {
509 		mtx_unlock_spin(&dt_lock);
510 		pmap_trm_free(pldt->ldt_base, pldt->ldt_len *
511 		    sizeof(union descriptor));
512 		free(pldt, M_SUBPROC);
513 	} else
514 		mtx_unlock_spin(&dt_lock);
515 }
516 
517 /*
518  * Note for the authors of compat layers (linux, etc): copyout() in
519  * the function below is not a problem since it presents data in
520  * arch-specific format (i.e. i386-specific in this case), not in
521  * the OS-specific one.
522  */
523 int
524 i386_get_ldt(struct thread *td, struct i386_ldt_args *uap)
525 {
526 	struct proc_ldt *pldt;
527 	char *data;
528 	u_int nldt, num;
529 	int error;
530 
531 #ifdef DEBUG
532 	printf("i386_get_ldt: start=%u num=%u descs=%p\n",
533 	    uap->start, uap->num, (void *)uap->descs);
534 #endif
535 
536 	num = min(uap->num, MAX_LD);
537 	data = malloc(num * sizeof(union descriptor), M_TEMP, M_WAITOK);
538 	mtx_lock_spin(&dt_lock);
539 	pldt = td->td_proc->p_md.md_ldt;
540 	nldt = pldt != NULL ? pldt->ldt_len : NLDT;
541 	if (uap->start >= nldt) {
542 		num = 0;
543 	} else {
544 		num = min(num, nldt - uap->start);
545 		bcopy(pldt != NULL ?
546 		    &((union descriptor *)(pldt->ldt_base))[uap->start] :
547 		    &ldt[uap->start], data, num * sizeof(union descriptor));
548 	}
549 	mtx_unlock_spin(&dt_lock);
550 	error = copyout(data, uap->descs, num * sizeof(union descriptor));
551 	if (error == 0)
552 		td->td_retval[0] = num;
553 	free(data, M_TEMP);
554 	return (error);
555 }
556 
557 int
558 i386_set_ldt(struct thread *td, struct i386_ldt_args *uap,
559     union descriptor *descs)
560 {
561 	struct mdproc *mdp;
562 	struct proc_ldt *pldt;
563 	union descriptor *dp;
564 	u_int largest_ld, i;
565 	int error;
566 
567 #ifdef DEBUG
568 	printf("i386_set_ldt: start=%u num=%u descs=%p\n",
569 	    uap->start, uap->num, (void *)uap->descs);
570 #endif
571 	error = 0;
572 	mdp = &td->td_proc->p_md;
573 
574 	if (descs == NULL) {
575 		/* Free descriptors */
576 		if (uap->start == 0 && uap->num == 0) {
577 			/*
578 			 * Treat this as a special case, so userland needn't
579 			 * know magic number NLDT.
580 			 */
581 			uap->start = NLDT;
582 			uap->num = MAX_LD - NLDT;
583 		}
584 		mtx_lock_spin(&dt_lock);
585 		if ((pldt = mdp->md_ldt) == NULL ||
586 		    uap->start >= pldt->ldt_len) {
587 			mtx_unlock_spin(&dt_lock);
588 			return (0);
589 		}
590 		largest_ld = uap->start + uap->num;
591 		if (largest_ld > pldt->ldt_len)
592 			largest_ld = pldt->ldt_len;
593 		for (i = uap->start; i < largest_ld; i++)
594 			atomic_store_rel_64(&((uint64_t *)(pldt->ldt_base))[i],
595 			    0);
596 		mtx_unlock_spin(&dt_lock);
597 		return (0);
598 	}
599 
600 	if (uap->start != LDT_AUTO_ALLOC || uap->num != 1) {
601 		/* verify range of descriptors to modify */
602 		largest_ld = uap->start + uap->num;
603 		if (uap->start >= MAX_LD || largest_ld > MAX_LD)
604 			return (EINVAL);
605 	}
606 
607 	/* Check descriptors for access violations */
608 	for (i = 0; i < uap->num; i++) {
609 		dp = &descs[i];
610 
611 		switch (dp->sd.sd_type) {
612 		case SDT_SYSNULL:	/* system null */
613 			dp->sd.sd_p = 0;
614 			break;
615 		case SDT_SYS286TSS: /* system 286 TSS available */
616 		case SDT_SYSLDT:    /* system local descriptor table */
617 		case SDT_SYS286BSY: /* system 286 TSS busy */
618 		case SDT_SYSTASKGT: /* system task gate */
619 		case SDT_SYS286IGT: /* system 286 interrupt gate */
620 		case SDT_SYS286TGT: /* system 286 trap gate */
621 		case SDT_SYSNULL2:  /* undefined by Intel */
622 		case SDT_SYS386TSS: /* system 386 TSS available */
623 		case SDT_SYSNULL3:  /* undefined by Intel */
624 		case SDT_SYS386BSY: /* system 386 TSS busy */
625 		case SDT_SYSNULL4:  /* undefined by Intel */
626 		case SDT_SYS386IGT: /* system 386 interrupt gate */
627 		case SDT_SYS386TGT: /* system 386 trap gate */
628 		case SDT_SYS286CGT: /* system 286 call gate */
629 		case SDT_SYS386CGT: /* system 386 call gate */
630 			return (EACCES);
631 
632 		/* memory segment types */
633 		case SDT_MEMEC:   /* memory execute only conforming */
634 		case SDT_MEMEAC:  /* memory execute only accessed conforming */
635 		case SDT_MEMERC:  /* memory execute read conforming */
636 		case SDT_MEMERAC: /* memory execute read accessed conforming */
637 			 /* Must be "present" if executable and conforming. */
638 			if (dp->sd.sd_p == 0)
639 				return (EACCES);
640 			break;
641 		case SDT_MEMRO:   /* memory read only */
642 		case SDT_MEMROA:  /* memory read only accessed */
643 		case SDT_MEMRW:   /* memory read write */
644 		case SDT_MEMRWA:  /* memory read write accessed */
645 		case SDT_MEMROD:  /* memory read only expand dwn limit */
646 		case SDT_MEMRODA: /* memory read only expand dwn lim accessed */
647 		case SDT_MEMRWD:  /* memory read write expand dwn limit */
648 		case SDT_MEMRWDA: /* memory read write expand dwn lim acessed */
649 		case SDT_MEME:    /* memory execute only */
650 		case SDT_MEMEA:   /* memory execute only accessed */
651 		case SDT_MEMER:   /* memory execute read */
652 		case SDT_MEMERA:  /* memory execute read accessed */
653 			break;
654 		default:
655 			return (EINVAL);
656 		}
657 
658 		/* Only user (ring-3) descriptors may be present. */
659 		if (dp->sd.sd_p != 0 && dp->sd.sd_dpl != SEL_UPL)
660 			return (EACCES);
661 	}
662 
663 	if (uap->start == LDT_AUTO_ALLOC && uap->num == 1) {
664 		/* Allocate a free slot */
665 		mtx_lock_spin(&dt_lock);
666 		if ((pldt = mdp->md_ldt) == NULL) {
667 			if ((error = i386_ldt_grow(td, NLDT + 1))) {
668 				mtx_unlock_spin(&dt_lock);
669 				return (error);
670 			}
671 			pldt = mdp->md_ldt;
672 		}
673 again:
674 		/*
675 		 * start scanning a bit up to leave room for NVidia and
676 		 * Wine, which still user the "Blat" method of allocation.
677 		 */
678 		dp = &((union descriptor *)(pldt->ldt_base))[NLDT];
679 		for (i = NLDT; i < pldt->ldt_len; ++i) {
680 			if (dp->sd.sd_type == SDT_SYSNULL)
681 				break;
682 			dp++;
683 		}
684 		if (i >= pldt->ldt_len) {
685 			if ((error = i386_ldt_grow(td, pldt->ldt_len+1))) {
686 				mtx_unlock_spin(&dt_lock);
687 				return (error);
688 			}
689 			goto again;
690 		}
691 		uap->start = i;
692 		error = i386_set_ldt_data(td, i, 1, descs);
693 		mtx_unlock_spin(&dt_lock);
694 	} else {
695 		largest_ld = uap->start + uap->num;
696 		mtx_lock_spin(&dt_lock);
697 		if (!(error = i386_ldt_grow(td, largest_ld))) {
698 			error = i386_set_ldt_data(td, uap->start, uap->num,
699 			    descs);
700 		}
701 		mtx_unlock_spin(&dt_lock);
702 	}
703 	if (error == 0)
704 		td->td_retval[0] = uap->start;
705 	return (error);
706 }
707 
708 static int
709 i386_set_ldt_data(struct thread *td, int start, int num,
710     union descriptor *descs)
711 {
712 	struct mdproc *mdp;
713 	struct proc_ldt *pldt;
714 	uint64_t *dst, *src;
715 	int i;
716 
717 	mtx_assert(&dt_lock, MA_OWNED);
718 
719 	mdp = &td->td_proc->p_md;
720 	pldt = mdp->md_ldt;
721 	dst = (uint64_t *)(pldt->ldt_base);
722 	src = (uint64_t *)descs;
723 
724 	/*
725 	 * Atomic(9) is used only to get 64bit atomic store with
726 	 * cmpxchg8b when available.  There is no op without release
727 	 * semantic.
728 	 */
729 	for (i = 0; i < num; i++)
730 		atomic_store_rel_64(&dst[start + i], src[i]);
731 	return (0);
732 }
733 
734 static int
735 i386_ldt_grow(struct thread *td, int len)
736 {
737 	struct mdproc *mdp;
738 	struct proc_ldt *new_ldt, *pldt;
739 	caddr_t old_ldt_base;
740 	int old_ldt_len;
741 
742 	mtx_assert(&dt_lock, MA_OWNED);
743 
744 	if (len > MAX_LD)
745 		return (ENOMEM);
746 	if (len < NLDT + 1)
747 		len = NLDT + 1;
748 
749 	mdp = &td->td_proc->p_md;
750 	old_ldt_base = NULL_LDT_BASE;
751 	old_ldt_len = 0;
752 
753 	/* Allocate a user ldt. */
754 	if ((pldt = mdp->md_ldt) == NULL || len > pldt->ldt_len) {
755 		new_ldt = user_ldt_alloc(mdp, len);
756 		if (new_ldt == NULL)
757 			return (ENOMEM);
758 		pldt = mdp->md_ldt;
759 
760 		if (pldt != NULL) {
761 			if (new_ldt->ldt_len <= pldt->ldt_len) {
762 				/*
763 				 * We just lost the race for allocation, so
764 				 * free the new object and return.
765 				 */
766 				mtx_unlock_spin(&dt_lock);
767 				pmap_trm_free(new_ldt->ldt_base,
768 				   new_ldt->ldt_len * sizeof(union descriptor));
769 				free(new_ldt, M_SUBPROC);
770 				mtx_lock_spin(&dt_lock);
771 				return (0);
772 			}
773 
774 			/*
775 			 * We have to substitute the current LDT entry for
776 			 * curproc with the new one since its size grew.
777 			 */
778 			old_ldt_base = pldt->ldt_base;
779 			old_ldt_len = pldt->ldt_len;
780 			pldt->ldt_sd = new_ldt->ldt_sd;
781 			pldt->ldt_base = new_ldt->ldt_base;
782 			pldt->ldt_len = new_ldt->ldt_len;
783 		} else
784 			mdp->md_ldt = pldt = new_ldt;
785 #ifdef SMP
786 		/*
787 		 * Signal other cpus to reload ldt.  We need to unlock dt_lock
788 		 * here because other CPU will contest on it since their
789 		 * curthreads won't hold the lock and will block when trying
790 		 * to acquire it.
791 		 */
792 		mtx_unlock_spin(&dt_lock);
793 		smp_rendezvous(NULL, set_user_ldt_rv, NULL,
794 		    td->td_proc->p_vmspace);
795 #else
796 		set_user_ldt_locked(&td->td_proc->p_md);
797 		mtx_unlock_spin(&dt_lock);
798 #endif
799 		if (old_ldt_base != NULL_LDT_BASE) {
800 			pmap_trm_free(old_ldt_base, old_ldt_len *
801 			    sizeof(union descriptor));
802 			free(new_ldt, M_SUBPROC);
803 		}
804 		mtx_lock_spin(&dt_lock);
805 	}
806 	return (0);
807 }
808