xref: /openbsd/sys/dev/pv/xen.c (revision fc61954a)
1 /*	$OpenBSD: xen.c,v 1.64 2016/10/06 17:00:25 mikeb Exp $	*/
2 
3 /*
4  * Copyright (c) 2015 Mike Belopuhov
5  *
6  * Permission to use, copy, modify, and distribute this software for any
7  * purpose with or without fee is hereby granted, provided that the above
8  * copyright notice and this permission notice appear in all copies.
9  *
10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17  */
18 
19 #include <sys/param.h>
20 
21 /* Xen requires locked atomic operations */
22 #ifndef MULTIPROCESSOR
23 #define _XENMPATOMICS
24 #define MULTIPROCESSOR
25 #endif
26 #include <sys/atomic.h>
27 #ifdef _XENMPATOMICS
28 #undef MULTIPROCESSOR
29 #undef _XENMPATOMICS
30 #endif
31 
32 #include <sys/systm.h>
33 #include <sys/proc.h>
34 #include <sys/signal.h>
35 #include <sys/signalvar.h>
36 #include <sys/malloc.h>
37 #include <sys/kernel.h>
38 #include <sys/device.h>
39 #include <sys/task.h>
40 #include <sys/syslog.h>
41 
42 #include <machine/bus.h>
43 #include <machine/cpu.h>
44 #include <machine/cpufunc.h>
45 
46 #include <uvm/uvm_extern.h>
47 
48 #include <machine/i82489var.h>
49 
50 #include <dev/rndvar.h>
51 
52 #include <dev/pv/pvvar.h>
53 #include <dev/pv/pvreg.h>
54 #include <dev/pv/xenreg.h>
55 #include <dev/pv/xenvar.h>
56 
57 struct xen_softc *xen_sc;
58 
59 int	xen_init_hypercall(struct xen_softc *);
60 int	xen_getfeatures(struct xen_softc *);
61 int	xen_init_info_page(struct xen_softc *);
62 int	xen_init_cbvec(struct xen_softc *);
63 int	xen_init_interrupts(struct xen_softc *);
64 int	xen_init_grant_tables(struct xen_softc *);
65 struct xen_gntent *
66 	xen_grant_table_grow(struct xen_softc *);
67 int	xen_grant_table_alloc(struct xen_softc *, grant_ref_t *);
68 void	xen_grant_table_free(struct xen_softc *, grant_ref_t);
69 void	xen_grant_table_enter(struct xen_softc *, grant_ref_t, paddr_t,
70 	    int, int);
71 void	xen_grant_table_remove(struct xen_softc *, grant_ref_t);
72 void	xen_disable_emulated_devices(struct xen_softc *);
73 
74 int 	xen_match(struct device *, void *, void *);
75 void	xen_attach(struct device *, struct device *, void *);
76 void	xen_deferred(struct device *);
77 void	xen_control(void *);
78 void	xen_resume(struct device *);
79 int	xen_activate(struct device *, int);
80 int	xen_probe_devices(struct xen_softc *);
81 
82 int	xen_bus_dmamap_create(bus_dma_tag_t, bus_size_t, int, bus_size_t,
83 	    bus_size_t, int, bus_dmamap_t *);
84 void	xen_bus_dmamap_destroy(bus_dma_tag_t, bus_dmamap_t);
85 int	xen_bus_dmamap_load(bus_dma_tag_t, bus_dmamap_t, void *, bus_size_t,
86 	    struct proc *, int);
87 int	xen_bus_dmamap_load_mbuf(bus_dma_tag_t, bus_dmamap_t, struct mbuf *,
88 	    int);
89 void	xen_bus_dmamap_unload(bus_dma_tag_t, bus_dmamap_t);
90 void	xen_bus_dmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_addr_t,
91 	    bus_size_t, int);
92 
93 int	xs_attach(struct xen_softc *);
94 
95 struct cfdriver xen_cd = {
96 	NULL, "xen", DV_DULL
97 };
98 
99 const struct cfattach xen_ca = {
100 	sizeof(struct xen_softc), xen_match, xen_attach, NULL, xen_activate
101 };
102 
103 struct bus_dma_tag xen_bus_dma_tag = {
104 	NULL,
105 	xen_bus_dmamap_create,
106 	xen_bus_dmamap_destroy,
107 	xen_bus_dmamap_load,
108 	xen_bus_dmamap_load_mbuf,
109 	NULL,
110 	NULL,
111 	xen_bus_dmamap_unload,
112 	xen_bus_dmamap_sync,
113 	_bus_dmamem_alloc,
114 	NULL,
115 	_bus_dmamem_free,
116 	_bus_dmamem_map,
117 	_bus_dmamem_unmap,
118 	NULL,
119 };
120 
121 int
122 xen_match(struct device *parent, void *match, void *aux)
123 {
124 	struct pv_attach_args *pva = aux;
125 	struct pvbus_hv *hv = &pva->pva_hv[PVBUS_XEN];
126 
127 	if (hv->hv_base == 0)
128 		return (0);
129 
130 	return (1);
131 }
132 
133 void
134 xen_attach(struct device *parent, struct device *self, void *aux)
135 {
136 	struct pv_attach_args *pva = (struct pv_attach_args *)aux;
137 	struct pvbus_hv *hv = &pva->pva_hv[PVBUS_XEN];
138 	struct xen_softc *sc = (struct xen_softc *)self;
139 
140 	sc->sc_base = hv->hv_base;
141 
142 	if (xen_init_hypercall(sc))
143 		return;
144 
145 	/* Wire it up to the global */
146 	xen_sc = sc;
147 
148 	if (xen_getfeatures(sc))
149 		return;
150 
151 	if (xen_init_info_page(sc))
152 		return;
153 
154 	xen_init_cbvec(sc);
155 
156 	if (xen_init_interrupts(sc))
157 		return;
158 
159 	if (xen_init_grant_tables(sc))
160 		return;
161 
162 	if (xs_attach(sc))
163 		return;
164 
165 	xen_probe_devices(sc);
166 
167 	/* pvbus(4) key/value interface */
168 	hv->hv_kvop = xs_kvop;
169 	hv->hv_arg = sc;
170 
171 	xen_disable_emulated_devices(sc);
172 
173 	config_mountroot(self, xen_deferred);
174 }
175 
176 void
177 xen_deferred(struct device *self)
178 {
179 	struct xen_softc *sc = (struct xen_softc *)self;
180 
181 	if (!(sc->sc_flags & XSF_CBVEC)) {
182 		DPRINTF("%s: callback vector hasn't been established\n",
183 		    sc->sc_dev.dv_xname);
184 		return;
185 	}
186 
187 	xen_intr_enable();
188 
189 	if (xs_watch(sc, "control", "shutdown", &sc->sc_ctltsk,
190 	    xen_control, sc))
191 		printf("%s: failed to setup shutdown control watch\n",
192 		    sc->sc_dev.dv_xname);
193 }
194 
195 void
196 xen_control(void *arg)
197 {
198 	struct xen_softc *sc = arg;
199 	struct xs_transaction xst;
200 	char action[128];
201 	int error;
202 
203 	memset(&xst, 0, sizeof(xst));
204 	xst.xst_id = 0;
205 	xst.xst_sc = sc->sc_xs;
206 
207 	error = xs_getprop(sc, "control", "shutdown", action, sizeof(action));
208 	if (error) {
209 		if (error != ENOENT)
210 			printf("%s: failed to process control event\n",
211 			    sc->sc_dev.dv_xname);
212 		return;
213 	}
214 
215 	if (strlen(action) == 0)
216 		return;
217 
218 	/* Acknowledge the event */
219 	xs_setprop(sc, "control", "shutdown", "", 0);
220 
221 	if (strcmp(action, "halt") == 0 || strcmp(action, "poweroff") == 0) {
222 		extern int allowpowerdown;
223 
224 		if (allowpowerdown == 0)
225 			return;
226 
227 		suspend_randomness();
228 
229 		log(LOG_KERN | LOG_NOTICE, "Shutting down in response to "
230 		    "request from Xen host\n");
231 		prsignal(initprocess, SIGUSR2);
232 	} else if (strcmp(action, "reboot") == 0) {
233 		extern int allowpowerdown;
234 
235 		if (allowpowerdown == 0)
236 			return;
237 
238 		suspend_randomness();
239 
240 		log(LOG_KERN | LOG_NOTICE, "Rebooting in response to request "
241 		    "from Xen host\n");
242 		prsignal(initprocess, SIGINT);
243 	} else if (strcmp(action, "crash") == 0) {
244 		panic("xen told us to do this");
245 	} else if (strcmp(action, "suspend") == 0) {
246 		/* Not implemented yet */
247 	} else {
248 		printf("%s: unknown shutdown event \"%s\"\n",
249 		    sc->sc_dev.dv_xname, action);
250 	}
251 }
252 
253 void
254 xen_resume(struct device *self)
255 {
256 }
257 
258 int
259 xen_activate(struct device *self, int act)
260 {
261 	int rv = 0;
262 
263 	switch (act) {
264 	case DVACT_RESUME:
265 		xen_resume(self);
266 		break;
267 	}
268 	return (rv);
269 }
270 
271 int
272 xen_init_hypercall(struct xen_softc *sc)
273 {
274 	extern void *xen_hypercall_page;
275 	uint32_t regs[4];
276 	paddr_t pa;
277 
278 	/* Get hypercall page configuration MSR */
279 	CPUID(sc->sc_base + CPUID_OFFSET_XEN_HYPERCALL,
280 	    regs[0], regs[1], regs[2], regs[3]);
281 
282 	/* We don't support more than one hypercall page */
283 	if (regs[0] != 1) {
284 		printf(": requested %d hypercall pages\n", regs[0]);
285 		return (-1);
286 	}
287 
288 	sc->sc_hc = &xen_hypercall_page;
289 
290 	if (!pmap_extract(pmap_kernel(), (vaddr_t)sc->sc_hc, &pa)) {
291 		printf(": hypercall page PA extraction failed\n");
292 		return (-1);
293 	}
294 	wrmsr(regs[1], pa);
295 
296 	return (0);
297 }
298 
299 int
300 xen_hypercall(struct xen_softc *sc, int op, int argc, ...)
301 {
302 	va_list ap;
303 	ulong argv[5];
304 	int i;
305 
306 	if (argc < 0 || argc > 5)
307 		return (-1);
308 	va_start(ap, argc);
309 	for (i = 0; i < argc; i++)
310 		argv[i] = (ulong)va_arg(ap, ulong);
311 	return (xen_hypercallv(sc, op, argc, argv));
312 }
313 
314 int
315 xen_hypercallv(struct xen_softc *sc, int op, int argc, ulong *argv)
316 {
317 	ulong hcall;
318 	int rv = 0;
319 
320 	hcall = (ulong)sc->sc_hc + op * 32;
321 
322 #if defined(XEN_DEBUG) && disabled
323 	{
324 		int i;
325 
326 		printf("hypercall %d", op);
327 		if (argc > 0) {
328 			printf(", args {");
329 			for (i = 0; i < argc; i++)
330 				printf(" %#lx", argv[i]);
331 			printf(" }\n");
332 		} else
333 			printf("\n");
334 	}
335 #endif
336 
337 	switch (argc) {
338 	case 0: {
339 		HYPERCALL_RES1;
340 		__asm__ volatile (			\
341 			  HYPERCALL_LABEL		\
342 			: HYPERCALL_OUT1		\
343 			: HYPERCALL_PTR(hcall)		\
344 			: HYPERCALL_CLOBBER		\
345 		);
346 		HYPERCALL_RET(rv);
347 		break;
348 	}
349 	case 1: {
350 		HYPERCALL_RES1; HYPERCALL_RES2;
351 		HYPERCALL_ARG1(argv[0]);
352 		__asm__ volatile (			\
353 			  HYPERCALL_LABEL		\
354 			: HYPERCALL_OUT1 HYPERCALL_OUT2	\
355 			: HYPERCALL_IN1			\
356 			, HYPERCALL_PTR(hcall)		\
357 			: HYPERCALL_CLOBBER		\
358 		);
359 		HYPERCALL_RET(rv);
360 		break;
361 	}
362 	case 2: {
363 		HYPERCALL_RES1; HYPERCALL_RES2; HYPERCALL_RES3;
364 		HYPERCALL_ARG1(argv[0]); HYPERCALL_ARG2(argv[1]);
365 		__asm__ volatile (			\
366 			  HYPERCALL_LABEL		\
367 			: HYPERCALL_OUT1 HYPERCALL_OUT2	\
368 			  HYPERCALL_OUT3		\
369 			: HYPERCALL_IN1	HYPERCALL_IN2	\
370 			, HYPERCALL_PTR(hcall)		\
371 			: HYPERCALL_CLOBBER		\
372 		);
373 		HYPERCALL_RET(rv);
374 		break;
375 	}
376 	case 3: {
377 		HYPERCALL_RES1; HYPERCALL_RES2; HYPERCALL_RES3;
378 		HYPERCALL_RES4;
379 		HYPERCALL_ARG1(argv[0]); HYPERCALL_ARG2(argv[1]);
380 		HYPERCALL_ARG3(argv[2]);
381 		__asm__ volatile (			\
382 			  HYPERCALL_LABEL		\
383 			: HYPERCALL_OUT1 HYPERCALL_OUT2	\
384 			  HYPERCALL_OUT3 HYPERCALL_OUT4	\
385 			: HYPERCALL_IN1	HYPERCALL_IN2	\
386 			  HYPERCALL_IN3			\
387 			, HYPERCALL_PTR(hcall)		\
388 			: HYPERCALL_CLOBBER		\
389 		);
390 		HYPERCALL_RET(rv);
391 		break;
392 	}
393 	case 4: {
394 		HYPERCALL_RES1; HYPERCALL_RES2; HYPERCALL_RES3;
395 		HYPERCALL_RES4; HYPERCALL_RES5;
396 		HYPERCALL_ARG1(argv[0]); HYPERCALL_ARG2(argv[1]);
397 		HYPERCALL_ARG3(argv[2]); HYPERCALL_ARG4(argv[3]);
398 		__asm__ volatile (			\
399 			  HYPERCALL_LABEL		\
400 			: HYPERCALL_OUT1 HYPERCALL_OUT2	\
401 			  HYPERCALL_OUT3 HYPERCALL_OUT4	\
402 			  HYPERCALL_OUT5		\
403 			: HYPERCALL_IN1	HYPERCALL_IN2	\
404 			  HYPERCALL_IN3	HYPERCALL_IN4	\
405 			, HYPERCALL_PTR(hcall)		\
406 			: HYPERCALL_CLOBBER		\
407 		);
408 		HYPERCALL_RET(rv);
409 		break;
410 	}
411 	case 5: {
412 		HYPERCALL_RES1; HYPERCALL_RES2; HYPERCALL_RES3;
413 		HYPERCALL_RES4; HYPERCALL_RES5; HYPERCALL_RES6;
414 		HYPERCALL_ARG1(argv[0]); HYPERCALL_ARG2(argv[1]);
415 		HYPERCALL_ARG3(argv[2]); HYPERCALL_ARG4(argv[3]);
416 		HYPERCALL_ARG5(argv[4]);
417 		__asm__ volatile (			\
418 			  HYPERCALL_LABEL		\
419 			: HYPERCALL_OUT1 HYPERCALL_OUT2	\
420 			  HYPERCALL_OUT3 HYPERCALL_OUT4	\
421 			  HYPERCALL_OUT5 HYPERCALL_OUT6	\
422 			: HYPERCALL_IN1	HYPERCALL_IN2	\
423 			  HYPERCALL_IN3	HYPERCALL_IN4	\
424 			  HYPERCALL_IN5			\
425 			, HYPERCALL_PTR(hcall)		\
426 			: HYPERCALL_CLOBBER		\
427 		);
428 		HYPERCALL_RET(rv);
429 		break;
430 	}
431 	default:
432 		DPRINTF("%s: wrong number of arguments: %d\n", __func__, argc);
433 		rv = -1;
434 		break;
435 	}
436 	return (rv);
437 }
438 
439 int
440 xen_getfeatures(struct xen_softc *sc)
441 {
442 	struct xen_feature_info xfi;
443 
444 	memset(&xfi, 0, sizeof(xfi));
445 	if (xen_hypercall(sc, XC_VERSION, 2, XENVER_get_features, &xfi) < 0) {
446 		printf(": failed to fetch features\n");
447 		return (-1);
448 	}
449 	sc->sc_features = xfi.submap;
450 #ifdef XEN_DEBUG
451 	printf(": features %b", sc->sc_features,
452 	    "\20\014DOM0\013PIRQ\012PVCLOCK\011CBVEC\010GNTFLAGS\007HMA"
453 	    "\006PTUPD\005PAE4G\004SUPERVISOR\003AUTOPMAP\002WDT\001WPT");
454 #else
455 	printf(": features %#x", sc->sc_features);
456 #endif
457 	return (0);
458 }
459 
460 #ifdef XEN_DEBUG
461 void
462 xen_print_info_page(void)
463 {
464 	struct xen_softc *sc = xen_sc;
465 	struct shared_info *s = sc->sc_ipg;
466 	struct vcpu_info *v;
467 	int i;
468 
469 	virtio_membar_sync();
470 	for (i = 0; i < XEN_LEGACY_MAX_VCPUS; i++) {
471 		v = &s->vcpu_info[i];
472 		if (!v->evtchn_upcall_pending && !v->evtchn_upcall_mask &&
473 		    !v->evtchn_pending_sel && !v->time.version &&
474 		    !v->time.tsc_timestamp && !v->time.system_time &&
475 		    !v->time.tsc_to_system_mul && !v->time.tsc_shift)
476 			continue;
477 		printf("vcpu%d:\n"
478 		    "   upcall_pending=%02x upcall_mask=%02x pending_sel=%#lx\n"
479 		    "   time version=%u tsc=%llu system=%llu\n"
480 		    "   time mul=%u shift=%d\n",
481 		    i, v->evtchn_upcall_pending, v->evtchn_upcall_mask,
482 		    v->evtchn_pending_sel, v->time.version,
483 		    v->time.tsc_timestamp, v->time.system_time,
484 		    v->time.tsc_to_system_mul, v->time.tsc_shift);
485 	}
486 	printf("pending events: ");
487 	for (i = 0; i < nitems(s->evtchn_pending); i++) {
488 		if (s->evtchn_pending[i] == 0)
489 			continue;
490 		printf(" %d:%#lx", i, s->evtchn_pending[i]);
491 	}
492 	printf("\nmasked events: ");
493 	for (i = 0; i < nitems(s->evtchn_mask); i++) {
494 		if (s->evtchn_mask[i] == 0xffffffffffffffffULL)
495 			continue;
496 		printf(" %d:%#lx", i, s->evtchn_mask[i]);
497 	}
498 	printf("\nwc ver=%u sec=%u nsec=%u\n", s->wc_version, s->wc_sec,
499 	    s->wc_nsec);
500 	printf("arch maxpfn=%lu framelist=%lu nmi=%lu\n", s->arch.max_pfn,
501 	    s->arch.pfn_to_mfn_frame_list, s->arch.nmi_reason);
502 }
503 #endif	/* XEN_DEBUG */
504 
505 int
506 xen_init_info_page(struct xen_softc *sc)
507 {
508 	struct xen_add_to_physmap xatp;
509 	paddr_t pa;
510 
511 	sc->sc_ipg = malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT | M_ZERO);
512 	if (sc->sc_ipg == NULL) {
513 		printf(": failed to allocate shared info page\n");
514 		return (-1);
515 	}
516 	if (!pmap_extract(pmap_kernel(), (vaddr_t)sc->sc_ipg, &pa)) {
517 		printf(": shared info page PA extraction failed\n");
518 		free(sc->sc_ipg, M_DEVBUF, PAGE_SIZE);
519 		return (-1);
520 	}
521 	xatp.domid = DOMID_SELF;
522 	xatp.idx = 0;
523 	xatp.space = XENMAPSPACE_shared_info;
524 	xatp.gpfn = atop(pa);
525 	if (xen_hypercall(sc, XC_MEMORY, 2, XENMEM_add_to_physmap, &xatp)) {
526 		printf(": failed to register shared info page\n");
527 		free(sc->sc_ipg, M_DEVBUF, PAGE_SIZE);
528 		return (-1);
529 	}
530 	return (0);
531 }
532 
533 int
534 xen_init_cbvec(struct xen_softc *sc)
535 {
536 	struct xen_hvm_param xhp;
537 
538 	if ((sc->sc_features & XENFEAT_CBVEC) == 0)
539 		return (ENOENT);
540 
541 	xhp.domid = DOMID_SELF;
542 	xhp.index = HVM_PARAM_CALLBACK_IRQ;
543 	xhp.value = HVM_CALLBACK_VECTOR(LAPIC_XEN_VECTOR);
544 	if (xen_hypercall(sc, XC_HVM, 2, HVMOP_set_param, &xhp)) {
545 		/* Will retry with the xspd(4) PCI interrupt */
546 		return (ENOENT);
547 	}
548 	DPRINTF(", idtvec %d", LAPIC_XEN_VECTOR);
549 
550 	sc->sc_flags |= XSF_CBVEC;
551 
552 	return (0);
553 }
554 
555 int
556 xen_init_interrupts(struct xen_softc *sc)
557 {
558 	int i;
559 
560 	sc->sc_irq = LAPIC_XEN_VECTOR;
561 
562 	/*
563 	 * Clear all pending events and mask all interrupts
564 	 */
565 	for (i = 0; i < nitems(sc->sc_ipg->evtchn_pending); i++) {
566 		sc->sc_ipg->evtchn_pending[i] = 0;
567 		sc->sc_ipg->evtchn_mask[i] = ~0UL;
568 	}
569 
570 	SLIST_INIT(&sc->sc_intrs);
571 
572 	return (0);
573 }
574 
575 static int
576 xen_evtchn_hypercall(struct xen_softc *sc, int cmd, void *arg, size_t len)
577 {
578 	struct evtchn_op compat;
579 	int error;
580 
581 	error = xen_hypercall(sc, XC_EVTCHN, 2, cmd, arg);
582 	if (error == -ENOXENSYS) {
583 		memset(&compat, 0, sizeof(compat));
584 		compat.cmd = cmd;
585 		memcpy(&compat.u, arg, len);
586 		error = xen_hypercall(sc, XC_OEVTCHN, 1, &compat);
587 	}
588 	return (error);
589 }
590 
591 static inline struct xen_intsrc *
592 xen_lookup_intsrc(struct xen_softc *sc, evtchn_port_t port)
593 {
594 	struct xen_intsrc *xi;
595 
596 	SLIST_FOREACH(xi, &sc->sc_intrs, xi_entry)
597 		if (xi->xi_port == port)
598 			break;
599 	return (xi);
600 }
601 
602 void
603 xen_intr_ack(void)
604 {
605 	struct xen_softc *sc = xen_sc;
606 	struct shared_info *s = sc->sc_ipg;
607 	struct cpu_info *ci = curcpu();
608 	struct vcpu_info *v = &s->vcpu_info[CPU_INFO_UNIT(ci)];
609 
610 	v->evtchn_upcall_pending = 0;
611 	virtio_membar_sync();
612 }
613 
614 void
615 xen_intr(void)
616 {
617 	struct xen_softc *sc = xen_sc;
618 	struct xen_intsrc *xi;
619 	struct shared_info *s = sc->sc_ipg;
620 	struct cpu_info *ci = curcpu();
621 	struct vcpu_info *v = &s->vcpu_info[CPU_INFO_UNIT(ci)];
622 	ulong pending, selector;
623 	int port, bit, row;
624 
625 	v->evtchn_upcall_pending = 0;
626 	selector = atomic_swap_ulong(&v->evtchn_pending_sel, 0);
627 
628 	for (row = 0; selector > 0; selector >>= 1, row++) {
629 		if ((selector & 1) == 0)
630 			continue;
631 		if ((sc->sc_ipg->evtchn_pending[row] &
632 		    ~(sc->sc_ipg->evtchn_mask[row])) == 0)
633 			continue;
634 		pending = atomic_swap_ulong(&sc->sc_ipg->evtchn_pending[row],
635 		    0) & ~(sc->sc_ipg->evtchn_mask[row]);
636 		for (bit = 0; pending > 0; pending >>= 1, bit++) {
637 			if ((pending & 1) == 0)
638 				continue;
639 			port = (row * LONG_BIT) + bit;
640 			if ((xi = xen_lookup_intsrc(sc, port)) == NULL) {
641 				printf("%s: unhandled interrupt on port %u\n",
642 				    sc->sc_dev.dv_xname, port);
643 				continue;
644 			}
645 			xi->xi_evcnt.ec_count++;
646 			task_add(xi->xi_taskq, &xi->xi_task);
647 		}
648 	}
649 }
650 
651 void
652 xen_intr_schedule(xen_intr_handle_t xih)
653 {
654 	struct xen_softc *sc = xen_sc;
655 	struct xen_intsrc *xi;
656 
657 	if ((xi = xen_lookup_intsrc(sc, (evtchn_port_t)xih)) != NULL)
658 		task_add(xi->xi_taskq, &xi->xi_task);
659 }
660 
661 void
662 xen_intr_signal(xen_intr_handle_t xih)
663 {
664 	struct xen_softc *sc = xen_sc;
665 	struct xen_intsrc *xi;
666 	struct evtchn_send es;
667 
668 	if ((xi = xen_lookup_intsrc(sc, (evtchn_port_t)xih)) != NULL) {
669 		es.port = xi->xi_port;
670 		xen_evtchn_hypercall(sc, EVTCHNOP_send, &es, sizeof(es));
671 	}
672 }
673 
674 int
675 xen_intr_establish(evtchn_port_t port, xen_intr_handle_t *xih, int domain,
676     void (*handler)(void *), void *arg, char *name)
677 {
678 	struct xen_softc *sc = xen_sc;
679 	struct xen_intsrc *xi;
680 	struct evtchn_alloc_unbound eau;
681 #ifdef notyet
682 	struct evtchn_bind_vcpu ebv;
683 #endif
684 #if defined(XEN_DEBUG) && disabled
685 	struct evtchn_status es;
686 #endif
687 
688 	if (port && xen_lookup_intsrc(sc, port)) {
689 		DPRINTF("%s: interrupt handler has already been established "
690 		    "for port %u\n", sc->sc_dev.dv_xname, port);
691 		return (-1);
692 	}
693 
694 	xi = malloc(sizeof(*xi), M_DEVBUF, M_NOWAIT | M_ZERO);
695 	if (xi == NULL)
696 		return (-1);
697 
698 	xi->xi_port = (evtchn_port_t)*xih;
699 
700 	xi->xi_taskq = taskq_create(name, 1, IPL_NET, TASKQ_MPSAFE);
701 	if (!xi->xi_taskq) {
702 		printf("%s: failed to create interrupt task for %s\n",
703 		    sc->sc_dev.dv_xname, name);
704 		free(xi, M_DEVBUF, sizeof(*xi));
705 		return (-1);
706 	}
707 	task_set(&xi->xi_task, handler, arg);
708 
709 	if (port == 0) {
710 		/* We're being asked to allocate a new event port */
711 		memset(&eau, 0, sizeof(eau));
712 		eau.dom = DOMID_SELF;
713 		eau.remote_dom = domain;
714 		if (xen_evtchn_hypercall(sc, EVTCHNOP_alloc_unbound, &eau,
715 		    sizeof(eau)) != 0) {
716 			DPRINTF("%s: failed to allocate new event port\n",
717 			    sc->sc_dev.dv_xname);
718 			free(xi, M_DEVBUF, sizeof(*xi));
719 			return (-1);
720 		}
721 		*xih = xi->xi_port = eau.port;
722 	} else {
723 		*xih = xi->xi_port = port;
724 		/*
725 		 * The Event Channel API didn't open this port, so it is not
726 		 * responsible for closing it automatically on unbind.
727 		 */
728 		xi->xi_noclose = 1;
729 	}
730 
731 #ifdef notyet
732 	/* Bind interrupt to VCPU#0 */
733 	memset(&ebv, 0, sizeof(ebv));
734 	ebv.port = xi->xi_port;
735 	ebv.vcpu = 0;
736 	if (xen_evtchn_hypercall(sc, EVTCHNOP_bind_vcpu, &ebv, sizeof(ebv))) {
737 		printf("%s: failed to bind interrupt on port %u to vcpu%d\n",
738 		    sc->sc_dev.dv_xname, ebv.port, ebv.vcpu);
739 	}
740 #endif
741 
742 	evcount_attach(&xi->xi_evcnt, name, &sc->sc_irq);
743 
744 	SLIST_INSERT_HEAD(&sc->sc_intrs, xi, xi_entry);
745 
746 	/* Mask the event port */
747 	set_bit(xi->xi_port, &sc->sc_ipg->evtchn_mask[0]);
748 
749 #if defined(XEN_DEBUG) && disabled
750 	memset(&es, 0, sizeof(es));
751 	es.dom = DOMID_SELF;
752 	es.port = xi->xi_port;
753 	if (xen_evtchn_hypercall(sc, EVTCHNOP_status, &es, sizeof(es))) {
754 		printf("%s: failed to obtain status for port %d\n",
755 		    sc->sc_dev.dv_xname, es.port);
756 	}
757 	printf("%s: port %u bound to vcpu%u", sc->sc_dev.dv_xname,
758 	    es.port, es.vcpu);
759 	if (es.status == EVTCHNSTAT_interdomain)
760 		printf(": domain %d port %u\n", es.u.interdomain.dom,
761 		    es.u.interdomain.port);
762 	else if (es.status == EVTCHNSTAT_unbound)
763 		printf(": domain %d\n", es.u.unbound.dom);
764 	else if (es.status == EVTCHNSTAT_pirq)
765 		printf(": pirq %u\n", es.u.pirq);
766 	else if (es.status == EVTCHNSTAT_virq)
767 		printf(": virq %u\n", es.u.virq);
768 	else
769 		printf("\n");
770 #endif
771 
772 	return (0);
773 }
774 
775 int
776 xen_intr_disestablish(xen_intr_handle_t xih)
777 {
778 	struct xen_softc *sc = xen_sc;
779 	evtchn_port_t port = (evtchn_port_t)xih;
780 	struct evtchn_close ec;
781 	struct xen_intsrc *xi;
782 
783 	if ((xi = xen_lookup_intsrc(sc, port)) == NULL)
784 		return (-1);
785 
786 	evcount_detach(&xi->xi_evcnt);
787 
788 	/* XXX not MP safe */
789 	SLIST_REMOVE(&sc->sc_intrs, xi, xen_intsrc, xi_entry);
790 
791 	taskq_destroy(xi->xi_taskq);
792 
793 	set_bit(xi->xi_port, &sc->sc_ipg->evtchn_mask[0]);
794 	clear_bit(xi->xi_port, &sc->sc_ipg->evtchn_pending[0]);
795 
796 	if (!xi->xi_noclose) {
797 		ec.port = xi->xi_port;
798 		if (xen_evtchn_hypercall(sc, EVTCHNOP_close, &ec, sizeof(ec))) {
799 			DPRINTF("%s: failed to close event port %u\n",
800 			    sc->sc_dev.dv_xname, xi->xi_port);
801 		}
802 	}
803 
804 	free(xi, M_DEVBUF, sizeof(*xi));
805 	return (0);
806 }
807 
808 void
809 xen_intr_enable(void)
810 {
811 	struct xen_softc *sc = xen_sc;
812 	struct xen_intsrc *xi;
813 	struct evtchn_unmask eu;
814 
815 	SLIST_FOREACH(xi, &sc->sc_intrs, xi_entry) {
816 		if (!xi->xi_masked) {
817 			eu.port = xi->xi_port;
818 			if (xen_evtchn_hypercall(sc, EVTCHNOP_unmask, &eu,
819 			    sizeof(eu)))
820 				printf("%s: unmasking port %u failed\n",
821 				    sc->sc_dev.dv_xname, xi->xi_port);
822 			virtio_membar_sync();
823 			if (test_bit(xi->xi_port, &sc->sc_ipg->evtchn_mask[0]))
824 				printf("%s: port %u is still masked\n",
825 				    sc->sc_dev.dv_xname, xi->xi_port);
826 		}
827 	}
828 }
829 
830 void
831 xen_intr_mask(xen_intr_handle_t xih)
832 {
833 	struct xen_softc *sc = xen_sc;
834 	evtchn_port_t port = (evtchn_port_t)xih;
835 	struct xen_intsrc *xi;
836 
837 	if ((xi = xen_lookup_intsrc(sc, port)) != NULL) {
838 		xi->xi_masked = 1;
839 		set_bit(xi->xi_port, &sc->sc_ipg->evtchn_mask[0]);
840 	}
841 }
842 
843 int
844 xen_intr_unmask(xen_intr_handle_t xih)
845 {
846 	struct xen_softc *sc = xen_sc;
847 	evtchn_port_t port = (evtchn_port_t)xih;
848 	struct xen_intsrc *xi;
849 	struct evtchn_unmask eu;
850 
851 	if ((xi = xen_lookup_intsrc(sc, port)) != NULL) {
852 		xi->xi_masked = 0;
853 		if (!test_bit(xi->xi_port, &sc->sc_ipg->evtchn_mask[0]))
854 			return (0);
855 		eu.port = xi->xi_port;
856 		return (xen_evtchn_hypercall(sc, EVTCHNOP_unmask, &eu,
857 		    sizeof(eu)));
858 	}
859 	return (0);
860 }
861 
862 int
863 xen_init_grant_tables(struct xen_softc *sc)
864 {
865 	struct gnttab_query_size gqs;
866 
867 	gqs.dom = DOMID_SELF;
868 	if (xen_hypercall(sc, XC_GNTTAB, 3, GNTTABOP_query_size, &gqs, 1)) {
869 		printf(": failed the query for grant table pages\n");
870 		return (-1);
871 	}
872 	if (gqs.nr_frames == 0 || gqs.nr_frames > gqs.max_nr_frames) {
873 		printf(": invalid number of grant table pages: %u/%u\n",
874 		    gqs.nr_frames, gqs.max_nr_frames);
875 		return (-1);
876 	}
877 
878 	sc->sc_gntmax = gqs.max_nr_frames;
879 
880 	sc->sc_gnt = mallocarray(sc->sc_gntmax + 1, sizeof(struct xen_gntent),
881 	    M_DEVBUF, M_ZERO | M_NOWAIT);
882 	if (sc->sc_gnt == NULL) {
883 		printf(": failed to allocate grant table lookup table\n");
884 		return (-1);
885 	}
886 
887 	mtx_init(&sc->sc_gntmtx, IPL_NET);
888 
889 	if (xen_grant_table_grow(sc) == NULL) {
890 		free(sc->sc_gnt, M_DEVBUF, sc->sc_gntmax *
891 		    sizeof(struct xen_gntent));
892 		return (-1);
893 	}
894 
895 	printf(", %u grant table frames", sc->sc_gntmax);
896 
897 	xen_bus_dma_tag._cookie = sc;
898 
899 	return (0);
900 }
901 
902 struct xen_gntent *
903 xen_grant_table_grow(struct xen_softc *sc)
904 {
905 	struct xen_add_to_physmap xatp;
906 	struct xen_gntent *ge;
907 	paddr_t pa;
908 
909 	if (sc->sc_gntcnt == sc->sc_gntmax) {
910 		printf("%s: grant table frame allotment limit reached\n",
911 		    sc->sc_dev.dv_xname);
912 		return (NULL);
913 	}
914 
915 	mtx_enter(&sc->sc_gntmtx);
916 
917 	ge = &sc->sc_gnt[sc->sc_gntcnt];
918 	ge->ge_table = km_alloc(PAGE_SIZE, &kv_any, &kp_zero, &kd_nowait);
919 	if (ge->ge_table == NULL) {
920 		free(ge, M_DEVBUF, sizeof(*ge));
921 		mtx_leave(&sc->sc_gntmtx);
922 		return (NULL);
923 	}
924 	if (!pmap_extract(pmap_kernel(), (vaddr_t)ge->ge_table, &pa)) {
925 		printf("%s: grant table page PA extraction failed\n",
926 		    sc->sc_dev.dv_xname);
927 		km_free(ge->ge_table, PAGE_SIZE, &kv_any, &kp_zero);
928 		free(ge, M_DEVBUF, sizeof(*ge));
929 		mtx_leave(&sc->sc_gntmtx);
930 		return (NULL);
931 	}
932 	xatp.domid = DOMID_SELF;
933 	xatp.idx = sc->sc_gntcnt;
934 	xatp.space = XENMAPSPACE_grant_table;
935 	xatp.gpfn = atop(pa);
936 	if (xen_hypercall(sc, XC_MEMORY, 2, XENMEM_add_to_physmap, &xatp)) {
937 		printf("%s: failed to add a grant table page\n",
938 		    sc->sc_dev.dv_xname);
939 		km_free(ge->ge_table, PAGE_SIZE, &kv_any, &kp_zero);
940 		free(ge, M_DEVBUF, sizeof(*ge));
941 		mtx_leave(&sc->sc_gntmtx);
942 		return (NULL);
943 	}
944 	ge->ge_start = sc->sc_gntcnt * GNTTAB_NEPG;
945 	/* First page has 8 reserved entries */
946 	ge->ge_reserved = ge->ge_start == 0 ? GNTTAB_NR_RESERVED_ENTRIES : 0;
947 	ge->ge_free = GNTTAB_NEPG - ge->ge_reserved;
948 	ge->ge_next = ge->ge_reserved;
949 	mtx_init(&ge->ge_mtx, IPL_NET);
950 
951 	sc->sc_gntcnt++;
952 	mtx_leave(&sc->sc_gntmtx);
953 
954 	return (ge);
955 }
956 
957 int
958 xen_grant_table_alloc(struct xen_softc *sc, grant_ref_t *ref)
959 {
960 	struct xen_gntent *ge;
961 	int i;
962 
963 	/* Start with a previously allocated table page */
964 	ge = &sc->sc_gnt[sc->sc_gntcnt - 1];
965 	if (ge->ge_free > 0) {
966 		mtx_enter(&ge->ge_mtx);
967 		if (ge->ge_free > 0)
968 			goto search;
969 		mtx_leave(&ge->ge_mtx);
970 	}
971 
972 	/* Try other existing table pages */
973 	for (i = 0; i < sc->sc_gntcnt; i++) {
974 		ge = &sc->sc_gnt[i];
975 		if (ge->ge_free == 0)
976 			continue;
977 		mtx_enter(&ge->ge_mtx);
978 		if (ge->ge_free > 0)
979 			goto search;
980 		mtx_leave(&ge->ge_mtx);
981 	}
982 
983  alloc:
984 	/* Allocate a new table page */
985 	if ((ge = xen_grant_table_grow(sc)) == NULL)
986 		return (-1);
987 
988 	mtx_enter(&ge->ge_mtx);
989 	if (ge->ge_free == 0) {
990 		/* We were not fast enough... */
991 		mtx_leave(&ge->ge_mtx);
992 		goto alloc;
993 	}
994 
995  search:
996 	for (i = ge->ge_next;
997 	     /* Math works here because GNTTAB_NEPG is a power of 2 */
998 	     i != ((ge->ge_next + GNTTAB_NEPG - 1) & (GNTTAB_NEPG - 1));
999 	     i++) {
1000 		if (i == GNTTAB_NEPG)
1001 			i = 0;
1002 		if (ge->ge_reserved && i < ge->ge_reserved)
1003 			continue;
1004 		if (ge->ge_table[i].flags != GTF_invalid &&
1005 		    ge->ge_table[i].frame != 0)
1006 			continue;
1007 		*ref = ge->ge_start + i;
1008 		/* XXX Mark as taken */
1009 		ge->ge_table[i].frame = 0xffffffff;
1010 		if ((ge->ge_next = i + 1) == GNTTAB_NEPG)
1011 			ge->ge_next = ge->ge_reserved;
1012 		ge->ge_free--;
1013 		mtx_leave(&ge->ge_mtx);
1014 		return (0);
1015 	}
1016 	mtx_leave(&ge->ge_mtx);
1017 
1018 	panic("page full, sc %p gnt %p (%d) ge %p", sc, sc->sc_gnt,
1019 	    sc->sc_gntcnt, ge);
1020 	return (-1);
1021 }
1022 
1023 void
1024 xen_grant_table_free(struct xen_softc *sc, grant_ref_t ref)
1025 {
1026 	struct xen_gntent *ge;
1027 
1028 #ifdef XEN_DEBUG
1029 	if (ref > sc->sc_gntcnt * GNTTAB_NEPG)
1030 		panic("unmanaged ref %u sc %p gnt %p (%d)", ref, sc,
1031 		    sc->sc_gnt, sc->sc_gntcnt);
1032 #endif
1033 	ge = &sc->sc_gnt[ref / GNTTAB_NEPG];
1034 	mtx_enter(&ge->ge_mtx);
1035 #ifdef XEN_DEBUG
1036 	if (ref < ge->ge_start || ref > ge->ge_start + GNTTAB_NEPG) {
1037 		mtx_leave(&ge->ge_mtx);
1038 		panic("out of bounds ref %u ge %p start %u sc %p gnt %p",
1039 		    ref, ge, ge->ge_start, sc, sc->sc_gnt);
1040 	}
1041 #endif
1042 	ref -= ge->ge_start;
1043 	if (ge->ge_table[ref].flags != GTF_invalid) {
1044 		mtx_leave(&ge->ge_mtx);
1045 #ifdef XEN_DEBUG
1046 		panic("ref %u is still in use, sc %p gnt %p", ref +
1047 		    ge->ge_start, sc, sc->sc_gnt);
1048 #else
1049 		printf("%s: reference %u is still in use\n",
1050 		    sc->sc_dev.dv_xname, ref + ge->ge_start);
1051 #endif
1052 	}
1053 	ge->ge_table[ref].frame = 0;
1054 	ge->ge_next = ref;
1055 	ge->ge_free++;
1056 	mtx_leave(&ge->ge_mtx);
1057 }
1058 
1059 void
1060 xen_grant_table_enter(struct xen_softc *sc, grant_ref_t ref, paddr_t pa,
1061     int domain, int flags)
1062 {
1063 	struct xen_gntent *ge;
1064 
1065 #ifdef XEN_DEBUG
1066 	if (ref > sc->sc_gntcnt * GNTTAB_NEPG)
1067 		panic("unmanaged ref %u sc %p gnt %p (%d)", ref, sc,
1068 		    sc->sc_gnt, sc->sc_gntcnt);
1069 #endif
1070 	ge = &sc->sc_gnt[ref / GNTTAB_NEPG];
1071 #ifdef XEN_DEBUG
1072 	if (ref < ge->ge_start || ref > ge->ge_start + GNTTAB_NEPG) {
1073 		panic("out of bounds ref %u ge %p start %u sc %p gnt %p",
1074 		    ref, ge, ge->ge_start, sc, sc->sc_gnt);
1075 	}
1076 #endif
1077 	ref -= ge->ge_start;
1078 	ge->ge_table[ref].frame = atop(pa);
1079 	ge->ge_table[ref].domid = domain;
1080 	virtio_membar_sync();
1081 	ge->ge_table[ref].flags = GTF_permit_access | flags;
1082 	virtio_membar_sync();
1083 }
1084 
1085 void
1086 xen_grant_table_remove(struct xen_softc *sc, grant_ref_t ref)
1087 {
1088 	struct xen_gntent *ge;
1089 	uint32_t flags, *ptr;
1090 	int loop;
1091 
1092 #ifdef XEN_DEBUG
1093 	if (ref > sc->sc_gntcnt * GNTTAB_NEPG)
1094 		panic("unmanaged ref %u sc %p gnt %p (%d)", ref, sc,
1095 		    sc->sc_gnt, sc->sc_gntcnt);
1096 #endif
1097 	ge = &sc->sc_gnt[ref / GNTTAB_NEPG];
1098 #ifdef XEN_DEBUG
1099 	if (ref < ge->ge_start || ref > ge->ge_start + GNTTAB_NEPG) {
1100 		panic("out of bounds ref %u ge %p start %u sc %p gnt %p",
1101 		    ref, ge, ge->ge_start, sc, sc->sc_gnt);
1102 	}
1103 #endif
1104 	ref -= ge->ge_start;
1105 	/* Invalidate the grant reference */
1106 	virtio_membar_sync();
1107 	ptr = (uint32_t *)&ge->ge_table[ref];
1108 	flags = (ge->ge_table[ref].flags & ~(GTF_reading|GTF_writing)) |
1109 	    (ge->ge_table[ref].domid << 16);
1110 	loop = 0;
1111 	while (atomic_cas_uint(ptr, flags, GTF_invalid) != flags) {
1112 		if (loop++ > 10000000) {
1113 			printf("%s: grant table reference %u is held "
1114 			    "by domain %d\n", sc->sc_dev.dv_xname, ref +
1115 			    ge->ge_start, ge->ge_table[ref].domid);
1116 			return;
1117 		}
1118 		CPU_BUSY_CYCLE();
1119 	}
1120 	ge->ge_table[ref].frame = 0xffffffff;
1121 }
1122 
1123 int
1124 xen_bus_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments,
1125     bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp)
1126 {
1127 	struct xen_softc *sc = t->_cookie;
1128 	struct xen_gntmap *gm;
1129 	int i, error;
1130 
1131 	if (maxsegsz < PAGE_SIZE)
1132 		return (EINVAL);
1133 
1134 	/* Allocate a dma map structure */
1135 	error = _bus_dmamap_create(t, size, nsegments, maxsegsz, boundary,
1136 	    flags, dmamp);
1137 	if (error)
1138 		return (error);
1139 	/* Allocate an array of grant table pa<->ref maps */
1140 	gm = mallocarray(nsegments, sizeof(struct xen_gntmap), M_DEVBUF,
1141 	    M_ZERO | ((flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK));
1142 	if (gm == NULL) {
1143 		_bus_dmamap_destroy(t, *dmamp);
1144 		*dmamp = NULL;
1145 		return (ENOMEM);
1146 	}
1147 	/* Wire it to the dma map */
1148 	(*dmamp)->_dm_cookie = gm;
1149 	/* Claim references from the grant table */
1150 	for (i = 0; i < (*dmamp)->_dm_segcnt; i++) {
1151 		if (xen_grant_table_alloc(sc, &gm[i].gm_ref)) {
1152 			xen_bus_dmamap_destroy(t, *dmamp);
1153 			*dmamp = NULL;
1154 			return (ENOBUFS);
1155 		}
1156 	}
1157 	return (0);
1158 }
1159 
1160 void
1161 xen_bus_dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map)
1162 {
1163 	struct xen_softc *sc = t->_cookie;
1164 	struct xen_gntmap *gm;
1165 	int i;
1166 
1167 	gm = map->_dm_cookie;
1168 	for (i = 0; i < map->_dm_segcnt; i++) {
1169 		if (gm[i].gm_ref == 0)
1170 			continue;
1171 		xen_grant_table_free(sc, gm[i].gm_ref);
1172 	}
1173 	free(gm, M_DEVBUF, map->_dm_segcnt * sizeof(struct xen_gntmap));
1174 	_bus_dmamap_destroy(t, map);
1175 }
1176 
1177 int
1178 xen_bus_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
1179     bus_size_t buflen, struct proc *p, int flags)
1180 {
1181 	struct xen_softc *sc = t->_cookie;
1182 	struct xen_gntmap *gm = map->_dm_cookie;
1183 	int i, domain, error;
1184 
1185 	domain = flags >> 16;
1186 	flags &= 0xffff;
1187 	error = _bus_dmamap_load(t, map, buf, buflen, p, flags);
1188 	if (error)
1189 		return (error);
1190 	for (i = 0; i < map->dm_nsegs; i++) {
1191 		xen_grant_table_enter(sc, gm[i].gm_ref, map->dm_segs[i].ds_addr,
1192 		    domain, flags & BUS_DMA_WRITE ? GTF_readonly : 0);
1193 		gm[i].gm_paddr = map->dm_segs[i].ds_addr;
1194 		map->dm_segs[i].ds_addr = gm[i].gm_ref;
1195 	}
1196 	return (0);
1197 }
1198 
1199 int
1200 xen_bus_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0,
1201     int flags)
1202 {
1203 	struct xen_softc *sc = t->_cookie;
1204 	struct xen_gntmap *gm = map->_dm_cookie;
1205 	int i, domain, error;
1206 
1207 	domain = flags >> 16;
1208 	flags &= 0xffff;
1209 	error = _bus_dmamap_load_mbuf(t, map, m0, flags);
1210 	if (error)
1211 		return (error);
1212 	for (i = 0; i < map->dm_nsegs; i++) {
1213 		xen_grant_table_enter(sc, gm[i].gm_ref, map->dm_segs[i].ds_addr,
1214 		    domain, flags & BUS_DMA_WRITE ? GTF_readonly : 0);
1215 		gm[i].gm_paddr = map->dm_segs[i].ds_addr;
1216 		map->dm_segs[i].ds_addr = gm[i].gm_ref;
1217 	}
1218 	return (0);
1219 }
1220 
1221 void
1222 xen_bus_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map)
1223 {
1224 	struct xen_softc *sc = t->_cookie;
1225 	struct xen_gntmap *gm = map->_dm_cookie;
1226 	int i;
1227 
1228 	for (i = 0; i < map->dm_nsegs; i++) {
1229 		if (gm[i].gm_paddr == 0)
1230 			continue;
1231 		xen_grant_table_remove(sc, gm[i].gm_ref);
1232 		map->dm_segs[i].ds_addr = gm[i].gm_paddr;
1233 		gm[i].gm_paddr = 0;
1234 	}
1235 	_bus_dmamap_unload(t, map);
1236 }
1237 
1238 void
1239 xen_bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t addr,
1240     bus_size_t size, int op)
1241 {
1242 	if ((op == (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) ||
1243 	    (op == (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE)))
1244 		virtio_membar_sync();
1245 }
1246 
1247 static int
1248 atoi(char *cp, int *res)
1249 {
1250 	*res = 0;
1251 	do {
1252 		if (*cp < '0' || *cp > '9')
1253 			return (-1);
1254 		*res *= 10;
1255 		*res += *cp - '0';
1256 	} while (*(++cp) != '\0');
1257 	return (0);
1258 }
1259 
1260 static int
1261 xen_attach_print(void *aux, const char *name)
1262 {
1263 	struct xen_attach_args *xa = aux;
1264 
1265 	if (name)
1266 		printf("\"%s\" at %s: %s", xa->xa_name, name, xa->xa_node);
1267 
1268 	return (UNCONF);
1269 }
1270 
1271 int
1272 xen_probe_devices(struct xen_softc *sc)
1273 {
1274 	struct xen_attach_args xa;
1275 	struct xs_transaction xst;
1276 	struct iovec *iovp1 = NULL, *iovp2 = NULL;
1277 	int i, j, error = 0, iov1_cnt = 0, iov2_cnt = 0;
1278 	char domid[16];
1279 	char path[256];
1280 
1281 	memset(&xst, 0, sizeof(xst));
1282 	xst.xst_id = 0;
1283 	xst.xst_sc = sc->sc_xs;
1284 	xst.xst_flags |= XST_POLL;
1285 
1286 	if ((error = xs_cmd(&xst, XS_LIST, "device", &iovp1, &iov1_cnt)) != 0)
1287 		return (error);
1288 
1289 	for (i = 0; i < iov1_cnt; i++) {
1290 		if (strcmp("suspend", (char *)iovp1[i].iov_base) == 0)
1291 			continue;
1292 		snprintf(path, sizeof(path), "device/%s",
1293 		    (char *)iovp1[i].iov_base);
1294 		if ((error = xs_cmd(&xst, XS_LIST, path, &iovp2,
1295 		    &iov2_cnt)) != 0) {
1296 			xs_resfree(&xst, iovp1, iov1_cnt);
1297 			return (error);
1298 		}
1299 		for (j = 0; j < iov2_cnt; j++) {
1300 			xa.xa_parent = sc;
1301 			xa.xa_dmat = &xen_bus_dma_tag;
1302 			strlcpy(xa.xa_name, (char *)iovp1[i].iov_base,
1303 			    sizeof(xa.xa_name));
1304 			snprintf(xa.xa_node, sizeof(xa.xa_node), "device/%s/%s",
1305 			    (char *)iovp1[i].iov_base,
1306 			    (char *)iovp2[j].iov_base);
1307 			if (xs_getprop(sc, xa.xa_node, "backend-id", domid,
1308 			    sizeof(domid)) ||
1309 			    xs_getprop(sc, xa.xa_node, "backend", xa.xa_backend,
1310 			    sizeof(xa.xa_backend))) {
1311 				printf("%s: failed to identify \"backend\" "
1312 				    "for \"%s\"\n", sc->sc_dev.dv_xname,
1313 				    xa.xa_node);
1314 			} else if (atoi(domid, &xa.xa_domid)) {
1315 				printf("%s: non-numeric backend domain id "
1316 				    "\"%s\" for \"%s\"\n", sc->sc_dev.dv_xname,
1317 				    domid, xa.xa_node);
1318 			}
1319 			config_found((struct device *)sc, &xa,
1320 			    xen_attach_print);
1321 		}
1322 		xs_resfree(&xst, iovp2, iov2_cnt);
1323 	}
1324 
1325 	return (error);
1326 }
1327 
1328 #include <machine/pio.h>
1329 
1330 #define	XMI_PORT		0x10
1331 #define XMI_MAGIC		0x49d2
1332 #define XMI_UNPLUG_IDE		0x01
1333 #define XMI_UNPLUG_NIC		0x02
1334 #define XMI_UNPLUG_IDESEC	0x04
1335 
1336 void
1337 xen_disable_emulated_devices(struct xen_softc *sc)
1338 {
1339 #if defined(__i386__) || defined(__amd64__)
1340 	ushort unplug = 0;
1341 
1342 	if (inw(XMI_PORT) != XMI_MAGIC) {
1343 		printf("%s: failed to disable emulated devices\n",
1344 		    sc->sc_dev.dv_xname);
1345 		return;
1346 	}
1347 	if (sc->sc_flags & XSF_UNPLUG_IDE)
1348 		unplug |= XMI_UNPLUG_IDE;
1349 	if (sc->sc_flags & XSF_UNPLUG_IDESEC)
1350 		unplug |= XMI_UNPLUG_IDESEC;
1351 	if (sc->sc_flags & XSF_UNPLUG_NIC)
1352 		unplug |= XMI_UNPLUG_NIC;
1353 	if (unplug)
1354 		outw(XMI_PORT, unplug);
1355 #endif	/* __i386__ || __amd64__ */
1356 }
1357