xref: /openbsd/sys/arch/amd64/amd64/bus_space.c (revision e12dfaa1)

/*	$OpenBSD: bus_space.c,v 1.27 2022/06/29 07:44:10 kettenis Exp $	*/
/*	$NetBSD: bus_space.c,v 1.2 2003/03/14 18:47:53 christos Exp $	*/

/*-
 * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Charles M. Hannum and by Jason R. Thorpe of the Numerical Aerospace
 * Simulation Facility, NASA Ames Research Center.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/extent.h>

#include <uvm/uvm_extern.h>

#include <machine/bus.h>

#include <dev/isa/isareg.h>
#include <machine/isa_machdep.h>

extern int pmap_initialized;

/*
 * Extent maps to manage I/O and memory space.  Allocate
 * storage for 16 regions in each, initially.  Later, ioport_malloc_safe
 * will indicate that it's safe to use malloc() to dynamically allocate
 * region descriptors.
 *
 * N.B. At least two regions are _always_ allocated from the iomem
 * extent map; (0 -> ISA hole) and (end of ISA hole -> end of RAM).
 *
 * The extent maps are not static!  Machine-dependent ISA and EISA
 * routines need access to them for bus address space allocation.
 */
static	long ioport_ex_storage[EXTENT_FIXED_STORAGE_SIZE(16) / sizeof(long)];
static	long iomem_ex_storage[EXTENT_FIXED_STORAGE_SIZE(16) / sizeof(long)];
struct	extent *ioport_ex;
struct	extent *iomem_ex;
static	int ioport_malloc_safe;

int	x86_mem_add_mapping(bus_addr_t, bus_size_t,
	    int, bus_space_handle_t *);

u_int8_t	x86_bus_space_io_read_1(bus_space_handle_t, bus_size_t);
u_int16_t	x86_bus_space_io_read_2(bus_space_handle_t, bus_size_t);
u_int32_t	x86_bus_space_io_read_4(bus_space_handle_t, bus_size_t);
u_int64_t	x86_bus_space_io_read_8(bus_space_handle_t, bus_size_t);

void		x86_bus_space_io_read_multi_1(bus_space_handle_t, bus_size_t,
		    u_int8_t *, bus_size_t);
void		x86_bus_space_io_read_multi_2(bus_space_handle_t, bus_size_t,
		    u_int16_t *, bus_size_t);
void		x86_bus_space_io_read_multi_4(bus_space_handle_t, bus_size_t,
		    u_int32_t *, bus_size_t);
void		x86_bus_space_io_read_multi_8(bus_space_handle_t, bus_size_t,
		    u_int64_t *, bus_size_t);

void		x86_bus_space_io_read_region_1(bus_space_handle_t, bus_size_t,
		    u_int8_t *, bus_size_t);
void		x86_bus_space_io_read_region_2(bus_space_handle_t, bus_size_t,
		    u_int16_t *, bus_size_t);
void		x86_bus_space_io_read_region_4(bus_space_handle_t, bus_size_t,
		    u_int32_t *, bus_size_t);
void		x86_bus_space_io_read_region_8(bus_space_handle_t, bus_size_t,
		    u_int64_t *, bus_size_t);

void		x86_bus_space_io_write_1(bus_space_handle_t, bus_size_t,
		    u_int8_t);
void		x86_bus_space_io_write_2(bus_space_handle_t, bus_size_t,
		    u_int16_t);
void		x86_bus_space_io_write_4(bus_space_handle_t, bus_size_t,
		    u_int32_t);
void		x86_bus_space_io_write_8(bus_space_handle_t, bus_size_t,
		    u_int64_t);

void		x86_bus_space_io_write_multi_1(bus_space_handle_t,
		    bus_size_t, const u_int8_t *, bus_size_t);
void		x86_bus_space_io_write_multi_2(bus_space_handle_t,
		    bus_size_t, const u_int16_t *, bus_size_t);
void		x86_bus_space_io_write_multi_4(bus_space_handle_t,
		    bus_size_t, const u_int32_t *, bus_size_t);
void		x86_bus_space_io_write_multi_8(bus_space_handle_t,
		    bus_size_t, const u_int64_t *, bus_size_t);

void		x86_bus_space_io_write_region_1(bus_space_handle_t,
		    bus_size_t, const u_int8_t *, bus_size_t);
void		x86_bus_space_io_write_region_2(bus_space_handle_t,
		    bus_size_t, const u_int16_t *, bus_size_t);
void		x86_bus_space_io_write_region_4(bus_space_handle_t,
		    bus_size_t, const u_int32_t *, bus_size_t);
void		x86_bus_space_io_write_region_8(bus_space_handle_t,
		    bus_size_t, const u_int64_t *, bus_size_t);

void		x86_bus_space_io_set_multi_1(bus_space_handle_t, bus_size_t,
		    u_int8_t, size_t);
void		x86_bus_space_io_set_multi_2(bus_space_handle_t, bus_size_t,
		    u_int16_t, size_t);
void		x86_bus_space_io_set_multi_4(bus_space_handle_t, bus_size_t,
		    u_int32_t, size_t);
void		x86_bus_space_io_set_multi_8(bus_space_handle_t, bus_size_t,
		    u_int64_t, size_t);

void		x86_bus_space_io_set_region_1(bus_space_handle_t, bus_size_t,
		    u_int8_t, size_t);
void		x86_bus_space_io_set_region_2(bus_space_handle_t, bus_size_t,
		    u_int16_t, size_t);
void		x86_bus_space_io_set_region_4(bus_space_handle_t, bus_size_t,
		    u_int32_t, size_t);
void		x86_bus_space_io_set_region_8(bus_space_handle_t, bus_size_t,
		    u_int64_t, size_t);

void		x86_bus_space_io_copy_1(bus_space_handle_t, bus_size_t,
		    bus_space_handle_t, bus_size_t, size_t);
void		x86_bus_space_io_copy_2(bus_space_handle_t, bus_size_t,
		    bus_space_handle_t, bus_size_t, size_t);
void		x86_bus_space_io_copy_4(bus_space_handle_t, bus_size_t,
		    bus_space_handle_t, bus_size_t, size_t);
void		x86_bus_space_io_copy_8(bus_space_handle_t, bus_size_t,
		    bus_space_handle_t, bus_size_t, size_t);

void *		x86_bus_space_io_vaddr(bus_space_handle_t);
paddr_t		x86_bus_space_io_mmap(bus_addr_t, off_t, int, int);

const struct x86_bus_space_ops x86_bus_space_io_ops = {
	x86_bus_space_io_read_1,
	x86_bus_space_io_read_2,
	x86_bus_space_io_read_4,
	x86_bus_space_io_read_8,
	x86_bus_space_io_read_multi_1,
	x86_bus_space_io_read_multi_2,
	x86_bus_space_io_read_multi_4,
	x86_bus_space_io_read_multi_8,
	x86_bus_space_io_read_region_1,
	x86_bus_space_io_read_region_2,
	x86_bus_space_io_read_region_4,
	x86_bus_space_io_read_region_8,
	x86_bus_space_io_write_1,
	x86_bus_space_io_write_2,
	x86_bus_space_io_write_4,
	x86_bus_space_io_write_8,
	x86_bus_space_io_write_multi_1,
	x86_bus_space_io_write_multi_2,
	x86_bus_space_io_write_multi_4,
	x86_bus_space_io_write_multi_8,
	x86_bus_space_io_write_region_1,
	x86_bus_space_io_write_region_2,
	x86_bus_space_io_write_region_4,
	x86_bus_space_io_write_region_8,
	x86_bus_space_io_set_multi_1,
	x86_bus_space_io_set_multi_2,
	x86_bus_space_io_set_multi_4,
	x86_bus_space_io_set_multi_8,
	x86_bus_space_io_set_region_1,
	x86_bus_space_io_set_region_2,
	x86_bus_space_io_set_region_4,
	x86_bus_space_io_set_region_8,
	x86_bus_space_io_copy_1,
	x86_bus_space_io_copy_2,
	x86_bus_space_io_copy_4,
	x86_bus_space_io_copy_8,
	x86_bus_space_io_vaddr,
	x86_bus_space_io_mmap
};

u_int8_t	x86_bus_space_mem_read_1(bus_space_handle_t, bus_size_t);
u_int16_t	x86_bus_space_mem_read_2(bus_space_handle_t, bus_size_t);
u_int32_t	x86_bus_space_mem_read_4(bus_space_handle_t, bus_size_t);
u_int64_t	x86_bus_space_mem_read_8(bus_space_handle_t, bus_size_t);

void		x86_bus_space_mem_read_multi_1(bus_space_handle_t, bus_size_t,
		    u_int8_t *, bus_size_t);
void		x86_bus_space_mem_read_multi_2(bus_space_handle_t, bus_size_t,
		    u_int16_t *, bus_size_t);
void		x86_bus_space_mem_read_multi_4(bus_space_handle_t, bus_size_t,
		    u_int32_t *, bus_size_t);
void		x86_bus_space_mem_read_multi_8(bus_space_handle_t, bus_size_t,
		    u_int64_t *, bus_size_t);

void		x86_bus_space_mem_read_region_1(bus_space_handle_t, bus_size_t,
		    u_int8_t *, bus_size_t);
void		x86_bus_space_mem_read_region_2(bus_space_handle_t, bus_size_t,
		    u_int16_t *, bus_size_t);
void		x86_bus_space_mem_read_region_4(bus_space_handle_t, bus_size_t,
		    u_int32_t *, bus_size_t);
void		x86_bus_space_mem_read_region_8(bus_space_handle_t, bus_size_t,
		    u_int64_t *, bus_size_t);

void		x86_bus_space_mem_write_1(bus_space_handle_t, bus_size_t,
		    u_int8_t);
void		x86_bus_space_mem_write_2(bus_space_handle_t, bus_size_t,
		    u_int16_t);
void		x86_bus_space_mem_write_4(bus_space_handle_t, bus_size_t,
		    u_int32_t);
void		x86_bus_space_mem_write_8(bus_space_handle_t, bus_size_t,
		    u_int64_t);

void		x86_bus_space_mem_write_multi_1(bus_space_handle_t,
		    bus_size_t, const u_int8_t *, bus_size_t);
void		x86_bus_space_mem_write_multi_2(bus_space_handle_t,
		    bus_size_t, const u_int16_t *, bus_size_t);
void		x86_bus_space_mem_write_multi_4(bus_space_handle_t,
		    bus_size_t, const u_int32_t *, bus_size_t);
void		x86_bus_space_mem_write_multi_8(bus_space_handle_t,
		    bus_size_t, const u_int64_t *, bus_size_t);

void		x86_bus_space_mem_write_region_1(bus_space_handle_t,
		    bus_size_t, const u_int8_t *, bus_size_t);
void		x86_bus_space_mem_write_region_2(bus_space_handle_t,
		    bus_size_t, const u_int16_t *, bus_size_t);
void		x86_bus_space_mem_write_region_4(bus_space_handle_t,
		    bus_size_t, const u_int32_t *, bus_size_t);
void		x86_bus_space_mem_write_region_8(bus_space_handle_t,
		    bus_size_t, const u_int64_t *, bus_size_t);

void		x86_bus_space_mem_set_multi_1(bus_space_handle_t, bus_size_t,
		    u_int8_t, size_t);
void		x86_bus_space_mem_set_multi_2(bus_space_handle_t, bus_size_t,
		    u_int16_t, size_t);
void		x86_bus_space_mem_set_multi_4(bus_space_handle_t, bus_size_t,
		    u_int32_t, size_t);
void		x86_bus_space_mem_set_multi_8(bus_space_handle_t, bus_size_t,
		    u_int64_t, size_t);

void		x86_bus_space_mem_set_region_1(bus_space_handle_t, bus_size_t,
		    u_int8_t, size_t);
void		x86_bus_space_mem_set_region_2(bus_space_handle_t, bus_size_t,
		    u_int16_t, size_t);
void		x86_bus_space_mem_set_region_4(bus_space_handle_t, bus_size_t,
		    u_int32_t, size_t);
void		x86_bus_space_mem_set_region_8(bus_space_handle_t, bus_size_t,
		    u_int64_t, size_t);

void		x86_bus_space_mem_copy_1(bus_space_handle_t, bus_size_t,
		    bus_space_handle_t, bus_size_t, size_t);
void		x86_bus_space_mem_copy_2(bus_space_handle_t, bus_size_t,
		    bus_space_handle_t, bus_size_t, size_t);
void		x86_bus_space_mem_copy_4(bus_space_handle_t, bus_size_t,
		    bus_space_handle_t, bus_size_t, size_t);
void		x86_bus_space_mem_copy_8(bus_space_handle_t, bus_size_t,
		    bus_space_handle_t, bus_size_t, size_t);

void *		x86_bus_space_mem_vaddr(bus_space_handle_t);

paddr_t		x86_bus_space_mem_mmap(bus_addr_t, off_t, int, int);

const struct x86_bus_space_ops x86_bus_space_mem_ops = {
	x86_bus_space_mem_read_1,
	x86_bus_space_mem_read_2,
	x86_bus_space_mem_read_4,
	x86_bus_space_mem_read_8,
	x86_bus_space_mem_read_multi_1,
	x86_bus_space_mem_read_multi_2,
	x86_bus_space_mem_read_multi_4,
	x86_bus_space_mem_read_multi_8,
	x86_bus_space_mem_read_region_1,
	x86_bus_space_mem_read_region_2,
	x86_bus_space_mem_read_region_4,
	x86_bus_space_mem_read_region_8,
	x86_bus_space_mem_write_1,
	x86_bus_space_mem_write_2,
	x86_bus_space_mem_write_4,
	x86_bus_space_mem_write_8,
	x86_bus_space_mem_write_multi_1,
	x86_bus_space_mem_write_multi_2,
	x86_bus_space_mem_write_multi_4,
	x86_bus_space_mem_write_multi_8,
	x86_bus_space_mem_write_region_1,
	x86_bus_space_mem_write_region_2,
	x86_bus_space_mem_write_region_4,
	x86_bus_space_mem_write_region_8,
	x86_bus_space_mem_set_multi_1,
	x86_bus_space_mem_set_multi_2,
	x86_bus_space_mem_set_multi_4,
	x86_bus_space_mem_set_multi_8,
	x86_bus_space_mem_set_region_1,
	x86_bus_space_mem_set_region_2,
	x86_bus_space_mem_set_region_4,
	x86_bus_space_mem_set_region_8,
	x86_bus_space_mem_copy_1,
	x86_bus_space_mem_copy_2,
	x86_bus_space_mem_copy_4,
	x86_bus_space_mem_copy_8,
	x86_bus_space_mem_vaddr,
	x86_bus_space_mem_mmap
};

void
x86_bus_space_init(void)
{
	/*
	 * Initialize the I/O port and I/O mem extent maps.
	 * Note: we don't have to check the return value since
	 * creation of a fixed extent map will never fail (since
	 * descriptor storage has already been allocated).
	 *
	 * N.B. The iomem extent manages _all_ physical addresses
	 * on the machine.  When the amount of RAM is found, the two
	 * extents of RAM are allocated from the map (0 -> ISA hole
	 * and end of ISA hole -> end of RAM).
	 */
	ioport_ex = extent_create("ioport", 0x0, 0xffff, M_DEVBUF,
	    (caddr_t)ioport_ex_storage, sizeof(ioport_ex_storage),
	    EX_NOCOALESCE|EX_NOWAIT);
	iomem_ex = extent_create("iomem", 0x0, 0xffffffffffff, M_DEVBUF,
	    (caddr_t)iomem_ex_storage, sizeof(iomem_ex_storage),
	    EX_NOCOALESCE|EX_NOWAIT);
}

void
x86_bus_space_mallocok(void)
{
	ioport_malloc_safe = 1;
}

int
bus_space_map(bus_space_tag_t t, bus_addr_t bpa, bus_size_t size, int flags,
    bus_space_handle_t *bshp)
{
	int error;
	struct extent *ex;

	/*
	 * Pick the appropriate extent map.
	 */
	if (t == X86_BUS_SPACE_IO) {
		ex = ioport_ex;
		if (flags & BUS_SPACE_MAP_LINEAR)
			return (EINVAL);
	} else if (t == X86_BUS_SPACE_MEM)
		ex = iomem_ex;
	else
		panic("bus_space_map: bad bus space tag");

	/*
	 * Before we go any further, let's make sure that this
	 * region is available.
	 */
	error = extent_alloc_region(ex, bpa, size,
	    EX_NOWAIT | (ioport_malloc_safe ? EX_MALLOCOK : 0));
	if (error)
		return (error);

	/*
	 * For I/O space, that's all she wrote.
	 */
	if (t == X86_BUS_SPACE_IO) {
		*bshp = bpa;
		return (0);
	}

	if (bpa >= IOM_BEGIN && (bpa + size) <= IOM_END) {
		*bshp = (bus_space_handle_t)ISA_HOLE_VADDR(bpa);
		return(0);
	}

	if (!pmap_initialized && bpa < 0x100000000) {
		*bshp = (bus_space_handle_t)PMAP_DIRECT_MAP(bpa);
		return(0);
	}

	/*
	 * For memory space, map the bus physical address to
	 * a kernel virtual address.
	 */
	error = x86_mem_add_mapping(bpa, size, flags, bshp);
	if (error) {
		if (extent_free(ex, bpa, size, EX_NOWAIT |
		    (ioport_malloc_safe ? EX_MALLOCOK : 0))) {
			printf("bus_space_map: pa 0x%lx, size 0x%lx\n",
			    bpa, size);
			printf("bus_space_map: can't free region\n");
		}
	}

	return (error);
}

int
_bus_space_map(bus_space_tag_t t, bus_addr_t bpa, bus_size_t size, int flags,
    bus_space_handle_t *bshp)
{

	/*
	 * For I/O space, just fill in the handle.
	 */
	if (t == X86_BUS_SPACE_IO) {
		*bshp = bpa;
		return (0);
	}

	/*
	 * For memory space, map the bus physical address to
	 * a kernel virtual address.
	 */
	return (x86_mem_add_mapping(bpa, size, flags, bshp));
}

int
bus_space_alloc(bus_space_tag_t t, bus_addr_t rstart, bus_addr_t rend,
    bus_size_t size, bus_size_t alignment, bus_size_t boundary, int flags,
    bus_addr_t *bpap, bus_space_handle_t *bshp)
{
	struct extent *ex;
	u_long bpa;
	int error;

	/*
	 * Pick the appropriate extent map.
	 */
	if (t == X86_BUS_SPACE_IO) {
		ex = ioport_ex;
	} else if (t == X86_BUS_SPACE_MEM)
		ex = iomem_ex;
	else
		panic("bus_space_alloc: bad bus space tag");

	/*
	 * Sanity check the allocation against the extent's boundaries.
	 */
	if (rstart < ex->ex_start || rend > ex->ex_end)
		panic("bus_space_alloc: bad region start/end");

	/*
	 * Do the requested allocation.
	 */
	error = extent_alloc_subregion(ex, rstart, rend, size, alignment,
	    0, boundary,
	    EX_FAST | EX_NOWAIT | (ioport_malloc_safe ?  EX_MALLOCOK : 0),
	    &bpa);

	if (error)
		return (error);

	/*
	 * For I/O space, that's all she wrote.
	 */
	if (t == X86_BUS_SPACE_IO) {
		*bshp = *bpap = bpa;
		return (0);
	}

	/*
	 * For memory space, map the bus physical address to
	 * a kernel virtual address.
	 */
	error = x86_mem_add_mapping(bpa, size, flags, bshp);
	if (error) {
		if (extent_free(iomem_ex, bpa, size, EX_NOWAIT |
		    (ioport_malloc_safe ? EX_MALLOCOK : 0))) {
			printf("bus_space_alloc: pa 0x%lx, size 0x%lx\n",
			    bpa, size);
			printf("bus_space_alloc: can't free region\n");
		}
	}

	*bpap = bpa;

	return (error);
}

int
x86_mem_add_mapping(bus_addr_t bpa, bus_size_t size, int flags,
    bus_space_handle_t *bshp)
{
	paddr_t pa, endpa;
	vaddr_t va;
	bus_size_t map_size;
	int pmap_flags = PMAP_NOCACHE;

	pa = trunc_page(bpa);
	endpa = round_page(bpa + size);

#ifdef DIAGNOSTIC
	if (endpa <= pa && endpa != 0)
		panic("bus_mem_add_mapping: overflow");
#endif

	map_size = endpa - pa;

	va = (vaddr_t)km_alloc(map_size, &kv_any, &kp_none, &kd_nowait);
	if (va == 0)
		return (ENOMEM);

	*bshp = (bus_space_handle_t)(va + (bpa & PGOFSET));

	if (flags & BUS_SPACE_MAP_CACHEABLE)
		pmap_flags = 0;
	else if (flags & BUS_SPACE_MAP_PREFETCHABLE)
		pmap_flags = PMAP_WC;

	for (; map_size > 0;
	    pa += PAGE_SIZE, va += PAGE_SIZE, map_size -= PAGE_SIZE)
		pmap_kenter_pa(va, pa | pmap_flags, PROT_READ | PROT_WRITE);
	pmap_update(pmap_kernel());

	return 0;
}

/*
 * void _bus_space_unmap(bus_space_tag bst, bus_space_handle bsh,
 *                        bus_size_t size, bus_addr_t *adrp)
 *
 *   This function unmaps memory- or io-space mapped by the function
 *   _bus_space_map().  This function works nearly as same as
 *   bus_space_unmap(), but this function does not ask kernel
 *   built-in extents and returns physical address of the bus space,
 *   for the convenience of the extra extent manager.
 */
void
_bus_space_unmap(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t size,
    bus_addr_t *adrp)
{
	u_long va, endva;
	bus_addr_t bpa;

	/*
	 * Find the correct bus physical address.
	 */
	if (t == X86_BUS_SPACE_IO) {
		bpa = bsh;
	} else if (t == X86_BUS_SPACE_MEM) {
		bpa = (bus_addr_t)ISA_PHYSADDR(bsh);
		if (IOM_BEGIN <= bpa && bpa <= IOM_END)
			goto ok;

		va = trunc_page(bsh);
		endva = round_page(bsh + size);

#ifdef DIAGNOSTIC
		if (endva <= va)
			panic("_bus_space_unmap: overflow");
#endif

		(void) pmap_extract(pmap_kernel(), va, &bpa);
		bpa += (bsh & PGOFSET);

		pmap_kremove(va, endva - va);
		pmap_update(pmap_kernel());

		/*
		 * Free the kernel virtual mapping.
		 */
		km_free((void *)va, endva - va, &kv_any, &kp_none);
	} else
		panic("bus_space_unmap: bad bus space tag");

ok:
	if (adrp != NULL)
		*adrp = bpa;
}

void
bus_space_unmap(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t size)
{
	struct extent *ex;
	u_long va, endva;
	bus_addr_t bpa;

	/*
	 * Find the correct extent and bus physical address.
	 */
	if (t == X86_BUS_SPACE_IO) {
		ex = ioport_ex;
		bpa = bsh;
	} else if (t == X86_BUS_SPACE_MEM) {
		ex = iomem_ex;
		bpa = (bus_addr_t)ISA_PHYSADDR(bsh);
		if (IOM_BEGIN <= bpa && bpa <= IOM_END)
			goto ok;

		if (bsh >= PMAP_DIRECT_BASE && bsh < PMAP_DIRECT_END) {
			bpa = PMAP_DIRECT_UNMAP(bsh);
			goto ok;
		}

		va = trunc_page(bsh);
		endva = round_page(bsh + size);

#ifdef DIAGNOSTIC
		if (endva <= va)
			panic("bus_space_unmap: overflow");
#endif

		(void)pmap_extract(pmap_kernel(), va, &bpa);
		bpa += (bsh & PGOFSET);

		pmap_kremove(va, endva - va);
		pmap_update(pmap_kernel());

		/*
		 * Free the kernel virtual mapping.
		 */
		km_free((void *)va, endva - va, &kv_any, &kp_none);
	} else
		panic("bus_space_unmap: bad bus space tag");

ok:
	if (extent_free(ex, bpa, size,
	    EX_NOWAIT | (ioport_malloc_safe ? EX_MALLOCOK : 0))) {
		printf("bus_space_unmap: %s 0x%lx, size 0x%lx\n",
		    (t == X86_BUS_SPACE_IO) ? "port" : "pa", bpa, size);
		printf("bus_space_unmap: can't free region\n");
	}
}

void
bus_space_free(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t size)
{

	/* bus_space_unmap() does all that we need to do. */
	bus_space_unmap(t, bsh, size);
}

int
bus_space_subregion(bus_space_tag_t t, bus_space_handle_t bsh,
    bus_size_t offset, bus_size_t size, bus_space_handle_t *nbshp)
{

	*nbshp = bsh + offset;
	return (0);
}

u_int8_t
x86_bus_space_io_read_1(bus_space_handle_t h, bus_size_t o)
{
	return (inb(h + o));
}

u_int16_t
x86_bus_space_io_read_2(bus_space_handle_t h, bus_size_t o)
{
	return (inw(h + o));
}

u_int32_t
x86_bus_space_io_read_4(bus_space_handle_t h, bus_size_t o)
{
	return (inl(h + o));
}

u_int64_t
x86_bus_space_io_read_8(bus_space_handle_t h, bus_size_t o)
{
	panic("bus_space_read_8: invalid bus space tag");
}

void
x86_bus_space_io_read_multi_1(bus_space_handle_t h, bus_size_t o,
    u_int8_t *ptr, bus_size_t cnt)
{
	insb(h + o, ptr, cnt);
}

void
x86_bus_space_io_read_multi_2(bus_space_handle_t h, bus_size_t o,
    u_int16_t *ptr, bus_size_t cnt)
{
	insw(h + o, ptr, cnt);
}

void
x86_bus_space_io_read_multi_4(bus_space_handle_t h, bus_size_t o,
    u_int32_t *ptr, bus_size_t cnt)
{
	insl(h + o, ptr, cnt);
}

void
x86_bus_space_io_read_multi_8(bus_space_handle_t h, bus_size_t o,
    u_int64_t *ptr, bus_size_t cnt)
{
	panic("bus_space_multi_8: invalid bus space tag");
}

void
x86_bus_space_io_read_region_1(bus_space_handle_t h,
    bus_size_t o, u_int8_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	int __x;
	u_int32_t port = h + o;
	__asm volatile(
	"1:	inb %w1,%%al				;"
	"	stosb					;"
	"	incl %1					;"
	"	loop 1b"				:
	    "=&a" (__x), "=d" (dummy1), "=D" (dummy2),
	    "=c" (dummy3)				:
	    "1" (port), "2" (ptr), "3" (cnt)	:
	    "memory");
}

void
x86_bus_space_io_read_region_2(bus_space_handle_t h,
    bus_size_t o, u_int16_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	int __x;
	u_int32_t port = h + o;
	__asm volatile(
	"1:	inw %w1,%%ax				;"
	"	stosw					;"
	"	addl $2,%1				;"
	"	loop 1b"				:
	    "=&a" (__x), "=d" (dummy1), "=D" (dummy2),
	    "=c" (dummy3)				:
	    "1" ((port)), "2" ((ptr)), "3" ((cnt))	:
	    "memory");
}

void
x86_bus_space_io_read_region_4(bus_space_handle_t h,
    bus_size_t o, u_int32_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	int __x;
	u_int32_t port = h + o;
	__asm volatile(
	"1:	inl %w1,%%eax				;"
	"	stosl					;"
	"	addl $4,%1				;"
	"	loop 1b"				:
	    "=&a" (__x), "=d" (dummy1), "=D" (dummy2),
	    "=c" (dummy3)				:
	    "1" (port), "2" (ptr), "3" (cnt)	:
	    "memory");
}

void
x86_bus_space_io_read_region_8(bus_space_handle_t h,
    bus_size_t o, u_int64_t *ptr, bus_size_t cnt)
{
	panic("bus_space_read_region_8: invalid bus space tag");
}

void
x86_bus_space_io_write_1(bus_space_handle_t h, bus_size_t o, u_int8_t v)
{
	outb(h + o, v);
}

void
x86_bus_space_io_write_2(bus_space_handle_t h, bus_size_t o, u_int16_t v)
{
	outw(h + o, v);
}

void
x86_bus_space_io_write_4(bus_space_handle_t h, bus_size_t o, u_int32_t v)
{
	outl(h + o, v);
}

void
x86_bus_space_io_write_8(bus_space_handle_t h, bus_size_t o, u_int64_t v)
{
	panic("bus_space_write_8: invalid bus space tag");
}

void
x86_bus_space_io_write_multi_1(bus_space_handle_t h,
    bus_size_t o, const u_int8_t *ptr, bus_size_t cnt)
{
	outsb(h + o, ptr, cnt);
}

void
x86_bus_space_io_write_multi_2(bus_space_handle_t h,
    bus_size_t o, const u_int16_t *ptr, bus_size_t cnt)
{
	outsw(h + o, ptr, cnt);
}

void
x86_bus_space_io_write_multi_4(bus_space_handle_t h,
    bus_size_t o, const u_int32_t *ptr, bus_size_t cnt)
{
	outsl(h + o, ptr, cnt);
}

void
x86_bus_space_io_write_multi_8(bus_space_handle_t h,
    bus_size_t o, const u_int64_t *ptr, bus_size_t cnt)
{
	panic("bus_space_write_multi_8: invalid bus space tag");
}

void
x86_bus_space_io_write_region_1(bus_space_handle_t h,
    bus_size_t o, const u_int8_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	int __x;
	u_int32_t port = h + o;
	__asm volatile(
	"1:	lodsb					;"
	"	outb %%al,%w1				;"
	"	incl %1					;"
	"	loop 1b"				:
	    "=&a" (__x), "=d" (dummy1), "=S" (dummy2),
	    "=c" (dummy3)				:
	    "1" (port), "2" (ptr), "3" (cnt)	:
	    "memory");
}

void
x86_bus_space_io_write_region_2(bus_space_handle_t h,
    bus_size_t o, const u_int16_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	int __x;
	u_int32_t port = h + o;
	__asm volatile(
	"1:	lodsw					;"
	"	outw %%ax,%w1				;"
	"	addl $2,%1				;"
	"	loop 1b"				:
	    "=&a" (__x), "=d" (dummy1), "=S" (dummy2),
	    "=c" (dummy3)				:
	    "1" (port), "2" (ptr), "3" (cnt)	:
	    "memory");
}

void
x86_bus_space_io_write_region_4(bus_space_handle_t h,
    bus_size_t o, const u_int32_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	int __x;
	u_int32_t port = h + o;
	__asm volatile(
	"1:	lodsl					;"
	"	outl %%eax,%w1				;"
	"	addl $4,%1				;"
	"	loop 1b"				:
	    "=&a" (__x), "=d" (dummy1), "=S" (dummy2),
	    "=c" (dummy3)				:
	    "1" (port), "2" (ptr), "3" (cnt)	:
	    "memory");
}

void
x86_bus_space_io_write_region_8(bus_space_handle_t h,
    bus_size_t o, const u_int64_t *ptr, bus_size_t cnt)
{
	panic("bus_space_write_region_8: invalid bus space tag");
}

void
x86_bus_space_io_set_multi_1(bus_space_handle_t h, bus_size_t o,
    u_int8_t v, size_t c)
{
	bus_addr_t addr = h + o;

	while (c--)
		outb(addr, v);
}

void
x86_bus_space_io_set_multi_2(bus_space_handle_t h, bus_size_t o,
    u_int16_t v, size_t c)
{
	bus_addr_t addr = h + o;

	while (c--)
		outw(addr, v);
}

void
x86_bus_space_io_set_multi_4(bus_space_handle_t h, bus_size_t o,
    u_int32_t v, size_t c)
{
	bus_addr_t addr = h + o;

	while (c--)
		outl(addr, v);
}

void
x86_bus_space_io_set_multi_8(bus_space_handle_t h, bus_size_t o,
    u_int64_t v, size_t c)
{
	panic("bus_space_set_multi_8: invalid bus space tag");
}

void
x86_bus_space_io_set_region_1(bus_space_handle_t h, bus_size_t o,
    u_int8_t v, size_t c)
{
	bus_addr_t addr = h + o;

	for (; c != 0; c--, addr++)
		outb(addr, v);
}

void
x86_bus_space_io_set_region_2(bus_space_handle_t h, bus_size_t o,
    u_int16_t v, size_t c)
{
	bus_addr_t addr = h + o;

	for (; c != 0; c--, addr += sizeof(v))
		outw(addr, v);
}

void
x86_bus_space_io_set_region_4(bus_space_handle_t h, bus_size_t o,
    u_int32_t v, size_t c)
{
	bus_addr_t addr = h + o;

	for (; c != 0; c--, addr += sizeof(v))
		outl(addr, v);
}

void
x86_bus_space_io_set_region_8(bus_space_handle_t h, bus_size_t o,
    u_int64_t v, size_t c)
{
	panic("bus_space_set_region_8: invalid bus space tag");
}

void
x86_bus_space_io_copy_1(bus_space_handle_t h1, bus_size_t o1,
    bus_space_handle_t h2, bus_size_t o2, size_t c)
{
	bus_addr_t addr1 = h1 + o1;
	bus_addr_t addr2 = h2 + o2;

	if (addr1 >= addr2) {
		/* src after dest: copy forward */
		for (; c != 0; c--, addr1++, addr2++)
			outb(addr2, inb(addr1));
	} else {
		/* dest after src: copy backwards */
		for (addr1 += (c - 1), addr2 += (c - 1);
		    c != 0; c--, addr1--, addr2--)
			outb(addr2, inb(addr1));
	}
}

void
x86_bus_space_io_copy_2(bus_space_handle_t h1, bus_size_t o1,
    bus_space_handle_t h2, bus_size_t o2, size_t c)
{
	bus_addr_t addr1 = h1 + o1;
	bus_addr_t addr2 = h2 + o2;

	if (addr1 >= addr2) {
		/* src after dest: copy forward */
		for (; c != 0; c--, addr1 += 2, addr2 += 2)
			outw(addr2, inw(addr1));
	} else {
		/* dest after src: copy backwards */
		for (addr1 += 2 * (c - 1), addr2 += 2 * (c - 1);
		    c != 0; c--, addr1 -= 2, addr2 -= 2)
			outw(addr2, inw(addr1));
	}
}

void
x86_bus_space_io_copy_4(bus_space_handle_t h1, bus_size_t o1,
    bus_space_handle_t h2, bus_size_t o2, size_t c)
{
	bus_addr_t addr1 = h1 + o1;
	bus_addr_t addr2 = h2 + o2;

	if (addr1 >= addr2) {
		/* src after dest: copy forward */
		for (; c != 0; c--, addr1 += 4, addr2 += 4)
			outl(addr2, inl(addr1));
	} else {
		/* dest after src: copy backwards */
		for (addr1 += 4 * (c - 1), addr2 += 4 * (c - 1);
		    c != 0; c--, addr1 -= 4, addr2 -= 4)
			outl(addr2, inl(addr1));
	}
}

void
x86_bus_space_io_copy_8(bus_space_handle_t h1, bus_size_t o1,
    bus_space_handle_t h2, bus_size_t o2, size_t c)
{
	panic("bus_space_set_region_8: invalid bus space tag");
}

void *
x86_bus_space_io_vaddr(bus_space_handle_t h)
{
	return (NULL);
}

paddr_t
x86_bus_space_io_mmap(bus_addr_t addr, off_t off, int prot, int flags)
{
	/* Can't mmap I/O space. */
	return (-1);
}

void
x86_bus_space_mem_write_1(bus_space_handle_t h, bus_size_t o, u_int8_t v)
{
	*(volatile u_int8_t *)(h + o) = v;
}

void
x86_bus_space_mem_write_2(bus_space_handle_t h, bus_size_t o, u_int16_t v)
{
	*(volatile u_int16_t *)(h + o) = v;
}

u_int8_t
x86_bus_space_mem_read_1(bus_space_handle_t h, bus_size_t o)
{
	return (*(volatile u_int8_t *)(h + o));
}

u_int16_t
x86_bus_space_mem_read_2(bus_space_handle_t h, bus_size_t o)
{
	return (*(volatile u_int16_t *)(h + o));
}

u_int32_t
x86_bus_space_mem_read_4(bus_space_handle_t h, bus_size_t o)
{
	return (*(volatile u_int32_t *)(h + o));
}

u_int64_t
x86_bus_space_mem_read_8(bus_space_handle_t h, bus_size_t o)
{
	return (*(volatile u_int64_t *)(h + o));
}

void
x86_bus_space_mem_read_multi_1(bus_space_handle_t h, bus_size_t o,
    u_int8_t *ptr, bus_size_t cnt)
{
	void *dummy1;
	int dummy2;
	void *dummy3;
	int __x;
	__asm volatile(
	"1:	movb (%2),%%al				;"
	"	stosb					;"
	"	loop 1b"				:
	    "=D" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
	    "0" ((ptr)), "1" ((cnt)), "2" (h + o)       :
	    "memory");
}

void
x86_bus_space_mem_read_multi_2(bus_space_handle_t h, bus_size_t o,
    u_int16_t *ptr, bus_size_t cnt)
{
	void *dummy1;
	int dummy2;
	void *dummy3;
	int __x;
	__asm volatile(
	"1:	movw (%2),%%ax				;"
	"	stosw					;"
	"	loop 1b"				:
	    "=D" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
	    "0" ((ptr)), "1" ((cnt)), "2" (h + o)       :
	    "memory");
}

void
x86_bus_space_mem_read_multi_4(bus_space_handle_t h, bus_size_t o,
    u_int32_t *ptr, bus_size_t cnt)
{
	void *dummy1;
	int dummy2;
	void *dummy3;
	int __x;
	__asm volatile(
	"1:	movl (%2),%%eax				;"
	"	stosl					;"
	"	loop 1b"				:
	    "=D" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
	    "0" ((ptr)), "1" ((cnt)), "2" (h + o)       :
	    "memory");
}

void
x86_bus_space_mem_read_multi_8(bus_space_handle_t h, bus_size_t o,
    u_int64_t *ptr, bus_size_t cnt)
{
	void *dummy1;
	int dummy2;
	void *dummy3;
	int __x;
	__asm volatile(
	"1:	movq (%2),%%rax				;"
	"	stosq					;"
	"	loop 1b"				:
	    "=D" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
	    "0" ((ptr)), "1" ((cnt)), "2" (h + o)       :
	    "memory");
}

void
x86_bus_space_mem_read_region_1(bus_space_handle_t h,
    bus_size_t o, u_int8_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	__asm volatile(
	"	repne					;"
	"	movsb"					:
	    "=S" (dummy1), "=D" (dummy2), "=c" (dummy3)	:
	    "0" (h + o), "1" (ptr), "2" (cnt)	:
	    "memory");
}

void
x86_bus_space_mem_read_region_2(bus_space_handle_t h,
    bus_size_t o, u_int16_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	__asm volatile(
	"	repne					;"
	"	movsw"					:
	    "=S" (dummy1), "=D" (dummy2), "=c" (dummy3)	:
	    "0" (h + o), "1" (ptr), "2" (cnt)	:
	    "memory");
}

void
x86_bus_space_mem_read_region_4(bus_space_handle_t h,
    bus_size_t o, u_int32_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	__asm volatile(
	"	repne					;"
	"	movsl"					:
	    "=S" (dummy1), "=D" (dummy2), "=c" (dummy3)	:
	    "0" (h + o), "1" (ptr), "2" (cnt)	:
	    "memory");
}

void
x86_bus_space_mem_read_region_8(bus_space_handle_t h,
    bus_size_t o, u_int64_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	__asm volatile(
	"	repne					;"
	"	movsq"					:
	    "=S" (dummy1), "=D" (dummy2), "=c" (dummy3)	:
	    "0" (h + o), "1" (ptr), "2" (cnt)	:
	    "memory");
}

void
x86_bus_space_mem_write_4(bus_space_handle_t h, bus_size_t o, u_int32_t v)
{
	*(volatile u_int32_t *)(h + o) = v;
}

void
x86_bus_space_mem_write_8(bus_space_handle_t h, bus_size_t o, u_int64_t v)
{
	*(volatile u_int64_t *)(h + o) = v;
}

void
x86_bus_space_mem_write_multi_1(bus_space_handle_t h,
    bus_size_t o, const u_int8_t *ptr, bus_size_t cnt)
{
	void *dummy1;
	int dummy2;
	void *dummy3;
	int __x;
	__asm volatile(
	"1:	lodsb					;"
	"	movb %%al,(%2)				;"
	"	loop 1b"				:
	    "=S" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
	    "0" (ptr), "1" (cnt), "2" (h + o));
}

void
x86_bus_space_mem_write_multi_2(bus_space_handle_t h,
    bus_size_t o, const u_int16_t *ptr, bus_size_t cnt)
{
	void *dummy1;
	int dummy2;
	void *dummy3;
	int __x;
	__asm volatile(
	"1:	lodsw					;"
	"	movw %%ax,(%2)				;"
	"	loop 1b"				:
	    "=S" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
	    "0" (ptr), "1" (cnt), "2" (h + o));
}

void
x86_bus_space_mem_write_multi_4(bus_space_handle_t h,
    bus_size_t o, const u_int32_t *ptr, bus_size_t cnt)
{
	void *dummy1;
	int dummy2;
	void *dummy3;
	int __x;
	__asm volatile(
	"1:	lodsl					;"
	"	movl %%eax,(%2)				;"
	"	loop 1b"				:
	    "=S" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
	    "0" (ptr), "1" (cnt), "2" (h + o));
}

void
x86_bus_space_mem_write_multi_8(bus_space_handle_t h,
    bus_size_t o, const u_int64_t *ptr, bus_size_t cnt)
{
	void *dummy1;
	int dummy2;
	void *dummy3;
	int __x;
	__asm volatile(
	"1:	lodsq					;"
	"	movq %%rax,(%2)				;"
	"	loop 1b"				:
	    "=S" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
	    "0" (ptr), "1" (cnt), "2" (h + o));
}

void
x86_bus_space_mem_write_region_1(bus_space_handle_t h,
    bus_size_t o, const u_int8_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	__asm volatile(
	"	repne					;"
	"	movsb"					:
	    "=D" (dummy1), "=S" (dummy2), "=c" (dummy3)	:
	    "0" (h + o), "1" (ptr), "2" (cnt)	:
	    "memory");
}

void
x86_bus_space_mem_write_region_2(bus_space_handle_t h,
    bus_size_t o, const u_int16_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	__asm volatile(
	"	repne					;"
	"	movsw"					:
	    "=D" (dummy1), "=S" (dummy2), "=c" (dummy3)	:
	    "0" (h + o), "1" (ptr), "2" (cnt)	:
	    "memory");
}

void
x86_bus_space_mem_write_region_4(bus_space_handle_t h,
    bus_size_t o, const u_int32_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	__asm volatile(
	"	repne					;"
	"	movsl"					:
	    "=D" (dummy1), "=S" (dummy2), "=c" (dummy3)	:
	    "0" (h + o), "1" (ptr), "2" (cnt)	:
	    "memory");
}

void
x86_bus_space_mem_write_region_8(bus_space_handle_t h,
    bus_size_t o, const u_int64_t *ptr, bus_size_t cnt)
{
	int dummy1;
	void *dummy2;
	int dummy3;
	__asm volatile(
	"	repne					;"
	"	movsq"					:
	    "=D" (dummy1), "=S" (dummy2), "=c" (dummy3)	:
	    "0" (h + o), "1" (ptr), "2" (cnt)	:
	    "memory");
}

void
x86_bus_space_mem_set_multi_1(bus_space_handle_t h, bus_size_t o,
    u_int8_t v, size_t c)
{
	bus_addr_t addr = h + o;

	while (c--)
		*(volatile u_int8_t *)(addr) = v;
}

void
x86_bus_space_mem_set_multi_2(bus_space_handle_t h, bus_size_t o,
    u_int16_t v, size_t c)
{
	bus_addr_t addr = h + o;

	while (c--)
		*(volatile u_int16_t *)(addr) = v;
}

void
x86_bus_space_mem_set_multi_4(bus_space_handle_t h, bus_size_t o,
    u_int32_t v, size_t c)
{
	bus_addr_t addr = h + o;

	while (c--)
		*(volatile u_int32_t *)(addr) = v;
}

void
x86_bus_space_mem_set_multi_8(bus_space_handle_t h, bus_size_t o,
    u_int64_t v, size_t c)
{
	bus_addr_t addr = h + o;

	while (c--)
		*(volatile u_int64_t *)(addr) = v;
}

void
x86_bus_space_mem_set_region_1(bus_space_handle_t h, bus_size_t o,
    u_int8_t v, size_t c)
{
	bus_addr_t addr = h + o;

	for (; c != 0; c--, addr++)
		*(volatile u_int8_t *)(addr) = v;
}

void
x86_bus_space_mem_set_region_2(bus_space_handle_t h, bus_size_t o,
    u_int16_t v, size_t c)
{
	bus_addr_t addr = h + o;

	for (; c != 0; c--, addr += sizeof(v))
		*(volatile u_int16_t *)(addr) = v;
}

void
x86_bus_space_mem_set_region_4(bus_space_handle_t h, bus_size_t o,
    u_int32_t v, size_t c)
{
	bus_addr_t addr = h + o;

	for (; c != 0; c--, addr += sizeof(v))
		*(volatile u_int32_t *)(addr) = v;
}

void
x86_bus_space_mem_set_region_8(bus_space_handle_t h, bus_size_t o,
    u_int64_t v, size_t c)
{
	bus_addr_t addr = h + o;

	for (; c != 0; c--, addr += sizeof(v))
		*(volatile u_int64_t *)(addr) = v;
}

void
x86_bus_space_mem_copy_1( bus_space_handle_t h1, bus_size_t o1,
    bus_space_handle_t h2, bus_size_t o2, size_t c)
{
	bus_addr_t addr1 = h1 + o1;
	bus_addr_t addr2 = h2 + o2;

	if (addr1 >= addr2) {
		/* src after dest: copy forward */
		for (; c != 0; c--, addr1++, addr2++)
			*(volatile u_int8_t *)(addr2) =
			    *(volatile u_int8_t *)(addr1);
	} else {
		/* dest after src: copy backwards */
		for (addr1 += (c - 1), addr2 += (c - 1);
		    c != 0; c--, addr1--, addr2--)
			*(volatile u_int8_t *)(addr2) =
			    *(volatile u_int8_t *)(addr1);
	}
}

void
x86_bus_space_mem_copy_2(bus_space_handle_t h1, bus_size_t o1,
    bus_space_handle_t h2, bus_size_t o2, size_t c)
{
	bus_addr_t addr1 = h1 + o1;
	bus_addr_t addr2 = h2 + o2;

	if (addr1 >= addr2) {
		/* src after dest: copy forward */
		for (; c != 0; c--, addr1 += 2, addr2 += 2)
			*(volatile u_int16_t *)(addr2) =
			    *(volatile u_int16_t *)(addr1);
	} else {
		/* dest after src: copy backwards */
		for (addr1 += 2 * (c - 1), addr2 += 2 * (c - 1);
		    c != 0; c--, addr1 -= 2, addr2 -= 2)
			*(volatile u_int16_t *)(addr2) =
			    *(volatile u_int16_t *)(addr1);
	}
}

void
x86_bus_space_mem_copy_4(bus_space_handle_t h1, bus_size_t o1,
    bus_space_handle_t h2, bus_size_t o2, size_t c)
{
	bus_addr_t addr1 = h1 + o1;
	bus_addr_t addr2 = h2 + o2;

	if (addr1 >= addr2) {
		/* src after dest: copy forward */
		for (; c != 0; c--, addr1 += 4, addr2 += 4)
			*(volatile u_int32_t *)(addr2) =
			    *(volatile u_int32_t *)(addr1);
	} else {
		/* dest after src: copy backwards */
		for (addr1 += 4 * (c - 1), addr2 += 4 * (c - 1);
		    c != 0; c--, addr1 -= 4, addr2 -= 4)
			*(volatile u_int32_t *)(addr2) =
			    *(volatile u_int32_t *)(addr1);
	}
}

void
x86_bus_space_mem_copy_8(bus_space_handle_t h1, bus_size_t o1,
    bus_space_handle_t h2, bus_size_t o2, size_t c)
{
	bus_addr_t addr1 = h1 + o1;
	bus_addr_t addr2 = h2 + o2;

	if (addr1 >= addr2) {
		/* src after dest: copy forward */
		for (; c != 0; c--, addr1 += 8, addr2 += 8)
			*(volatile u_int64_t *)(addr2) =
			    *(volatile u_int64_t *)(addr1);
	} else {
		/* dest after src: copy backwards */
		for (addr1 += 8 * (c - 1), addr2 += 8 * (c - 1);
		    c != 0; c--, addr1 -= 8, addr2 -= 8)
			*(volatile u_int64_t *)(addr2) =
			    *(volatile u_int64_t *)(addr1);
	}
}

void *
x86_bus_space_mem_vaddr(bus_space_handle_t h)
{
	return ((void *)h);
}

paddr_t
x86_bus_space_mem_mmap(bus_addr_t addr, off_t off, int prot, int flags)
{
	/*
	 * "addr" is the base address of the device we're mapping.
	 * "off" is the offset into that device.
	 *
	 * Note we are called for each "page" in the device that
	 * the upper layers want to map.
	 */
	return (addr + off);
}