xref: /openbsd/sys/dev/ic/qlw.c (revision a454aff3)

/*	$OpenBSD: qlw.c,v 1.48 2022/04/16 19:19:59 naddy Exp $ */

/*
 * Copyright (c) 2011 David Gwynne <dlg@openbsd.org>
 * Copyright (c) 2013, 2014 Jonathan Matthew <jmatthew@openbsd.org>
 * Copyright (c) 2014 Mark Kettenis <kettenis@openbsd.org>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/atomic.h>
#include <sys/device.h>
#include <sys/ioctl.h>
#include <sys/malloc.h>
#include <sys/kernel.h>
#include <sys/mutex.h>
#include <sys/rwlock.h>
#include <sys/sensors.h>
#include <sys/queue.h>

#include <machine/bus.h>

#include <scsi/scsi_all.h>
#include <scsi/scsiconf.h>

#include <dev/ic/qlwreg.h>
#include <dev/ic/qlwvar.h>

#ifndef SMALL_KERNEL
#ifndef QLW_DEBUG
#define QLW_DEBUG
#endif
#endif

#ifdef QLW_DEBUG
#define DPRINTF(m, f...) do { if ((qlwdebug & (m)) == (m)) printf(f); } \
    while (0)
#define QLW_D_MBOX		0x01
#define QLW_D_INTR		0x02
#define QLW_D_PORT		0x04
#define QLW_D_IO		0x08
#define QLW_D_IOCB		0x10
int qlwdebug = QLW_D_PORT | QLW_D_INTR | QLW_D_MBOX;
#else
#define DPRINTF(m, f...)
#endif

struct cfdriver qlw_cd = {
	NULL,
	"qlw",
	DV_DULL
};

void		qlw_scsi_cmd(struct scsi_xfer *);

u_int16_t	qlw_read(struct qlw_softc *, bus_size_t);
void		qlw_write(struct qlw_softc *, bus_size_t, u_int16_t);
void		qlw_host_cmd(struct qlw_softc *sc, u_int16_t);

int		qlw_mbox(struct qlw_softc *, int, int);
void		qlw_mbox_putaddr(u_int16_t *, struct qlw_dmamem *);
u_int16_t	qlw_read_mbox(struct qlw_softc *, int);
void		qlw_write_mbox(struct qlw_softc *, int, u_int16_t);

int		qlw_config_bus(struct qlw_softc *, int);
int		qlw_config_target(struct qlw_softc *, int, int);
void		qlw_update_bus(struct qlw_softc *, int);
void		qlw_update_target(struct qlw_softc *, int, int);
void		qlw_update_task(void *);

void		qlw_handle_intr(struct qlw_softc *, u_int16_t, u_int16_t);
void		qlw_set_ints(struct qlw_softc *, int);
int		qlw_read_isr(struct qlw_softc *, u_int16_t *, u_int16_t *);
void		qlw_clear_isr(struct qlw_softc *, u_int16_t);

void		qlw_update(struct qlw_softc *, int);
void		qlw_put_marker(struct qlw_softc *, int, void *);
void		qlw_put_cmd(struct qlw_softc *, void *, struct scsi_xfer *,
		    struct qlw_ccb *);
void		qlw_put_cont(struct qlw_softc *, void *, struct scsi_xfer *,
		    struct qlw_ccb *, int);
struct qlw_ccb *qlw_handle_resp(struct qlw_softc *, u_int16_t);
void		qlw_get_header(struct qlw_softc *, struct qlw_iocb_hdr *,
		    int *, int *);
void		qlw_put_header(struct qlw_softc *, struct qlw_iocb_hdr *,
		    int, int);
void		qlw_put_data_seg(struct qlw_softc *, struct qlw_iocb_seg *,
		    bus_dmamap_t, int);

int		qlw_softreset(struct qlw_softc *);
void		qlw_dma_burst_enable(struct qlw_softc *);

int		qlw_async(struct qlw_softc *, u_int16_t);

int		qlw_load_firmware_words(struct qlw_softc *, const u_int16_t *,
		    u_int16_t);
int		qlw_load_firmware(struct qlw_softc *);
int		qlw_read_nvram(struct qlw_softc *);
void		qlw_parse_nvram_1040(struct qlw_softc *, int);
void		qlw_parse_nvram_1080(struct qlw_softc *, int);
void		qlw_init_defaults(struct qlw_softc *, int);

struct qlw_dmamem *qlw_dmamem_alloc(struct qlw_softc *, size_t);
void		qlw_dmamem_free(struct qlw_softc *, struct qlw_dmamem *);

int		qlw_alloc_ccbs(struct qlw_softc *);
void		qlw_free_ccbs(struct qlw_softc *);
void		*qlw_get_ccb(void *);
void		qlw_put_ccb(void *, void *);

#ifdef QLW_DEBUG
void		qlw_dump_iocb(struct qlw_softc *, void *, int);
void		qlw_dump_iocb_segs(struct qlw_softc *, void *, int);
#else
#define qlw_dump_iocb(sc, h, fl)	do { /* nothing */ } while (0)
#define qlw_dump_iocb_segs(sc, h, fl)	do { /* nothing */ } while (0)
#endif

static inline int
qlw_xs_bus(struct qlw_softc *sc, struct scsi_xfer *xs)
{
	/*
	 * sc_scsibus[0] == NULL -> bus 0 probing during config_found().
	 * sc_scsibus[0] == xs->sc_link->bus -> bus 0 normal operation.
	 * sc_scsibus[1] == NULL -> bus 1 probing during config_found().
	 * sc_scsibus[1] == xs->sc_link->bus -> bus 1 normal operation.
	 */
	if ((sc->sc_scsibus[0] == NULL) ||
	    (xs->sc_link->bus == sc->sc_scsibus[0]))
		return 0;
	else
		return 1;
}

static inline u_int16_t
qlw_swap16(struct qlw_softc *sc, u_int16_t value)
{
	if (sc->sc_isp_gen == QLW_GEN_ISP1000)
		return htobe16(value);
	else
		return htole16(value);
}

static inline u_int32_t
qlw_swap32(struct qlw_softc *sc, u_int32_t value)
{
	if (sc->sc_isp_gen == QLW_GEN_ISP1000)
		return htobe32(value);
	else
		return htole32(value);
}

static inline u_int16_t
qlw_queue_read(struct qlw_softc *sc, bus_size_t offset)
{
	return qlw_read(sc, sc->sc_mbox_base + offset);
}

static inline void
qlw_queue_write(struct qlw_softc *sc, bus_size_t offset, u_int16_t value)
{
	qlw_write(sc, sc->sc_mbox_base + offset, value);
}

const struct scsi_adapter qlw_switch = {
	qlw_scsi_cmd, NULL, NULL, NULL, NULL
};

int
qlw_attach(struct qlw_softc *sc)
{
	struct scsibus_attach_args saa;
	void (*parse_nvram)(struct qlw_softc *, int);
	int reset_delay;
	int bus;

	task_set(&sc->sc_update_task, qlw_update_task, sc);

	switch (sc->sc_isp_gen) {
	case QLW_GEN_ISP1000:
		sc->sc_nvram_size = 0;
		break;
	case QLW_GEN_ISP1040:
		sc->sc_nvram_size = 128;
		sc->sc_nvram_minversion = 2;
		parse_nvram = qlw_parse_nvram_1040;
		break;
	case QLW_GEN_ISP1080:
	case QLW_GEN_ISP12160:
		sc->sc_nvram_size = 256;
		sc->sc_nvram_minversion = 1;
		parse_nvram = qlw_parse_nvram_1080;
		break;

	default:
		printf("unknown isp type\n");
		return (ENXIO);
	}

	/* after reset, mbox registers 1-3 should contain the string "ISP   " */
	if (qlw_read_mbox(sc, 1) != 0x4953 ||
	    qlw_read_mbox(sc, 2) != 0x5020 ||
	    qlw_read_mbox(sc, 3) != 0x2020) {
		/* try releasing the risc processor */
		qlw_host_cmd(sc, QLW_HOST_CMD_RELEASE);
	}

	qlw_host_cmd(sc, QLW_HOST_CMD_PAUSE);
	if (qlw_softreset(sc) != 0) {
		printf("softreset failed\n");
		return (ENXIO);
	}

	for (bus = 0; bus < sc->sc_numbusses; bus++)
		qlw_init_defaults(sc, bus);

	if (qlw_read_nvram(sc) == 0) {
		for (bus = 0; bus < sc->sc_numbusses; bus++)
			parse_nvram(sc, bus);
	}

#ifndef ISP_NOFIRMWARE
	if (sc->sc_firmware && qlw_load_firmware(sc)) {
		printf("firmware load failed\n");
		return (ENXIO);
	}
#endif

	/* execute firmware */
	sc->sc_mbox[0] = QLW_MBOX_EXEC_FIRMWARE;
	sc->sc_mbox[1] = QLW_CODE_ORG;
	if (qlw_mbox(sc, 0x0003, 0x0001)) {
		printf("ISP couldn't exec firmware: %x\n", sc->sc_mbox[0]);
		return (ENXIO);
	}

	delay(250000);		/* from isp(4) */

	sc->sc_mbox[0] = QLW_MBOX_ABOUT_FIRMWARE;
	if (qlw_mbox(sc, QLW_MBOX_ABOUT_FIRMWARE_IN,
	    QLW_MBOX_ABOUT_FIRMWARE_OUT)) {
		printf("ISP not talking after firmware exec: %x\n",
		    sc->sc_mbox[0]);
		return (ENXIO);
	}
	/* The ISP1000 firmware we use doesn't return a version number. */
	if (sc->sc_isp_gen == QLW_GEN_ISP1000 && sc->sc_firmware) {
		sc->sc_mbox[1] = 1;
		sc->sc_mbox[2] = 37;
		sc->sc_mbox[3] = 0;
		sc->sc_mbox[6] = 0;
	}
	printf("%s: firmware rev %d.%d.%d, attrs 0x%x\n", DEVNAME(sc),
	    sc->sc_mbox[1], sc->sc_mbox[2], sc->sc_mbox[3], sc->sc_mbox[6]);

	/* work out how many ccbs to allocate */
	sc->sc_mbox[0] = QLW_MBOX_GET_FIRMWARE_STATUS;
	if (qlw_mbox(sc, 0x0001, 0x0007)) {
		printf("couldn't get firmware status: %x\n", sc->sc_mbox[0]);
		return (ENXIO);
	}
	sc->sc_maxrequests = sc->sc_mbox[2];
	if (sc->sc_maxrequests > 512)
		sc->sc_maxrequests = 512;
	for (bus = 0; bus < sc->sc_numbusses; bus++) {
		if (sc->sc_max_queue_depth[bus] > sc->sc_maxrequests)
			sc->sc_max_queue_depth[bus] = sc->sc_maxrequests;
	}

	/*
	 * On some 1020/1040 variants the response queue is limited to
	 * 256 entries.  We don't really need all that many anyway.
	 */
	sc->sc_maxresponses = sc->sc_maxrequests / 2;
	if (sc->sc_maxresponses < 64)
		sc->sc_maxresponses = 64;

	/* We may need up to 3 request entries per SCSI command. */
	sc->sc_maxccbs = sc->sc_maxrequests / 3;

	/* Allegedly the FIFO is busted on the 1040A. */
	if (sc->sc_isp_type == QLW_ISP1040A)
		sc->sc_isp_config &= ~QLW_PCI_FIFO_MASK;
	qlw_write(sc, QLW_CFG1, sc->sc_isp_config);

	if (sc->sc_isp_config & QLW_BURST_ENABLE)
		qlw_dma_burst_enable(sc);

	sc->sc_mbox[0] = QLW_MBOX_SET_FIRMWARE_FEATURES;
	sc->sc_mbox[1] = 0;
	if (sc->sc_fw_features & QLW_FW_FEATURE_LVD_NOTIFY)
		sc->sc_mbox[1] |= QLW_FW_FEATURE_LVD_NOTIFY;
	if (sc->sc_mbox[1] != 0 && qlw_mbox(sc, 0x0003, 0x0001)) {
		printf("couldn't set firmware features: %x\n", sc->sc_mbox[0]);
		return (ENXIO);
	}

	sc->sc_mbox[0] = QLW_MBOX_SET_CLOCK_RATE;
	sc->sc_mbox[1] = sc->sc_clock;
	if (qlw_mbox(sc, 0x0003, 0x0001)) {
		printf("couldn't set clock rate: %x\n", sc->sc_mbox[0]);
		return (ENXIO);
	}

	sc->sc_mbox[0] = QLW_MBOX_SET_RETRY_COUNT;
	sc->sc_mbox[1] = sc->sc_retry_count[0];
	sc->sc_mbox[2] = sc->sc_retry_delay[0];
	sc->sc_mbox[6] = sc->sc_retry_count[1];
	sc->sc_mbox[7] = sc->sc_retry_delay[1];
	if (qlw_mbox(sc, 0x00c7, 0x0001)) {
		printf("couldn't set retry count: %x\n", sc->sc_mbox[0]);
		return (ENXIO);
	}

	sc->sc_mbox[0] = QLW_MBOX_SET_ASYNC_DATA_SETUP;
	sc->sc_mbox[1] = sc->sc_async_data_setup[0];
	sc->sc_mbox[2] = sc->sc_async_data_setup[1];
	if (qlw_mbox(sc, 0x0007, 0x0001)) {
		printf("couldn't set async data setup: %x\n", sc->sc_mbox[0]);
		return (ENXIO);
	}

	sc->sc_mbox[0] = QLW_MBOX_SET_ACTIVE_NEGATION;
	sc->sc_mbox[1] = sc->sc_req_ack_active_neg[0] << 5;
	sc->sc_mbox[1] |= sc->sc_data_line_active_neg[0] << 4;
	sc->sc_mbox[2] = sc->sc_req_ack_active_neg[1] << 5;
	sc->sc_mbox[2] |= sc->sc_data_line_active_neg[1] << 4;
	if (qlw_mbox(sc, 0x0007, 0x0001)) {
		printf("couldn't set active negation: %x\n", sc->sc_mbox[0]);
		return (ENXIO);
	}

	sc->sc_mbox[0] = QLW_MBOX_SET_TAG_AGE_LIMIT;
	sc->sc_mbox[1] = sc->sc_tag_age_limit[0];
	sc->sc_mbox[2] = sc->sc_tag_age_limit[1];
	if (qlw_mbox(sc, 0x0007, 0x0001)) {
		printf("couldn't set tag age limit: %x\n", sc->sc_mbox[0]);
		return (ENXIO);
	}

	sc->sc_mbox[0] = QLW_MBOX_SET_SELECTION_TIMEOUT;
	sc->sc_mbox[1] = sc->sc_selection_timeout[0];
	sc->sc_mbox[2] = sc->sc_selection_timeout[1];
	if (qlw_mbox(sc, 0x0007, 0x0001)) {
		printf("couldn't set selection timeout: %x\n", sc->sc_mbox[0]);
		return (ENXIO);
	}

	for (bus = 0; bus < sc->sc_numbusses; bus++) {
		if (qlw_config_bus(sc, bus))
			return (ENXIO);
	}

	if (qlw_alloc_ccbs(sc)) {
		/* error already printed */
		return (ENOMEM);
	}

	sc->sc_mbox[0] = QLW_MBOX_INIT_REQ_QUEUE;
	sc->sc_mbox[1] = sc->sc_maxrequests;
	qlw_mbox_putaddr(sc->sc_mbox, sc->sc_requests);
	sc->sc_mbox[4] = 0;
	if (qlw_mbox(sc, 0x00df, 0x0001)) {
		printf("couldn't init request queue: %x\n", sc->sc_mbox[0]);
		goto free_ccbs;
	}

	sc->sc_mbox[0] = QLW_MBOX_INIT_RSP_QUEUE;
	sc->sc_mbox[1] = sc->sc_maxresponses;
	qlw_mbox_putaddr(sc->sc_mbox, sc->sc_responses);
	sc->sc_mbox[5] = 0;
	if (qlw_mbox(sc, 0x00ef, 0x0001)) {
		printf("couldn't init response queue: %x\n", sc->sc_mbox[0]);
		goto free_ccbs;
	}

	reset_delay = 0;
	for (bus = 0; bus < sc->sc_numbusses; bus++) {
		sc->sc_mbox[0] = QLW_MBOX_BUS_RESET;
		sc->sc_mbox[1] = sc->sc_reset_delay[bus];
		sc->sc_mbox[2] = bus;
		if (qlw_mbox(sc, 0x0007, 0x0001)) {
			printf("couldn't reset bus: %x\n", sc->sc_mbox[0]);
			goto free_ccbs;
		}
		sc->sc_marker_required[bus] = 1;
		sc->sc_update_required[bus] = 0xffff;

		if (sc->sc_reset_delay[bus] > reset_delay)
			reset_delay = sc->sc_reset_delay[bus];
	}

	/* wait for the busses to settle */
	delay(reset_delay * 1000000);

	saa.saa_adapter = &qlw_switch;
	saa.saa_adapter_softc = sc;
	saa.saa_adapter_buswidth = QLW_MAX_TARGETS;
	saa.saa_luns = QLW_MAX_LUNS;
	saa.saa_pool = &sc->sc_iopool;
	saa.saa_quirks = saa.saa_flags = 0;
	saa.saa_wwpn = saa.saa_wwnn = 0;
	for (bus = 0; bus < sc->sc_numbusses; bus++) {
		saa.saa_adapter_target = sc->sc_initiator[bus];
		saa.saa_openings = sc->sc_max_queue_depth[bus];

		sc->sc_scsibus[bus] = (struct scsibus_softc *)
		    config_found(&sc->sc_dev, &saa, scsiprint);

		qlw_update_bus(sc, bus);
	}

	sc->sc_running = 1;
	return(0);

free_ccbs:
	qlw_free_ccbs(sc);
	return (ENXIO);
}

int
qlw_detach(struct qlw_softc *sc, int flags)
{
	return (0);
}

int
qlw_config_bus(struct qlw_softc *sc, int bus)
{
	int target, err;

	sc->sc_mbox[0] = QLW_MBOX_SET_INITIATOR_ID;
	sc->sc_mbox[1] = (bus << 7) | sc->sc_initiator[bus];

	if (qlw_mbox(sc, 0x0003, 0x0001)) {
		printf("couldn't set initiator id: %x\n", sc->sc_mbox[0]);
		return (ENXIO);
	}

	for (target = 0; target < QLW_MAX_TARGETS; target++) {
		err = qlw_config_target(sc, bus, target);
		if (err)
			return (err);
	}

	return (0);
}

int
qlw_config_target(struct qlw_softc *sc, int bus, int target)
{
	int lun;

	sc->sc_mbox[0] = QLW_MBOX_SET_TARGET_PARAMETERS;
	sc->sc_mbox[1] = (((bus << 7) | target) << 8);
	sc->sc_mbox[2] = sc->sc_target[bus][target].qt_params;
	sc->sc_mbox[2] &= QLW_TARGET_SAFE;
	sc->sc_mbox[2] |= QLW_TARGET_NARROW | QLW_TARGET_ASYNC;
	sc->sc_mbox[3] = 0;

	if (qlw_mbox(sc, 0x000f, 0x0001)) {
		printf("couldn't set target parameters: %x\n", sc->sc_mbox[0]);
		return (ENXIO);
	}

	for (lun = 0; lun < QLW_MAX_LUNS; lun++) {
		sc->sc_mbox[0] = QLW_MBOX_SET_DEVICE_QUEUE;
		sc->sc_mbox[1] = (((bus << 7) | target) << 8) | lun;
		sc->sc_mbox[2] = sc->sc_max_queue_depth[bus];
		sc->sc_mbox[3] = sc->sc_target[bus][target].qt_exec_throttle;
		if (qlw_mbox(sc, 0x000f, 0x0001)) {
			printf("couldn't set lun parameters: %x\n",
			    sc->sc_mbox[0]);
			return (ENXIO);
		}
	}

	return (0);
}

void
qlw_update_bus(struct qlw_softc *sc, int bus)
{
	int target;

	for (target = 0; target < QLW_MAX_TARGETS; target++)
		qlw_update_target(sc, bus, target);
}

void
qlw_update_target(struct qlw_softc *sc, int bus, int target)
{
	struct scsi_link *link;
	int lun;

	if ((sc->sc_update_required[bus] & (1 << target)) == 0)
		return;
	atomic_clearbits_int(&sc->sc_update_required[bus], (1 << target));

	link = scsi_get_link(sc->sc_scsibus[bus], target, 0);
	if (link == NULL)
		return;

	sc->sc_mbox[0] = QLW_MBOX_SET_TARGET_PARAMETERS;
	sc->sc_mbox[1] = (((bus << 7) | target) << 8);
	sc->sc_mbox[2] = sc->sc_target[bus][target].qt_params;
	sc->sc_mbox[2] |= QLW_TARGET_RENEG;
	sc->sc_mbox[2] &= ~QLW_TARGET_QFRZ;
	if (link->quirks & SDEV_NOSYNC)
		sc->sc_mbox[2] &= ~QLW_TARGET_SYNC;
	if (link->quirks & SDEV_NOWIDE)
		sc->sc_mbox[2] &= ~QLW_TARGET_WIDE;
	if (link->quirks & SDEV_NOTAGS)
		sc->sc_mbox[2] &= ~QLW_TARGET_TAGS;

	sc->sc_mbox[3] = sc->sc_target[bus][target].qt_sync_period;
	sc->sc_mbox[3] |= (sc->sc_target[bus][target].qt_sync_offset << 8);

	if (qlw_mbox(sc, 0x000f, 0x0001)) {
		printf("couldn't set target parameters: %x\n", sc->sc_mbox[0]);
		return;
	}

	/* XXX do PPR detection */

	for (lun = 0; lun < QLW_MAX_LUNS; lun++) {
		sc->sc_mbox[0] = QLW_MBOX_SET_DEVICE_QUEUE;
		sc->sc_mbox[1] = (((bus << 7) | target) << 8) | lun;
		sc->sc_mbox[2] = sc->sc_max_queue_depth[bus];
		sc->sc_mbox[3] = sc->sc_target[bus][target].qt_exec_throttle;
		if (qlw_mbox(sc, 0x000f, 0x0001)) {
			printf("couldn't set lun parameters: %x\n",
			    sc->sc_mbox[0]);
			return;
		}
	}
}

void
qlw_update_task(void *xsc)
{
	struct qlw_softc *sc = xsc;
	int bus;

	for (bus = 0; bus < sc->sc_numbusses; bus++)
		qlw_update_bus(sc, bus);
}

struct qlw_ccb *
qlw_handle_resp(struct qlw_softc *sc, u_int16_t id)
{
	struct qlw_ccb *ccb;
	struct qlw_iocb_hdr *hdr;
	struct qlw_iocb_status *status;
	struct scsi_xfer *xs;
	u_int32_t handle;
	int entry_type;
	int flags;
	int bus;

	ccb = NULL;
	hdr = QLW_DMA_KVA(sc->sc_responses) + (id * QLW_QUEUE_ENTRY_SIZE);

	bus_dmamap_sync(sc->sc_dmat,
	    QLW_DMA_MAP(sc->sc_responses), id * QLW_QUEUE_ENTRY_SIZE,
	    QLW_QUEUE_ENTRY_SIZE, BUS_DMASYNC_POSTREAD);

	qlw_get_header(sc, hdr, &entry_type, &flags);
	switch (entry_type) {
	case QLW_IOCB_STATUS:
		status = (struct qlw_iocb_status *)hdr;
		handle = qlw_swap32(sc, status->handle);
		if (handle > sc->sc_maxccbs) {
			panic("bad completed command handle: %d (> %d)",
			    handle, sc->sc_maxccbs);
		}

		ccb = &sc->sc_ccbs[handle];
		xs = ccb->ccb_xs;
		if (xs == NULL) {
			DPRINTF(QLW_D_INTR, "%s: got status for inactive"
			    " ccb %d\n", DEVNAME(sc), handle);
			qlw_dump_iocb(sc, hdr, QLW_D_INTR);
			ccb = NULL;
			break;
		}
		if (xs->io != ccb) {
			panic("completed command handle doesn't match xs "
			    "(handle %d, ccb %p, xs->io %p)", handle, ccb,
			    xs->io);
		}

		if (xs->datalen > 0) {
			bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, 0,
			    ccb->ccb_dmamap->dm_mapsize,
			    (xs->flags & SCSI_DATA_IN) ? BUS_DMASYNC_POSTREAD :
			    BUS_DMASYNC_POSTWRITE);
			bus_dmamap_unload(sc->sc_dmat, ccb->ccb_dmamap);
		}

		bus = qlw_xs_bus(sc, xs);
		xs->status = qlw_swap16(sc, status->scsi_status);
		switch (qlw_swap16(sc, status->completion)) {
		case QLW_IOCB_STATUS_COMPLETE:
			if (qlw_swap16(sc, status->scsi_status) &
			    QLW_SCSI_STATUS_SENSE_VALID) {
				memcpy(&xs->sense, status->sense_data,
				    sizeof(xs->sense));
				xs->error = XS_SENSE;
			} else {
				xs->error = XS_NOERROR;
			}
			xs->resid = 0;
			break;

		case QLW_IOCB_STATUS_INCOMPLETE:
			if (flags & QLW_STATE_GOT_TARGET) {
				xs->error = XS_DRIVER_STUFFUP;
			} else {
				xs->error = XS_SELTIMEOUT;
			}
			break;

		case QLW_IOCB_STATUS_DMA_ERROR:
			DPRINTF(QLW_D_INTR, "%s: dma error\n", DEVNAME(sc));
			/* set resid apparently? */
			break;

		case QLW_IOCB_STATUS_RESET:
			DPRINTF(QLW_D_INTR, "%s: reset destroyed command\n",
			    DEVNAME(sc));
			sc->sc_marker_required[bus] = 1;
			xs->error = XS_RESET;
			break;

		case QLW_IOCB_STATUS_ABORTED:
			DPRINTF(QLW_D_INTR, "%s: aborted\n", DEVNAME(sc));
			sc->sc_marker_required[bus] = 1;
			xs->error = XS_DRIVER_STUFFUP;
			break;

		case QLW_IOCB_STATUS_TIMEOUT:
			DPRINTF(QLW_D_INTR, "%s: command timed out\n",
			    DEVNAME(sc));
			xs->error = XS_TIMEOUT;
			break;

		case QLW_IOCB_STATUS_DATA_OVERRUN:
		case QLW_IOCB_STATUS_DATA_UNDERRUN:
			xs->resid = qlw_swap32(sc, status->resid);
			xs->error = XS_NOERROR;
			break;

		case QLW_IOCB_STATUS_QUEUE_FULL:
			DPRINTF(QLW_D_INTR, "%s: queue full\n", DEVNAME(sc));
			xs->error = XS_BUSY;
			break;

		case QLW_IOCB_STATUS_WIDE_FAILED:
			DPRINTF(QLW_D_INTR, "%s: wide failed\n", DEVNAME(sc));
			xs->sc_link->quirks |= SDEV_NOWIDE;
			atomic_setbits_int(&sc->sc_update_required[bus],
			    1 << xs->sc_link->target);
			task_add(systq, &sc->sc_update_task);
			xs->resid = qlw_swap32(sc, status->resid);
			xs->error = XS_NOERROR;
			break;

		case QLW_IOCB_STATUS_SYNCXFER_FAILED:
			DPRINTF(QLW_D_INTR, "%s: sync failed\n", DEVNAME(sc));
			xs->sc_link->quirks |= SDEV_NOSYNC;
			atomic_setbits_int(&sc->sc_update_required[bus],
			    1 << xs->sc_link->target);
			task_add(systq, &sc->sc_update_task);
			xs->resid = qlw_swap32(sc, status->resid);
			xs->error = XS_NOERROR;
			break;

		default:
			DPRINTF(QLW_D_INTR, "%s: unexpected completion"
			    " status %x\n", DEVNAME(sc),
			    qlw_swap16(sc, status->completion));
			qlw_dump_iocb(sc, hdr, QLW_D_INTR);
			xs->error = XS_DRIVER_STUFFUP;
			break;
		}
		break;

	default:
		DPRINTF(QLW_D_INTR, "%s: unexpected response entry type %x\n",
		    DEVNAME(sc), entry_type);
		qlw_dump_iocb(sc, hdr, QLW_D_INTR);
		break;
	}

	return (ccb);
}

void
qlw_handle_intr(struct qlw_softc *sc, u_int16_t isr, u_int16_t info)
{
	int i;
	u_int16_t rspin;
	struct qlw_ccb *ccb;

	switch (isr) {
	case QLW_INT_TYPE_ASYNC:
		qlw_async(sc, info);
		qlw_clear_isr(sc, isr);
		break;

	case QLW_INT_TYPE_IO:
		qlw_clear_isr(sc, isr);
		rspin = qlw_queue_read(sc, QLW_RESP_IN);
		if (rspin == sc->sc_last_resp_id) {
			/* seems to happen a lot on 2200s when mbox commands
			 * complete but it doesn't want to give us the register
			 * semaphore, or something.
			 *
			 * if we're waiting on a mailbox command, don't ack
			 * the interrupt yet.
			 */
			if (sc->sc_mbox_pending) {
				DPRINTF(QLW_D_MBOX, "%s: ignoring premature"
				    " mbox int\n", DEVNAME(sc));
				return;
			}

			break;
		}

		if (sc->sc_responses == NULL)
			break;

		DPRINTF(QLW_D_IO, "%s: response queue %x=>%x\n",
		    DEVNAME(sc), sc->sc_last_resp_id, rspin);

		do {
			ccb = qlw_handle_resp(sc, sc->sc_last_resp_id);
			if (ccb)
				scsi_done(ccb->ccb_xs);

			sc->sc_last_resp_id++;
			sc->sc_last_resp_id %= sc->sc_maxresponses;
		} while (sc->sc_last_resp_id != rspin);

		qlw_queue_write(sc, QLW_RESP_OUT, rspin);
		break;

	case QLW_INT_TYPE_MBOX:
		if (sc->sc_mbox_pending) {
			if (info == QLW_MBOX_COMPLETE) {
				for (i = 1; i < nitems(sc->sc_mbox); i++) {
					sc->sc_mbox[i] = qlw_read_mbox(sc, i);
				}
			} else {
				sc->sc_mbox[0] = info;
			}
			wakeup(sc->sc_mbox);
		} else {
			DPRINTF(QLW_D_MBOX, "%s: unexpected mbox interrupt:"
			    " %x\n", DEVNAME(sc), info);
		}
		qlw_clear_isr(sc, isr);
		break;

	default:
		/* maybe log something? */
		break;
	}
}

int
qlw_intr(void *xsc)
{
	struct qlw_softc *sc = xsc;
	u_int16_t isr;
	u_int16_t info;

	if (qlw_read_isr(sc, &isr, &info) == 0)
		return (0);

	qlw_handle_intr(sc, isr, info);
	return (1);
}

void
qlw_scsi_cmd(struct scsi_xfer *xs)
{
	struct scsi_link	*link = xs->sc_link;
	struct qlw_softc	*sc = link->bus->sb_adapter_softc;
	struct qlw_ccb		*ccb;
	struct qlw_iocb_req0	*iocb;
	struct qlw_ccb_list	list;
	u_int16_t		req, rspin;
	int			offset, error, done;
	bus_dmamap_t		dmap;
	int			bus;
	int			seg;

	if (xs->cmdlen > sizeof(iocb->cdb)) {
		DPRINTF(QLW_D_IO, "%s: cdb too big (%d)\n", DEVNAME(sc),
		    xs->cmdlen);
		memset(&xs->sense, 0, sizeof(xs->sense));
		xs->sense.error_code = SSD_ERRCODE_VALID | SSD_ERRCODE_CURRENT;
		xs->sense.flags = SKEY_ILLEGAL_REQUEST;
		xs->sense.add_sense_code = 0x20;
		xs->error = XS_SENSE;
		scsi_done(xs);
		return;
	}

	ccb = xs->io;
	dmap = ccb->ccb_dmamap;
	if (xs->datalen > 0) {
		error = bus_dmamap_load(sc->sc_dmat, dmap, xs->data,
		    xs->datalen, NULL, (xs->flags & SCSI_NOSLEEP) ?
		    BUS_DMA_NOWAIT : BUS_DMA_WAITOK);
		if (error) {
			xs->error = XS_DRIVER_STUFFUP;
			scsi_done(xs);
			return;
		}

		bus_dmamap_sync(sc->sc_dmat, dmap, 0,
		    dmap->dm_mapsize,
		    (xs->flags & SCSI_DATA_IN) ? BUS_DMASYNC_PREREAD :
		    BUS_DMASYNC_PREWRITE);
	}

	mtx_enter(&sc->sc_queue_mtx);

	/* put in a sync marker if required */
	bus = qlw_xs_bus(sc, xs);
	if (sc->sc_marker_required[bus]) {
		req = sc->sc_next_req_id++;
		if (sc->sc_next_req_id == sc->sc_maxrequests)
			sc->sc_next_req_id = 0;

		DPRINTF(QLW_D_IO, "%s: writing marker at request %d\n",
		    DEVNAME(sc), req);
		offset = (req * QLW_QUEUE_ENTRY_SIZE);
		iocb = QLW_DMA_KVA(sc->sc_requests) + offset;
		bus_dmamap_sync(sc->sc_dmat, QLW_DMA_MAP(sc->sc_requests),
		    offset, QLW_QUEUE_ENTRY_SIZE, BUS_DMASYNC_POSTWRITE);
		qlw_put_marker(sc, bus, iocb);
		bus_dmamap_sync(sc->sc_dmat, QLW_DMA_MAP(sc->sc_requests),
		    offset, QLW_QUEUE_ENTRY_SIZE, BUS_DMASYNC_PREWRITE);
		qlw_queue_write(sc, QLW_REQ_IN, sc->sc_next_req_id);
		sc->sc_marker_required[bus] = 0;
	}

	req = sc->sc_next_req_id++;
	if (sc->sc_next_req_id == sc->sc_maxrequests)
		sc->sc_next_req_id = 0;

	offset = (req * QLW_QUEUE_ENTRY_SIZE);
	iocb = QLW_DMA_KVA(sc->sc_requests) + offset;
	bus_dmamap_sync(sc->sc_dmat, QLW_DMA_MAP(sc->sc_requests), offset,
	    QLW_QUEUE_ENTRY_SIZE, BUS_DMASYNC_POSTWRITE);

	ccb->ccb_xs = xs;

	DPRINTF(QLW_D_IO, "%s: writing cmd at request %d\n", DEVNAME(sc), req);
	qlw_put_cmd(sc, iocb, xs, ccb);
	seg = QLW_IOCB_SEGS_PER_CMD;

	bus_dmamap_sync(sc->sc_dmat, QLW_DMA_MAP(sc->sc_requests), offset,
	    QLW_QUEUE_ENTRY_SIZE, BUS_DMASYNC_PREWRITE);

	while (seg < ccb->ccb_dmamap->dm_nsegs) {
		req = sc->sc_next_req_id++;
		if (sc->sc_next_req_id == sc->sc_maxrequests)
			sc->sc_next_req_id = 0;

		offset = (req * QLW_QUEUE_ENTRY_SIZE);
		iocb = QLW_DMA_KVA(sc->sc_requests) + offset;
		bus_dmamap_sync(sc->sc_dmat, QLW_DMA_MAP(sc->sc_requests), offset,
		    QLW_QUEUE_ENTRY_SIZE, BUS_DMASYNC_POSTWRITE);

		DPRINTF(QLW_D_IO, "%s: writing cont at request %d\n", DEVNAME(sc), req);
		qlw_put_cont(sc, iocb, xs, ccb, seg);
		seg += QLW_IOCB_SEGS_PER_CONT;

		bus_dmamap_sync(sc->sc_dmat, QLW_DMA_MAP(sc->sc_requests), offset,
		    QLW_QUEUE_ENTRY_SIZE, BUS_DMASYNC_PREWRITE);
	}

	qlw_queue_write(sc, QLW_REQ_IN, sc->sc_next_req_id);

	if (!ISSET(xs->flags, SCSI_POLL)) {
		mtx_leave(&sc->sc_queue_mtx);
		return;
	}

	done = 0;
	SIMPLEQ_INIT(&list);
	do {
		u_int16_t isr, info;

		delay(100);

		if (qlw_read_isr(sc, &isr, &info) == 0) {
			continue;
		}

		if (isr != QLW_INT_TYPE_IO) {
			qlw_handle_intr(sc, isr, info);
			continue;
		}

		qlw_clear_isr(sc, isr);

		rspin = qlw_queue_read(sc, QLW_RESP_IN);
		while (rspin != sc->sc_last_resp_id) {
			ccb = qlw_handle_resp(sc, sc->sc_last_resp_id);

			sc->sc_last_resp_id++;
			if (sc->sc_last_resp_id == sc->sc_maxresponses)
				sc->sc_last_resp_id = 0;

			if (ccb != NULL)
				SIMPLEQ_INSERT_TAIL(&list, ccb, ccb_link);
			if (ccb == xs->io)
				done = 1;
		}
		qlw_queue_write(sc, QLW_RESP_OUT, rspin);
	} while (done == 0);

	mtx_leave(&sc->sc_queue_mtx);

	while ((ccb = SIMPLEQ_FIRST(&list)) != NULL) {
		SIMPLEQ_REMOVE_HEAD(&list, ccb_link);
		scsi_done(ccb->ccb_xs);
	}
}

u_int16_t
qlw_read(struct qlw_softc *sc, bus_size_t offset)
{
	u_int16_t v;
	v = bus_space_read_2(sc->sc_iot, sc->sc_ioh, offset);
	bus_space_barrier(sc->sc_iot, sc->sc_ioh, offset, 2,
	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
	return (v);
}

void
qlw_write(struct qlw_softc *sc, bus_size_t offset, u_int16_t value)
{
	bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, value);
	bus_space_barrier(sc->sc_iot, sc->sc_ioh, offset, 2,
	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
}

u_int16_t
qlw_read_mbox(struct qlw_softc *sc, int mbox)
{
	/* could range-check mboxes according to chip type? */
	return (qlw_read(sc, sc->sc_mbox_base + (mbox * 2)));
}

void
qlw_write_mbox(struct qlw_softc *sc, int mbox, u_int16_t value)
{
	qlw_write(sc, sc->sc_mbox_base + (mbox * 2), value);
}

void
qlw_host_cmd(struct qlw_softc *sc, u_int16_t cmd)
{
	qlw_write(sc, sc->sc_host_cmd_ctrl, cmd << QLW_HOST_CMD_SHIFT);
}

#define MBOX_COMMAND_TIMEOUT	4000

int
qlw_mbox(struct qlw_softc *sc, int maskin, int maskout)
{
	int i;
	int result = 0;
	int rv;

	sc->sc_mbox_pending = 1;
	for (i = 0; i < nitems(sc->sc_mbox); i++) {
		if (maskin & (1 << i)) {
			qlw_write_mbox(sc, i, sc->sc_mbox[i]);
		}
	}
	qlw_host_cmd(sc, QLW_HOST_CMD_SET_HOST_INT);

	if (sc->sc_running == 0) {
		for (i = 0; i < MBOX_COMMAND_TIMEOUT && result == 0; i++) {
			u_int16_t isr, info;

			delay(100);

			if (qlw_read_isr(sc, &isr, &info) == 0)
				continue;

			switch (isr) {
			case QLW_INT_TYPE_MBOX:
				result = info;
				break;

			default:
				qlw_handle_intr(sc, isr, info);
				break;
			}
		}
	} else {
		tsleep_nsec(sc->sc_mbox, PRIBIO, "qlw_mbox", INFSLP);
		result = sc->sc_mbox[0];
	}

	switch (result) {
	case QLW_MBOX_COMPLETE:
		for (i = 1; i < nitems(sc->sc_mbox); i++) {
			sc->sc_mbox[i] = (maskout & (1 << i)) ?
			    qlw_read_mbox(sc, i) : 0;
		}
		rv = 0;
		break;

	case 0:
		/* timed out; do something? */
		DPRINTF(QLW_D_MBOX, "%s: mbox timed out\n", DEVNAME(sc));
		rv = 1;
		break;

	default:
		sc->sc_mbox[0] = result;
		rv = result;
		break;
	}

	qlw_clear_isr(sc, QLW_INT_TYPE_MBOX);
	sc->sc_mbox_pending = 0;
	return (rv);
}

void
qlw_mbox_putaddr(u_int16_t *mbox, struct qlw_dmamem *mem)
{
	mbox[2] = (QLW_DMA_DVA(mem) >> 16) & 0xffff;
	mbox[3] = (QLW_DMA_DVA(mem) >> 0) & 0xffff;
	mbox[6] = (QLW_DMA_DVA(mem) >> 48) & 0xffff;
	mbox[7] = (QLW_DMA_DVA(mem) >> 32) & 0xffff;
}

void
qlw_set_ints(struct qlw_softc *sc, int enabled)
{
	u_int16_t v = enabled ? (QLW_INT_REQ | QLW_RISC_INT_REQ) : 0;
	qlw_write(sc, QLW_INT_CTRL, v);
}

int
qlw_read_isr(struct qlw_softc *sc, u_int16_t *isr, u_int16_t *info)
{
	u_int16_t int_status;

	if (qlw_read(sc, QLW_SEMA) & QLW_SEMA_LOCK) {
		*info = qlw_read_mbox(sc, 0);
		if (*info & QLW_MBOX_HAS_STATUS)
			*isr = QLW_INT_TYPE_MBOX;
		else
			*isr = QLW_INT_TYPE_ASYNC;
	} else {
		int_status = qlw_read(sc, QLW_INT_STATUS);
		if ((int_status & (QLW_INT_REQ | QLW_RISC_INT_REQ)) == 0)
			return (0);

		*isr = QLW_INT_TYPE_IO;
	}

	return (1);
}

void
qlw_clear_isr(struct qlw_softc *sc, u_int16_t isr)
{
	qlw_host_cmd(sc, QLW_HOST_CMD_CLR_RISC_INT);
	switch (isr) {
	case QLW_INT_TYPE_MBOX:
	case QLW_INT_TYPE_ASYNC:
		qlw_write(sc, QLW_SEMA, 0);
		break;
	default:
		break;
	}
}

int
qlw_softreset(struct qlw_softc *sc)
{
	int i;

	qlw_set_ints(sc, 0);

	/* reset */
	qlw_write(sc, QLW_INT_CTRL, QLW_RESET);
	delay(100);
	/* clear data and control dma engines? */

	/* wait for soft reset to clear */
	for (i = 0; i < 1000; i++) {
		if ((qlw_read(sc, QLW_INT_CTRL) & QLW_RESET) == 0)
			break;

		delay(100);
	}

	if (i == 1000) {
		DPRINTF(QLW_D_INTR, "%s: reset didn't clear\n", DEVNAME(sc));
		qlw_set_ints(sc, 0);
		return (ENXIO);
	}

	qlw_write(sc, QLW_CFG1, 0);

	/* reset risc processor */
	qlw_host_cmd(sc, QLW_HOST_CMD_RESET);
	delay(100);
	qlw_write(sc, QLW_SEMA, 0);
	qlw_host_cmd(sc, QLW_HOST_CMD_RELEASE);

	/* reset queue pointers */
	qlw_queue_write(sc, QLW_REQ_IN, 0);
	qlw_queue_write(sc, QLW_REQ_OUT, 0);
	qlw_queue_write(sc, QLW_RESP_IN, 0);
	qlw_queue_write(sc, QLW_RESP_OUT, 0);

	qlw_set_ints(sc, 1);
	qlw_host_cmd(sc, QLW_HOST_CMD_BIOS);

	/* do a basic mailbox operation to check we're alive */
	sc->sc_mbox[0] = QLW_MBOX_NOP;
	if (qlw_mbox(sc, 0x0001, 0x0001)) {
		DPRINTF(QLW_D_INTR, "%s: ISP not responding after reset\n",
		    DEVNAME(sc));
		return (ENXIO);
	}

	return (0);
}

void
qlw_dma_burst_enable(struct qlw_softc *sc)
{
	if (sc->sc_isp_gen == QLW_GEN_ISP1000 ||
	    sc->sc_isp_gen == QLW_GEN_ISP1040) {
		qlw_write(sc, QLW_CDMA_CFG,
		    qlw_read(sc, QLW_CDMA_CFG) | QLW_DMA_BURST_ENABLE);
		qlw_write(sc, QLW_DDMA_CFG,
		    qlw_read(sc, QLW_DDMA_CFG) | QLW_DMA_BURST_ENABLE);
	} else {
		qlw_host_cmd(sc, QLW_HOST_CMD_PAUSE);
		qlw_write(sc, QLW_CFG1,
		    qlw_read(sc, QLW_CFG1) | QLW_DMA_BANK);
		qlw_write(sc, QLW_CDMA_CFG_1080,
		    qlw_read(sc, QLW_CDMA_CFG_1080) | QLW_DMA_BURST_ENABLE);
		qlw_write(sc, QLW_DDMA_CFG_1080,
		    qlw_read(sc, QLW_DDMA_CFG_1080) | QLW_DMA_BURST_ENABLE);
		qlw_write(sc, QLW_CFG1,
		    qlw_read(sc, QLW_CFG1) & ~QLW_DMA_BANK);
		qlw_host_cmd(sc, QLW_HOST_CMD_RELEASE);
	}
}

void
qlw_update(struct qlw_softc *sc, int task)
{
	/* do things */
}

int
qlw_async(struct qlw_softc *sc, u_int16_t info)
{
	int bus;

	switch (info) {
	case QLW_ASYNC_BUS_RESET:
		DPRINTF(QLW_D_PORT, "%s: bus reset\n", DEVNAME(sc));
		bus = qlw_read_mbox(sc, 6);
		sc->sc_marker_required[bus] = 1;
		break;

#if 0
	case QLW_ASYNC_SYSTEM_ERROR:
		qla_update(sc, QLW_UPDATE_SOFTRESET);
		break;

	case QLW_ASYNC_REQ_XFER_ERROR:
		qla_update(sc, QLW_UPDATE_SOFTRESET);
		break;

	case QLW_ASYNC_RSP_XFER_ERROR:
		qla_update(sc, QLW_UPDATE_SOFTRESET);
		break;
#endif

	case QLW_ASYNC_SCSI_CMD_COMPLETE:
		/* shouldn't happen, we disable fast posting */
		break;

	case QLW_ASYNC_CTIO_COMPLETE:
		/* definitely shouldn't happen, we don't do target mode */
		break;

	default:
		DPRINTF(QLW_D_INTR, "%s: unknown async %x\n", DEVNAME(sc),
		    info);
		break;
	}
	return (1);
}

#ifdef QLW_DEBUG
void
qlw_dump_iocb(struct qlw_softc *sc, void *buf, int flags)
{
	u_int8_t *iocb = buf;
	int l;
	int b;

	if ((qlwdebug & flags) == 0)
		return;

	printf("%s: iocb:\n", DEVNAME(sc));
	for (l = 0; l < 4; l++) {
		for (b = 0; b < 16; b++) {
			printf(" %2.2x", iocb[(l*16)+b]);
		}
		printf("\n");
	}
}

void
qlw_dump_iocb_segs(struct qlw_softc *sc, void *segs, int n)
{
	u_int8_t *buf = segs;
	int s, b;
	if ((qlwdebug & QLW_D_IOCB) == 0)
		return;

	printf("%s: iocb segs:\n", DEVNAME(sc));
	for (s = 0; s < n; s++) {
		for (b = 0; b < sizeof(struct qlw_iocb_seg); b++) {
			printf(" %2.2x", buf[(s*(sizeof(struct qlw_iocb_seg)))
			    + b]);
		}
		printf("\n");
	}
}
#endif

/*
 * The PCI bus is little-endian whereas SBus is big-endian.  This
 * leads to some differences in byte twisting of DMA transfers of
 * request and response queue entries.  Most fields can be treated as
 * 16-bit or 32-bit with the endianness of the bus, but the header
 * fields end up being swapped by the ISP1000's SBus interface.
 */

void
qlw_get_header(struct qlw_softc *sc, struct qlw_iocb_hdr *hdr,
    int *type, int *flags)
{
	if (sc->sc_isp_gen == QLW_GEN_ISP1000) {
		*type = hdr->entry_count;
		*flags = hdr->seqno;
	} else {
		*type = hdr->entry_type;
		*flags = hdr->flags;
	}
}

void
qlw_put_header(struct qlw_softc *sc, struct qlw_iocb_hdr *hdr,
    int type, int count)
{
	if (sc->sc_isp_gen == QLW_GEN_ISP1000) {
		hdr->entry_type = count;
		hdr->entry_count = type;
		hdr->seqno = 0;
		hdr->flags = 0;
	} else {
		hdr->entry_type = type;
		hdr->entry_count = count;
		hdr->seqno = 0;
		hdr->flags = 0;
	}
}

void
qlw_put_data_seg(struct qlw_softc *sc, struct qlw_iocb_seg *seg,
    bus_dmamap_t dmap, int num)
{
	seg->seg_addr = qlw_swap32(sc, dmap->dm_segs[num].ds_addr);
	seg->seg_len = qlw_swap32(sc, dmap->dm_segs[num].ds_len);
}

void
qlw_put_marker(struct qlw_softc *sc, int bus, void *buf)
{
	struct qlw_iocb_marker *marker = buf;

	qlw_put_header(sc, &marker->hdr, QLW_IOCB_MARKER, 1);

	/* could be more specific here; isp(4) isn't */
	marker->device = qlw_swap16(sc, (bus << 7) << 8);
	marker->modifier = qlw_swap16(sc, QLW_IOCB_MARKER_SYNC_ALL);
	qlw_dump_iocb(sc, buf, QLW_D_IOCB);
}

void
qlw_put_cmd(struct qlw_softc *sc, void *buf, struct scsi_xfer *xs,
    struct qlw_ccb *ccb)
{
	struct qlw_iocb_req0 *req = buf;
	int entry_count = 1;
	u_int16_t dir;
	int seg, nsegs;
	int seg_count;
	int timeout = 0;
	int bus, target, lun;

	if (xs->datalen == 0) {
		dir = QLW_IOCB_CMD_NO_DATA;
		seg_count = 1;
	} else {
		dir = xs->flags & SCSI_DATA_IN ? QLW_IOCB_CMD_READ_DATA :
		    QLW_IOCB_CMD_WRITE_DATA;
		seg_count = ccb->ccb_dmamap->dm_nsegs;
		nsegs = ccb->ccb_dmamap->dm_nsegs - QLW_IOCB_SEGS_PER_CMD;
		while (nsegs > 0) {
			entry_count++;
			nsegs -= QLW_IOCB_SEGS_PER_CONT;
		}
		for (seg = 0; seg < ccb->ccb_dmamap->dm_nsegs; seg++) {
			if (seg >= QLW_IOCB_SEGS_PER_CMD)
				break;
			qlw_put_data_seg(sc, &req->segs[seg],
			    ccb->ccb_dmamap, seg);
		}
	}

	if (sc->sc_running && (xs->sc_link->quirks & SDEV_NOTAGS) == 0)
		dir |= QLW_IOCB_CMD_SIMPLE_QUEUE;

	qlw_put_header(sc, &req->hdr, QLW_IOCB_CMD_TYPE_0, entry_count);

	/*
	 * timeout is in seconds.  make sure it's at least 1 if a timeout
	 * was specified in xs
	 */
	if (xs->timeout != 0)
		timeout = MAX(1, xs->timeout/1000);

	req->flags = qlw_swap16(sc, dir);
	req->seg_count = qlw_swap16(sc, seg_count);
	req->timeout = qlw_swap16(sc, timeout);

	bus = qlw_xs_bus(sc, xs);
	target = xs->sc_link->target;
	lun = xs->sc_link->lun;
	req->device = qlw_swap16(sc, (((bus << 7) | target) << 8) | lun);

	memcpy(req->cdb, &xs->cmd, xs->cmdlen);
	req->ccblen = qlw_swap16(sc, xs->cmdlen);

	req->handle = qlw_swap32(sc, ccb->ccb_id);

	qlw_dump_iocb(sc, buf, QLW_D_IOCB);
}

void
qlw_put_cont(struct qlw_softc *sc, void *buf, struct scsi_xfer *xs,
    struct qlw_ccb *ccb, int seg0)
{
	struct qlw_iocb_cont0 *cont = buf;
	int seg;

	qlw_put_header(sc, &cont->hdr, QLW_IOCB_CONT_TYPE_0, 1);

	for (seg = seg0; seg < ccb->ccb_dmamap->dm_nsegs; seg++) {
		if ((seg - seg0) >= QLW_IOCB_SEGS_PER_CONT)
			break;
		qlw_put_data_seg(sc, &cont->segs[seg - seg0],
		    ccb->ccb_dmamap, seg);
	}
}

#ifndef ISP_NOFIRMWARE
int
qlw_load_firmware_words(struct qlw_softc *sc, const u_int16_t *src,
    u_int16_t dest)
{
	u_int16_t i;

	for (i = 0; i < src[3]; i++) {
		sc->sc_mbox[0] = QLW_MBOX_WRITE_RAM_WORD;
		sc->sc_mbox[1] = i + dest;
		sc->sc_mbox[2] = src[i];
		if (qlw_mbox(sc, 0x07, 0x01)) {
			printf("firmware load failed\n");
			return (1);
		}
	}

	sc->sc_mbox[0] = QLW_MBOX_VERIFY_CSUM;
	sc->sc_mbox[1] = dest;
	if (qlw_mbox(sc, 0x0003, 0x0003)) {
		printf("verification of chunk at %x failed: %x\n",
		    dest, sc->sc_mbox[1]);
		return (1);
	}

	return (0);
}

int
qlw_load_firmware(struct qlw_softc *sc)
{
	return qlw_load_firmware_words(sc, sc->sc_firmware, QLW_CODE_ORG);
}

#endif	/* !ISP_NOFIRMWARE */

int
qlw_read_nvram(struct qlw_softc *sc)
{
	u_int16_t data[sizeof(sc->sc_nvram) >> 1];
	u_int16_t req, cmd, val;
	u_int8_t csum;
	int i, bit;
	int reqcmd;
	int nbits;

	if (sc->sc_nvram_size == 0)
		return (1);

	if (sc->sc_nvram_size == 128) {
		reqcmd = (QLW_NVRAM_CMD_READ << 6);
		nbits = 8;
	} else {
		reqcmd = (QLW_NVRAM_CMD_READ << 8);
		nbits = 10;
	}

	qlw_write(sc, QLW_NVRAM, QLW_NVRAM_CHIP_SEL);
	delay(10);
	qlw_write(sc, QLW_NVRAM, QLW_NVRAM_CHIP_SEL | QLW_NVRAM_CLOCK);
	delay(10);

	for (i = 0; i < (sc->sc_nvram_size >> 1); i++) {
		req = i | reqcmd;

		/* write each bit out through the nvram register */
		for (bit = nbits; bit >= 0; bit--) {
			cmd = QLW_NVRAM_CHIP_SEL;
			if ((req >> bit) & 1) {
				cmd |= QLW_NVRAM_DATA_OUT;
			}
			qlw_write(sc, QLW_NVRAM, cmd);
			delay(10);
			qlw_read(sc, QLW_NVRAM);

			qlw_write(sc, QLW_NVRAM, cmd | QLW_NVRAM_CLOCK);
			delay(10);
			qlw_read(sc, QLW_NVRAM);

			qlw_write(sc, QLW_NVRAM, cmd);
			delay(10);
			qlw_read(sc, QLW_NVRAM);
		}

		/* read the result back */
		val = 0;
		for (bit = 0; bit < 16; bit++) {
			val <<= 1;
			qlw_write(sc, QLW_NVRAM, QLW_NVRAM_CHIP_SEL |
			    QLW_NVRAM_CLOCK);
			delay(10);
			if (qlw_read(sc, QLW_NVRAM) & QLW_NVRAM_DATA_IN)
				val |= 1;
			delay(10);

			qlw_write(sc, QLW_NVRAM, QLW_NVRAM_CHIP_SEL);
			delay(10);
			qlw_read(sc, QLW_NVRAM);
		}

		qlw_write(sc, QLW_NVRAM, 0);
		delay(10);
		qlw_read(sc, QLW_NVRAM);

		data[i] = letoh16(val);
	}

	csum = 0;
	for (i = 0; i < (sc->sc_nvram_size >> 1); i++) {
		csum += data[i] & 0xff;
		csum += data[i] >> 8;
	}

	memcpy(&sc->sc_nvram, data, sizeof(sc->sc_nvram));
	/* id field should be 'ISP ', version should high enough */
	if (sc->sc_nvram.id[0] != 'I' || sc->sc_nvram.id[1] != 'S' ||
	    sc->sc_nvram.id[2] != 'P' || sc->sc_nvram.id[3] != ' ' ||
	    sc->sc_nvram.nvram_version < sc->sc_nvram_minversion ||
	    (csum != 0)) {
		printf("%s: nvram corrupt\n", DEVNAME(sc));
		return (1);
	}
	return (0);
}

void
qlw_parse_nvram_1040(struct qlw_softc *sc, int bus)
{
	struct qlw_nvram_1040 *nv = (struct qlw_nvram_1040 *)&sc->sc_nvram;
	int target;

	KASSERT(bus == 0);

	if (!ISSET(sc->sc_flags, QLW_FLAG_INITIATOR))
		sc->sc_initiator[0] = (nv->config1 >> 4);

	sc->sc_retry_count[0] = nv->retry_count;
	sc->sc_retry_delay[0] = nv->retry_delay;
	sc->sc_reset_delay[0] = nv->reset_delay;
	sc->sc_tag_age_limit[0] = nv->tag_age_limit;
	sc->sc_selection_timeout[0] = letoh16(nv->selection_timeout);
	sc->sc_max_queue_depth[0] = letoh16(nv->max_queue_depth);
	sc->sc_async_data_setup[0] = (nv->config2 & 0x0f);
	sc->sc_req_ack_active_neg[0] = ((nv->config2 & 0x10) >> 4);
	sc->sc_data_line_active_neg[0] = ((nv->config2 & 0x20) >> 5);

	for (target = 0; target < QLW_MAX_TARGETS; target++) {
		struct qlw_target *qt = &sc->sc_target[0][target];

		qt->qt_params = (nv->target[target].parameter << 8);
		qt->qt_exec_throttle = nv->target[target].execution_throttle;
		qt->qt_sync_period = nv->target[target].sync_period;
		qt->qt_sync_offset = nv->target[target].flags & 0x0f;
	}
}

void
qlw_parse_nvram_1080(struct qlw_softc *sc, int bus)
{
	struct qlw_nvram_1080 *nvram = (struct qlw_nvram_1080 *)&sc->sc_nvram;
	struct qlw_nvram_bus *nv = &nvram->bus[bus];
	int target;

	sc->sc_isp_config = nvram->isp_config;
	sc->sc_fw_features = nvram->fw_features;

	if (!ISSET(sc->sc_flags, QLW_FLAG_INITIATOR))
		sc->sc_initiator[bus] = (nv->config1 & 0x0f);

	sc->sc_retry_count[bus] = nv->retry_count;
	sc->sc_retry_delay[bus] = nv->retry_delay;
	sc->sc_reset_delay[bus] = nv->reset_delay;
	sc->sc_selection_timeout[bus] = letoh16(nv->selection_timeout);
	sc->sc_max_queue_depth[bus] = letoh16(nv->max_queue_depth);
	sc->sc_async_data_setup[bus] = (nv->config2 & 0x0f);
	sc->sc_req_ack_active_neg[bus] = ((nv->config2 & 0x10) >> 4);
	sc->sc_data_line_active_neg[bus] = ((nv->config2 & 0x20) >> 5);

	for (target = 0; target < QLW_MAX_TARGETS; target++) {
		struct qlw_target *qt = &sc->sc_target[bus][target];

		qt->qt_params = (nv->target[target].parameter << 8);
		qt->qt_exec_throttle = nv->target[target].execution_throttle;
		qt->qt_sync_period = nv->target[target].sync_period;
		if (sc->sc_isp_gen == QLW_GEN_ISP12160)
			qt->qt_sync_offset = nv->target[target].flags & 0x1f;
		else
			qt->qt_sync_offset = nv->target[target].flags & 0x0f;
	}
}

void
qlw_init_defaults(struct qlw_softc *sc, int bus)
{
	int target;

	switch (sc->sc_isp_gen) {
	case QLW_GEN_ISP1000:
		break;
	case QLW_GEN_ISP1040:
		sc->sc_isp_config = QLW_BURST_ENABLE | QLW_PCI_FIFO_64;
		break;
	case QLW_GEN_ISP1080:
	case QLW_GEN_ISP12160:
		sc->sc_isp_config = QLW_BURST_ENABLE | QLW_PCI_FIFO_128;
		sc->sc_fw_features = QLW_FW_FEATURE_LVD_NOTIFY;
		break;
	}

	sc->sc_retry_count[bus] = 0;
	sc->sc_retry_delay[bus] = 0;
	sc->sc_reset_delay[bus] = 3;
	sc->sc_tag_age_limit[bus] = 8;
	sc->sc_selection_timeout[bus] = 250;
	sc->sc_max_queue_depth[bus] = 32;
	if (sc->sc_clock > 40)
		sc->sc_async_data_setup[bus] = 9;
	else
		sc->sc_async_data_setup[bus] = 6;
	sc->sc_req_ack_active_neg[bus] = 1;
	sc->sc_data_line_active_neg[bus] = 1;

	for (target = 0; target < QLW_MAX_TARGETS; target++) {
		struct qlw_target *qt = &sc->sc_target[bus][target];

		qt->qt_params = QLW_TARGET_DEFAULT;
		qt->qt_exec_throttle = 16;
		qt->qt_sync_period = 10;
		qt->qt_sync_offset = 12;
	}
}

struct qlw_dmamem *
qlw_dmamem_alloc(struct qlw_softc *sc, size_t size)
{
	struct qlw_dmamem *m;
	int nsegs;

	m = malloc(sizeof(*m), M_DEVBUF, M_NOWAIT | M_ZERO);
	if (m == NULL)
		return (NULL);

	m->qdm_size = size;

	if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
	    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &m->qdm_map) != 0)
		goto qdmfree;

	if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &m->qdm_seg, 1,
	    &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO) != 0)
		goto destroy;

	if (bus_dmamem_map(sc->sc_dmat, &m->qdm_seg, nsegs, size, &m->qdm_kva,
	    BUS_DMA_NOWAIT) != 0)
		goto free;

	if (bus_dmamap_load(sc->sc_dmat, m->qdm_map, m->qdm_kva, size, NULL,
	    BUS_DMA_NOWAIT) != 0)
		goto unmap;

	return (m);

unmap:
	bus_dmamem_unmap(sc->sc_dmat, m->qdm_kva, m->qdm_size);
free:
	bus_dmamem_free(sc->sc_dmat, &m->qdm_seg, 1);
destroy:
	bus_dmamap_destroy(sc->sc_dmat, m->qdm_map);
qdmfree:
	free(m, M_DEVBUF, sizeof(*m));

	return (NULL);
}

void
qlw_dmamem_free(struct qlw_softc *sc, struct qlw_dmamem *m)
{
	bus_dmamap_unload(sc->sc_dmat, m->qdm_map);
	bus_dmamem_unmap(sc->sc_dmat, m->qdm_kva, m->qdm_size);
	bus_dmamem_free(sc->sc_dmat, &m->qdm_seg, 1);
	bus_dmamap_destroy(sc->sc_dmat, m->qdm_map);
	free(m, M_DEVBUF, sizeof(*m));
}

int
qlw_alloc_ccbs(struct qlw_softc *sc)
{
	struct qlw_ccb		*ccb;
	u_int8_t		*cmd;
	int			i;

	SIMPLEQ_INIT(&sc->sc_ccb_free);
	mtx_init(&sc->sc_ccb_mtx, IPL_BIO);
	mtx_init(&sc->sc_queue_mtx, IPL_BIO);

	sc->sc_ccbs = mallocarray(sc->sc_maxccbs, sizeof(struct qlw_ccb),
	    M_DEVBUF, M_WAITOK | M_CANFAIL | M_ZERO);
	if (sc->sc_ccbs == NULL) {
		printf("%s: unable to allocate ccbs\n", DEVNAME(sc));
		return (1);
	}

	sc->sc_requests = qlw_dmamem_alloc(sc, sc->sc_maxrequests *
	    QLW_QUEUE_ENTRY_SIZE);
	if (sc->sc_requests == NULL) {
		printf("%s: unable to allocate ccb dmamem\n", DEVNAME(sc));
		goto free_ccbs;
	}
	sc->sc_responses = qlw_dmamem_alloc(sc, sc->sc_maxresponses *
	    QLW_QUEUE_ENTRY_SIZE);
	if (sc->sc_responses == NULL) {
		printf("%s: unable to allocate rcb dmamem\n", DEVNAME(sc));
		goto free_req;
	}

	cmd = QLW_DMA_KVA(sc->sc_requests);
	memset(cmd, 0, QLW_QUEUE_ENTRY_SIZE * sc->sc_maxccbs);
	for (i = 0; i < sc->sc_maxccbs; i++) {
		ccb = &sc->sc_ccbs[i];

		if (bus_dmamap_create(sc->sc_dmat, MAXPHYS,
		    QLW_MAX_SEGS, MAXPHYS, 0,
		    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
		    &ccb->ccb_dmamap) != 0) {
			printf("%s: unable to create dma map\n", DEVNAME(sc));
			goto free_maps;
		}

		ccb->ccb_sc = sc;
		ccb->ccb_id = i;

		qlw_put_ccb(sc, ccb);
	}

	scsi_iopool_init(&sc->sc_iopool, sc, qlw_get_ccb, qlw_put_ccb);
	return (0);

free_maps:
	while ((ccb = qlw_get_ccb(sc)) != NULL)
		bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);

	qlw_dmamem_free(sc, sc->sc_responses);
free_req:
	qlw_dmamem_free(sc, sc->sc_requests);
free_ccbs:
	free(sc->sc_ccbs, M_DEVBUF, 0);

	return (1);
}

void
qlw_free_ccbs(struct qlw_softc *sc)
{
	struct qlw_ccb		*ccb;

	scsi_iopool_destroy(&sc->sc_iopool);
	while ((ccb = qlw_get_ccb(sc)) != NULL)
		bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
	qlw_dmamem_free(sc, sc->sc_responses);
	qlw_dmamem_free(sc, sc->sc_requests);
	free(sc->sc_ccbs, M_DEVBUF, 0);
}

void *
qlw_get_ccb(void *xsc)
{
	struct qlw_softc	*sc = xsc;
	struct qlw_ccb		*ccb;

	mtx_enter(&sc->sc_ccb_mtx);
	ccb = SIMPLEQ_FIRST(&sc->sc_ccb_free);
	if (ccb != NULL) {
		SIMPLEQ_REMOVE_HEAD(&sc->sc_ccb_free, ccb_link);
	}
	mtx_leave(&sc->sc_ccb_mtx);
	return (ccb);
}

void
qlw_put_ccb(void *xsc, void *io)
{
	struct qlw_softc	*sc = xsc;
	struct qlw_ccb		*ccb = io;

	ccb->ccb_xs = NULL;
	mtx_enter(&sc->sc_ccb_mtx);
	SIMPLEQ_INSERT_HEAD(&sc->sc_ccb_free, ccb, ccb_link);
	mtx_leave(&sc->sc_ccb_mtx);
}