xref: /freebsd/sys/dev/qat_c2xxx/qat_ae.c (revision 685dc743)

/* SPDX-License-Identifier: BSD-2-Clause AND BSD-3-Clause */
/*	$NetBSD: qat_ae.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $	*/

/*
 * Copyright (c) 2019 Internet Initiative Japan, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 *   Copyright(c) 2007-2019 Intel Corporation. All rights reserved.
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>
#if 0
__KERNEL_RCSID(0, "$NetBSD: qat_ae.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $");
#endif

#include <sys/param.h>
#include <sys/bus.h>
#include <sys/firmware.h>
#include <sys/limits.h>
#include <sys/systm.h>

#include <machine/bus.h>

#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>

#include "qatreg.h"
#include "qatvar.h"
#include "qat_aevar.h"

static int	qat_ae_write_4(struct qat_softc *, u_char, bus_size_t,
		    uint32_t);
static int	qat_ae_read_4(struct qat_softc *, u_char, bus_size_t,
		    uint32_t *);
static void	qat_ae_ctx_indr_write(struct qat_softc *, u_char, uint32_t,
		    bus_size_t, uint32_t);
static int	qat_ae_ctx_indr_read(struct qat_softc *, u_char, uint32_t,
		    bus_size_t, uint32_t *);

static u_short	qat_aereg_get_10bit_addr(enum aereg_type, u_short);
static int	qat_aereg_rel_data_write(struct qat_softc *, u_char, u_char,
		    enum aereg_type, u_short, uint32_t);
static int	qat_aereg_rel_data_read(struct qat_softc *, u_char, u_char,
		    enum aereg_type, u_short, uint32_t *);
static int	qat_aereg_rel_rdxfer_write(struct qat_softc *, u_char, u_char,
		    enum aereg_type, u_short, uint32_t);
static int	qat_aereg_rel_wrxfer_write(struct qat_softc *, u_char, u_char,
		    enum aereg_type, u_short, uint32_t);
static int	qat_aereg_rel_nn_write(struct qat_softc *, u_char, u_char,
		    enum aereg_type, u_short, uint32_t);
static int	qat_aereg_abs_to_rel(struct qat_softc *, u_char, u_short,
		    u_short *, u_char *);
static int	qat_aereg_abs_data_write(struct qat_softc *, u_char,
		    enum aereg_type, u_short, uint32_t);

static void	qat_ae_enable_ctx(struct qat_softc *, u_char, u_int);
static void	qat_ae_disable_ctx(struct qat_softc *, u_char, u_int);
static void	qat_ae_write_ctx_mode(struct qat_softc *, u_char, u_char);
static void	qat_ae_write_nn_mode(struct qat_softc *, u_char, u_char);
static void	qat_ae_write_lm_mode(struct qat_softc *, u_char,
		    enum aereg_type, u_char);
static void	qat_ae_write_shared_cs_mode0(struct qat_softc *, u_char,
		    u_char);
static void	qat_ae_write_shared_cs_mode(struct qat_softc *, u_char, u_char);
static int	qat_ae_set_reload_ustore(struct qat_softc *, u_char, u_int, int,
		    u_int);

static enum qat_ae_status qat_ae_get_status(struct qat_softc *, u_char);
static int	qat_ae_is_active(struct qat_softc *, u_char);
static int	qat_ae_wait_num_cycles(struct qat_softc *, u_char, int, int);

static int	qat_ae_clear_reset(struct qat_softc *);
static int	qat_ae_check(struct qat_softc *);
static int	qat_ae_reset_timestamp(struct qat_softc *);
static void	qat_ae_clear_xfer(struct qat_softc *);
static int	qat_ae_clear_gprs(struct qat_softc *);

static void	qat_ae_get_shared_ustore_ae(u_char, u_char *);
static u_int	qat_ae_ucode_parity64(uint64_t);
static uint64_t	qat_ae_ucode_set_ecc(uint64_t);
static int	qat_ae_ucode_write(struct qat_softc *, u_char, u_int, u_int,
		    const uint64_t *);
static int	qat_ae_ucode_read(struct qat_softc *, u_char, u_int, u_int,
		    uint64_t *);
static u_int	qat_ae_concat_ucode(uint64_t *, u_int, u_int, u_int, u_int *);
static int	qat_ae_exec_ucode(struct qat_softc *, u_char, u_char,
		    uint64_t *, u_int, int, u_int, u_int *);
static int	qat_ae_exec_ucode_init_lm(struct qat_softc *, u_char, u_char,
		    int *, uint64_t *, u_int,
		    u_int *, u_int *, u_int *, u_int *, u_int *);
static int	qat_ae_restore_init_lm_gprs(struct qat_softc *, u_char, u_char,
		    u_int, u_int, u_int, u_int, u_int);
static int	qat_ae_get_inst_num(int);
static int	qat_ae_batch_put_lm(struct qat_softc *, u_char,
		    struct qat_ae_batch_init_list *, size_t);
static int	qat_ae_write_pc(struct qat_softc *, u_char, u_int, u_int);

static u_int	qat_aefw_csum(char *, int);
static const char *qat_aefw_uof_string(struct qat_softc *, size_t);
static struct uof_chunk_hdr *qat_aefw_uof_find_chunk(struct qat_softc *,
		    const char *, struct uof_chunk_hdr *);

static int	qat_aefw_load_mof(struct qat_softc *);
static void	qat_aefw_unload_mof(struct qat_softc *);
static int	qat_aefw_load_mmp(struct qat_softc *);
static void	qat_aefw_unload_mmp(struct qat_softc *);

static int	qat_aefw_mof_find_uof0(struct qat_softc *,
		    struct mof_uof_hdr *, struct mof_uof_chunk_hdr *,
		    u_int, size_t, const char *,
		    size_t *, void **);
static int	qat_aefw_mof_find_uof(struct qat_softc *);
static int	qat_aefw_mof_parse(struct qat_softc *);

static int	qat_aefw_uof_parse_image(struct qat_softc *,
		    struct qat_uof_image *, struct uof_chunk_hdr *uch);
static int	qat_aefw_uof_parse_images(struct qat_softc *);
static int	qat_aefw_uof_parse(struct qat_softc *);

static int	qat_aefw_alloc_auth_dmamem(struct qat_softc *, char *, size_t,
		    struct qat_dmamem *);
static int	qat_aefw_auth(struct qat_softc *, struct qat_dmamem *);
static int	qat_aefw_suof_load(struct qat_softc *sc,
		    struct qat_dmamem *dma);
static int	qat_aefw_suof_parse_image(struct qat_softc *,
		    struct qat_suof_image *, struct suof_chunk_hdr *);
static int	qat_aefw_suof_parse(struct qat_softc *);
static int	qat_aefw_suof_write(struct qat_softc *);

static int	qat_aefw_uof_assign_image(struct qat_softc *, struct qat_ae *,
		    struct qat_uof_image *);
static int	qat_aefw_uof_init_ae(struct qat_softc *, u_char);
static int	qat_aefw_uof_init(struct qat_softc *);

static int	qat_aefw_init_memory_one(struct qat_softc *,
		    struct uof_init_mem *);
static void	qat_aefw_free_lm_init(struct qat_softc *, u_char);
static int	qat_aefw_init_ustore(struct qat_softc *);
static int	qat_aefw_init_reg(struct qat_softc *, u_char, u_char,
		    enum aereg_type, u_short, u_int);
static int	qat_aefw_init_reg_sym_expr(struct qat_softc *, u_char,
		    struct qat_uof_image *);
static int	qat_aefw_init_memory(struct qat_softc *);
static int	qat_aefw_init_globals(struct qat_softc *);
static uint64_t	qat_aefw_get_uof_inst(struct qat_softc *,
		    struct qat_uof_page *, u_int);
static int	qat_aefw_do_pagein(struct qat_softc *, u_char,
		    struct qat_uof_page *);
static int	qat_aefw_uof_write_one(struct qat_softc *,
		    struct qat_uof_image *);
static int	qat_aefw_uof_write(struct qat_softc *);

static int
qat_ae_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset,
	uint32_t value)
{
	int times = TIMEOUT_AE_CSR;

	do {
		qat_ae_local_write_4(sc, ae, offset, value);
		if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) &
		    LOCAL_CSR_STATUS_STATUS) == 0)
			return 0;

	} while (times--);

	device_printf(sc->sc_dev,
	    "couldn't write AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset);
	return EFAULT;
}

static int
qat_ae_read_4(struct qat_softc *sc, u_char ae, bus_size_t offset,
	uint32_t *value)
{
	int times = TIMEOUT_AE_CSR;
	uint32_t v;

	do {
		v = qat_ae_local_read_4(sc, ae, offset);
		if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) &
		    LOCAL_CSR_STATUS_STATUS) == 0) {
			*value = v;
			return 0;
		}
	} while (times--);

	device_printf(sc->sc_dev,
	    "couldn't read AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset);
	return EFAULT;
}

static void
qat_ae_ctx_indr_write(struct qat_softc *sc, u_char ae, uint32_t ctx_mask,
    bus_size_t offset, uint32_t value)
{
	int ctx;
	uint32_t ctxptr;

	MPASS(offset == CTX_FUTURE_COUNT_INDIRECT ||
	    offset == FUTURE_COUNT_SIGNAL_INDIRECT ||
	    offset == CTX_STS_INDIRECT ||
	    offset == CTX_WAKEUP_EVENTS_INDIRECT ||
	    offset == CTX_SIG_EVENTS_INDIRECT ||
	    offset == LM_ADDR_0_INDIRECT ||
	    offset == LM_ADDR_1_INDIRECT ||
	    offset == INDIRECT_LM_ADDR_0_BYTE_INDEX ||
	    offset == INDIRECT_LM_ADDR_1_BYTE_INDEX);

	qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr);
	for (ctx = 0; ctx < MAX_AE_CTX; ctx++) {
		if ((ctx_mask & (1 << ctx)) == 0)
			continue;
		qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx);
		qat_ae_write_4(sc, ae, offset, value);
	}
	qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr);
}

static int
qat_ae_ctx_indr_read(struct qat_softc *sc, u_char ae, uint32_t ctx,
    bus_size_t offset, uint32_t *value)
{
	int error;
	uint32_t ctxptr;

	MPASS(offset == CTX_FUTURE_COUNT_INDIRECT ||
	    offset == FUTURE_COUNT_SIGNAL_INDIRECT ||
	    offset == CTX_STS_INDIRECT ||
	    offset == CTX_WAKEUP_EVENTS_INDIRECT ||
	    offset == CTX_SIG_EVENTS_INDIRECT ||
	    offset == LM_ADDR_0_INDIRECT ||
	    offset == LM_ADDR_1_INDIRECT ||
	    offset == INDIRECT_LM_ADDR_0_BYTE_INDEX ||
	    offset == INDIRECT_LM_ADDR_1_BYTE_INDEX);

	/* save the ctx ptr */
	qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr);
	if ((ctxptr & CSR_CTX_POINTER_CONTEXT) !=
	    (ctx & CSR_CTX_POINTER_CONTEXT))
		qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx);

	error = qat_ae_read_4(sc, ae, offset, value);

	/* restore ctx ptr */
	if ((ctxptr & CSR_CTX_POINTER_CONTEXT) !=
	    (ctx & CSR_CTX_POINTER_CONTEXT))
		qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr);

	return error;
}

static u_short
qat_aereg_get_10bit_addr(enum aereg_type regtype, u_short reg)
{
	u_short addr;

	switch (regtype) {
	case AEREG_GPA_ABS:
	case AEREG_GPB_ABS:
		addr = (reg & 0x7f) | 0x80;
		break;
	case AEREG_GPA_REL:
	case AEREG_GPB_REL:
		addr = reg & 0x1f;
		break;
	case AEREG_SR_RD_REL:
	case AEREG_SR_WR_REL:
	case AEREG_SR_REL:
		addr = 0x180 | (reg & 0x1f);
		break;
	case AEREG_SR_INDX:
		addr = 0x140 | ((reg & 0x3) << 1);
		break;
	case AEREG_DR_RD_REL:
	case AEREG_DR_WR_REL:
	case AEREG_DR_REL:
		addr = 0x1c0 | (reg & 0x1f);
		break;
	case AEREG_DR_INDX:
		addr = 0x100 | ((reg & 0x3) << 1);
		break;
	case AEREG_NEIGH_INDX:
		addr = 0x241 | ((reg & 0x3) << 1);
		break;
	case AEREG_NEIGH_REL:
		addr = 0x280 | (reg & 0x1f);
		break;
	case AEREG_LMEM0:
		addr = 0x200;
		break;
	case AEREG_LMEM1:
		addr = 0x220;
		break;
	case AEREG_NO_DEST:
		addr = 0x300 | (reg & 0xff);
		break;
	default:
		addr = AEREG_BAD_REGADDR;
		break;
	}
	return (addr);
}

static int
qat_aereg_rel_data_write(struct qat_softc *sc, u_char ae, u_char ctx,
    enum aereg_type regtype, u_short relreg, uint32_t value)
{
	uint16_t srchi, srclo, destaddr, data16hi, data16lo;
	uint64_t inst[] = {
		0x0F440000000ull,	/* immed_w1[reg, val_hi16] */
		0x0F040000000ull,	/* immed_w0[reg, val_lo16] */
		0x0F0000C0300ull,	/* nop */
		0x0E000010000ull	/* ctx_arb[kill] */
	};
	const int ninst = nitems(inst);
	const int imm_w1 = 0, imm_w0 = 1;
	unsigned int ctxen;
	uint16_t mask;

	/* This logic only works for GPRs and LM index registers,
	   not NN or XFER registers! */
	MPASS(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL ||
	    regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1);

	if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL)) {
		/* determine the context mode */
		qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
		if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
			/* 4-ctx mode */
			if (ctx & 0x1)
				return EINVAL;
			mask = 0x1f;
		} else {
			/* 8-ctx mode */
			mask = 0x0f;
		}
		if (relreg & ~mask)
			return EINVAL;
	}
	if ((destaddr = qat_aereg_get_10bit_addr(regtype, relreg)) ==
	    AEREG_BAD_REGADDR) {
		return EINVAL;
	}

	data16lo = 0xffff & value;
	data16hi = 0xffff & (value >> 16);
	srchi = qat_aereg_get_10bit_addr(AEREG_NO_DEST,
		(uint16_t)(0xff & data16hi));
	srclo = qat_aereg_get_10bit_addr(AEREG_NO_DEST,
		(uint16_t)(0xff & data16lo));

	switch (regtype) {
	case AEREG_GPA_REL:	/* A rel source */
		inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) |
		    ((srchi & 0x3ff) << 10) | (destaddr & 0x3ff);
		inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) |
		    ((srclo & 0x3ff) << 10) | (destaddr & 0x3ff);
		break;
	default:
		inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) |
		    ((destaddr & 0x3ff) << 10) | (srchi & 0x3ff);
		inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) |
		    ((destaddr & 0x3ff) << 10) | (srclo & 0x3ff);
		break;
	}

	return qat_ae_exec_ucode(sc, ae, ctx, inst, ninst, 1, ninst * 5, NULL);
}

static int
qat_aereg_rel_data_read(struct qat_softc *sc, u_char ae, u_char ctx,
    enum aereg_type regtype, u_short relreg, uint32_t *value)
{
	uint64_t inst, savucode;
	uint32_t ctxen, misc, nmisc, savctx, ctxarbctl, ulo, uhi;
	u_int uaddr, ustore_addr;
	int error;
	u_short mask, regaddr;
	u_char nae;

	MPASS(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL ||
	    regtype == AEREG_SR_REL || regtype == AEREG_SR_RD_REL ||
	    regtype == AEREG_DR_REL || regtype == AEREG_DR_RD_REL ||
	    regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1);

	if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL) ||
	    (regtype == AEREG_SR_REL) || (regtype == AEREG_SR_RD_REL) ||
	    (regtype == AEREG_DR_REL) || (regtype == AEREG_DR_RD_REL))
	{
		/* determine the context mode */
		qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
		if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
			/* 4-ctx mode */
			if (ctx & 0x1)
				return EINVAL;
			mask = 0x1f;
		} else {
			/* 8-ctx mode */
			mask = 0x0f;
		}
		if (relreg & ~mask)
			return EINVAL;
	}
	if ((regaddr = qat_aereg_get_10bit_addr(regtype, relreg)) ==
	    AEREG_BAD_REGADDR) {
		return EINVAL;
	}

	/* instruction -- alu[--, --, B, reg] */
	switch (regtype) {
	case AEREG_GPA_REL:
		/* A rel source */
		inst = 0xA070000000ull | (regaddr & 0x3ff);
		break;
	default:
		inst = (0xA030000000ull | ((regaddr & 0x3ff) << 10));
		break;
	}

	/* backup shared control store bit, and force AE to
	 * none-shared mode before executing ucode snippet */
	qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
	if (misc & AE_MISC_CONTROL_SHARE_CS) {
		qat_ae_get_shared_ustore_ae(ae, &nae);
		if ((1 << nae) & sc->sc_ae_mask && qat_ae_is_active(sc, nae))
			return EBUSY;
	}

	nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS;
	qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc);

	/* read current context */
	qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx);
	qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl);

	qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
	/* prevent clearing the W1C bits: the breakpoint bit,
	ECC error bit, and Parity error bit */
	ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;

	/* change the context */
	if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO))
		qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
		    ctx & ACTIVE_CTX_STATUS_ACNO);
	/* save a ustore location */
	if ((error = qat_ae_ucode_read(sc, ae, 0, 1, &savucode)) != 0) {
		/* restore AE_MISC_CONTROL csr */
		qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);

		/* restore the context */
		if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) {
			qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
			    savctx & ACTIVE_CTX_STATUS_ACNO);
		}
		qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl);

		return (error);
	}

	/* turn off ustore parity */
	qat_ae_write_4(sc, ae, CTX_ENABLES,
	    ctxen & (~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE));

	/* save ustore-addr csr */
	qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr);

	/* write the ALU instruction to ustore, enable ecs bit */
	uaddr = 0 | USTORE_ADDRESS_ECS;

	/* set the uaddress */
	qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);
	inst = qat_ae_ucode_set_ecc(inst);

	ulo = (uint32_t)(inst & 0xffffffff);
	uhi = (uint32_t)(inst >> 32);

	qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo);

	/* this will auto increment the address */
	qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi);

	/* set the uaddress */
	qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);

	/* delay for at least 8 cycles */
	qat_ae_wait_num_cycles(sc, ae, 0x8, 0);

	/* read ALU output -- the instruction should have been executed
	prior to clearing the ECS in putUwords */
	qat_ae_read_4(sc, ae, ALU_OUT, value);

	/* restore ustore-addr csr */
	qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr);

	/* restore the ustore */
	error = qat_ae_ucode_write(sc, ae, 0, 1, &savucode);

	/* restore the context */
	if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) {
		qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
		    savctx & ACTIVE_CTX_STATUS_ACNO);
	}

	qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl);

	/* restore AE_MISC_CONTROL csr */
	qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);

	qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);

	return error;
}

static int
qat_aereg_rel_rdxfer_write(struct qat_softc *sc, u_char ae, u_char ctx,
    enum aereg_type regtype, u_short relreg, uint32_t value)
{
	bus_size_t addr;
	int error;
	uint32_t ctxen;
	u_short mask;
	u_short dr_offset;

	MPASS(regtype == AEREG_SR_REL || regtype == AEREG_DR_REL ||
	    regtype == AEREG_SR_RD_REL || regtype == AEREG_DR_RD_REL);

	error = qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
	if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
		if (ctx & 0x1) {
			device_printf(sc->sc_dev,
			    "bad ctx argument in 4-ctx mode,ctx=0x%x\n", ctx);
			return EINVAL;
		}
		mask = 0x1f;
		dr_offset = 0x20;

	} else {
		mask = 0x0f;
		dr_offset = 0x10;
	}

	if (relreg & ~mask)
		return EINVAL;

	addr = relreg + (ctx << 0x5);

	switch (regtype) {
	case AEREG_SR_REL:
	case AEREG_SR_RD_REL:
		qat_ae_xfer_write_4(sc, ae, addr, value);
		break;
	case AEREG_DR_REL:
	case AEREG_DR_RD_REL:
		qat_ae_xfer_write_4(sc, ae, addr + dr_offset, value);
		break;
	default:
		error = EINVAL;
	}

	return error;
}

static int
qat_aereg_rel_wrxfer_write(struct qat_softc *sc, u_char ae, u_char ctx,
    enum aereg_type regtype, u_short relreg, uint32_t value)
{

	panic("notyet");

	return 0;
}

static int
qat_aereg_rel_nn_write(struct qat_softc *sc, u_char ae, u_char ctx,
    enum aereg_type regtype, u_short relreg, uint32_t value)
{

	panic("notyet");

	return 0;
}

static int
qat_aereg_abs_to_rel(struct qat_softc *sc, u_char ae,
	u_short absreg, u_short *relreg, u_char *ctx)
{
	uint32_t ctxen;

	qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
	if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
		/* 4-ctx mode */
		*relreg = absreg & 0x1f;
		*ctx = (absreg >> 0x4) & 0x6;
	} else {
		/* 8-ctx mode */
		*relreg = absreg & 0x0f;
		*ctx = (absreg >> 0x4) & 0x7;
	}

	return 0;
}

static int
qat_aereg_abs_data_write(struct qat_softc *sc, u_char ae,
	enum aereg_type regtype, u_short absreg, uint32_t value)
{
	int error;
	u_short relreg;
	u_char ctx;

	qat_aereg_abs_to_rel(sc, ae, absreg, &relreg, &ctx);

	switch (regtype) {
	case AEREG_GPA_ABS:
		MPASS(absreg < MAX_GPR_REG);
		error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL,
		    relreg, value);
		break;
	case AEREG_GPB_ABS:
		MPASS(absreg < MAX_GPR_REG);
		error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL,
		    relreg, value);
		break;
	case AEREG_DR_RD_ABS:
		MPASS(absreg < MAX_XFER_REG);
		error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_DR_RD_REL,
		    relreg, value);
		break;
	case AEREG_SR_RD_ABS:
		MPASS(absreg < MAX_XFER_REG);
		error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_SR_RD_REL,
		    relreg, value);
		break;
	case AEREG_DR_WR_ABS:
		MPASS(absreg < MAX_XFER_REG);
		error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_DR_WR_REL,
		    relreg, value);
		break;
	case AEREG_SR_WR_ABS:
		MPASS(absreg < MAX_XFER_REG);
		error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_SR_WR_REL,
		    relreg, value);
		break;
	case AEREG_NEIGH_ABS:
		MPASS(absreg < MAX_NN_REG);
		if (absreg >= MAX_NN_REG)
			return EINVAL;
		error = qat_aereg_rel_nn_write(sc, ae, ctx, AEREG_NEIGH_REL,
		    relreg, value);
		break;
	default:
		panic("Invalid Register Type");
	}

	return error;
}

static void
qat_ae_enable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask)
{
	uint32_t ctxen;

	qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
	ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;

	if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
		ctx_mask &= 0x55;
	} else {
		ctx_mask &= 0xff;
	}

	ctxen |= __SHIFTIN(ctx_mask, CTX_ENABLES_ENABLE);
	qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);
}

static void
qat_ae_disable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask)
{
	uint32_t ctxen;

	qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
	ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;
	ctxen &= ~(__SHIFTIN(ctx_mask & AE_ALL_CTX, CTX_ENABLES_ENABLE));
	qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);
}

static void
qat_ae_write_ctx_mode(struct qat_softc *sc, u_char ae, u_char mode)
{
	uint32_t val, nval;

	qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
	val &= CTX_ENABLES_IGNORE_W1C_MASK;

	if (mode == 4)
		nval = val | CTX_ENABLES_INUSE_CONTEXTS;
	else
		nval = val & ~CTX_ENABLES_INUSE_CONTEXTS;

	if (val != nval)
		qat_ae_write_4(sc, ae, CTX_ENABLES, nval);
}

static void
qat_ae_write_nn_mode(struct qat_softc *sc, u_char ae, u_char mode)
{
	uint32_t val, nval;

	qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
	val &= CTX_ENABLES_IGNORE_W1C_MASK;

	if (mode)
		nval = val | CTX_ENABLES_NN_MODE;
	else
		nval = val & ~CTX_ENABLES_NN_MODE;

	if (val != nval)
		qat_ae_write_4(sc, ae, CTX_ENABLES, nval);
}

static void
qat_ae_write_lm_mode(struct qat_softc *sc, u_char ae,
	enum aereg_type lm, u_char mode)
{
	uint32_t val, nval;
	uint32_t bit;

	qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
	val &= CTX_ENABLES_IGNORE_W1C_MASK;

	switch (lm) {
	case AEREG_LMEM0:
		bit = CTX_ENABLES_LMADDR_0_GLOBAL;
		break;
	case AEREG_LMEM1:
		bit = CTX_ENABLES_LMADDR_1_GLOBAL;
		break;
	default:
		panic("invalid lmem reg type");
		break;
	}

	if (mode)
		nval = val | bit;
	else
		nval = val & ~bit;

	if (val != nval)
		qat_ae_write_4(sc, ae, CTX_ENABLES, nval);
}

static void
qat_ae_write_shared_cs_mode0(struct qat_softc *sc, u_char ae, u_char mode)
{
	uint32_t val, nval;

	qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val);

	if (mode == 1)
		nval = val | AE_MISC_CONTROL_SHARE_CS;
	else
		nval = val & ~AE_MISC_CONTROL_SHARE_CS;

	if (val != nval)
		qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nval);
}

static void
qat_ae_write_shared_cs_mode(struct qat_softc *sc, u_char ae, u_char mode)
{
	u_char nae;

	qat_ae_get_shared_ustore_ae(ae, &nae);

	qat_ae_write_shared_cs_mode0(sc, ae, mode);

	if ((sc->sc_ae_mask & (1 << nae))) {
		qat_ae_write_shared_cs_mode0(sc, nae, mode);
	}
}

static int
qat_ae_set_reload_ustore(struct qat_softc *sc, u_char ae,
	u_int reload_size, int shared_mode, u_int ustore_dram_addr)
{
	uint32_t val, cs_reload;

	switch (reload_size) {
	case 0:
		cs_reload = 0x0;
		break;
	case QAT_2K:
		cs_reload = 0x1;
		break;
	case QAT_4K:
		cs_reload = 0x2;
		break;
	case QAT_8K:
		cs_reload = 0x3;
		break;
	default:
		return EINVAL;
	}

	if (cs_reload)
		QAT_AE(sc, ae).qae_ustore_dram_addr = ustore_dram_addr;

	QAT_AE(sc, ae).qae_reload_size = reload_size;

	qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val);
	val &= ~(AE_MISC_CONTROL_ONE_CTX_RELOAD |
	    AE_MISC_CONTROL_CS_RELOAD | AE_MISC_CONTROL_SHARE_CS);
	val |= __SHIFTIN(cs_reload, AE_MISC_CONTROL_CS_RELOAD) |
	    __SHIFTIN(shared_mode, AE_MISC_CONTROL_ONE_CTX_RELOAD);
	qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val);

	return 0;
}

static enum qat_ae_status
qat_ae_get_status(struct qat_softc *sc, u_char ae)
{
	int error;
	uint32_t val = 0;

	error = qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
	if (error || val & CTX_ENABLES_ENABLE)
		return QAT_AE_ENABLED;

	qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val);
	if (val & ACTIVE_CTX_STATUS_ABO)
		return QAT_AE_ACTIVE;

	return QAT_AE_DISABLED;
}


static int
qat_ae_is_active(struct qat_softc *sc, u_char ae)
{
	uint32_t val;

	if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED)
		return 1;

	qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val);
	if (val & ACTIVE_CTX_STATUS_ABO)
		return 1;
	else
		return 0;
}

/* returns 1 if actually waited for specified number of cycles */
static int
qat_ae_wait_num_cycles(struct qat_softc *sc, u_char ae, int cycles, int check)
{
	uint32_t cnt, actx;
	int pcnt, ccnt, elapsed, times;

	qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt);
	pcnt = cnt & 0xffff;

	times = TIMEOUT_AE_CHECK;
	do {
		qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt);
		ccnt = cnt & 0xffff;

		elapsed = ccnt - pcnt;
		if (elapsed == 0) {
			times--;
		}
		if (times <= 0) {
			device_printf(sc->sc_dev,
			    "qat_ae_wait_num_cycles timeout\n");
			return -1;
		}

		if (elapsed < 0)
			elapsed += 0x10000;

		if (elapsed >= CYCLES_FROM_READY2EXE && check) {
			if (qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS,
			    &actx) == 0) {
				if ((actx & ACTIVE_CTX_STATUS_ABO) == 0)
					return 0;
			}
		}
	} while (cycles > elapsed);

	if (check && qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &actx) == 0) {
		if ((actx & ACTIVE_CTX_STATUS_ABO) == 0)
			return 0;
	}

	return 1;
}

int
qat_ae_init(struct qat_softc *sc)
{
	int error;
	uint32_t mask, val = 0;
	u_char ae;

	/* XXX adf_initSysMemInfo */

	/* XXX Disable clock gating for some chip if debug mode */

	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
		struct qat_ae *qae = &sc->sc_ae[ae];
		if (!(mask & 1))
			continue;

		qae->qae_ustore_size = USTORE_SIZE;

		qae->qae_free_addr = 0;
		qae->qae_free_size = USTORE_SIZE;
		qae->qae_live_ctx_mask = AE_ALL_CTX;
		qae->qae_ustore_dram_addr = 0;
		qae->qae_reload_size = 0;
	}

	/* XXX Enable attention interrupt */

	error = qat_ae_clear_reset(sc);
	if (error)
		return error;

	qat_ae_clear_xfer(sc);

	if (!sc->sc_hw.qhw_fw_auth) {
		error = qat_ae_clear_gprs(sc);
		if (error)
			return error;
	}

	/* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */
	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
		if (!(mask & 1))
			continue;
		qat_ae_read_4(sc, ae, SIGNATURE_ENABLE, &val);
		val |= 0x1;
		qat_ae_write_4(sc, ae, SIGNATURE_ENABLE, val);
	}

	error = qat_ae_clear_reset(sc);
	if (error)
		return error;

	/* XXX XXX XXX Clean MMP memory if mem scrub is supported */
	/* halMem_ScrubMMPMemory */

	return 0;
}

int
qat_ae_start(struct qat_softc *sc)
{
	int error;
	u_char ae;

	for (ae = 0; ae < sc->sc_ae_num; ae++) {
		if ((sc->sc_ae_mask & (1 << ae)) == 0)
			continue;

		error = qat_aefw_start(sc, ae, 0xff);
		if (error)
			return error;
	}

	return 0;
}

void
qat_ae_cluster_intr(void *arg)
{
	/* Nothing to implement until we support SRIOV. */
	printf("qat_ae_cluster_intr\n");
}

static int
qat_ae_clear_reset(struct qat_softc *sc)
{
	int error;
	uint32_t times, reset, clock, reg, mask;
	u_char ae;

	reset = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET);
	reset &= ~(__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK));
	reset &= ~(__SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK));
	times = TIMEOUT_AE_RESET;
	do {
		qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_RESET, reset);
		if ((times--) == 0) {
			device_printf(sc->sc_dev, "couldn't reset AEs\n");
			return EBUSY;
		}
		reg = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET);
	} while ((__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK) |
	    __SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK))
	    & reg);

	/* Enable clock for AE and QAT */
	clock = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_CLK_EN);
	clock |= __SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_CLK_EN_AE_MASK);
	clock |= __SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_CLK_EN_ACCEL_MASK);
	qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_CLK_EN, clock);

	error = qat_ae_check(sc);
	if (error)
		return error;

	/*
	 * Set undefined power-up/reset states to reasonable default values...
	 * just to make sure we're starting from a known point
	 */
	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
		if (!(mask & 1))
			continue;

		/* init the ctx_enable */
		qat_ae_write_4(sc, ae, CTX_ENABLES,
		    CTX_ENABLES_INIT);

		/* initialize the PCs */
		qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
		    CTX_STS_INDIRECT,
		    UPC_MASK & CTX_STS_INDIRECT_UPC_INIT);

		/* init the ctx_arb */
		qat_ae_write_4(sc, ae, CTX_ARB_CNTL,
		    CTX_ARB_CNTL_INIT);

		/* enable cc */
		qat_ae_write_4(sc, ae, CC_ENABLE,
		    CC_ENABLE_INIT);
		qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
		    CTX_WAKEUP_EVENTS_INDIRECT,
		    CTX_WAKEUP_EVENTS_INDIRECT_INIT);
		qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
		    CTX_SIG_EVENTS_INDIRECT,
		    CTX_SIG_EVENTS_INDIRECT_INIT);
	}

	if ((sc->sc_ae_mask != 0) &&
	    sc->sc_flags & QAT_FLAG_ESRAM_ENABLE_AUTO_INIT) {
		/* XXX XXX XXX init eSram only when this is boot time */
	}

	if ((sc->sc_ae_mask != 0) &&
	    sc->sc_flags & QAT_FLAG_SHRAM_WAIT_READY) {
		/* XXX XXX XXX wait shram to complete initialization */
	}

	qat_ae_reset_timestamp(sc);

	return 0;
}

static int
qat_ae_check(struct qat_softc *sc)
{
	int error, times, ae;
	uint32_t cnt, pcnt, mask;

	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
		if (!(mask & 1))
			continue;

		times = TIMEOUT_AE_CHECK;
		error = qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt);
		if (error) {
			device_printf(sc->sc_dev,
			    "couldn't access AE %d CSR\n", ae);
			return error;
		}
		pcnt = cnt & 0xffff;

		while (1) {
			error = qat_ae_read_4(sc, ae,
			    PROFILE_COUNT, &cnt);
			if (error) {
				device_printf(sc->sc_dev,
				    "couldn't access AE %d CSR\n", ae);
				return error;
			}
			cnt &= 0xffff;
			if (cnt == pcnt)
				times--;
			else
				break;
			if (times <= 0) {
				device_printf(sc->sc_dev,
				    "AE %d CSR is useless\n", ae);
				return EFAULT;
			}
		}
	}

	return 0;
}

static int
qat_ae_reset_timestamp(struct qat_softc *sc)
{
	uint32_t misc, mask;
	u_char ae;

	/* stop the timestamp timers */
	misc = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_MISC);
	if (misc & CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN) {
		qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC,
		    misc & (~CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN));
	}

	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
		if (!(mask & 1))
			continue;
		qat_ae_write_4(sc, ae, TIMESTAMP_LOW, 0);
		qat_ae_write_4(sc, ae, TIMESTAMP_HIGH, 0);
	}

	/* start timestamp timers */
	qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC,
	    misc | CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN);

	return 0;
}

static void
qat_ae_clear_xfer(struct qat_softc *sc)
{
	u_int mask, reg;
	u_char ae;

	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
		if (!(mask & 1))
			continue;

		for (reg = 0; reg < MAX_GPR_REG; reg++) {
			qat_aereg_abs_data_write(sc, ae, AEREG_SR_RD_ABS,
			    reg, 0);
			qat_aereg_abs_data_write(sc, ae, AEREG_DR_RD_ABS,
			    reg, 0);
		}
	}
}

static int
qat_ae_clear_gprs(struct qat_softc *sc)
{
	uint32_t val;
	uint32_t saved_ctx = 0;
	int times = TIMEOUT_AE_CHECK, rv;
	u_char ae;
	u_int mask;

	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
		if (!(mask & 1))
			continue;

		/* turn off share control store bit */
		val = qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val);
		val &= ~AE_MISC_CONTROL_SHARE_CS;
		qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val);

		/* turn off ucode parity */
		/* make sure nn_mode is set to self */
		qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
		val &= CTX_ENABLES_IGNORE_W1C_MASK;
		val |= CTX_ENABLES_NN_MODE;
		val &= ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE;
		qat_ae_write_4(sc, ae, CTX_ENABLES, val);

		/* copy instructions to ustore */
		qat_ae_ucode_write(sc, ae, 0, nitems(ae_clear_gprs_inst),
		    ae_clear_gprs_inst);

		/* set PC */
		qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_STS_INDIRECT,
		    UPC_MASK & CTX_STS_INDIRECT_UPC_INIT);

		/* save current context */
		qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &saved_ctx);
		/* change the active context */
		/* start the context from ctx 0 */
		qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 0);

		/* wakeup-event voluntary */
		qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
		    CTX_WAKEUP_EVENTS_INDIRECT,
		    CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY);
		/* clean signals */
		qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
		    CTX_SIG_EVENTS_INDIRECT, 0);
		qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0);

		qat_ae_enable_ctx(sc, ae, AE_ALL_CTX);
	}

	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
		if (!(mask & 1))
			continue;
		/* wait for AE to finish */
		do {
			rv = qat_ae_wait_num_cycles(sc, ae, AE_EXEC_CYCLE, 1);
		} while (rv && times--);
		if (times <= 0) {
			device_printf(sc->sc_dev,
			    "qat_ae_clear_gprs timeout");
			return ETIMEDOUT;
		}
		qat_ae_disable_ctx(sc, ae, AE_ALL_CTX);
		/* change the active context */
		qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
		    saved_ctx & ACTIVE_CTX_STATUS_ACNO);
		/* init the ctx_enable */
		qat_ae_write_4(sc, ae, CTX_ENABLES, CTX_ENABLES_INIT);
		/* initialize the PCs */
		qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
		    CTX_STS_INDIRECT, UPC_MASK & CTX_STS_INDIRECT_UPC_INIT);
		/* init the ctx_arb */
		qat_ae_write_4(sc, ae, CTX_ARB_CNTL, CTX_ARB_CNTL_INIT);
		/* enable cc */
		qat_ae_write_4(sc, ae, CC_ENABLE, CC_ENABLE_INIT);
		qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
		    CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_INIT);
		qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_SIG_EVENTS_INDIRECT,
		    CTX_SIG_EVENTS_INDIRECT_INIT);
	}

	return 0;
}

static void
qat_ae_get_shared_ustore_ae(u_char ae, u_char *nae)
{
	if (ae & 0x1)
		*nae = ae - 1;
	else
		*nae = ae + 1;
}

static u_int
qat_ae_ucode_parity64(uint64_t ucode)
{

	ucode ^= ucode >> 1;
	ucode ^= ucode >> 2;
	ucode ^= ucode >> 4;
	ucode ^= ucode >> 8;
	ucode ^= ucode >> 16;
	ucode ^= ucode >> 32;

	return ((u_int)(ucode & 1));
}

static uint64_t
qat_ae_ucode_set_ecc(uint64_t ucode)
{
	static const uint64_t
		bit0mask=0xff800007fffULL, bit1mask=0x1f801ff801fULL,
		bit2mask=0xe387e0781e1ULL, bit3mask=0x7cb8e388e22ULL,
		bit4mask=0xaf5b2c93244ULL, bit5mask=0xf56d5525488ULL,
		bit6mask=0xdaf69a46910ULL;

	/* clear the ecc bits */
	ucode &= ~(0x7fULL << USTORE_ECC_BIT_0);

	ucode |= (uint64_t)qat_ae_ucode_parity64(bit0mask & ucode) <<
	    USTORE_ECC_BIT_0;
	ucode |= (uint64_t)qat_ae_ucode_parity64(bit1mask & ucode) <<
	    USTORE_ECC_BIT_1;
	ucode |= (uint64_t)qat_ae_ucode_parity64(bit2mask & ucode) <<
	    USTORE_ECC_BIT_2;
	ucode |= (uint64_t)qat_ae_ucode_parity64(bit3mask & ucode) <<
	    USTORE_ECC_BIT_3;
	ucode |= (uint64_t)qat_ae_ucode_parity64(bit4mask & ucode) <<
	    USTORE_ECC_BIT_4;
	ucode |= (uint64_t)qat_ae_ucode_parity64(bit5mask & ucode) <<
	    USTORE_ECC_BIT_5;
	ucode |= (uint64_t)qat_ae_ucode_parity64(bit6mask & ucode) <<
	    USTORE_ECC_BIT_6;

	return (ucode);
}

static int
qat_ae_ucode_write(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst,
	const uint64_t *ucode)
{
	uint64_t tmp;
	uint32_t ustore_addr, ulo, uhi;
	int i;

	qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr);
	uaddr |= USTORE_ADDRESS_ECS;

	qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);
	for (i = 0; i < ninst; i++) {
		tmp = qat_ae_ucode_set_ecc(ucode[i]);
		ulo = (uint32_t)(tmp & 0xffffffff);
		uhi = (uint32_t)(tmp >> 32);

		qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo);
		/* this will auto increment the address */
		qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi);
	}
	qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr);

	return 0;
}

static int
qat_ae_ucode_read(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst,
    uint64_t *ucode)
{
	uint32_t misc, ustore_addr, ulo, uhi;
	u_int ii;
	u_char nae;

	if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED)
		return EBUSY;

	/* determine whether it neighbour AE runs in shared control store
	 * status */
	qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
	if (misc & AE_MISC_CONTROL_SHARE_CS) {
		qat_ae_get_shared_ustore_ae(ae, &nae);
		if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae))
			return EBUSY;
	}

	/* if reloadable, then get it all from dram-ustore */
	if (__SHIFTOUT(misc, AE_MISC_CONTROL_CS_RELOAD))
		panic("notyet"); /* XXX getReloadUwords */

	/* disable SHARE_CS bit to workaround silicon bug */
	qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc & 0xfffffffb);

	MPASS(uaddr + ninst <= USTORE_SIZE);

	/* save ustore-addr csr */
	qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr);

	uaddr |= USTORE_ADDRESS_ECS;	/* enable ecs bit */
	for (ii = 0; ii < ninst; ii++) {
		qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);

		uaddr++;
		qat_ae_read_4(sc, ae, USTORE_DATA_LOWER, &ulo);
		qat_ae_read_4(sc, ae, USTORE_DATA_UPPER, &uhi);
		ucode[ii] = uhi;
		ucode[ii] = (ucode[ii] << 32) | ulo;
	}

	/* restore SHARE_CS bit to workaround silicon bug */
	qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);
	qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr);

	return 0;
}

static u_int
qat_ae_concat_ucode(uint64_t *ucode, u_int ninst, u_int size, u_int addr,
    u_int *value)
{
	const uint64_t *inst_arr;
	u_int ninst0, curvalue;
	int ii, vali, fixup, usize = 0;

	if (size == 0)
		return 0;

	ninst0 = ninst;
	vali = 0;
	curvalue = value[vali++];

	switch (size) {
	case 0x1:
		inst_arr = ae_inst_1b;
		usize = nitems(ae_inst_1b);
		break;
	case 0x2:
		inst_arr = ae_inst_2b;
		usize = nitems(ae_inst_2b);
		break;
	case 0x3:
		inst_arr = ae_inst_3b;
		usize = nitems(ae_inst_3b);
		break;
	default:
		inst_arr = ae_inst_4b;
		usize = nitems(ae_inst_4b);
		break;
	}

	fixup = ninst;
	for (ii = 0; ii < usize; ii++)
		ucode[ninst++] = inst_arr[ii];

	INSERT_IMMED_GPRA_CONST(ucode[fixup], (addr));
	fixup++;
	INSERT_IMMED_GPRA_CONST(ucode[fixup], 0);
	fixup++;
	INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0));
	fixup++;
	INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16));
	/* XXX fixup++ ? */

	if (size <= 0x4)
		return (ninst - ninst0);

	size -= sizeof(u_int);
	while (size >= sizeof(u_int)) {
		curvalue = value[vali++];
		fixup = ninst;
		ucode[ninst++] = ae_inst_4b[0x2];
		ucode[ninst++] = ae_inst_4b[0x3];
		ucode[ninst++] = ae_inst_4b[0x8];
		INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16));
		fixup++;
		INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0));
		/* XXX fixup++ ? */

		addr += sizeof(u_int);
		size -= sizeof(u_int);
	}
	/* call this function recusive when the left size less than 4 */
	ninst +=
	    qat_ae_concat_ucode(ucode, ninst, size, addr, value + vali);

	return (ninst - ninst0);
}

static int
qat_ae_exec_ucode(struct qat_softc *sc, u_char ae, u_char ctx,
    uint64_t *ucode, u_int ninst, int cond_code_off, u_int max_cycles,
    u_int *endpc)
{
	int error = 0, share_cs = 0;
	uint64_t savucode[MAX_EXEC_INST];
	uint32_t indr_lm_addr_0, indr_lm_addr_1;
	uint32_t indr_lm_addr_byte_0, indr_lm_addr_byte_1;
	uint32_t indr_future_cnt_sig;
	uint32_t indr_sig, active_sig;
	uint32_t wakeup_ev, savpc, savcc, savctx, ctxarbctl;
	uint32_t misc, nmisc, ctxen;
	u_char nae;

	MPASS(ninst <= USTORE_SIZE);

	if (qat_ae_is_active(sc, ae))
		return EBUSY;

	/* save current LM addr */
	qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_0_INDIRECT, &indr_lm_addr_0);
	qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_1_INDIRECT, &indr_lm_addr_1);
	qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX,
	    &indr_lm_addr_byte_0);
	qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX,
	    &indr_lm_addr_byte_1);

	/* backup shared control store bit, and force AE to
	   none-shared mode before executing ucode snippet */
	qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
	if (misc & AE_MISC_CONTROL_SHARE_CS) {
		share_cs = 1;
		qat_ae_get_shared_ustore_ae(ae, &nae);
		if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae))
			return EBUSY;
	}
	nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS;
	qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc);

	/* save current states: */
	if (ninst <= MAX_EXEC_INST) {
		error = qat_ae_ucode_read(sc, ae, 0, ninst, savucode);
		if (error) {
			qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);
			return error;
		}
	}

	/* save wakeup-events */
	qat_ae_ctx_indr_read(sc, ae, ctx, CTX_WAKEUP_EVENTS_INDIRECT,
	    &wakeup_ev);
	/* save PC */
	qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, &savpc);
	savpc &= UPC_MASK;

	/* save ctx enables */
	qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
	ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;
	/* save conditional-code */
	qat_ae_read_4(sc, ae, CC_ENABLE, &savcc);
	/* save current context */
	qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx);
	qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl);

	/* save indirect csrs */
	qat_ae_ctx_indr_read(sc, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT,
	    &indr_future_cnt_sig);
	qat_ae_ctx_indr_read(sc, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &indr_sig);
	qat_ae_read_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, &active_sig);

	/* turn off ucode parity */
	qat_ae_write_4(sc, ae, CTX_ENABLES,
	    ctxen & ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE);

	/* copy instructions to ustore */
	qat_ae_ucode_write(sc, ae, 0, ninst, ucode);
	/* set PC */
	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, 0);
	/* change the active context */
	qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
	    ctx & ACTIVE_CTX_STATUS_ACNO);

	if (cond_code_off) {
		/* disable conditional-code*/
		qat_ae_write_4(sc, ae, CC_ENABLE, savcc & 0xffffdfff);
	}

	/* wakeup-event voluntary */
	qat_ae_ctx_indr_write(sc, ae, 1 << ctx,
	    CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY);

	/* clean signals */
	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT, 0);
	qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0);

	/* enable context */
	qat_ae_enable_ctx(sc, ae, 1 << ctx);

	/* wait for it to finish */
	if (qat_ae_wait_num_cycles(sc, ae, max_cycles, 1) != 0)
		error = ETIMEDOUT;

	/* see if we need to get the current PC */
	if (endpc != NULL) {
		uint32_t ctx_status;

		qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT,
		    &ctx_status);
		*endpc = ctx_status & UPC_MASK;
	}
#if 0
	{
		uint32_t ctx_status;

		qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT,
		    &ctx_status);
		printf("%s: endpc 0x%08x\n", __func__,
		    ctx_status & UPC_MASK);
	}
#endif

	/* retore to previous states: */
	/* disable context */
	qat_ae_disable_ctx(sc, ae, 1 << ctx);
	if (ninst <= MAX_EXEC_INST) {
		/* instructions */
		qat_ae_ucode_write(sc, ae, 0, ninst, savucode);
	}
	/* wakeup-events */
	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_WAKEUP_EVENTS_INDIRECT,
	    wakeup_ev);
	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, savpc);

	/* only restore shared control store bit,
	   other bit might be changed by AE code snippet */
	qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
	if (share_cs)
		nmisc = misc | AE_MISC_CONTROL_SHARE_CS;
	else
		nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS;
	qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc);
	/* conditional-code */
	qat_ae_write_4(sc, ae, CC_ENABLE, savcc);
	/* change the active context */
	qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
	    savctx & ACTIVE_CTX_STATUS_ACNO);
	/* restore the nxt ctx to run */
	qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl);
	/* restore current LM addr */
	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_0_INDIRECT,
	    indr_lm_addr_0);
	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_1_INDIRECT,
	    indr_lm_addr_1);
	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX,
	    indr_lm_addr_byte_0);
	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX,
	    indr_lm_addr_byte_1);

	/* restore indirect csrs */
	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, FUTURE_COUNT_SIGNAL_INDIRECT,
	    indr_future_cnt_sig);
	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT,
	    indr_sig);
	qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, active_sig);

	/* ctx-enables */
	qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);

	return error;
}

static int
qat_ae_exec_ucode_init_lm(struct qat_softc *sc, u_char ae, u_char ctx,
    int *first_exec, uint64_t *ucode, u_int ninst,
    u_int *gpr_a0, u_int *gpr_a1, u_int *gpr_a2, u_int *gpr_b0, u_int *gpr_b1)
{

	if (*first_exec) {
		qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0);
		qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1);
		qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2);
		qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0);
		qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1);
		*first_exec = 0;
	}

	return qat_ae_exec_ucode(sc, ae, ctx, ucode, ninst, 1, ninst * 5, NULL);
}

static int
qat_ae_restore_init_lm_gprs(struct qat_softc *sc, u_char ae, u_char ctx,
    u_int gpr_a0, u_int gpr_a1, u_int gpr_a2, u_int gpr_b0, u_int gpr_b1)
{
	qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0);
	qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1);
	qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2);
	qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0);
	qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1);

	return 0;
}

static int
qat_ae_get_inst_num(int lmsize)
{
	int ninst, left;

	if (lmsize == 0)
		return 0;

	left = lmsize % sizeof(u_int);

	if (left) {
		ninst = nitems(ae_inst_1b) +
		    qat_ae_get_inst_num(lmsize - left);
	} else {
		/* 3 instruction is needed for further code */
		ninst = (lmsize - sizeof(u_int)) * 3 / 4 + nitems(ae_inst_4b);
	}

	return (ninst);
}

static int
qat_ae_batch_put_lm(struct qat_softc *sc, u_char ae,
    struct qat_ae_batch_init_list *qabi_list, size_t nqabi)
{
	struct qat_ae_batch_init *qabi;
	size_t alloc_ninst, ninst;
	uint64_t *ucode;
	u_int gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1;
	int insnsz, error = 0, execed = 0, first_exec = 1;

	if (STAILQ_FIRST(qabi_list) == NULL)
		return 0;

	alloc_ninst = min(USTORE_SIZE, nqabi);
	ucode = qat_alloc_mem(sizeof(uint64_t) * alloc_ninst);

	ninst = 0;
	STAILQ_FOREACH(qabi, qabi_list, qabi_next) {
		insnsz = qat_ae_get_inst_num(qabi->qabi_size);
		if (insnsz + ninst > alloc_ninst) {
			/* add ctx_arb[kill] */
			ucode[ninst++] = 0x0E000010000ull;
			execed = 1;

			error = qat_ae_exec_ucode_init_lm(sc, ae, 0,
			    &first_exec, ucode, ninst,
			    &gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1);
			if (error) {
				qat_ae_restore_init_lm_gprs(sc, ae, 0,
				    gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1);
				qat_free_mem(ucode);
				return error;
			}
			/* run microExec to execute the microcode */
			ninst = 0;
		}
		ninst += qat_ae_concat_ucode(ucode, ninst,
		    qabi->qabi_size, qabi->qabi_addr, qabi->qabi_value);
	}

	if (ninst > 0) {
		ucode[ninst++] = 0x0E000010000ull;
		execed = 1;

		error = qat_ae_exec_ucode_init_lm(sc, ae, 0,
		    &first_exec, ucode, ninst,
		    &gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1);
	}
	if (execed) {
		qat_ae_restore_init_lm_gprs(sc, ae, 0,
		    gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1);
	}

	qat_free_mem(ucode);

	return error;
}

static int
qat_ae_write_pc(struct qat_softc *sc, u_char ae, u_int ctx_mask, u_int upc)
{

	if (qat_ae_is_active(sc, ae))
		return EBUSY;

	qat_ae_ctx_indr_write(sc, ae, ctx_mask, CTX_STS_INDIRECT,
	    UPC_MASK & upc);
	return 0;
}

static inline u_int
qat_aefw_csum_calc(u_int reg, int ch)
{
	int i;
	u_int topbit = CRC_BITMASK(CRC_WIDTH - 1);
	u_int inbyte = (u_int)((reg >> 0x18) ^ ch);

	reg ^= inbyte << (CRC_WIDTH - 0x8);
	for (i = 0; i < 0x8; i++) {
		if (reg & topbit)
			reg = (reg << 1) ^ CRC_POLY;
		else
			reg <<= 1;
	}

	return (reg & CRC_WIDTHMASK(CRC_WIDTH));
}

static u_int
qat_aefw_csum(char *buf, int size)
{
	u_int csum = 0;

	while (size--) {
		csum = qat_aefw_csum_calc(csum, *buf++);
	}

	return csum;
}

static const char *
qat_aefw_uof_string(struct qat_softc *sc, size_t offset)
{
	if (offset >= sc->sc_aefw_uof.qafu_str_tab_size)
		return NULL;
	if (sc->sc_aefw_uof.qafu_str_tab == NULL)
		return NULL;

	return (const char *)((uintptr_t)sc->sc_aefw_uof.qafu_str_tab + offset);
}

static struct uof_chunk_hdr *
qat_aefw_uof_find_chunk(struct qat_softc *sc,
	const char *id, struct uof_chunk_hdr *cur)
{
	struct uof_obj_hdr *uoh = sc->sc_aefw_uof.qafu_obj_hdr;
	struct uof_chunk_hdr *uch;
	int i;

	uch = (struct uof_chunk_hdr *)(uoh + 1);
	for (i = 0; i < uoh->uoh_num_chunks; i++, uch++) {
		if (uch->uch_offset + uch->uch_size > sc->sc_aefw_uof.qafu_size)
			return NULL;

		if (cur < uch && !strncmp(uch->uch_id, id, UOF_OBJ_ID_LEN))
			return uch;
	}

	return NULL;
}

static int
qat_aefw_load_mof(struct qat_softc *sc)
{
	const struct firmware *fw;

	fw = firmware_get(sc->sc_hw.qhw_mof_fwname);
	if (fw == NULL) {
		device_printf(sc->sc_dev, "couldn't load MOF firmware %s\n",
		    sc->sc_hw.qhw_mof_fwname);
		return ENXIO;
	}

	sc->sc_fw_mof = qat_alloc_mem(fw->datasize);
	sc->sc_fw_mof_size = fw->datasize;
	memcpy(sc->sc_fw_mof, fw->data, fw->datasize);
	firmware_put(fw, FIRMWARE_UNLOAD);
	return 0;
}

static void
qat_aefw_unload_mof(struct qat_softc *sc)
{
	if (sc->sc_fw_mof != NULL) {
		qat_free_mem(sc->sc_fw_mof);
		sc->sc_fw_mof = NULL;
	}
}

static int
qat_aefw_load_mmp(struct qat_softc *sc)
{
	const struct firmware *fw;

	fw = firmware_get(sc->sc_hw.qhw_mmp_fwname);
	if (fw == NULL) {
		device_printf(sc->sc_dev, "couldn't load MOF firmware %s\n",
		    sc->sc_hw.qhw_mmp_fwname);
		return ENXIO;
	}

	sc->sc_fw_mmp = qat_alloc_mem(fw->datasize);
	sc->sc_fw_mmp_size = fw->datasize;
	memcpy(sc->sc_fw_mmp, fw->data, fw->datasize);
	firmware_put(fw, FIRMWARE_UNLOAD);
	return 0;
}

static void
qat_aefw_unload_mmp(struct qat_softc *sc)
{
	if (sc->sc_fw_mmp != NULL) {
		qat_free_mem(sc->sc_fw_mmp);
		sc->sc_fw_mmp = NULL;
	}
}

static int
qat_aefw_mof_find_uof0(struct qat_softc *sc,
	struct mof_uof_hdr *muh, struct mof_uof_chunk_hdr *head,
	u_int nchunk, size_t size, const char *id,
	size_t *fwsize, void **fwptr)
{
	int i;
	char *uof_name;

	for (i = 0; i < nchunk; i++) {
		struct mof_uof_chunk_hdr *much = &head[i];

		if (strncmp(much->much_id, id, MOF_OBJ_ID_LEN))
			return EINVAL;

		if (much->much_offset + much->much_size > size)
			return EINVAL;

		if (sc->sc_mof.qmf_sym_size <= much->much_name)
			return EINVAL;

		uof_name = (char *)((uintptr_t)sc->sc_mof.qmf_sym +
		    much->much_name);

		if (!strcmp(uof_name, sc->sc_fw_uof_name)) {
			*fwptr = (void *)((uintptr_t)muh +
			    (uintptr_t)much->much_offset);
			*fwsize = (size_t)much->much_size;
			return 0;
		}
	}

	return ENOENT;
}

static int
qat_aefw_mof_find_uof(struct qat_softc *sc)
{
	struct mof_uof_hdr *uof_hdr, *suof_hdr;
	u_int nuof_chunks = 0, nsuof_chunks = 0;
	int error;

	uof_hdr = sc->sc_mof.qmf_uof_objs;
	suof_hdr = sc->sc_mof.qmf_suof_objs;

	if (uof_hdr != NULL) {
		if (uof_hdr->muh_max_chunks < uof_hdr->muh_num_chunks) {
			return EINVAL;
		}
		nuof_chunks = uof_hdr->muh_num_chunks;
	}
	if (suof_hdr != NULL) {
		if (suof_hdr->muh_max_chunks < suof_hdr->muh_num_chunks)
			return EINVAL;
		nsuof_chunks = suof_hdr->muh_num_chunks;
	}

	if (nuof_chunks + nsuof_chunks == 0)
		return EINVAL;

	if (uof_hdr != NULL) {
		error = qat_aefw_mof_find_uof0(sc, uof_hdr,
		    (struct mof_uof_chunk_hdr *)(uof_hdr + 1), nuof_chunks,
		    sc->sc_mof.qmf_uof_objs_size, UOF_IMAG,
		    &sc->sc_fw_uof_size, &sc->sc_fw_uof);
		if (error && error != ENOENT)
			return error;
	}

	if (suof_hdr != NULL) {
		error = qat_aefw_mof_find_uof0(sc, suof_hdr,
		    (struct mof_uof_chunk_hdr *)(suof_hdr + 1), nsuof_chunks,
		    sc->sc_mof.qmf_suof_objs_size, SUOF_IMAG,
		    &sc->sc_fw_suof_size, &sc->sc_fw_suof);
		if (error && error != ENOENT)
			return error;
	}

	if (sc->sc_fw_uof == NULL && sc->sc_fw_suof == NULL)
		return ENOENT;

	return 0;
}

static int
qat_aefw_mof_parse(struct qat_softc *sc)
{
	const struct mof_file_hdr *mfh;
	const struct mof_file_chunk_hdr *mfch;
	size_t size;
	u_int csum;
	int error, i;

	size = sc->sc_fw_mof_size;

	if (size < sizeof(struct mof_file_hdr))
		return EINVAL;
	size -= sizeof(struct mof_file_hdr);

	mfh = sc->sc_fw_mof;

	if (mfh->mfh_fid != MOF_FID)
		return EINVAL;

	csum = qat_aefw_csum((char *)((uintptr_t)sc->sc_fw_mof +
	    offsetof(struct mof_file_hdr, mfh_min_ver)),
	    sc->sc_fw_mof_size -
	    offsetof(struct mof_file_hdr, mfh_min_ver));
	if (mfh->mfh_csum != csum)
		return EINVAL;

	if (mfh->mfh_min_ver != MOF_MIN_VER ||
	    mfh->mfh_maj_ver != MOF_MAJ_VER)
		return EINVAL;

	if (mfh->mfh_max_chunks < mfh->mfh_num_chunks)
		return EINVAL;

	if (size < sizeof(struct mof_file_chunk_hdr) * mfh->mfh_num_chunks)
		return EINVAL;
	mfch = (const struct mof_file_chunk_hdr *)(mfh + 1);

	for (i = 0; i < mfh->mfh_num_chunks; i++, mfch++) {
		if (mfch->mfch_offset + mfch->mfch_size > sc->sc_fw_mof_size)
			return EINVAL;

		if (!strncmp(mfch->mfch_id, SYM_OBJS, MOF_OBJ_ID_LEN)) {
			if (sc->sc_mof.qmf_sym != NULL)
				return EINVAL;

			sc->sc_mof.qmf_sym =
			    (void *)((uintptr_t)sc->sc_fw_mof +
			    (uintptr_t)mfch->mfch_offset + sizeof(u_int));
			sc->sc_mof.qmf_sym_size =
			    *(u_int *)((uintptr_t)sc->sc_fw_mof +
			    (uintptr_t)mfch->mfch_offset);

			if (sc->sc_mof.qmf_sym_size % sizeof(u_int) != 0)
				return EINVAL;
			if (mfch->mfch_size != sc->sc_mof.qmf_sym_size +
			    sizeof(u_int) || mfch->mfch_size == 0)
				return EINVAL;
			if (*(char *)((uintptr_t)sc->sc_mof.qmf_sym +
			    sc->sc_mof.qmf_sym_size - 1) != '\0')
				return EINVAL;

		} else if (!strncmp(mfch->mfch_id, UOF_OBJS, MOF_OBJ_ID_LEN)) {
			if (sc->sc_mof.qmf_uof_objs != NULL)
				return EINVAL;

			sc->sc_mof.qmf_uof_objs =
			    (void *)((uintptr_t)sc->sc_fw_mof +
			    (uintptr_t)mfch->mfch_offset);
			sc->sc_mof.qmf_uof_objs_size = mfch->mfch_size;

		} else if (!strncmp(mfch->mfch_id, SUOF_OBJS, MOF_OBJ_ID_LEN)) {
			if (sc->sc_mof.qmf_suof_objs != NULL)
				return EINVAL;

			sc->sc_mof.qmf_suof_objs =
			    (void *)((uintptr_t)sc->sc_fw_mof +
			    (uintptr_t)mfch->mfch_offset);
			sc->sc_mof.qmf_suof_objs_size = mfch->mfch_size;
		}
	}

	if (sc->sc_mof.qmf_sym == NULL ||
	    (sc->sc_mof.qmf_uof_objs == NULL &&
	    sc->sc_mof.qmf_suof_objs == NULL))
		return EINVAL;

	error = qat_aefw_mof_find_uof(sc);
	if (error)
		return error;
	return 0;
}

static int
qat_aefw_uof_parse_image(struct qat_softc *sc,
	struct qat_uof_image *qui, struct uof_chunk_hdr *uch)
{
	struct uof_image *image;
	struct uof_code_page *page;
	uintptr_t base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr;
	size_t lim = uch->uch_offset + uch->uch_size, size;
	int i, p;

	size = uch->uch_size;
	if (size < sizeof(struct uof_image))
		return EINVAL;
	size -= sizeof(struct uof_image);

	qui->qui_image = image =
	    (struct uof_image *)(base + uch->uch_offset);

#define ASSIGN_OBJ_TAB(np, typep, type, base, off, lim)			\
do {									\
	u_int nent;							\
	nent = ((struct uof_obj_table *)((base) + (off)))->uot_nentries;\
	if ((lim) < off + sizeof(struct uof_obj_table) +		\
	    sizeof(type) * nent)					\
		return EINVAL;						\
	*(np) = nent;							\
	if (nent > 0)							\
		*(typep) = (type)((struct uof_obj_table *)		\
		    ((base) + (off)) + 1);				\
	else								\
		*(typep) = NULL;					\
} while (0)

	ASSIGN_OBJ_TAB(&qui->qui_num_ae_reg, &qui->qui_ae_reg,
	    struct uof_ae_reg *, base, image->ui_reg_tab, lim);
	ASSIGN_OBJ_TAB(&qui->qui_num_init_reg_sym, &qui->qui_init_reg_sym,
	    struct uof_init_reg_sym *, base, image->ui_init_reg_sym_tab, lim);
	ASSIGN_OBJ_TAB(&qui->qui_num_sbreak, &qui->qui_sbreak,
	    struct qui_sbreak *, base, image->ui_sbreak_tab, lim);

	if (size < sizeof(struct uof_code_page) * image->ui_num_pages)
		return EINVAL;
	if (nitems(qui->qui_pages) < image->ui_num_pages)
		return EINVAL;

	page = (struct uof_code_page *)(image + 1);

	for (p = 0; p < image->ui_num_pages; p++, page++) {
		struct qat_uof_page *qup = &qui->qui_pages[p];
		struct uof_code_area *uca;

		qup->qup_page_num = page->ucp_page_num;
		qup->qup_def_page = page->ucp_def_page;
		qup->qup_page_region = page->ucp_page_region;
		qup->qup_beg_vaddr = page->ucp_beg_vaddr;
		qup->qup_beg_paddr = page->ucp_beg_paddr;

		ASSIGN_OBJ_TAB(&qup->qup_num_uc_var, &qup->qup_uc_var,
		    struct uof_uword_fixup *, base,
		    page->ucp_uc_var_tab, lim);
		ASSIGN_OBJ_TAB(&qup->qup_num_imp_var, &qup->qup_imp_var,
		    struct uof_import_var *, base,
		    page->ucp_imp_var_tab, lim);
		ASSIGN_OBJ_TAB(&qup->qup_num_imp_expr, &qup->qup_imp_expr,
		    struct uof_uword_fixup *, base,
		    page->ucp_imp_expr_tab, lim);
		ASSIGN_OBJ_TAB(&qup->qup_num_neigh_reg, &qup->qup_neigh_reg,
		    struct uof_uword_fixup *, base,
		    page->ucp_neigh_reg_tab, lim);

		if (lim < page->ucp_code_area + sizeof(struct uof_code_area))
			return EINVAL;

		uca = (struct uof_code_area *)(base + page->ucp_code_area);
		qup->qup_num_micro_words = uca->uca_num_micro_words;

		ASSIGN_OBJ_TAB(&qup->qup_num_uw_blocks, &qup->qup_uw_blocks,
		    struct qat_uof_uword_block *, base,
		    uca->uca_uword_block_tab, lim);

		for (i = 0; i < qup->qup_num_uw_blocks; i++) {
			u_int uwordoff = ((struct uof_uword_block *)(
			    &qup->qup_uw_blocks[i]))->uub_uword_offset;

			if (lim < uwordoff)
				return EINVAL;

			qup->qup_uw_blocks[i].quub_micro_words =
			    (base + uwordoff);
		}
	}

#undef ASSIGN_OBJ_TAB

	return 0;
}

static int
qat_aefw_uof_parse_images(struct qat_softc *sc)
{
	struct uof_chunk_hdr *uch = NULL;
	int i, error;

	for (i = 0; i < MAX_NUM_AE * MAX_AE_CTX; i++) {
		uch = qat_aefw_uof_find_chunk(sc, UOF_IMAG, uch);
		if (uch == NULL)
			break;

		if (i >= nitems(sc->sc_aefw_uof.qafu_imgs))
			return ENOENT;

		error = qat_aefw_uof_parse_image(sc, &sc->sc_aefw_uof.qafu_imgs[i], uch);
		if (error)
			return error;

		sc->sc_aefw_uof.qafu_num_imgs++;
	}

	return 0;
}

static int
qat_aefw_uof_parse(struct qat_softc *sc)
{
	struct uof_file_hdr *ufh;
	struct uof_file_chunk_hdr *ufch;
	struct uof_obj_hdr *uoh;
	struct uof_chunk_hdr *uch;
	void *uof = NULL;
	size_t size, uof_size, hdr_size;
	uintptr_t base;
	u_int csum;
	int i;

	size = sc->sc_fw_uof_size;
	if (size < MIN_UOF_SIZE)
		return EINVAL;
	size -= sizeof(struct uof_file_hdr);

	ufh = sc->sc_fw_uof;

	if (ufh->ufh_id != UOF_FID)
		return EINVAL;
	if (ufh->ufh_min_ver != UOF_MIN_VER || ufh->ufh_maj_ver != UOF_MAJ_VER)
		return EINVAL;

	if (ufh->ufh_max_chunks < ufh->ufh_num_chunks)
		return EINVAL;
	if (size < sizeof(struct uof_file_chunk_hdr) * ufh->ufh_num_chunks)
		return EINVAL;
	ufch = (struct uof_file_chunk_hdr *)(ufh + 1);

	uof_size = 0;
	for (i = 0; i < ufh->ufh_num_chunks; i++, ufch++) {
		if (ufch->ufch_offset + ufch->ufch_size > sc->sc_fw_uof_size)
			return EINVAL;

		if (!strncmp(ufch->ufch_id, UOF_OBJS, UOF_OBJ_ID_LEN)) {
			if (uof != NULL)
				return EINVAL;

			uof =
			    (void *)((uintptr_t)sc->sc_fw_uof +
			    ufch->ufch_offset);
			uof_size = ufch->ufch_size;

			csum = qat_aefw_csum(uof, uof_size);
			if (csum != ufch->ufch_csum)
				return EINVAL;
		}
	}

	if (uof == NULL)
		return ENOENT;

	size = uof_size;
	if (size < sizeof(struct uof_obj_hdr))
		return EINVAL;
	size -= sizeof(struct uof_obj_hdr);

	uoh = uof;

	if (size < sizeof(struct uof_chunk_hdr) * uoh->uoh_num_chunks)
		return EINVAL;

	/* Check if the UOF objects are compatible with the chip */
	if ((uoh->uoh_cpu_type & sc->sc_hw.qhw_prod_type) == 0)
		return ENOTSUP;

	if (uoh->uoh_min_cpu_ver > sc->sc_rev ||
	    uoh->uoh_max_cpu_ver < sc->sc_rev)
		return ENOTSUP;

	sc->sc_aefw_uof.qafu_size = uof_size;
	sc->sc_aefw_uof.qafu_obj_hdr = uoh;

	base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr;

	/* map uof string-table */
	uch = qat_aefw_uof_find_chunk(sc, UOF_STRT, NULL);
	if (uch != NULL) {
		hdr_size = offsetof(struct uof_str_tab, ust_strings);
		sc->sc_aefw_uof.qafu_str_tab =
		    (void *)(base + uch->uch_offset + hdr_size);
		sc->sc_aefw_uof.qafu_str_tab_size = uch->uch_size - hdr_size;
	}

	/* get ustore mem inits table -- should be only one */
	uch = qat_aefw_uof_find_chunk(sc, UOF_IMEM, NULL);
	if (uch != NULL) {
		if (uch->uch_size < sizeof(struct uof_obj_table))
			return EINVAL;
		sc->sc_aefw_uof.qafu_num_init_mem = ((struct uof_obj_table *)(base +
		    uch->uch_offset))->uot_nentries;
		if (sc->sc_aefw_uof.qafu_num_init_mem) {
			sc->sc_aefw_uof.qafu_init_mem =
			    (struct uof_init_mem *)(base + uch->uch_offset +
			    sizeof(struct uof_obj_table));
			sc->sc_aefw_uof.qafu_init_mem_size =
			    uch->uch_size - sizeof(struct uof_obj_table);
		}
	}

	uch = qat_aefw_uof_find_chunk(sc, UOF_MSEG, NULL);
	if (uch != NULL) {
		if (uch->uch_size < sizeof(struct uof_obj_table) +
		    sizeof(struct uof_var_mem_seg))
			return EINVAL;
		sc->sc_aefw_uof.qafu_var_mem_seg =
		    (struct uof_var_mem_seg *)(base + uch->uch_offset +
		    sizeof(struct uof_obj_table));
	}

	return qat_aefw_uof_parse_images(sc);
}

static int
qat_aefw_suof_parse_image(struct qat_softc *sc, struct qat_suof_image *qsi,
    struct suof_chunk_hdr *sch)
{
	struct qat_aefw_suof *qafs = &sc->sc_aefw_suof;
	struct simg_ae_mode *ae_mode;
	u_int maj_ver;

	qsi->qsi_simg_buf = qafs->qafs_suof_buf + sch->sch_offset +
	    sizeof(struct suof_obj_hdr);
	qsi->qsi_simg_len =
	    ((struct suof_obj_hdr *)
	    (qafs->qafs_suof_buf + sch->sch_offset))->soh_img_length;

	qsi->qsi_css_header = qsi->qsi_simg_buf;
	qsi->qsi_css_key = qsi->qsi_css_header + sizeof(struct css_hdr);
	qsi->qsi_css_signature = qsi->qsi_css_key +
	    CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN;
	qsi->qsi_css_simg = qsi->qsi_css_signature + CSS_SIGNATURE_LEN;

	ae_mode = (struct simg_ae_mode *)qsi->qsi_css_simg;
	qsi->qsi_ae_mask = ae_mode->sam_ae_mask;
	qsi->qsi_simg_name = (u_long)&ae_mode->sam_simg_name;
	qsi->qsi_appmeta_data = (u_long)&ae_mode->sam_appmeta_data;
	qsi->qsi_fw_type = ae_mode->sam_fw_type;

	if (ae_mode->sam_dev_type != sc->sc_hw.qhw_prod_type)
		return EINVAL;

	maj_ver = (QAT_PID_MAJOR_REV | (sc->sc_rev & QAT_PID_MINOR_REV)) & 0xff;
	if ((maj_ver > ae_mode->sam_devmax_ver) ||
	    (maj_ver < ae_mode->sam_devmin_ver)) {
		return EINVAL;
	}

	return 0;
}

static int
qat_aefw_suof_parse(struct qat_softc *sc)
{
	struct suof_file_hdr *sfh;
	struct suof_chunk_hdr *sch;
	struct qat_aefw_suof *qafs = &sc->sc_aefw_suof;
	struct qat_suof_image *qsi;
	size_t size;
	u_int csum;
	int ae0_img = MAX_AE;
	int i, error;

	size = sc->sc_fw_suof_size;
	if (size < sizeof(struct suof_file_hdr))
		return EINVAL;

	sfh = sc->sc_fw_suof;

	if (sfh->sfh_file_id != SUOF_FID)
		return EINVAL;
	if (sfh->sfh_fw_type != 0)
		return EINVAL;
	if (sfh->sfh_num_chunks <= 1)
		return EINVAL;
	if (sfh->sfh_min_ver != SUOF_MIN_VER ||
	    sfh->sfh_maj_ver != SUOF_MAJ_VER)
		return EINVAL;

	csum = qat_aefw_csum((char *)&sfh->sfh_min_ver,
	    size - offsetof(struct suof_file_hdr, sfh_min_ver));
	if (csum != sfh->sfh_check_sum)
		return EINVAL;

	size -= sizeof(struct suof_file_hdr);

	qafs->qafs_file_id = SUOF_FID;
	qafs->qafs_suof_buf = sc->sc_fw_suof;
	qafs->qafs_suof_size = sc->sc_fw_suof_size;
	qafs->qafs_check_sum = sfh->sfh_check_sum;
	qafs->qafs_min_ver = sfh->sfh_min_ver;
	qafs->qafs_maj_ver = sfh->sfh_maj_ver;
	qafs->qafs_fw_type = sfh->sfh_fw_type;

	if (size < sizeof(struct suof_chunk_hdr))
		return EINVAL;
	sch = (struct suof_chunk_hdr *)(sfh + 1);
	size -= sizeof(struct suof_chunk_hdr);

	if (size < sizeof(struct suof_str_tab))
		return EINVAL;
	size -= offsetof(struct suof_str_tab, sst_strings);

	qafs->qafs_sym_size = ((struct suof_str_tab *)
	    (qafs->qafs_suof_buf + sch->sch_offset))->sst_tab_length;
	if (size < qafs->qafs_sym_size)
		return EINVAL;
	qafs->qafs_sym_str = qafs->qafs_suof_buf + sch->sch_offset +
	    offsetof(struct suof_str_tab, sst_strings);

	qafs->qafs_num_simgs = sfh->sfh_num_chunks - 1;
	if (qafs->qafs_num_simgs == 0)
		return EINVAL;

	qsi = qat_alloc_mem(
	    sizeof(struct qat_suof_image) * qafs->qafs_num_simgs);
	qafs->qafs_simg = qsi;

	for (i = 0; i < qafs->qafs_num_simgs; i++) {
		error = qat_aefw_suof_parse_image(sc, &qsi[i], &sch[i + 1]);
		if (error)
			return error;
		if ((qsi[i].qsi_ae_mask & 0x1) != 0)
			ae0_img = i;
	}

	if (ae0_img != qafs->qafs_num_simgs - 1) {
		struct qat_suof_image last_qsi;

		memcpy(&last_qsi, &qsi[qafs->qafs_num_simgs - 1],
		    sizeof(struct qat_suof_image));
		memcpy(&qsi[qafs->qafs_num_simgs - 1], &qsi[ae0_img],
		    sizeof(struct qat_suof_image));
		memcpy(&qsi[ae0_img], &last_qsi,
		    sizeof(struct qat_suof_image));
	}

	return 0;
}

static int
qat_aefw_alloc_auth_dmamem(struct qat_softc *sc, char *image, size_t size,
    struct qat_dmamem *dma)
{
	struct css_hdr *css = (struct css_hdr *)image;
	struct auth_chunk *auth_chunk;
	struct fw_auth_desc *auth_desc;
	size_t mapsize, simg_offset = sizeof(struct auth_chunk);
	bus_size_t bus_addr;
	uintptr_t virt_addr;
	int error;

	if (size > AE_IMG_OFFSET + CSS_MAX_IMAGE_LEN)
		return EINVAL;

	mapsize = (css->css_fw_type == CSS_AE_FIRMWARE) ?
	    CSS_AE_SIMG_LEN + simg_offset :
	    size + CSS_FWSK_PAD_LEN + simg_offset;
	error = qat_alloc_dmamem(sc, dma, 1, mapsize, PAGE_SIZE);
	if (error)
		return error;

	memset(dma->qdm_dma_vaddr, 0, mapsize);

	auth_chunk = dma->qdm_dma_vaddr;
	auth_chunk->ac_chunk_size = mapsize;
	auth_chunk->ac_chunk_bus_addr = dma->qdm_dma_seg.ds_addr;

	virt_addr = (uintptr_t)dma->qdm_dma_vaddr;
	virt_addr += simg_offset;
	bus_addr = auth_chunk->ac_chunk_bus_addr;
	bus_addr += simg_offset;

	auth_desc = &auth_chunk->ac_fw_auth_desc;
	auth_desc->fad_css_hdr_high = (uint64_t)bus_addr >> 32;
	auth_desc->fad_css_hdr_low = bus_addr;

	memcpy((void *)virt_addr, image, sizeof(struct css_hdr));
	/* pub key */
	virt_addr += sizeof(struct css_hdr);
	bus_addr += sizeof(struct css_hdr);
	image += sizeof(struct css_hdr);

	auth_desc->fad_fwsk_pub_high = (uint64_t)bus_addr >> 32;
	auth_desc->fad_fwsk_pub_low = bus_addr;

	memcpy((void *)virt_addr, image, CSS_FWSK_MODULUS_LEN);
	memset((void *)(virt_addr + CSS_FWSK_MODULUS_LEN), 0, CSS_FWSK_PAD_LEN);
	memcpy((void *)(virt_addr + CSS_FWSK_MODULUS_LEN + CSS_FWSK_PAD_LEN),
	    image + CSS_FWSK_MODULUS_LEN, sizeof(uint32_t));

	virt_addr += CSS_FWSK_PUB_LEN;
	bus_addr += CSS_FWSK_PUB_LEN;
	image += CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN;

	auth_desc->fad_signature_high = (uint64_t)bus_addr >> 32;
	auth_desc->fad_signature_low = bus_addr;

	memcpy((void *)virt_addr, image, CSS_SIGNATURE_LEN);

	virt_addr += CSS_SIGNATURE_LEN;
	bus_addr += CSS_SIGNATURE_LEN;
	image += CSS_SIGNATURE_LEN;

	auth_desc->fad_img_high = (uint64_t)bus_addr >> 32;
	auth_desc->fad_img_low = bus_addr;
	auth_desc->fad_img_len = size - AE_IMG_OFFSET;

	memcpy((void *)virt_addr, image, auth_desc->fad_img_len);

	if (css->css_fw_type == CSS_AE_FIRMWARE) {
		auth_desc->fad_img_ae_mode_data_high = auth_desc->fad_img_high;
		auth_desc->fad_img_ae_mode_data_low = auth_desc->fad_img_low;

		bus_addr += sizeof(struct simg_ae_mode);

		auth_desc->fad_img_ae_init_data_high = (uint64_t)bus_addr >> 32;
		auth_desc->fad_img_ae_init_data_low = bus_addr;

		bus_addr += SIMG_AE_INIT_SEQ_LEN;

		auth_desc->fad_img_ae_insts_high = (uint64_t)bus_addr >> 32;
		auth_desc->fad_img_ae_insts_low = bus_addr;
	} else {
		auth_desc->fad_img_ae_insts_high = auth_desc->fad_img_high;
		auth_desc->fad_img_ae_insts_low = auth_desc->fad_img_low;
	}

	bus_dmamap_sync(dma->qdm_dma_tag, dma->qdm_dma_map,
	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);

	return 0;
}

static int
qat_aefw_auth(struct qat_softc *sc, struct qat_dmamem *dma)
{
	bus_addr_t addr;
	uint32_t fcu, sts;
	int retry = 0;

	addr = dma->qdm_dma_seg.ds_addr;
	qat_cap_global_write_4(sc, FCU_DRAM_ADDR_HI, (uint64_t)addr >> 32);
	qat_cap_global_write_4(sc, FCU_DRAM_ADDR_LO, addr);
	qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_AUTH);

	do {
		DELAY(FW_AUTH_WAIT_PERIOD * 1000);
		fcu = qat_cap_global_read_4(sc, FCU_STATUS);
		sts = __SHIFTOUT(fcu, FCU_STATUS_STS);
		if (sts == FCU_STATUS_STS_VERI_FAIL)
			goto fail;
		if (fcu & FCU_STATUS_AUTHFWLD &&
		    sts == FCU_STATUS_STS_VERI_DONE) {
			return 0;
		}
	} while (retry++ < FW_AUTH_MAX_RETRY);

fail:
	device_printf(sc->sc_dev,
	   "firmware authentication error: status 0x%08x retry %d\n",
	   fcu, retry);
	return EINVAL;
}

static int
qat_aefw_suof_load(struct qat_softc *sc, struct qat_dmamem *dma)
{
	struct simg_ae_mode *ae_mode;
	uint32_t fcu, sts, loaded;
	u_int mask;
	u_char ae;
	int retry = 0;

	ae_mode = (struct simg_ae_mode *)((uintptr_t)dma->qdm_dma_vaddr +
	    sizeof(struct auth_chunk) + sizeof(struct css_hdr) +
	    CSS_FWSK_PUB_LEN + CSS_SIGNATURE_LEN);

	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
		if (!(mask & 1))
			continue;
		if (!((ae_mode->sam_ae_mask >> ae) & 0x1))
			continue;
		if (qat_ae_is_active(sc, ae)) {
			device_printf(sc->sc_dev, "AE %d is active\n", ae);
			return EINVAL;
		}
		qat_cap_global_write_4(sc, FCU_CTRL,
		    FCU_CTRL_CMD_LOAD | __SHIFTIN(ae, FCU_CTRL_AE));
		do {
			DELAY(FW_AUTH_WAIT_PERIOD * 1000);
			fcu = qat_cap_global_read_4(sc, FCU_STATUS);
			sts = __SHIFTOUT(fcu, FCU_STATUS_STS);
			loaded = __SHIFTOUT(fcu, FCU_STATUS_LOADED_AE);
			if (sts == FCU_STATUS_STS_LOAD_DONE &&
			    (loaded & (1 << ae))) {
				break;
			}
		} while (retry++ < FW_AUTH_MAX_RETRY);

		if (retry > FW_AUTH_MAX_RETRY) {
			device_printf(sc->sc_dev,
			    "firmware load timeout: status %08x\n", fcu);
			return EINVAL;
		}
	}

	return 0;
}

static int
qat_aefw_suof_write(struct qat_softc *sc)
{
	struct qat_suof_image *qsi;
	int i, error = 0;

	for (i = 0; i < sc->sc_aefw_suof.qafs_num_simgs; i++) {
		qsi = &sc->sc_aefw_suof.qafs_simg[i];
		error = qat_aefw_alloc_auth_dmamem(sc, qsi->qsi_simg_buf,
		    qsi->qsi_simg_len, &qsi->qsi_dma);
		if (error)
			return error;
		error = qat_aefw_auth(sc, &qsi->qsi_dma);
		if (error) {
			qat_free_dmamem(sc, &qsi->qsi_dma);
			return error;
		}
		error = qat_aefw_suof_load(sc, &qsi->qsi_dma);
		if (error) {
			qat_free_dmamem(sc, &qsi->qsi_dma);
			return error;
		}
		qat_free_dmamem(sc, &qsi->qsi_dma);
	}
	qat_free_mem(sc->sc_aefw_suof.qafs_simg);

	return 0;
}

static int
qat_aefw_uof_assign_image(struct qat_softc *sc, struct qat_ae *qae,
	struct qat_uof_image *qui)
{
	struct qat_ae_slice *slice;
	int i, npages, nregions;

	if (qae->qae_num_slices >= nitems(qae->qae_slices))
		return ENOENT;

	if (qui->qui_image->ui_ae_mode &
	    (AE_MODE_RELOAD_CTX_SHARED | AE_MODE_SHARED_USTORE)) {
		/* XXX */
		device_printf(sc->sc_dev,
		    "shared ae mode is not supported yet\n");
		return ENOTSUP;
	}

	qae->qae_shareable_ustore = 0; /* XXX */
	qae->qae_effect_ustore_size = USTORE_SIZE;

	slice = &qae->qae_slices[qae->qae_num_slices];

	slice->qas_image = qui;
	slice->qas_assigned_ctx_mask = qui->qui_image->ui_ctx_assigned;

	nregions = qui->qui_image->ui_num_page_regions;
	npages = qui->qui_image->ui_num_pages;

	if (nregions > nitems(slice->qas_regions))
		return ENOENT;
	if (npages > nitems(slice->qas_pages))
		return ENOENT;

	for (i = 0; i < nregions; i++) {
		STAILQ_INIT(&slice->qas_regions[i].qar_waiting_pages);
	}
	for (i = 0; i < npages; i++) {
		struct qat_ae_page *page = &slice->qas_pages[i];
		int region;

		page->qap_page = &qui->qui_pages[i];
		region = page->qap_page->qup_page_region;
		if (region >= nregions)
			return EINVAL;

		page->qap_region = &slice->qas_regions[region];
	}

	qae->qae_num_slices++;

	return 0;
}

static int
qat_aefw_uof_init_ae(struct qat_softc *sc, u_char ae)
{
	struct uof_image *image;
	struct qat_ae *qae = &(QAT_AE(sc, ae));
	int s;
	u_char nn_mode;

	for (s = 0; s < qae->qae_num_slices; s++) {
		if (qae->qae_slices[s].qas_image == NULL)
			continue;

		image = qae->qae_slices[s].qas_image->qui_image;
		qat_ae_write_ctx_mode(sc, ae,
		    __SHIFTOUT(image->ui_ae_mode, AE_MODE_CTX_MODE));

		nn_mode = __SHIFTOUT(image->ui_ae_mode, AE_MODE_NN_MODE);
		if (nn_mode != AE_MODE_NN_MODE_DONTCARE)
			qat_ae_write_nn_mode(sc, ae, nn_mode);

		qat_ae_write_lm_mode(sc, ae, AEREG_LMEM0,
		    __SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM0));
		qat_ae_write_lm_mode(sc, ae, AEREG_LMEM1,
		    __SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM1));

		qat_ae_write_shared_cs_mode(sc, ae,
		    __SHIFTOUT(image->ui_ae_mode, AE_MODE_SHARED_USTORE));
		qat_ae_set_reload_ustore(sc, ae, image->ui_reloadable_size,
		    __SHIFTOUT(image->ui_ae_mode, AE_MODE_RELOAD_CTX_SHARED),
		    qae->qae_reloc_ustore_dram);
	}

	return 0;
}

static int
qat_aefw_uof_init(struct qat_softc *sc)
{
	int ae, i, error;
	uint32_t mask;

	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
		struct qat_ae *qae;

		if (!(mask & 1))
			continue;

		qae = &(QAT_AE(sc, ae));

		for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) {
			if ((sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_ae_assigned &
			    (1 << ae)) == 0)
				continue;

			error = qat_aefw_uof_assign_image(sc, qae,
			    &sc->sc_aefw_uof.qafu_imgs[i]);
			if (error)
				return error;
		}

		/* XXX UcLo_initNumUwordUsed */

		qae->qae_reloc_ustore_dram = UINT_MAX; /* XXX */

		error = qat_aefw_uof_init_ae(sc, ae);
		if (error)
			return error;
	}

	return 0;
}

int
qat_aefw_load(struct qat_softc *sc)
{
	int error;

	error = qat_aefw_load_mof(sc);
	if (error)
		return error;

	error = qat_aefw_load_mmp(sc);
	if (error)
		return error;

	error = qat_aefw_mof_parse(sc);
	if (error) {
		device_printf(sc->sc_dev, "couldn't parse mof: %d\n", error);
		return error;
	}

	if (sc->sc_hw.qhw_fw_auth) {
		error = qat_aefw_suof_parse(sc);
		if (error) {
			device_printf(sc->sc_dev, "couldn't parse suof: %d\n",
			    error);
			return error;
		}

		error = qat_aefw_suof_write(sc);
		if (error) {
			device_printf(sc->sc_dev,
			    "could not write firmware: %d\n", error);
			return error;
		}

	} else {
		error = qat_aefw_uof_parse(sc);
		if (error) {
			device_printf(sc->sc_dev, "couldn't parse uof: %d\n",
			    error);
			return error;
		}

		error = qat_aefw_uof_init(sc);
		if (error) {
			device_printf(sc->sc_dev,
			    "couldn't init for aefw: %d\n", error);
			return error;
		}

		error = qat_aefw_uof_write(sc);
		if (error) {
			device_printf(sc->sc_dev,
			    "Could not write firmware: %d\n", error);
			return error;
		}
	}

	return 0;
}

void
qat_aefw_unload(struct qat_softc *sc)
{
	qat_aefw_unload_mmp(sc);
	qat_aefw_unload_mof(sc);
}

int
qat_aefw_start(struct qat_softc *sc, u_char ae, u_int ctx_mask)
{
	uint32_t fcu;
	int retry = 0;

	if (sc->sc_hw.qhw_fw_auth) {
		qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_START);
		do {
			DELAY(FW_AUTH_WAIT_PERIOD * 1000);
			fcu = qat_cap_global_read_4(sc, FCU_STATUS);
			if (fcu & FCU_STATUS_DONE)
				return 0;
		} while (retry++ < FW_AUTH_MAX_RETRY);

		device_printf(sc->sc_dev,
		    "firmware start timeout: status %08x\n", fcu);
		return EINVAL;
	} else {
		qat_ae_ctx_indr_write(sc, ae, (~ctx_mask) & AE_ALL_CTX,
		    CTX_WAKEUP_EVENTS_INDIRECT,
		    CTX_WAKEUP_EVENTS_INDIRECT_SLEEP);
		qat_ae_enable_ctx(sc, ae, ctx_mask);
	}

	return 0;
}

static int
qat_aefw_init_memory_one(struct qat_softc *sc, struct uof_init_mem *uim)
{
	struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
	struct qat_ae_batch_init_list *qabi_list;
	struct uof_mem_val_attr *memattr;
	size_t *curinit;
	u_long ael;
	int i;
	const char *sym;
	char *ep;

	memattr = (struct uof_mem_val_attr *)(uim + 1);

	switch (uim->uim_region) {
	case LMEM_REGION:
		if ((uim->uim_addr + uim->uim_num_bytes) > MAX_LMEM_REG * 4) {
			device_printf(sc->sc_dev,
			    "Invalid lmem addr or bytes\n");
			return ENOBUFS;
		}
		if (uim->uim_scope != UOF_SCOPE_LOCAL)
			return EINVAL;
		sym = qat_aefw_uof_string(sc, uim->uim_sym_name);
		ael = strtoul(sym, &ep, 10);
		if (ep == sym || ael > MAX_AE)
			return EINVAL;
		if ((sc->sc_ae_mask & (1 << ael)) == 0)
			return 0; /* ae is fused out */

		curinit = &qafu->qafu_num_lm_init[ael];
		qabi_list = &qafu->qafu_lm_init[ael];

		for (i = 0; i < uim->uim_num_val_attr; i++, memattr++) {
			struct qat_ae_batch_init *qabi;

			qabi = qat_alloc_mem(sizeof(struct qat_ae_batch_init));
			if (*curinit == 0)
				STAILQ_INIT(qabi_list);
			STAILQ_INSERT_TAIL(qabi_list, qabi, qabi_next);

			qabi->qabi_ae = (u_int)ael;
			qabi->qabi_addr =
			    uim->uim_addr + memattr->umva_byte_offset;
			qabi->qabi_value = &memattr->umva_value;
			qabi->qabi_size = 4;
			qafu->qafu_num_lm_init_inst[ael] +=
			    qat_ae_get_inst_num(qabi->qabi_size);
			(*curinit)++;
			if (*curinit >= MAX_LMEM_REG) {
				device_printf(sc->sc_dev,
				    "Invalid lmem val attr\n");
				return ENOBUFS;
			}
		}
		break;
	case SRAM_REGION:
	case DRAM_REGION:
	case DRAM1_REGION:
	case SCRATCH_REGION:
	case UMEM_REGION:
		/* XXX */
		/* fallthrough */
	default:
		device_printf(sc->sc_dev,
		    "unsupported memory region to init: %d\n",
		    uim->uim_region);
		return ENOTSUP;
	}

	return 0;
}

static void
qat_aefw_free_lm_init(struct qat_softc *sc, u_char ae)
{
	struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
	struct qat_ae_batch_init *qabi;

	while ((qabi = STAILQ_FIRST(&qafu->qafu_lm_init[ae])) != NULL) {
		STAILQ_REMOVE_HEAD(&qafu->qafu_lm_init[ae], qabi_next);
		qat_free_mem(qabi);
	}

	qafu->qafu_num_lm_init[ae] = 0;
	qafu->qafu_num_lm_init_inst[ae] = 0;
}

static int
qat_aefw_init_ustore(struct qat_softc *sc)
{
	uint64_t *fill;
	uint32_t dont_init;
	int a, i, p;
	int error = 0;
	int usz, end, start;
	u_char ae, nae;

	fill = qat_alloc_mem(MAX_USTORE * sizeof(uint64_t));

	for (a = 0; a < sc->sc_aefw_uof.qafu_num_imgs; a++) {
		struct qat_uof_image *qui = &sc->sc_aefw_uof.qafu_imgs[a];
		struct uof_image *ui = qui->qui_image;

		for (i = 0; i < MAX_USTORE; i++)
			memcpy(&fill[i], ui->ui_fill_pattern, sizeof(uint64_t));
		/*
		 * Compute do_not_init value as a value that will not be equal
		 * to fill data when cast to an int
		 */
		dont_init = 0;
		if (dont_init == (uint32_t)fill[0])
			dont_init = 0xffffffff;

		for (p = 0; p < ui->ui_num_pages; p++) {
			struct qat_uof_page *qup = &qui->qui_pages[p];
			if (!qup->qup_def_page)
				continue;

			for (i = qup->qup_beg_paddr;
			    i < qup->qup_beg_paddr + qup->qup_num_micro_words;
			    i++ ) {
				fill[i] = (uint64_t)dont_init;
			}
		}

		for (ae = 0; ae < sc->sc_ae_num; ae++) {
			MPASS(ae < UOF_MAX_NUM_OF_AE);
			if ((ui->ui_ae_assigned & (1 << ae)) == 0)
				continue;

			if (QAT_AE(sc, ae).qae_shareable_ustore && (ae & 1)) {
				qat_ae_get_shared_ustore_ae(ae, &nae);
				if (ui->ui_ae_assigned & (1 << ae))
					continue;
			}
			usz = QAT_AE(sc, ae).qae_effect_ustore_size;

			/* initialize the areas not going to be overwritten */
			end = -1;
			do {
				/* find next uword that needs to be initialized */
				for (start = end + 1; start < usz; start++) {
					if ((uint32_t)fill[start] != dont_init)
						break;
				}
				/* see if there are no more such uwords */
				if (start >= usz)
					break;
				for (end = start + 1; end < usz; end++) {
					if ((uint32_t)fill[end] == dont_init)
						break;
				}
				if (QAT_AE(sc, ae).qae_shareable_ustore) {
					error = ENOTSUP; /* XXX */
					goto out;
				} else {
					error = qat_ae_ucode_write(sc, ae,
					    start, end - start, &fill[start]);
					if (error) {
						goto out;
					}
				}

			} while (end < usz);
		}
	}

out:
	qat_free_mem(fill);
	return error;
}

static int
qat_aefw_init_reg(struct qat_softc *sc, u_char ae, u_char ctx_mask,
    enum aereg_type regtype, u_short regaddr, u_int value)
{
	int error = 0;
	u_char ctx;

	switch (regtype) {
	case AEREG_GPA_REL:
	case AEREG_GPB_REL:
	case AEREG_SR_REL:
	case AEREG_SR_RD_REL:
	case AEREG_SR_WR_REL:
	case AEREG_DR_REL:
	case AEREG_DR_RD_REL:
	case AEREG_DR_WR_REL:
	case AEREG_NEIGH_REL:
		/* init for all valid ctx */
		for (ctx = 0; ctx < MAX_AE_CTX; ctx++) {
			if ((ctx_mask & (1 << ctx)) == 0)
				continue;
			error = qat_aereg_rel_data_write(sc, ae, ctx, regtype,
			    regaddr, value);
		}
		break;
	case AEREG_GPA_ABS:
	case AEREG_GPB_ABS:
	case AEREG_SR_ABS:
	case AEREG_SR_RD_ABS:
	case AEREG_SR_WR_ABS:
	case AEREG_DR_ABS:
	case AEREG_DR_RD_ABS:
	case AEREG_DR_WR_ABS:
		error = qat_aereg_abs_data_write(sc, ae, regtype,
		    regaddr, value);
		break;
	default:
		error = EINVAL;
		break;
	}

	return error;
}

static int
qat_aefw_init_reg_sym_expr(struct qat_softc *sc, u_char ae,
    struct qat_uof_image *qui)
{
	u_int i, expres;
	u_char ctx_mask;

	for (i = 0; i < qui->qui_num_init_reg_sym; i++) {
		struct uof_init_reg_sym *uirs = &qui->qui_init_reg_sym[i];

		if (uirs->uirs_value_type == EXPR_VAL) {
			/* XXX */
			device_printf(sc->sc_dev,
			    "does not support initializing EXPR_VAL\n");
			return ENOTSUP;
		} else {
			expres = uirs->uirs_value;
		}

		switch (uirs->uirs_init_type) {
		case INIT_REG:
			if (__SHIFTOUT(qui->qui_image->ui_ae_mode,
			    AE_MODE_CTX_MODE) == MAX_AE_CTX) {
				ctx_mask = 0xff; /* 8-ctx mode */
			} else {
				ctx_mask = 0x55; /* 4-ctx mode */
			}
			qat_aefw_init_reg(sc, ae, ctx_mask,
			    (enum aereg_type)uirs->uirs_reg_type,
			    (u_short)uirs->uirs_addr_offset, expres);
			break;
		case INIT_REG_CTX:
			if (__SHIFTOUT(qui->qui_image->ui_ae_mode,
			    AE_MODE_CTX_MODE) == MAX_AE_CTX) {
				ctx_mask = 0xff; /* 8-ctx mode */
			} else {
				ctx_mask = 0x55; /* 4-ctx mode */
			}
			if (((1 << uirs->uirs_ctx) & ctx_mask) == 0)
				return EINVAL;
			qat_aefw_init_reg(sc, ae, 1 << uirs->uirs_ctx,
			    (enum aereg_type)uirs->uirs_reg_type,
			    (u_short)uirs->uirs_addr_offset, expres);
			break;
		case INIT_EXPR:
		case INIT_EXPR_ENDIAN_SWAP:
		default:
			device_printf(sc->sc_dev,
			    "does not support initializing init_type %d\n",
			    uirs->uirs_init_type);
			return ENOTSUP;
		}
	}

	return 0;
}

static int
qat_aefw_init_memory(struct qat_softc *sc)
{
	struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
	size_t uimsz, initmemsz = qafu->qafu_init_mem_size;
	struct uof_init_mem *uim;
	int error, i;
	u_char ae;

	uim = qafu->qafu_init_mem;
	for (i = 0; i < qafu->qafu_num_init_mem; i++) {
		uimsz = sizeof(struct uof_init_mem) +
		    sizeof(struct uof_mem_val_attr) * uim->uim_num_val_attr;
		if (uimsz > initmemsz) {
			device_printf(sc->sc_dev,
			    "invalid uof_init_mem or uof_mem_val_attr size\n");
			return EINVAL;
		}

		if (uim->uim_num_bytes > 0) {
			error = qat_aefw_init_memory_one(sc, uim);
			if (error) {
				device_printf(sc->sc_dev,
				    "Could not init ae memory: %d\n", error);
				return error;
			}
		}
		uim = (struct uof_init_mem *)((uintptr_t)uim + uimsz);
		initmemsz -= uimsz;
	}

	/* run Batch put LM API */
	for (ae = 0; ae < MAX_AE; ae++) {
		error = qat_ae_batch_put_lm(sc, ae, &qafu->qafu_lm_init[ae],
		    qafu->qafu_num_lm_init_inst[ae]);
		if (error)
			device_printf(sc->sc_dev, "Could not put lm\n");

		qat_aefw_free_lm_init(sc, ae);
	}

	error = qat_aefw_init_ustore(sc);

	/* XXX run Batch put LM API */

	return error;
}

static int
qat_aefw_init_globals(struct qat_softc *sc)
{
	struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
	int error, i, p, s;
	u_char ae;

	/* initialize the memory segments */
	if (qafu->qafu_num_init_mem > 0) {
		error = qat_aefw_init_memory(sc);
		if (error)
			return error;
	} else {
		error = qat_aefw_init_ustore(sc);
		if (error)
			return error;
	}

	/* XXX bind import variables with ivd values */

	/* XXX bind the uC global variables
	 * local variables will done on-the-fly */
	for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) {
		for (p = 0; p < sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_num_pages; p++) {
			struct qat_uof_page *qup =
			    &sc->sc_aefw_uof.qafu_imgs[i].qui_pages[p];
			if (qup->qup_num_uw_blocks &&
			    (qup->qup_num_uc_var || qup->qup_num_imp_var)) {
				device_printf(sc->sc_dev,
				    "not support uC global variables\n");
				return ENOTSUP;
			}
		}
	}

	for (ae = 0; ae < sc->sc_ae_num; ae++) {
		struct qat_ae *qae = &(QAT_AE(sc, ae));

		for (s = 0; s < qae->qae_num_slices; s++) {
			struct qat_ae_slice *qas = &qae->qae_slices[s];

			if (qas->qas_image == NULL)
				continue;

			error =
			    qat_aefw_init_reg_sym_expr(sc, ae, qas->qas_image);
			if (error)
				return error;
		}
	}

	return 0;
}

static uint64_t
qat_aefw_get_uof_inst(struct qat_softc *sc, struct qat_uof_page *qup,
    u_int addr)
{
	uint64_t uinst = 0;
	u_int i;

	/* find the block */
	for (i = 0; i < qup->qup_num_uw_blocks; i++) {
		struct qat_uof_uword_block *quub = &qup->qup_uw_blocks[i];

		if ((addr >= quub->quub_start_addr) &&
		    (addr <= (quub->quub_start_addr +
		    (quub->quub_num_words - 1)))) {
			/* unpack n bytes and assigned to the 64-bit uword value.
			note: the microwords are stored as packed bytes.
			*/
			addr -= quub->quub_start_addr;
			addr *= AEV2_PACKED_UWORD_BYTES;
			memcpy(&uinst,
			    (void *)((uintptr_t)quub->quub_micro_words + addr),
			    AEV2_PACKED_UWORD_BYTES);
			uinst = uinst & UWORD_MASK;

			return uinst;
		}
	}

	return INVLD_UWORD;
}

static int
qat_aefw_do_pagein(struct qat_softc *sc, u_char ae, struct qat_uof_page *qup)
{
	struct qat_ae *qae = &(QAT_AE(sc, ae));
	uint64_t fill, *ucode_cpybuf;
	u_int error, i, upaddr, ninst, cpylen;

	if (qup->qup_num_uc_var || qup->qup_num_neigh_reg ||
	    qup->qup_num_imp_var || qup->qup_num_imp_expr) {
		device_printf(sc->sc_dev,
		    "does not support fixup locals\n");
		return ENOTSUP;
	}

	ucode_cpybuf = qat_alloc_mem(UWORD_CPYBUF_SIZE * sizeof(uint64_t));

	/* XXX get fill-pattern from an image -- they are all the same */
	memcpy(&fill, sc->sc_aefw_uof.qafu_imgs[0].qui_image->ui_fill_pattern,
	    sizeof(uint64_t));

	upaddr = qup->qup_beg_paddr;
	ninst = qup->qup_num_micro_words;
	while (ninst > 0) {
		cpylen = min(ninst, UWORD_CPYBUF_SIZE);

		/* load the buffer */
		for (i = 0; i < cpylen; i++) {
			/* keep below code structure in case there are
			 * different handling for shared secnarios */
			if (!qae->qae_shareable_ustore) {
				/* qat_aefw_get_uof_inst() takes an address that
				 * is relative to the start of the page.
				 * So we don't need to add in the physical
				 * offset of the page. */
				if (qup->qup_page_region != 0) {
					/* XXX */
					device_printf(sc->sc_dev,
					    "region != 0 is not supported\n");
					qat_free_mem(ucode_cpybuf);
					return ENOTSUP;
				} else {
					/* for mixing case, it should take
					 * physical address */
					ucode_cpybuf[i] = qat_aefw_get_uof_inst(
					    sc, qup, upaddr + i);
					if (ucode_cpybuf[i] == INVLD_UWORD) {
					    /* fill hole in the uof */
					    ucode_cpybuf[i] = fill;
					}
				}
			} else {
				/* XXX */
				qat_free_mem(ucode_cpybuf);
				return ENOTSUP;
			}
		}

		/* copy the buffer to ustore */
		if (!qae->qae_shareable_ustore) {
			error = qat_ae_ucode_write(sc, ae, upaddr, cpylen,
			    ucode_cpybuf);
			if (error)
				return error;
		} else {
			/* XXX */
			qat_free_mem(ucode_cpybuf);
			return ENOTSUP;
		}
		upaddr += cpylen;
		ninst -= cpylen;
	}

	qat_free_mem(ucode_cpybuf);

	return 0;
}

static int
qat_aefw_uof_write_one(struct qat_softc *sc, struct qat_uof_image *qui)
{
	struct uof_image *ui = qui->qui_image;
	struct qat_ae_page *qap;
	u_int s, p, c;
	int error;
	u_char ae, ctx_mask;

	if (__SHIFTOUT(ui->ui_ae_mode, AE_MODE_CTX_MODE) == MAX_AE_CTX)
		ctx_mask = 0xff; /* 8-ctx mode */
	else
		ctx_mask = 0x55; /* 4-ctx mode */

	/* load the default page and set assigned CTX PC
	 * to the entrypoint address */
	for (ae = 0; ae < sc->sc_ae_num; ae++) {
		struct qat_ae *qae = &(QAT_AE(sc, ae));
		struct qat_ae_slice *qas;
		u_int metadata;

		MPASS(ae < UOF_MAX_NUM_OF_AE);

		if ((ui->ui_ae_assigned & (1 << ae)) == 0)
			continue;

		/* find the slice to which this image is assigned */
		for (s = 0; s < qae->qae_num_slices; s++) {
			qas = &qae->qae_slices[s];
			if (ui->ui_ctx_assigned & qas->qas_assigned_ctx_mask)
				break;
		}
		if (s >= qae->qae_num_slices)
			continue;

		qas = &qae->qae_slices[s];

		for (p = 0; p < ui->ui_num_pages; p++) {
			qap = &qas->qas_pages[p];

			/* Only load pages loaded by default */
			if (!qap->qap_page->qup_def_page)
				continue;

			error = qat_aefw_do_pagein(sc, ae, qap->qap_page);
			if (error)
				return error;
		}

		metadata = qas->qas_image->qui_image->ui_app_metadata;
		if (metadata != 0xffffffff && bootverbose) {
			device_printf(sc->sc_dev,
			    "loaded firmware: %s\n",
			    qat_aefw_uof_string(sc, metadata));
		}

		/* Assume starting page is page 0 */
		qap = &qas->qas_pages[0];
		for (c = 0; c < MAX_AE_CTX; c++) {
			if (ctx_mask & (1 << c))
				qas->qas_cur_pages[c] = qap;
			else
				qas->qas_cur_pages[c] = NULL;
		}

		/* set the live context */
		qae->qae_live_ctx_mask = ui->ui_ctx_assigned;

		/* set context PC to the image entrypoint address */
		error = qat_ae_write_pc(sc, ae, ui->ui_ctx_assigned,
		    ui->ui_entry_address);
		if (error)
			return error;
	}

	/* XXX store the checksum for convenience */

	return 0;
}

static int
qat_aefw_uof_write(struct qat_softc *sc)
{
	int error = 0;
	int i;

	error = qat_aefw_init_globals(sc);
	if (error) {
		device_printf(sc->sc_dev,
		    "Could not initialize globals\n");
		return error;
	}

	for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) {
		error = qat_aefw_uof_write_one(sc,
		    &sc->sc_aefw_uof.qafu_imgs[i]);
		if (error)
			break;
	}

	/* XXX UcLo_computeFreeUstore */

	return error;
}