disk/nvme/nvme_admin.c

/*
 * Copyright (c) 2016 The DragonFly Project.  All rights reserved.
 *
 * This code is derived from software contributed to The DragonFly Project
 * by Matthew Dillon <dillon@backplane.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name of The DragonFly Project nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific, prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
/*
 * Administration thread
 *
 * - Handles resetting, features, iteration of namespaces, and disk
 *   attachments.  Most admin operations are serialized by the admin thread.
 *
 * - Ioctls as well as any BIOs which require more sophisticated processing
 *   are handed to this thread as well.
 *
 * - Can freeze/resume other queues for various purposes.
 */

#include "nvme.h"

static void nvme_admin_thread(void *arg);
static int nvme_admin_state_identify_ctlr(nvme_softc_t *sc);
static int nvme_admin_state_make_queues(nvme_softc_t *sc);
static int nvme_admin_state_identify_ns(nvme_softc_t *sc);
static int nvme_admin_state_operating(nvme_softc_t *sc);
static int nvme_admin_state_failed(nvme_softc_t *sc);

/*
 * Start the admin thread and block until it says it is running.
 */
int
nvme_start_admin_thread(nvme_softc_t *sc)
{
	int error;

	lockinit(&sc->admin_lk, "admlk", 0, 0);
	lockinit(&sc->ioctl_lk, "nvioc", 0, 0);
	sc->admin_signal = 0;

	error = bus_setup_intr(sc->dev, sc->irq[0], INTR_MPSAFE,
			       nvme_intr, &sc->comqueues[0],
			       &sc->irq_handle[0], NULL);
	if (error) {
		device_printf(sc->dev, "unable to install interrupt\n");
		return error;
	}
	lockmgr(&sc->admin_lk, LK_EXCLUSIVE);
	kthread_create(nvme_admin_thread, sc, &sc->admintd, "nvme_admin");
	while ((sc->admin_signal & ADMIN_SIG_RUNNING) == 0)
		lksleep(&sc->admin_signal, &sc->admin_lk, 0, "nvwbeg", 0);
	lockmgr(&sc->admin_lk, LK_RELEASE);

	return 0;
}

/*
 * Stop the admin thread and block until it says it is done.
 */
void
nvme_stop_admin_thread(nvme_softc_t *sc)
{
	uint32_t i;

	atomic_set_int(&sc->admin_signal, ADMIN_SIG_STOP);

	/*
	 * We have to wait for the admin thread to finish its probe
	 * before shutting it down.  Break out if the admin thread
	 * never managed to even start.
	 */
	lockmgr(&sc->admin_lk, LK_EXCLUSIVE);
	while ((sc->admin_signal & ADMIN_SIG_PROBED) == 0) {
		if ((sc->admin_signal & ADMIN_SIG_RUNNING) == 0)
			break;
		lksleep(&sc->admin_signal, &sc->admin_lk, 0, "nvwend", 0);
	}
	lockmgr(&sc->admin_lk, LK_RELEASE);

	/*
	 * Disconnect our disks while the admin thread is still running,
	 * ensuring that the poll works even if interrupts are broken.
	 * Otherwise we could deadlock in the devfs core.
	 */
	for (i = 0; i < NVME_MAX_NAMESPACES; ++i) {
		nvme_softns_t *nsc;

		if ((nsc = sc->nscary[i]) != NULL) {
			nvme_disk_detach(nsc);

			kfree(nsc, M_NVME);
			sc->nscary[i] = NULL;
		}
	}

	/*
	 * Ask the admin thread to shut-down.
	 */
	lockmgr(&sc->admin_lk, LK_EXCLUSIVE);
	wakeup(&sc->admin_signal);
	while (sc->admin_signal & ADMIN_SIG_RUNNING)
		lksleep(&sc->admin_signal, &sc->admin_lk, 0, "nvwend", 0);
	lockmgr(&sc->admin_lk, LK_RELEASE);
	if (sc->irq_handle[0]) {
		bus_teardown_intr(sc->dev, sc->irq[0], sc->irq_handle[0]);
		sc->irq_handle[0] = NULL;
	}
	lockuninit(&sc->ioctl_lk);
	lockuninit(&sc->admin_lk);

	/*
	 * Thread might be running on another cpu, give it time to actually
	 * exit before returning in case the caller is about to unload the
	 * module.  Otherwise we don't need this.
	 */
	nvme_os_sleep(1);
}

static
void
nvme_admin_thread(void *arg)
{
	nvme_softc_t *sc = arg;
	uint32_t i;

	lockmgr(&sc->admin_lk, LK_EXCLUSIVE);
	atomic_set_int(&sc->admin_signal, ADMIN_SIG_RUNNING);
	wakeup(&sc->admin_signal);

	sc->admin_func = nvme_admin_state_identify_ctlr;

	while ((sc->admin_signal & ADMIN_SIG_STOP) == 0) {
		for (i = 0; i <= sc->niocomqs; ++i) {
			nvme_comqueue_t *comq = &sc->comqueues[i];

			if (comq->nqe == 0)	/* not configured */
				continue;

			lockmgr(&comq->lk, LK_EXCLUSIVE);
			nvme_poll_completions(comq, &comq->lk);
			lockmgr(&comq->lk, LK_RELEASE);
		}
		if (sc->admin_signal & ADMIN_SIG_REQUEUE) {
			atomic_clear_int(&sc->admin_signal, ADMIN_SIG_REQUEUE);
			nvme_disk_requeues(sc);
		}
		if (sc->admin_func(sc) == 0 &&
		    (sc->admin_signal & ADMIN_SIG_RUN_MASK) == 0) {
			lksleep(&sc->admin_signal, &sc->admin_lk, 0,
				"nvidle", hz);
		}
	}

	/*
	 * Cleanup state.
	 *
	 * Note that we actually issue delete queue commands here.  The NVME
	 * spec says that for a normal shutdown the I/O queues should be
	 * deleted prior to issuing the shutdown in the CONFIG register.
	 */
	for (i = 1; i <= sc->niosubqs; ++i) {
		nvme_delete_subqueue(sc, i);
		nvme_free_subqueue(sc, i);
	}
	for (i = 1; i <= sc->niocomqs; ++i) {
		nvme_delete_comqueue(sc, i);
		nvme_free_comqueue(sc, i);
	}

	/*
	 * Signal that we are done.
	 */
	atomic_clear_int(&sc->admin_signal, ADMIN_SIG_RUNNING);
	wakeup(&sc->admin_signal);
	lockmgr(&sc->admin_lk, LK_RELEASE);
}

/*
 * Identify the controller
 */
static
int
nvme_admin_state_identify_ctlr(nvme_softc_t *sc)
{
	nvme_request_t *req;
	nvme_ident_ctlr_data_t *rp;
	int status;
	uint64_t mempgsize;
	char serial[20+16];
	char model[40+16];

	/*
	 * Identify Controller
	 */
	mempgsize = NVME_CAP_MEMPG_MIN_GET(sc->cap);

	req = nvme_get_admin_request(sc, NVME_OP_IDENTIFY);
	req->cmd.identify.cns = NVME_CNS_CTLR;
	req->cmd.identify.cntid = 0;
	bzero(req->info, sizeof(*req->info));
	nvme_submit_request(req);
	status = nvme_wait_request(req, hz);
	/* XXX handle status */

	sc->idctlr = req->info->idctlr;
	nvme_put_request(req);

	rp = &sc->idctlr;

	KKASSERT(sizeof(sc->idctlr.serialno) == 20);
	KKASSERT(sizeof(sc->idctlr.modelno) == 40);
	bzero(serial, sizeof(serial));
	bzero(model, sizeof(model));
	bcopy(rp->serialno, serial, sizeof(rp->serialno));
	bcopy(rp->modelno, model, sizeof(rp->modelno));
	string_cleanup(serial, 0);
	string_cleanup(model, 0);

	device_printf(sc->dev, "Model %s BaseSerial %s nscount=%d\n",
		      model, serial, rp->ns_count);

	sc->admin_func = nvme_admin_state_make_queues;

	return 1;
}

/*
 * Request and create the I/O queues.  Figure out CPU mapping optimizations.
 */
static
int
nvme_admin_state_make_queues(nvme_softc_t *sc)
{
	nvme_request_t *req;
	uint16_t niosubqs;
	uint16_t niocomqs;
	uint32_t i;
	uint16_t qno;
	int status;
	int error;

	/*
	 * Calculate how many I/O queues (non-inclusive of admin queue)
	 * we want to have, up to 65535.  dw0 in the response returns the
	 * number of queues the controller gives us.  Submission and
	 * Completion queues are specified separately.
	 *
	 * This driver runs optimally with 4 submission queues and one
	 * completion queue per cpu (rdhipri, rdlopri, wrhipri, wrlopri),
	 *
	 * +1 for dumps			XXX future
	 * +1 for async events		XXX future
	 */
	req = nvme_get_admin_request(sc, NVME_OP_SET_FEATURES);

	niosubqs = ncpus * 2 + 0;
	niocomqs = ncpus + 0;
	if (niosubqs > NVME_MAX_QUEUES)
		niosubqs = NVME_MAX_QUEUES;
	if (niocomqs > NVME_MAX_QUEUES)
		niocomqs = NVME_MAX_QUEUES;
	device_printf(sc->dev, "Request %u/%u queues, ", niosubqs, niocomqs);

	req->cmd.setfeat.flags = NVME_FID_NUMQUEUES;
	req->cmd.setfeat.numqs.nsqr = niosubqs - 1;	/* 0's based 0=1 */
	req->cmd.setfeat.numqs.ncqr = niocomqs - 1;	/* 0's based 0=1 */

	nvme_submit_request(req);

	/*
	 * Get response and set our operations mode.
	 */
	status = nvme_wait_request(req, hz);
	/* XXX handle status */

	if (status == 0) {
		sc->niosubqs = 1 + (req->res.setfeat.dw0 & 0xFFFFU);
		sc->niocomqs = 1 + ((req->res.setfeat.dw0 >> 16) & 0xFFFFU);
	} else {
		sc->niosubqs = 0;
		sc->niocomqs = 0;
	}
	kprintf("Returns %u/%u queues, ", sc->niosubqs, sc->niocomqs);

	nvme_put_request(req);

	sc->dumpqno = 0;
	sc->eventqno = 0;

	if (sc->niosubqs >= ncpus * 2 + 0 && sc->niocomqs >= ncpus + 0) {
		/*
		 * If we got all the queues we wanted do a full-bore setup of
		 * qmap[cpu][type].
		 *
		 * Remember that subq 0 / comq 0 is the admin queue.
		 */
		kprintf("optimal map\n");
		qno = 1;
		for (i = 0; i < ncpus; ++i) {
			int cpuqno = sc->cputovect[i];

			KKASSERT(cpuqno != 0);
			sc->qmap[i][0] = qno + 0;
			sc->qmap[i][1] = qno + 1;
			sc->subqueues[qno + 0].comqid = cpuqno;
			sc->subqueues[qno + 1].comqid = cpuqno;
			qno += 2;
		}
		sc->niosubqs = ncpus * 2 + 0;
		sc->niocomqs = ncpus + 0;
	} else if (sc->niosubqs >= ncpus && sc->niocomqs >= ncpus) {
		/*
		 * We have enough to give each cpu its own submission
		 * and completion queue.
		 *
		 * leave dumpqno and eventqno set to the admin queue.
		 */
		kprintf("nominal map 1:1 cpu\n");
		for (i = 0; i < ncpus; ++i) {
			qno = sc->cputovect[i];
			KKASSERT(qno != 0);
			sc->qmap[i][0] = qno;
			sc->qmap[i][1] = qno;
			sc->subqueues[qno].comqid = qno;
		}
		sc->niosubqs = ncpus;
		sc->niocomqs = ncpus;
	} else if (sc->niosubqs >= 2 && sc->niocomqs >= 2) {
		/*
		 * We have enough queues to separate and prioritize reads
		 * and writes, but all cpus have to share the same submission
		 * queues.  Completion queues are split up between cpus
		 * as much as possible.
		 *
		 * leave dumpqno and eventqno set to the admin queue.
		 */
		kprintf("rw-sep map\n");
		qno = 1;
		for (i = 0; i < ncpus; ++i) {
			int cpuqno = sc->cputovect[i];

			KKASSERT(qno != 0);
			sc->qmap[i][0] = qno + 0;	/* read lopri */
			sc->qmap[i][1] = qno + 1;	/* read hipri */
			if (i <= 0)
				sc->subqueues[qno + 0].comqid = cpuqno;
			if (i <= 1)
				sc->subqueues[qno + 1].comqid = cpuqno;
			/* do not increment qno */
		}
		sc->niosubqs = 2;
		sc->niocomqs = 2;
	} else if (sc->niosubqs >= 2) {
		/*
		 * We have enough to have separate read and write queues.
		 */
		kprintf("basic map\n");
		qno = 1;
		for (i = 0; i < ncpus; ++i) {
			int cpuqno = sc->cputovect[i];

			KKASSERT(qno != 0);
			sc->qmap[i][0] = qno + 0;	/* read */
			sc->qmap[i][1] = qno + 1;	/* write */
			if (i <= 0)
				sc->subqueues[qno + 0].comqid = cpuqno;
			if (i <= 1)
				sc->subqueues[qno + 1].comqid = cpuqno;
		}
		sc->niosubqs = 2;
		sc->niocomqs = 1;
	} else {
		/*
		 * Minimal configuration, all cpus and I/O types use the
		 * same queue.  Sad day.
		 */
		kprintf("minimal map\n");
		sc->dumpqno = 0;
		sc->eventqno = 0;
		for (i = 0; i < ncpus; ++i) {
			sc->qmap[i][0] = 1;
			sc->qmap[i][1] = 1;
		}
		sc->subqueues[1].comqid = 1;
		sc->niosubqs = 1;
		sc->niocomqs = 1;
	}

	/*
	 * Create all I/O submission and completion queues.  The I/O
	 * queues start at 1 and are inclusive of niosubqs and niocomqs.
	 *
	 * NOTE: Completion queues must be created before submission queues.
	 *	 That is, the completion queue specified when creating a
	 *	 submission queue must already exist.
	 */
	error = 0;
	for (i = 1; i <= sc->niocomqs; ++i) {
		error += nvme_alloc_comqueue(sc, i);
		if (error) {
			device_printf(sc->dev, "Unable to allocate comqs\n");
			break;
		}
		error += nvme_create_comqueue(sc, i);
	}
	for (i = 1; i <= sc->niosubqs; ++i) {
		error += nvme_alloc_subqueue(sc, i);
		if (error) {
			device_printf(sc->dev, "Unable to allocate subqs\n");
			break;
		}
		error += nvme_create_subqueue(sc, i);
	}

	if (error) {
		device_printf(sc->dev, "Failed to initialize device!\n");
		sc->admin_func = nvme_admin_state_failed;
	} else {
		sc->admin_func = nvme_admin_state_identify_ns;
	}

	/*
	 * Basically interrupt coalescing is worthless if we care about
	 * performance, at least on the Intel 750.  Setting the threshold
	 * has no effect if time is set to 0.  The smallest time that can
	 * be set is a value of 1 (== 100uS), which is much too long.  That
	 * is only 10,000 interrupts/sec/cpu and on the Intel 750 it totally
	 * destroys sequential performance.
	 */
	req = nvme_get_admin_request(sc, NVME_OP_SET_FEATURES);

	device_printf(sc->dev, "Interrupt Coalesce: 100uS / 4 qentries\n");

	req->cmd.setfeat.flags = NVME_FID_INTCOALESCE;
	req->cmd.setfeat.intcoal.thr = 0;
	req->cmd.setfeat.intcoal.time = 0;

	nvme_submit_request(req);
	status = nvme_wait_request(req, hz);
	if (status) {
		device_printf(sc->dev,
			      "Interrupt coalesce failed status=%d\n",
			      status);
	}
	nvme_put_request(req);

	return 1;
}

/*
 * Identify available namespaces, iterate, and attach to disks.
 */
static
int
nvme_admin_state_identify_ns(nvme_softc_t *sc)
{
	nvme_request_t *req;
	nvme_ident_ns_list_t *rp;
	int status;
	uint32_t i;
	uint32_t j;

	if (bootverbose) {
		if (sc->idctlr.admin_cap & NVME_ADMIN_NSMANAGE)
			device_printf(sc->dev,
				      "Namespace management supported\n");
		else
			device_printf(sc->dev,
				      "Namespace management not supported\n");
	}
#if 0
	/*
	 * Identify Controllers		TODO TODO TODO
	 */
	if (sc->idctlr.admin_cap & NVME_ADMIN_NSMANAGE) {
		req = nvme_get_admin_request(sc, NVME_OP_IDENTIFY);
		req->cmd.identify.cns = NVME_CNS_ANY_CTLR_LIST;
		req->cmd.identify.cntid = 0;
		bzero(req->info, sizeof(*req->info));
		nvme_submit_request(req);
		status = nvme_wait_request(req, hz);
		kprintf("nsquery status %08x\n", status);

#if 0
		for (i = 0; i < req->info->ctlrlist.idcount; ++i) {
			kprintf("CTLR %04x\n", req->info->ctlrlist.ctlrids[i]);
		}
#endif
		nvme_put_request(req);
	}
#endif

	rp = kmalloc(sizeof(*rp), M_NVME, M_WAITOK | M_ZERO);
	if (sc->idctlr.admin_cap & NVME_ADMIN_NSMANAGE) {
		/*
		 * Namespace management supported, query active namespaces.
		 */
		req = nvme_get_admin_request(sc, NVME_OP_IDENTIFY);
		req->cmd.identify.cns = NVME_CNS_ACT_NSLIST;
		req->cmd.identify.cntid = 0;
		bzero(req->info, sizeof(*req->info));
		nvme_submit_request(req);
		status = nvme_wait_request(req, hz);
		kprintf("nsquery status %08x\n", status);
		/* XXX handle status */

		cpu_lfence();
		*rp = req->info->nslist;
		nvme_put_request(req);
	} else {
		/*
		 * Namespace management not supported, assume nsids 1..N.
		 */
		for (i = 1; i <= sc->idctlr.ns_count && i <= 1024; ++i)
			rp->nsids[i-1] = i;
	}

	/*
	 * Identify each Namespace
	 */
	for (i = 0; i < 1024; ++i) {
		nvme_softns_t *nsc;
		nvme_lba_fmt_data_t *lbafmt;

		if (rp->nsids[i] == 0)
			continue;
		req = nvme_get_admin_request(sc, NVME_OP_IDENTIFY);
		req->cmd.identify.cns = NVME_CNS_ACT_NS;
		req->cmd.identify.cntid = 0;
		req->cmd.identify.head.nsid = rp->nsids[i];
		bzero(req->info, sizeof(*req->info));
		nvme_submit_request(req);
		status = nvme_wait_request(req, hz);
		if (status != 0) {
			kprintf("NS FAILED %08x\n", status);
			continue;
		}

		for (j = 0; j < NVME_MAX_NAMESPACES; ++j) {
			if (sc->nscary[j] &&
			    sc->nscary[j]->nsid == rp->nsids[i])
				break;
		}
		if (j == NVME_MAX_NAMESPACES) {
			j = i;
			if (sc->nscary[j] != NULL) {
				for (j = NVME_MAX_NAMESPACES - 1; j >= 0; --j) {
					if (sc->nscary[j] == NULL)
						break;
				}
			}
		}
		if (j < 0) {
			device_printf(sc->dev, "not enough room in nscary for "
					       "namespace %08x\n", rp->nsids[i]);
			nvme_put_request(req);
			continue;
		}
		nsc = sc->nscary[j];
		if (nsc == NULL) {
			nsc = kmalloc(sizeof(*nsc), M_NVME, M_WAITOK | M_ZERO);
			nsc->unit = nvme_alloc_disk_unit();
			sc->nscary[j] = nsc;
		}
		if (sc->nscmax <= j)
			sc->nscmax = j + 1;
		nsc->sc = sc;
		nsc->nsid = rp->nsids[i];
		nsc->state = NVME_NSC_STATE_UNATTACHED;
		nsc->idns = req->info->idns;
		bioq_init(&nsc->bioq);
		lockinit(&nsc->lk, "nvnsc", 0, 0);

		nvme_put_request(req);

		j = NVME_FLBAS_SEL_GET(nsc->idns.flbas);
		lbafmt = &nsc->idns.lba_fmt[j];
		nsc->blksize = 1 << lbafmt->sect_size;

		/*
		 * Attach the namespace
		 */
		nvme_disk_attach(nsc);
	}
	kfree(rp, M_NVME);

	sc->admin_func = nvme_admin_state_operating;
	return 1;
}

static
int
nvme_admin_state_operating(nvme_softc_t *sc)
{
	if ((sc->admin_signal & ADMIN_SIG_PROBED) == 0) {
		atomic_set_int(&sc->admin_signal, ADMIN_SIG_PROBED);
		wakeup(&sc->admin_signal);
	}

	return 0;
}

static
int
nvme_admin_state_failed(nvme_softc_t *sc)
{
	if ((sc->admin_signal & ADMIN_SIG_PROBED) == 0) {
		atomic_set_int(&sc->admin_signal, ADMIN_SIG_PROBED);
		wakeup(&sc->admin_signal);
	}

	return 0;
}