197a077a0SMatthew Dillon /*
297a077a0SMatthew Dillon * Copyright (c) 2016 The DragonFly Project. All rights reserved.
397a077a0SMatthew Dillon *
497a077a0SMatthew Dillon * This code is derived from software contributed to The DragonFly Project
597a077a0SMatthew Dillon * by Matthew Dillon <dillon@backplane.com>
697a077a0SMatthew Dillon *
797a077a0SMatthew Dillon * Redistribution and use in source and binary forms, with or without
897a077a0SMatthew Dillon * modification, are permitted provided that the following conditions
997a077a0SMatthew Dillon * are met:
1097a077a0SMatthew Dillon *
1197a077a0SMatthew Dillon * 1. Redistributions of source code must retain the above copyright
1297a077a0SMatthew Dillon * notice, this list of conditions and the following disclaimer.
1397a077a0SMatthew Dillon * 2. Redistributions in binary form must reproduce the above copyright
1497a077a0SMatthew Dillon * notice, this list of conditions and the following disclaimer in
1597a077a0SMatthew Dillon * the documentation and/or other materials provided with the
1697a077a0SMatthew Dillon * distribution.
1797a077a0SMatthew Dillon * 3. Neither the name of The DragonFly Project nor the names of its
1897a077a0SMatthew Dillon * contributors may be used to endorse or promote products derived
1997a077a0SMatthew Dillon * from this software without specific, prior written permission.
2097a077a0SMatthew Dillon *
2197a077a0SMatthew Dillon * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
2297a077a0SMatthew Dillon * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
2397a077a0SMatthew Dillon * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
2497a077a0SMatthew Dillon * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
2597a077a0SMatthew Dillon * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
2697a077a0SMatthew Dillon * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
2797a077a0SMatthew Dillon * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
2897a077a0SMatthew Dillon * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
2997a077a0SMatthew Dillon * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
3097a077a0SMatthew Dillon * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
3197a077a0SMatthew Dillon * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
3297a077a0SMatthew Dillon * SUCH DAMAGE.
3397a077a0SMatthew Dillon */
3497a077a0SMatthew Dillon /*
3597a077a0SMatthew Dillon * Administration thread
3697a077a0SMatthew Dillon *
3797a077a0SMatthew Dillon * - Handles resetting, features, iteration of namespaces, and disk
3897a077a0SMatthew Dillon * attachments. Most admin operations are serialized by the admin thread.
3997a077a0SMatthew Dillon *
4097a077a0SMatthew Dillon * - Ioctls as well as any BIOs which require more sophisticated processing
4197a077a0SMatthew Dillon * are handed to this thread as well.
4297a077a0SMatthew Dillon *
4397a077a0SMatthew Dillon * - Can freeze/resume other queues for various purposes.
4497a077a0SMatthew Dillon */
4597a077a0SMatthew Dillon
4697a077a0SMatthew Dillon #include "nvme.h"
4797a077a0SMatthew Dillon
4897a077a0SMatthew Dillon static void nvme_admin_thread(void *arg);
497e782064SMatthew Dillon static int nvme_admin_state_identify_ctlr(nvme_softc_t *sc);
507e782064SMatthew Dillon static int nvme_admin_state_make_queues(nvme_softc_t *sc);
517e782064SMatthew Dillon static int nvme_admin_state_identify_ns(nvme_softc_t *sc);
527e782064SMatthew Dillon static int nvme_admin_state_operating(nvme_softc_t *sc);
537e782064SMatthew Dillon static int nvme_admin_state_failed(nvme_softc_t *sc);
5497a077a0SMatthew Dillon
5597a077a0SMatthew Dillon /*
5697a077a0SMatthew Dillon * Start the admin thread and block until it says it is running.
5797a077a0SMatthew Dillon */
5897a077a0SMatthew Dillon int
nvme_start_admin_thread(nvme_softc_t * sc)5997a077a0SMatthew Dillon nvme_start_admin_thread(nvme_softc_t *sc)
6097a077a0SMatthew Dillon {
6192a276ebSSepherosa Ziehau int error, intr_flags;
6297a077a0SMatthew Dillon
6397a077a0SMatthew Dillon lockinit(&sc->admin_lk, "admlk", 0, 0);
6443844926SMatthew Dillon lockinit(&sc->ioctl_lk, "nvioc", 0, 0);
6597a077a0SMatthew Dillon sc->admin_signal = 0;
6697a077a0SMatthew Dillon
6792a276ebSSepherosa Ziehau intr_flags = INTR_MPSAFE;
6892a276ebSSepherosa Ziehau if (sc->nirqs == 1) {
6992a276ebSSepherosa Ziehau /* This interrupt processes data CQs too */
7092a276ebSSepherosa Ziehau intr_flags |= INTR_HIFREQ;
7192a276ebSSepherosa Ziehau }
7292a276ebSSepherosa Ziehau
7392a276ebSSepherosa Ziehau error = bus_setup_intr(sc->dev, sc->irq[0], intr_flags,
7418d2384bSMatthew Dillon nvme_intr, &sc->comqueues[0],
7518d2384bSMatthew Dillon &sc->irq_handle[0], NULL);
7697a077a0SMatthew Dillon if (error) {
7797a077a0SMatthew Dillon device_printf(sc->dev, "unable to install interrupt\n");
7897a077a0SMatthew Dillon return error;
7997a077a0SMatthew Dillon }
8097a077a0SMatthew Dillon lockmgr(&sc->admin_lk, LK_EXCLUSIVE);
8197a077a0SMatthew Dillon kthread_create(nvme_admin_thread, sc, &sc->admintd, "nvme_admin");
8297a077a0SMatthew Dillon while ((sc->admin_signal & ADMIN_SIG_RUNNING) == 0)
8397a077a0SMatthew Dillon lksleep(&sc->admin_signal, &sc->admin_lk, 0, "nvwbeg", 0);
8497a077a0SMatthew Dillon lockmgr(&sc->admin_lk, LK_RELEASE);
8597a077a0SMatthew Dillon
8697a077a0SMatthew Dillon return 0;
8797a077a0SMatthew Dillon }
8897a077a0SMatthew Dillon
8997a077a0SMatthew Dillon /*
9097a077a0SMatthew Dillon * Stop the admin thread and block until it says it is done.
9197a077a0SMatthew Dillon */
9297a077a0SMatthew Dillon void
nvme_stop_admin_thread(nvme_softc_t * sc)9397a077a0SMatthew Dillon nvme_stop_admin_thread(nvme_softc_t *sc)
9497a077a0SMatthew Dillon {
957e782064SMatthew Dillon uint32_t i;
967e782064SMatthew Dillon
9797a077a0SMatthew Dillon atomic_set_int(&sc->admin_signal, ADMIN_SIG_STOP);
9897a077a0SMatthew Dillon
997e782064SMatthew Dillon /*
1007e782064SMatthew Dillon * We have to wait for the admin thread to finish its probe
101911f2d4fSMatthew Dillon * before shutting it down. Break out if the admin thread
102911f2d4fSMatthew Dillon * never managed to even start.
1037e782064SMatthew Dillon */
1047e782064SMatthew Dillon lockmgr(&sc->admin_lk, LK_EXCLUSIVE);
105911f2d4fSMatthew Dillon while ((sc->admin_signal & ADMIN_SIG_PROBED) == 0) {
106911f2d4fSMatthew Dillon if ((sc->admin_signal & ADMIN_SIG_RUNNING) == 0)
107911f2d4fSMatthew Dillon break;
1087e782064SMatthew Dillon lksleep(&sc->admin_signal, &sc->admin_lk, 0, "nvwend", 0);
109911f2d4fSMatthew Dillon }
1107e782064SMatthew Dillon lockmgr(&sc->admin_lk, LK_RELEASE);
1117e782064SMatthew Dillon
1127e782064SMatthew Dillon /*
1137e782064SMatthew Dillon * Disconnect our disks while the admin thread is still running,
1147e782064SMatthew Dillon * ensuring that the poll works even if interrupts are broken.
1157e782064SMatthew Dillon * Otherwise we could deadlock in the devfs core.
1167e782064SMatthew Dillon */
1177e782064SMatthew Dillon for (i = 0; i < NVME_MAX_NAMESPACES; ++i) {
1187e782064SMatthew Dillon nvme_softns_t *nsc;
1197e782064SMatthew Dillon
1207e782064SMatthew Dillon if ((nsc = sc->nscary[i]) != NULL) {
1217e782064SMatthew Dillon nvme_disk_detach(nsc);
1227e782064SMatthew Dillon
1237e782064SMatthew Dillon kfree(nsc, M_NVME);
1247e782064SMatthew Dillon sc->nscary[i] = NULL;
1257e782064SMatthew Dillon }
1267e782064SMatthew Dillon }
1277e782064SMatthew Dillon
1287e782064SMatthew Dillon /*
1297e782064SMatthew Dillon * Ask the admin thread to shut-down.
1307e782064SMatthew Dillon */
13197a077a0SMatthew Dillon lockmgr(&sc->admin_lk, LK_EXCLUSIVE);
13297a077a0SMatthew Dillon wakeup(&sc->admin_signal);
13397a077a0SMatthew Dillon while (sc->admin_signal & ADMIN_SIG_RUNNING)
13497a077a0SMatthew Dillon lksleep(&sc->admin_signal, &sc->admin_lk, 0, "nvwend", 0);
13597a077a0SMatthew Dillon lockmgr(&sc->admin_lk, LK_RELEASE);
13618d2384bSMatthew Dillon if (sc->irq_handle[0]) {
13718d2384bSMatthew Dillon bus_teardown_intr(sc->dev, sc->irq[0], sc->irq_handle[0]);
13818d2384bSMatthew Dillon sc->irq_handle[0] = NULL;
13997a077a0SMatthew Dillon }
14043844926SMatthew Dillon lockuninit(&sc->ioctl_lk);
14197a077a0SMatthew Dillon lockuninit(&sc->admin_lk);
14297a077a0SMatthew Dillon
14397a077a0SMatthew Dillon /*
14497a077a0SMatthew Dillon * Thread might be running on another cpu, give it time to actually
14597a077a0SMatthew Dillon * exit before returning in case the caller is about to unload the
14697a077a0SMatthew Dillon * module. Otherwise we don't need this.
14797a077a0SMatthew Dillon */
14897a077a0SMatthew Dillon nvme_os_sleep(1);
14997a077a0SMatthew Dillon }
15097a077a0SMatthew Dillon
15197a077a0SMatthew Dillon static
15297a077a0SMatthew Dillon void
nvme_admin_thread(void * arg)15397a077a0SMatthew Dillon nvme_admin_thread(void *arg)
15497a077a0SMatthew Dillon {
15597a077a0SMatthew Dillon nvme_softc_t *sc = arg;
15697a077a0SMatthew Dillon uint32_t i;
15797a077a0SMatthew Dillon
15897a077a0SMatthew Dillon lockmgr(&sc->admin_lk, LK_EXCLUSIVE);
15997a077a0SMatthew Dillon atomic_set_int(&sc->admin_signal, ADMIN_SIG_RUNNING);
16097a077a0SMatthew Dillon wakeup(&sc->admin_signal);
16197a077a0SMatthew Dillon
16297a077a0SMatthew Dillon sc->admin_func = nvme_admin_state_identify_ctlr;
16397a077a0SMatthew Dillon
16497a077a0SMatthew Dillon while ((sc->admin_signal & ADMIN_SIG_STOP) == 0) {
16518d2384bSMatthew Dillon for (i = 0; i <= sc->niocomqs; ++i) {
16697a077a0SMatthew Dillon nvme_comqueue_t *comq = &sc->comqueues[i];
16797a077a0SMatthew Dillon
16897a077a0SMatthew Dillon if (comq->nqe == 0) /* not configured */
16997a077a0SMatthew Dillon continue;
17097a077a0SMatthew Dillon
17197a077a0SMatthew Dillon lockmgr(&comq->lk, LK_EXCLUSIVE);
17297a077a0SMatthew Dillon nvme_poll_completions(comq, &comq->lk);
17397a077a0SMatthew Dillon lockmgr(&comq->lk, LK_RELEASE);
17497a077a0SMatthew Dillon }
17511759406SMatthew Dillon if (sc->admin_signal & ADMIN_SIG_REQUEUE) {
17611759406SMatthew Dillon atomic_clear_int(&sc->admin_signal, ADMIN_SIG_REQUEUE);
17711759406SMatthew Dillon nvme_disk_requeues(sc);
17811759406SMatthew Dillon }
17911759406SMatthew Dillon if (sc->admin_func(sc) == 0 &&
18011759406SMatthew Dillon (sc->admin_signal & ADMIN_SIG_RUN_MASK) == 0) {
1817e782064SMatthew Dillon lksleep(&sc->admin_signal, &sc->admin_lk, 0,
1827e782064SMatthew Dillon "nvidle", hz);
1837e782064SMatthew Dillon }
18497a077a0SMatthew Dillon }
18597a077a0SMatthew Dillon
18697a077a0SMatthew Dillon /*
18797a077a0SMatthew Dillon * Cleanup state.
18811759406SMatthew Dillon *
18911759406SMatthew Dillon * Note that we actually issue delete queue commands here. The NVME
19011759406SMatthew Dillon * spec says that for a normal shutdown the I/O queues should be
19111759406SMatthew Dillon * deleted prior to issuing the shutdown in the CONFIG register.
19297a077a0SMatthew Dillon */
19311759406SMatthew Dillon for (i = 1; i <= sc->niosubqs; ++i) {
19411759406SMatthew Dillon nvme_delete_subqueue(sc, i);
19597a077a0SMatthew Dillon nvme_free_subqueue(sc, i);
19611759406SMatthew Dillon }
19711759406SMatthew Dillon for (i = 1; i <= sc->niocomqs; ++i) {
19811759406SMatthew Dillon nvme_delete_comqueue(sc, i);
19997a077a0SMatthew Dillon nvme_free_comqueue(sc, i);
20011759406SMatthew Dillon }
20197a077a0SMatthew Dillon
20297a077a0SMatthew Dillon /*
20397a077a0SMatthew Dillon * Signal that we are done.
20497a077a0SMatthew Dillon */
20597a077a0SMatthew Dillon atomic_clear_int(&sc->admin_signal, ADMIN_SIG_RUNNING);
20697a077a0SMatthew Dillon wakeup(&sc->admin_signal);
20797a077a0SMatthew Dillon lockmgr(&sc->admin_lk, LK_RELEASE);
20897a077a0SMatthew Dillon }
20997a077a0SMatthew Dillon
21097a077a0SMatthew Dillon /*
21197a077a0SMatthew Dillon * Identify the controller
21297a077a0SMatthew Dillon */
21397a077a0SMatthew Dillon static
2147e782064SMatthew Dillon int
nvme_admin_state_identify_ctlr(nvme_softc_t * sc)21597a077a0SMatthew Dillon nvme_admin_state_identify_ctlr(nvme_softc_t *sc)
21697a077a0SMatthew Dillon {
21797a077a0SMatthew Dillon nvme_request_t *req;
21897a077a0SMatthew Dillon nvme_ident_ctlr_data_t *rp;
21997a077a0SMatthew Dillon int status;
22097a077a0SMatthew Dillon uint64_t mempgsize;
22197a077a0SMatthew Dillon char serial[20+16];
22297a077a0SMatthew Dillon char model[40+16];
22397a077a0SMatthew Dillon
22497a077a0SMatthew Dillon /*
22597a077a0SMatthew Dillon * Identify Controller
22697a077a0SMatthew Dillon */
22797a077a0SMatthew Dillon mempgsize = NVME_CAP_MEMPG_MIN_GET(sc->cap);
22897a077a0SMatthew Dillon
22997a077a0SMatthew Dillon req = nvme_get_admin_request(sc, NVME_OP_IDENTIFY);
23097a077a0SMatthew Dillon req->cmd.identify.cns = NVME_CNS_CTLR;
23197a077a0SMatthew Dillon req->cmd.identify.cntid = 0;
23297a077a0SMatthew Dillon bzero(req->info, sizeof(*req->info));
23397a077a0SMatthew Dillon nvme_submit_request(req);
234049f03b7SMatthew Dillon status = nvme_wait_request(req);
23597a077a0SMatthew Dillon /* XXX handle status */
23697a077a0SMatthew Dillon
23797a077a0SMatthew Dillon sc->idctlr = req->info->idctlr;
23897a077a0SMatthew Dillon nvme_put_request(req);
23997a077a0SMatthew Dillon
24097a077a0SMatthew Dillon rp = &sc->idctlr;
24197a077a0SMatthew Dillon
24297a077a0SMatthew Dillon KKASSERT(sizeof(sc->idctlr.serialno) == 20);
24397a077a0SMatthew Dillon KKASSERT(sizeof(sc->idctlr.modelno) == 40);
24497a077a0SMatthew Dillon bzero(serial, sizeof(serial));
24597a077a0SMatthew Dillon bzero(model, sizeof(model));
24697a077a0SMatthew Dillon bcopy(rp->serialno, serial, sizeof(rp->serialno));
24797a077a0SMatthew Dillon bcopy(rp->modelno, model, sizeof(rp->modelno));
24897a077a0SMatthew Dillon string_cleanup(serial, 0);
24997a077a0SMatthew Dillon string_cleanup(model, 0);
25097a077a0SMatthew Dillon
25197a077a0SMatthew Dillon device_printf(sc->dev, "Model %s BaseSerial %s nscount=%d\n",
25297a077a0SMatthew Dillon model, serial, rp->ns_count);
25397a077a0SMatthew Dillon
25497a077a0SMatthew Dillon sc->admin_func = nvme_admin_state_make_queues;
2557e782064SMatthew Dillon
2567e782064SMatthew Dillon return 1;
25797a077a0SMatthew Dillon }
25897a077a0SMatthew Dillon
259235fb4acSMatthew Dillon #define COMQFIXUP(msix, ncomqs) ((((msix) - 1) % ncomqs) + 1)
260235fb4acSMatthew Dillon
26197a077a0SMatthew Dillon /*
26297a077a0SMatthew Dillon * Request and create the I/O queues. Figure out CPU mapping optimizations.
26397a077a0SMatthew Dillon */
26497a077a0SMatthew Dillon static
2657e782064SMatthew Dillon int
nvme_admin_state_make_queues(nvme_softc_t * sc)26697a077a0SMatthew Dillon nvme_admin_state_make_queues(nvme_softc_t *sc)
26797a077a0SMatthew Dillon {
26897a077a0SMatthew Dillon nvme_request_t *req;
269*b5b25080SMatthew Dillon uint16_t niosubqs, subq_err_idx;
270*b5b25080SMatthew Dillon uint16_t niocomqs, comq_err_idx;
27197a077a0SMatthew Dillon uint32_t i;
27297a077a0SMatthew Dillon uint16_t qno;
27397a077a0SMatthew Dillon int status;
27497a077a0SMatthew Dillon int error;
27597a077a0SMatthew Dillon
27697a077a0SMatthew Dillon /*
27797a077a0SMatthew Dillon * Calculate how many I/O queues (non-inclusive of admin queue)
27897a077a0SMatthew Dillon * we want to have, up to 65535. dw0 in the response returns the
27997a077a0SMatthew Dillon * number of queues the controller gives us. Submission and
28097a077a0SMatthew Dillon * Completion queues are specified separately.
28197a077a0SMatthew Dillon *
28297a077a0SMatthew Dillon * This driver runs optimally with 4 submission queues and one
28397a077a0SMatthew Dillon * completion queue per cpu (rdhipri, rdlopri, wrhipri, wrlopri),
28418d2384bSMatthew Dillon *
28534885004SMatthew Dillon * +1 for dumps XXX future
28634885004SMatthew Dillon * +1 for async events XXX future
2872d746837SMatthew Dillon *
2882d746837SMatthew Dillon * NOTE: Set one less than the #define because we use 1...N for I/O
2892d746837SMatthew Dillon * queues (queue 0 is used for the admin queue). Easier this
2902d746837SMatthew Dillon * way.
29197a077a0SMatthew Dillon */
29297a077a0SMatthew Dillon req = nvme_get_admin_request(sc, NVME_OP_SET_FEATURES);
29397a077a0SMatthew Dillon
29434885004SMatthew Dillon niosubqs = ncpus * 2 + 0;
29534885004SMatthew Dillon niocomqs = ncpus + 0;
2962d746837SMatthew Dillon if (niosubqs >= NVME_MAX_QUEUES)
2972d746837SMatthew Dillon niosubqs = NVME_MAX_QUEUES - 1;
2982d746837SMatthew Dillon if (niocomqs >= NVME_MAX_QUEUES)
2992d746837SMatthew Dillon niocomqs = NVME_MAX_QUEUES - 1;
3002d746837SMatthew Dillon
3012d746837SMatthew Dillon /*
3022d746837SMatthew Dillon * If there are insufficient MSI-X vectors or we use a normal
3032d746837SMatthew Dillon * interrupt, the completion queues are going to wind up being
3042d746837SMatthew Dillon * polled by a single admin interrupt. Limit the number of
3052d746837SMatthew Dillon * completion queues in this case to something reasonable.
3062d746837SMatthew Dillon */
3072d746837SMatthew Dillon if (sc->nirqs == 1 && niocomqs > 4) {
3082d746837SMatthew Dillon niocomqs = 4;
3092d746837SMatthew Dillon device_printf(sc->dev, "no MSI-X support, limit comqs to %d\n",
3102d746837SMatthew Dillon niocomqs);
3112d746837SMatthew Dillon }
3122d746837SMatthew Dillon
31397a077a0SMatthew Dillon device_printf(sc->dev, "Request %u/%u queues, ", niosubqs, niocomqs);
31497a077a0SMatthew Dillon
31597a077a0SMatthew Dillon req->cmd.setfeat.flags = NVME_FID_NUMQUEUES;
31697a077a0SMatthew Dillon req->cmd.setfeat.numqs.nsqr = niosubqs - 1; /* 0's based 0=1 */
31797a077a0SMatthew Dillon req->cmd.setfeat.numqs.ncqr = niocomqs - 1; /* 0's based 0=1 */
31897a077a0SMatthew Dillon
31997a077a0SMatthew Dillon nvme_submit_request(req);
32097a077a0SMatthew Dillon
32197a077a0SMatthew Dillon /*
3222d746837SMatthew Dillon * Get response and set our operations mode. Limit the returned
3232d746837SMatthew Dillon * queue counts to no more than we requested (some chipsets may
3242d746837SMatthew Dillon * return more than the requested number of queues while others
3252d746837SMatthew Dillon * will not).
32697a077a0SMatthew Dillon */
327049f03b7SMatthew Dillon status = nvme_wait_request(req);
32897a077a0SMatthew Dillon /* XXX handle status */
32997a077a0SMatthew Dillon
33097a077a0SMatthew Dillon if (status == 0) {
33197a077a0SMatthew Dillon sc->niosubqs = 1 + (req->res.setfeat.dw0 & 0xFFFFU);
33297a077a0SMatthew Dillon sc->niocomqs = 1 + ((req->res.setfeat.dw0 >> 16) & 0xFFFFU);
3332d746837SMatthew Dillon if (sc->niosubqs > niosubqs)
3342d746837SMatthew Dillon sc->niosubqs = niosubqs;
3352d746837SMatthew Dillon if (sc->niocomqs > niocomqs)
3362d746837SMatthew Dillon sc->niocomqs = niocomqs;
33797a077a0SMatthew Dillon } else {
33897a077a0SMatthew Dillon sc->niosubqs = 0;
33997a077a0SMatthew Dillon sc->niocomqs = 0;
34097a077a0SMatthew Dillon }
34197a077a0SMatthew Dillon kprintf("Returns %u/%u queues, ", sc->niosubqs, sc->niocomqs);
34297a077a0SMatthew Dillon
34397a077a0SMatthew Dillon nvme_put_request(req);
34497a077a0SMatthew Dillon
345*b5b25080SMatthew Dillon tryagain:
34634885004SMatthew Dillon sc->dumpqno = 0;
34734885004SMatthew Dillon sc->eventqno = 0;
34834885004SMatthew Dillon
34934885004SMatthew Dillon if (sc->niosubqs >= ncpus * 2 + 0 && sc->niocomqs >= ncpus + 0) {
35097a077a0SMatthew Dillon /*
35197a077a0SMatthew Dillon * If we got all the queues we wanted do a full-bore setup of
35297a077a0SMatthew Dillon * qmap[cpu][type].
35328a5c21eSMatthew Dillon *
35428a5c21eSMatthew Dillon * Remember that subq 0 / comq 0 is the admin queue.
35597a077a0SMatthew Dillon */
35697a077a0SMatthew Dillon kprintf("optimal map\n");
35734885004SMatthew Dillon qno = 1;
35897a077a0SMatthew Dillon for (i = 0; i < ncpus; ++i) {
359235fb4acSMatthew Dillon int cpuqno = COMQFIXUP(sc->cputovect[i], ncpus);
36028a5c21eSMatthew Dillon
36128a5c21eSMatthew Dillon KKASSERT(cpuqno != 0);
36297a077a0SMatthew Dillon sc->qmap[i][0] = qno + 0;
36397a077a0SMatthew Dillon sc->qmap[i][1] = qno + 1;
36418d2384bSMatthew Dillon sc->subqueues[qno + 0].comqid = cpuqno;
36518d2384bSMatthew Dillon sc->subqueues[qno + 1].comqid = cpuqno;
36634885004SMatthew Dillon qno += 2;
36797a077a0SMatthew Dillon }
36834885004SMatthew Dillon sc->niosubqs = ncpus * 2 + 0;
36934885004SMatthew Dillon sc->niocomqs = ncpus + 0;
37018d2384bSMatthew Dillon } else if (sc->niosubqs >= ncpus && sc->niocomqs >= ncpus) {
37118d2384bSMatthew Dillon /*
37218d2384bSMatthew Dillon * We have enough to give each cpu its own submission
37318d2384bSMatthew Dillon * and completion queue.
37428a5c21eSMatthew Dillon *
37528a5c21eSMatthew Dillon * leave dumpqno and eventqno set to the admin queue.
37618d2384bSMatthew Dillon */
37718d2384bSMatthew Dillon kprintf("nominal map 1:1 cpu\n");
37818d2384bSMatthew Dillon for (i = 0; i < ncpus; ++i) {
37918d2384bSMatthew Dillon qno = sc->cputovect[i];
38028a5c21eSMatthew Dillon KKASSERT(qno != 0);
38128a5c21eSMatthew Dillon sc->qmap[i][0] = qno;
38228a5c21eSMatthew Dillon sc->qmap[i][1] = qno;
383235fb4acSMatthew Dillon sc->subqueues[qno].comqid = COMQFIXUP(qno, ncpus);
38418d2384bSMatthew Dillon }
38518d2384bSMatthew Dillon sc->niosubqs = ncpus;
38618d2384bSMatthew Dillon sc->niocomqs = ncpus;
38734885004SMatthew Dillon } else if (sc->niosubqs >= 2 && sc->niocomqs >= 2) {
38897a077a0SMatthew Dillon /*
38914676c8aSMatthew Dillon * prioritize trying to distribute available queues to
39014676c8aSMatthew Dillon * cpus, don't separate read and write.
39118d2384bSMatthew Dillon *
39228a5c21eSMatthew Dillon * leave dumpqno and eventqno set to the admin queue.
39397a077a0SMatthew Dillon */
39414676c8aSMatthew Dillon kprintf("rw-sep map (%d, %d)\n", sc->niosubqs, sc->niocomqs);
39597a077a0SMatthew Dillon for (i = 0; i < ncpus; ++i) {
39614676c8aSMatthew Dillon int cpuqno = COMQFIXUP(sc->cputovect[i], sc->niocomqs);
39714676c8aSMatthew Dillon int qno = COMQFIXUP((i + 1), sc->niosubqs);
39828a5c21eSMatthew Dillon
39928a5c21eSMatthew Dillon KKASSERT(qno != 0);
40014676c8aSMatthew Dillon sc->qmap[i][0] = qno; /* read */
40114676c8aSMatthew Dillon sc->qmap[i][1] = qno; /* write */
40214676c8aSMatthew Dillon sc->subqueues[qno].comqid = cpuqno;
40328a5c21eSMatthew Dillon /* do not increment qno */
40497a077a0SMatthew Dillon }
40514676c8aSMatthew Dillon #if 0
40634885004SMatthew Dillon sc->niosubqs = 2;
40734885004SMatthew Dillon sc->niocomqs = 2;
40814676c8aSMatthew Dillon #endif
40997a077a0SMatthew Dillon } else if (sc->niosubqs >= 2) {
41097a077a0SMatthew Dillon /*
41197a077a0SMatthew Dillon * We have enough to have separate read and write queues.
41297a077a0SMatthew Dillon */
41397a077a0SMatthew Dillon kprintf("basic map\n");
41497a077a0SMatthew Dillon qno = 1;
41597a077a0SMatthew Dillon for (i = 0; i < ncpus; ++i) {
416235fb4acSMatthew Dillon int cpuqno = COMQFIXUP(sc->cputovect[i], 1);
41728a5c21eSMatthew Dillon
41828a5c21eSMatthew Dillon KKASSERT(qno != 0);
41934885004SMatthew Dillon sc->qmap[i][0] = qno + 0; /* read */
42034885004SMatthew Dillon sc->qmap[i][1] = qno + 1; /* write */
42128a5c21eSMatthew Dillon if (i <= 0)
42218d2384bSMatthew Dillon sc->subqueues[qno + 0].comqid = cpuqno;
42328a5c21eSMatthew Dillon if (i <= 1)
42418d2384bSMatthew Dillon sc->subqueues[qno + 1].comqid = cpuqno;
42597a077a0SMatthew Dillon }
42697a077a0SMatthew Dillon sc->niosubqs = 2;
42797a077a0SMatthew Dillon sc->niocomqs = 1;
42897a077a0SMatthew Dillon } else {
42997a077a0SMatthew Dillon /*
43097a077a0SMatthew Dillon * Minimal configuration, all cpus and I/O types use the
43128a5c21eSMatthew Dillon * same queue. Sad day.
43297a077a0SMatthew Dillon */
43397a077a0SMatthew Dillon kprintf("minimal map\n");
43497a077a0SMatthew Dillon sc->dumpqno = 0;
43597a077a0SMatthew Dillon sc->eventqno = 0;
43697a077a0SMatthew Dillon for (i = 0; i < ncpus; ++i) {
43797a077a0SMatthew Dillon sc->qmap[i][0] = 1;
43897a077a0SMatthew Dillon sc->qmap[i][1] = 1;
43997a077a0SMatthew Dillon }
44097a077a0SMatthew Dillon sc->subqueues[1].comqid = 1;
44197a077a0SMatthew Dillon sc->niosubqs = 1;
44297a077a0SMatthew Dillon sc->niocomqs = 1;
44397a077a0SMatthew Dillon }
44497a077a0SMatthew Dillon
44597a077a0SMatthew Dillon /*
44697a077a0SMatthew Dillon * Create all I/O submission and completion queues. The I/O
44797a077a0SMatthew Dillon * queues start at 1 and are inclusive of niosubqs and niocomqs.
44897a077a0SMatthew Dillon *
44997a077a0SMatthew Dillon * NOTE: Completion queues must be created before submission queues.
45097a077a0SMatthew Dillon * That is, the completion queue specified when creating a
45197a077a0SMatthew Dillon * submission queue must already exist.
45297a077a0SMatthew Dillon */
45397a077a0SMatthew Dillon error = 0;
45497a077a0SMatthew Dillon for (i = 1; i <= sc->niocomqs; ++i) {
45597a077a0SMatthew Dillon error += nvme_alloc_comqueue(sc, i);
45697a077a0SMatthew Dillon if (error) {
457*b5b25080SMatthew Dillon device_printf(sc->dev, "Unable to alloc comq %d/%d\n",
458*b5b25080SMatthew Dillon i, sc->niocomqs);
45997a077a0SMatthew Dillon break;
46097a077a0SMatthew Dillon }
46197a077a0SMatthew Dillon error += nvme_create_comqueue(sc, i);
4621014e37cSMatthew Dillon if (error) {
463*b5b25080SMatthew Dillon device_printf(sc->dev, "Unable to create comq %d/%d\n",
464*b5b25080SMatthew Dillon i, sc->niocomqs);
465*b5b25080SMatthew Dillon ++i; /* also delete this one below */
4661014e37cSMatthew Dillon break;
4671014e37cSMatthew Dillon }
46897a077a0SMatthew Dillon }
469*b5b25080SMatthew Dillon comq_err_idx = i;
470*b5b25080SMatthew Dillon
47197a077a0SMatthew Dillon for (i = 1; i <= sc->niosubqs; ++i) {
47297a077a0SMatthew Dillon error += nvme_alloc_subqueue(sc, i);
47397a077a0SMatthew Dillon if (error) {
474*b5b25080SMatthew Dillon device_printf(sc->dev, "Unable to alloc subq %d/%d\n",
475*b5b25080SMatthew Dillon i, sc->niosubqs);
47697a077a0SMatthew Dillon break;
47797a077a0SMatthew Dillon }
47897a077a0SMatthew Dillon error += nvme_create_subqueue(sc, i);
4791014e37cSMatthew Dillon if (error) {
480*b5b25080SMatthew Dillon device_printf(sc->dev, "Unable to create subq %d/%d\n",
481*b5b25080SMatthew Dillon i, sc->niosubqs);
482*b5b25080SMatthew Dillon ++i; /* also delete this one below */
4831014e37cSMatthew Dillon break;
4841014e37cSMatthew Dillon }
48597a077a0SMatthew Dillon }
486*b5b25080SMatthew Dillon subq_err_idx = i;
48797a077a0SMatthew Dillon
488*b5b25080SMatthew Dillon /*
489*b5b25080SMatthew Dillon * If we are unable to allocate and create the number of queues
490*b5b25080SMatthew Dillon * the device told us it could handle.
491*b5b25080SMatthew Dillon */
49297a077a0SMatthew Dillon if (error) {
49397a077a0SMatthew Dillon device_printf(sc->dev, "Failed to initialize device!\n");
494*b5b25080SMatthew Dillon for (i = subq_err_idx - 1; i >= 1; --i) {
495*b5b25080SMatthew Dillon nvme_delete_subqueue(sc, i);
496*b5b25080SMatthew Dillon nvme_free_subqueue(sc, i);
497*b5b25080SMatthew Dillon }
498*b5b25080SMatthew Dillon for (i = comq_err_idx - 1; i >= 1; --i) {
499*b5b25080SMatthew Dillon nvme_delete_comqueue(sc, i);
500*b5b25080SMatthew Dillon nvme_free_comqueue(sc, i);
501*b5b25080SMatthew Dillon }
50297a077a0SMatthew Dillon sc->admin_func = nvme_admin_state_failed;
503*b5b25080SMatthew Dillon if (sc->niosubqs > 1 || sc->niocomqs > 1) {
504*b5b25080SMatthew Dillon int trywith = 1;
505*b5b25080SMatthew Dillon
506*b5b25080SMatthew Dillon device_printf(sc->dev,
507*b5b25080SMatthew Dillon "Retrying with fewer queues (%d/%d) "
508*b5b25080SMatthew Dillon "just in case the device lied to us\n",
509*b5b25080SMatthew Dillon trywith, trywith);
510*b5b25080SMatthew Dillon if (sc->niosubqs > trywith)
511*b5b25080SMatthew Dillon sc->niosubqs = trywith;
512*b5b25080SMatthew Dillon if (sc->niocomqs > trywith)
513*b5b25080SMatthew Dillon sc->niocomqs = trywith;
514*b5b25080SMatthew Dillon goto tryagain;
515*b5b25080SMatthew Dillon }
51697a077a0SMatthew Dillon } else {
51797a077a0SMatthew Dillon sc->admin_func = nvme_admin_state_identify_ns;
51897a077a0SMatthew Dillon }
5197e782064SMatthew Dillon
52023bba4b5SMatthew Dillon /*
5212d746837SMatthew Dillon * Disable interrupt coalescing. It is basically worthless because
5222d746837SMatthew Dillon * setting the threshold has no effect when time is set to 0, and the
5232d746837SMatthew Dillon * smallest time that can be set is 1 (== 100uS), which is too long.
5242d746837SMatthew Dillon * Sequential performance is destroyed (on e.g. the Intel 750).
5252d746837SMatthew Dillon * So kill it.
52623bba4b5SMatthew Dillon */
52723bba4b5SMatthew Dillon req = nvme_get_admin_request(sc, NVME_OP_SET_FEATURES);
52823bba4b5SMatthew Dillon device_printf(sc->dev, "Interrupt Coalesce: 100uS / 4 qentries\n");
52923bba4b5SMatthew Dillon
53023bba4b5SMatthew Dillon req->cmd.setfeat.flags = NVME_FID_INTCOALESCE;
53123bba4b5SMatthew Dillon req->cmd.setfeat.intcoal.thr = 0;
53223bba4b5SMatthew Dillon req->cmd.setfeat.intcoal.time = 0;
53323bba4b5SMatthew Dillon
53423bba4b5SMatthew Dillon nvme_submit_request(req);
535049f03b7SMatthew Dillon status = nvme_wait_request(req);
53623bba4b5SMatthew Dillon if (status) {
53723bba4b5SMatthew Dillon device_printf(sc->dev,
53823bba4b5SMatthew Dillon "Interrupt coalesce failed status=%d\n",
53923bba4b5SMatthew Dillon status);
54023bba4b5SMatthew Dillon }
54123bba4b5SMatthew Dillon nvme_put_request(req);
54223bba4b5SMatthew Dillon
5437e782064SMatthew Dillon return 1;
54497a077a0SMatthew Dillon }
54597a077a0SMatthew Dillon
54697a077a0SMatthew Dillon /*
54797a077a0SMatthew Dillon * Identify available namespaces, iterate, and attach to disks.
54897a077a0SMatthew Dillon */
54997a077a0SMatthew Dillon static
5507e782064SMatthew Dillon int
nvme_admin_state_identify_ns(nvme_softc_t * sc)55197a077a0SMatthew Dillon nvme_admin_state_identify_ns(nvme_softc_t *sc)
55297a077a0SMatthew Dillon {
55397a077a0SMatthew Dillon nvme_request_t *req;
55470394f3fSMatthew Dillon nvme_ident_ns_list_t *rp;
55597a077a0SMatthew Dillon int status;
556b9045046SMatthew Dillon int i;
557b9045046SMatthew Dillon int j;
55897a077a0SMatthew Dillon
55970394f3fSMatthew Dillon if (bootverbose) {
56070394f3fSMatthew Dillon if (sc->idctlr.admin_cap & NVME_ADMIN_NSMANAGE)
56170394f3fSMatthew Dillon device_printf(sc->dev,
56270394f3fSMatthew Dillon "Namespace management supported\n");
56370394f3fSMatthew Dillon else
56470394f3fSMatthew Dillon device_printf(sc->dev,
56570394f3fSMatthew Dillon "Namespace management not supported\n");
56670394f3fSMatthew Dillon }
56770394f3fSMatthew Dillon #if 0
56897a077a0SMatthew Dillon /*
56970394f3fSMatthew Dillon * Identify Controllers TODO TODO TODO
57070394f3fSMatthew Dillon */
57170394f3fSMatthew Dillon if (sc->idctlr.admin_cap & NVME_ADMIN_NSMANAGE) {
57270394f3fSMatthew Dillon req = nvme_get_admin_request(sc, NVME_OP_IDENTIFY);
57370394f3fSMatthew Dillon req->cmd.identify.cns = NVME_CNS_ANY_CTLR_LIST;
57470394f3fSMatthew Dillon req->cmd.identify.cntid = 0;
57570394f3fSMatthew Dillon bzero(req->info, sizeof(*req->info));
57670394f3fSMatthew Dillon nvme_submit_request(req);
577049f03b7SMatthew Dillon status = nvme_wait_request(req);
57870394f3fSMatthew Dillon kprintf("nsquery status %08x\n", status);
57970394f3fSMatthew Dillon
58070394f3fSMatthew Dillon #if 0
58170394f3fSMatthew Dillon for (i = 0; i < req->info->ctlrlist.idcount; ++i) {
58270394f3fSMatthew Dillon kprintf("CTLR %04x\n", req->info->ctlrlist.ctlrids[i]);
58370394f3fSMatthew Dillon }
58470394f3fSMatthew Dillon #endif
58570394f3fSMatthew Dillon nvme_put_request(req);
58670394f3fSMatthew Dillon }
58770394f3fSMatthew Dillon #endif
58870394f3fSMatthew Dillon
58970394f3fSMatthew Dillon rp = kmalloc(sizeof(*rp), M_NVME, M_WAITOK | M_ZERO);
59070394f3fSMatthew Dillon if (sc->idctlr.admin_cap & NVME_ADMIN_NSMANAGE) {
59170394f3fSMatthew Dillon /*
59270394f3fSMatthew Dillon * Namespace management supported, query active namespaces.
59397a077a0SMatthew Dillon */
59497a077a0SMatthew Dillon req = nvme_get_admin_request(sc, NVME_OP_IDENTIFY);
59597a077a0SMatthew Dillon req->cmd.identify.cns = NVME_CNS_ACT_NSLIST;
59697a077a0SMatthew Dillon req->cmd.identify.cntid = 0;
59797a077a0SMatthew Dillon bzero(req->info, sizeof(*req->info));
59897a077a0SMatthew Dillon nvme_submit_request(req);
599049f03b7SMatthew Dillon status = nvme_wait_request(req);
60070394f3fSMatthew Dillon kprintf("nsquery status %08x\n", status);
60197a077a0SMatthew Dillon /* XXX handle status */
60297a077a0SMatthew Dillon
60370394f3fSMatthew Dillon cpu_lfence();
60470394f3fSMatthew Dillon *rp = req->info->nslist;
60597a077a0SMatthew Dillon nvme_put_request(req);
60670394f3fSMatthew Dillon } else {
60770394f3fSMatthew Dillon /*
60870394f3fSMatthew Dillon * Namespace management not supported, assume nsids 1..N.
609b9045046SMatthew Dillon * (note: (i) limited to 1024).
61070394f3fSMatthew Dillon */
611b9045046SMatthew Dillon for (i = 1; i <= (int)sc->idctlr.ns_count && i <= 1024; ++i)
61270394f3fSMatthew Dillon rp->nsids[i-1] = i;
61370394f3fSMatthew Dillon }
61497a077a0SMatthew Dillon
61597a077a0SMatthew Dillon /*
61697a077a0SMatthew Dillon * Identify each Namespace
61797a077a0SMatthew Dillon */
61870394f3fSMatthew Dillon for (i = 0; i < 1024; ++i) {
61997a077a0SMatthew Dillon nvme_softns_t *nsc;
62097a077a0SMatthew Dillon nvme_lba_fmt_data_t *lbafmt;
62197a077a0SMatthew Dillon
62270394f3fSMatthew Dillon if (rp->nsids[i] == 0)
62397a077a0SMatthew Dillon continue;
62497a077a0SMatthew Dillon req = nvme_get_admin_request(sc, NVME_OP_IDENTIFY);
62597a077a0SMatthew Dillon req->cmd.identify.cns = NVME_CNS_ACT_NS;
62697a077a0SMatthew Dillon req->cmd.identify.cntid = 0;
62770394f3fSMatthew Dillon req->cmd.identify.head.nsid = rp->nsids[i];
62897a077a0SMatthew Dillon bzero(req->info, sizeof(*req->info));
62997a077a0SMatthew Dillon nvme_submit_request(req);
630049f03b7SMatthew Dillon status = nvme_wait_request(req);
63170394f3fSMatthew Dillon if (status != 0) {
63270394f3fSMatthew Dillon kprintf("NS FAILED %08x\n", status);
63397a077a0SMatthew Dillon continue;
63470394f3fSMatthew Dillon }
63597a077a0SMatthew Dillon
63697a077a0SMatthew Dillon for (j = 0; j < NVME_MAX_NAMESPACES; ++j) {
63797a077a0SMatthew Dillon if (sc->nscary[j] &&
63870394f3fSMatthew Dillon sc->nscary[j]->nsid == rp->nsids[i])
63997a077a0SMatthew Dillon break;
64097a077a0SMatthew Dillon }
64197a077a0SMatthew Dillon if (j == NVME_MAX_NAMESPACES) {
64297a077a0SMatthew Dillon j = i;
64397a077a0SMatthew Dillon if (sc->nscary[j] != NULL) {
64497a077a0SMatthew Dillon for (j = NVME_MAX_NAMESPACES - 1; j >= 0; --j) {
64597a077a0SMatthew Dillon if (sc->nscary[j] == NULL)
64697a077a0SMatthew Dillon break;
64797a077a0SMatthew Dillon }
64897a077a0SMatthew Dillon }
64997a077a0SMatthew Dillon }
65097a077a0SMatthew Dillon if (j < 0) {
65197a077a0SMatthew Dillon device_printf(sc->dev, "not enough room in nscary for "
65270394f3fSMatthew Dillon "namespace %08x\n", rp->nsids[i]);
65397a077a0SMatthew Dillon nvme_put_request(req);
65497a077a0SMatthew Dillon continue;
65597a077a0SMatthew Dillon }
65697a077a0SMatthew Dillon nsc = sc->nscary[j];
65797a077a0SMatthew Dillon if (nsc == NULL) {
65897a077a0SMatthew Dillon nsc = kmalloc(sizeof(*nsc), M_NVME, M_WAITOK | M_ZERO);
6597d057aeaSMatthew Dillon nsc->unit = nvme_alloc_disk_unit();
66097a077a0SMatthew Dillon sc->nscary[j] = nsc;
66197a077a0SMatthew Dillon }
66211759406SMatthew Dillon if (sc->nscmax <= j)
66311759406SMatthew Dillon sc->nscmax = j + 1;
66497a077a0SMatthew Dillon nsc->sc = sc;
66570394f3fSMatthew Dillon nsc->nsid = rp->nsids[i];
66697a077a0SMatthew Dillon nsc->state = NVME_NSC_STATE_UNATTACHED;
66797a077a0SMatthew Dillon nsc->idns = req->info->idns;
66811759406SMatthew Dillon bioq_init(&nsc->bioq);
66911759406SMatthew Dillon lockinit(&nsc->lk, "nvnsc", 0, 0);
67097a077a0SMatthew Dillon
67197a077a0SMatthew Dillon nvme_put_request(req);
67297a077a0SMatthew Dillon
67397a077a0SMatthew Dillon j = NVME_FLBAS_SEL_GET(nsc->idns.flbas);
67497a077a0SMatthew Dillon lbafmt = &nsc->idns.lba_fmt[j];
67597a077a0SMatthew Dillon nsc->blksize = 1 << lbafmt->sect_size;
67697a077a0SMatthew Dillon
67797a077a0SMatthew Dillon /*
67897a077a0SMatthew Dillon * Attach the namespace
67997a077a0SMatthew Dillon */
68097a077a0SMatthew Dillon nvme_disk_attach(nsc);
68197a077a0SMatthew Dillon }
68270394f3fSMatthew Dillon kfree(rp, M_NVME);
68397a077a0SMatthew Dillon
68497a077a0SMatthew Dillon sc->admin_func = nvme_admin_state_operating;
6857e782064SMatthew Dillon return 1;
68697a077a0SMatthew Dillon }
68797a077a0SMatthew Dillon
68897a077a0SMatthew Dillon static
6897e782064SMatthew Dillon int
nvme_admin_state_operating(nvme_softc_t * sc)69097a077a0SMatthew Dillon nvme_admin_state_operating(nvme_softc_t *sc)
69197a077a0SMatthew Dillon {
6927e782064SMatthew Dillon if ((sc->admin_signal & ADMIN_SIG_PROBED) == 0) {
6937e782064SMatthew Dillon atomic_set_int(&sc->admin_signal, ADMIN_SIG_PROBED);
6947e782064SMatthew Dillon wakeup(&sc->admin_signal);
6957e782064SMatthew Dillon }
6967e782064SMatthew Dillon
6977e782064SMatthew Dillon return 0;
69897a077a0SMatthew Dillon }
69997a077a0SMatthew Dillon
70097a077a0SMatthew Dillon static
7017e782064SMatthew Dillon int
nvme_admin_state_failed(nvme_softc_t * sc)70297a077a0SMatthew Dillon nvme_admin_state_failed(nvme_softc_t *sc)
70397a077a0SMatthew Dillon {
7047e782064SMatthew Dillon if ((sc->admin_signal & ADMIN_SIG_PROBED) == 0) {
7057e782064SMatthew Dillon atomic_set_int(&sc->admin_signal, ADMIN_SIG_PROBED);
7067e782064SMatthew Dillon wakeup(&sc->admin_signal);
7077e782064SMatthew Dillon }
7087e782064SMatthew Dillon
7097e782064SMatthew Dillon return 0;
71097a077a0SMatthew Dillon }
711