1bb0ec6b3SJim Harris /*- 2bb0ec6b3SJim Harris * Copyright (C) 2012 Intel Corporation 3bb0ec6b3SJim Harris * All rights reserved. 4bb0ec6b3SJim Harris * 5bb0ec6b3SJim Harris * Redistribution and use in source and binary forms, with or without 6bb0ec6b3SJim Harris * modification, are permitted provided that the following conditions 7bb0ec6b3SJim Harris * are met: 8bb0ec6b3SJim Harris * 1. Redistributions of source code must retain the above copyright 9bb0ec6b3SJim Harris * notice, this list of conditions and the following disclaimer. 10bb0ec6b3SJim Harris * 2. Redistributions in binary form must reproduce the above copyright 11bb0ec6b3SJim Harris * notice, this list of conditions and the following disclaimer in the 12bb0ec6b3SJim Harris * documentation and/or other materials provided with the distribution. 13bb0ec6b3SJim Harris * 14bb0ec6b3SJim Harris * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15bb0ec6b3SJim Harris * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16bb0ec6b3SJim Harris * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17bb0ec6b3SJim Harris * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18bb0ec6b3SJim Harris * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19bb0ec6b3SJim Harris * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20bb0ec6b3SJim Harris * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21bb0ec6b3SJim Harris * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22bb0ec6b3SJim Harris * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23bb0ec6b3SJim Harris * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24bb0ec6b3SJim Harris * SUCH DAMAGE. 25bb0ec6b3SJim Harris */ 26bb0ec6b3SJim Harris 27bb0ec6b3SJim Harris #include <sys/cdefs.h> 28bb0ec6b3SJim Harris __FBSDID("$FreeBSD$"); 29bb0ec6b3SJim Harris 30bb0ec6b3SJim Harris #include <sys/param.h> 31bb0ec6b3SJim Harris #include <sys/bio.h> 32bb0ec6b3SJim Harris #include <sys/conf.h> 33bb0ec6b3SJim Harris #include <sys/fcntl.h> 34bb0ec6b3SJim Harris #include <sys/kthread.h> 35bb0ec6b3SJim Harris #include <sys/module.h> 36bb0ec6b3SJim Harris #include <sys/proc.h> 37bb0ec6b3SJim Harris #include <sys/syscallsubr.h> 38bb0ec6b3SJim Harris #include <sys/sysctl.h> 39bb0ec6b3SJim Harris #include <sys/sysproto.h> 40bb0ec6b3SJim Harris #include <sys/systm.h> 41bb0ec6b3SJim Harris #include <sys/unistd.h> 42bb0ec6b3SJim Harris 43bb0ec6b3SJim Harris #include <geom/geom.h> 44bb0ec6b3SJim Harris 45bb0ec6b3SJim Harris #include "nvme_private.h" 46bb0ec6b3SJim Harris 47bb0ec6b3SJim Harris struct nvme_io_test_thread { 48bb0ec6b3SJim Harris 49bb0ec6b3SJim Harris uint32_t idx; 50bb0ec6b3SJim Harris struct nvme_namespace *ns; 51bb0ec6b3SJim Harris enum nvme_nvm_opcode opc; 52bb0ec6b3SJim Harris struct timeval start; 53bb0ec6b3SJim Harris void *buf; 54bb0ec6b3SJim Harris uint32_t size; 55bb0ec6b3SJim Harris uint32_t time; 56bb0ec6b3SJim Harris uint32_t io_completed; 57bb0ec6b3SJim Harris }; 58bb0ec6b3SJim Harris 59bb0ec6b3SJim Harris struct nvme_io_test_internal { 60bb0ec6b3SJim Harris 61bb0ec6b3SJim Harris struct nvme_namespace *ns; 62bb0ec6b3SJim Harris enum nvme_nvm_opcode opc; 63bb0ec6b3SJim Harris struct timeval start; 64bb0ec6b3SJim Harris uint32_t time; 65bb0ec6b3SJim Harris uint32_t size; 66bb0ec6b3SJim Harris uint32_t td_active; 67bb0ec6b3SJim Harris uint32_t td_idx; 68bb0ec6b3SJim Harris uint32_t flags; 69bb0ec6b3SJim Harris uint32_t io_completed[NVME_TEST_MAX_THREADS]; 70bb0ec6b3SJim Harris }; 71bb0ec6b3SJim Harris 72bb0ec6b3SJim Harris static void 73bb0ec6b3SJim Harris nvme_ns_bio_test_cb(struct bio *bio) 74bb0ec6b3SJim Harris { 75bb0ec6b3SJim Harris struct mtx *mtx; 76bb0ec6b3SJim Harris 77bb0ec6b3SJim Harris mtx = mtx_pool_find(mtxpool_sleep, bio); 78bb0ec6b3SJim Harris mtx_lock(mtx); 79bb0ec6b3SJim Harris wakeup(bio); 80bb0ec6b3SJim Harris mtx_unlock(mtx); 81bb0ec6b3SJim Harris } 82bb0ec6b3SJim Harris 83bb0ec6b3SJim Harris static void 84bb0ec6b3SJim Harris nvme_ns_bio_test(void *arg) 85bb0ec6b3SJim Harris { 86bb0ec6b3SJim Harris struct nvme_io_test_internal *io_test = arg; 87bb0ec6b3SJim Harris struct cdevsw *csw; 88bb0ec6b3SJim Harris struct mtx *mtx; 89bb0ec6b3SJim Harris struct bio *bio; 90bb0ec6b3SJim Harris struct cdev *dev; 91bb0ec6b3SJim Harris void *buf; 92bb0ec6b3SJim Harris struct timeval t; 93bb0ec6b3SJim Harris uint64_t offset; 94bb0ec6b3SJim Harris uint32_t idx, io_completed = 0; 95bb0ec6b3SJim Harris #if __FreeBSD_version >= 900017 96bb0ec6b3SJim Harris int ref; 97bb0ec6b3SJim Harris #endif 98bb0ec6b3SJim Harris 99bb0ec6b3SJim Harris buf = malloc(io_test->size, M_NVME, M_NOWAIT); 100bb0ec6b3SJim Harris idx = atomic_fetchadd_int(&io_test->td_idx, 1); 101bb0ec6b3SJim Harris dev = io_test->ns->cdev; 102bb0ec6b3SJim Harris 103bb0ec6b3SJim Harris offset = idx * 2048 * nvme_ns_get_sector_size(io_test->ns); 104bb0ec6b3SJim Harris 105bb0ec6b3SJim Harris while (1) { 106bb0ec6b3SJim Harris 107bb0ec6b3SJim Harris bio = g_alloc_bio(); 108bb0ec6b3SJim Harris 109bb0ec6b3SJim Harris memset(bio, 0, sizeof(*bio)); 110bb0ec6b3SJim Harris bio->bio_cmd = (io_test->opc == NVME_OPC_READ) ? 111bb0ec6b3SJim Harris BIO_READ : BIO_WRITE; 112bb0ec6b3SJim Harris bio->bio_done = nvme_ns_bio_test_cb; 113bb0ec6b3SJim Harris bio->bio_dev = dev; 114bb0ec6b3SJim Harris bio->bio_offset = offset; 115bb0ec6b3SJim Harris bio->bio_data = buf; 116bb0ec6b3SJim Harris bio->bio_bcount = io_test->size; 117bb0ec6b3SJim Harris 118bb0ec6b3SJim Harris if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) { 119bb0ec6b3SJim Harris #if __FreeBSD_version >= 900017 120bb0ec6b3SJim Harris csw = dev_refthread(dev, &ref); 121bb0ec6b3SJim Harris #else 122bb0ec6b3SJim Harris csw = dev_refthread(dev); 123bb0ec6b3SJim Harris #endif 124bb0ec6b3SJim Harris } else 125bb0ec6b3SJim Harris csw = dev->si_devsw; 126bb0ec6b3SJim Harris 127bb0ec6b3SJim Harris mtx = mtx_pool_find(mtxpool_sleep, bio); 128bb0ec6b3SJim Harris mtx_lock(mtx); 129bb0ec6b3SJim Harris (*csw->d_strategy)(bio); 130bb0ec6b3SJim Harris msleep(bio, mtx, PRIBIO, "biotestwait", 0); 131bb0ec6b3SJim Harris mtx_unlock(mtx); 132bb0ec6b3SJim Harris 133bb0ec6b3SJim Harris if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) { 134bb0ec6b3SJim Harris #if __FreeBSD_version >= 900017 135bb0ec6b3SJim Harris dev_relthread(dev, ref); 136bb0ec6b3SJim Harris #else 137bb0ec6b3SJim Harris dev_relthread(dev); 138bb0ec6b3SJim Harris #endif 139bb0ec6b3SJim Harris } 140bb0ec6b3SJim Harris 141bb0ec6b3SJim Harris if ((bio->bio_flags & BIO_ERROR) || (bio->bio_resid > 0)) 142bb0ec6b3SJim Harris break; 143bb0ec6b3SJim Harris 144bb0ec6b3SJim Harris g_destroy_bio(bio); 145bb0ec6b3SJim Harris 146bb0ec6b3SJim Harris io_completed++; 147bb0ec6b3SJim Harris 148bb0ec6b3SJim Harris getmicrouptime(&t); 149bb0ec6b3SJim Harris timevalsub(&t, &io_test->start); 150bb0ec6b3SJim Harris 151bb0ec6b3SJim Harris if (t.tv_sec >= io_test->time) 152bb0ec6b3SJim Harris break; 153bb0ec6b3SJim Harris 154bb0ec6b3SJim Harris offset += io_test->size; 155bb0ec6b3SJim Harris if ((offset + io_test->size) > nvme_ns_get_size(io_test->ns)) 156bb0ec6b3SJim Harris offset = 0; 157bb0ec6b3SJim Harris } 158bb0ec6b3SJim Harris 159bb0ec6b3SJim Harris io_test->io_completed[idx] = io_completed; 160bb0ec6b3SJim Harris wakeup_one(io_test); 161bb0ec6b3SJim Harris 162bb0ec6b3SJim Harris free(buf, M_NVME); 163bb0ec6b3SJim Harris 164bb0ec6b3SJim Harris atomic_subtract_int(&io_test->td_active, 1); 165bb0ec6b3SJim Harris mb(); 166bb0ec6b3SJim Harris 167bb0ec6b3SJim Harris #if __FreeBSD_version >= 800000 168bb0ec6b3SJim Harris kthread_exit(); 169bb0ec6b3SJim Harris #else 170bb0ec6b3SJim Harris kthread_exit(0); 171bb0ec6b3SJim Harris #endif 172bb0ec6b3SJim Harris } 173bb0ec6b3SJim Harris 174bb0ec6b3SJim Harris static void 175cf81529cSJim Harris nvme_ns_io_test_cb(void *arg, const struct nvme_completion *cpl) 176bb0ec6b3SJim Harris { 177bb0ec6b3SJim Harris struct nvme_io_test_thread *tth = arg; 178bb0ec6b3SJim Harris struct timeval t; 179bb0ec6b3SJim Harris 180bb0ec6b3SJim Harris tth->io_completed++; 181bb0ec6b3SJim Harris 182cf81529cSJim Harris if (nvme_completion_is_error(cpl)) { 183bb0ec6b3SJim Harris printf("%s: error occurred\n", __func__); 184bb0ec6b3SJim Harris wakeup_one(tth); 185bb0ec6b3SJim Harris return; 186bb0ec6b3SJim Harris } 187bb0ec6b3SJim Harris 188bb0ec6b3SJim Harris getmicrouptime(&t); 189bb0ec6b3SJim Harris timevalsub(&t, &tth->start); 190bb0ec6b3SJim Harris 191bb0ec6b3SJim Harris if (t.tv_sec >= tth->time) { 192bb0ec6b3SJim Harris wakeup_one(tth); 193bb0ec6b3SJim Harris return; 194bb0ec6b3SJim Harris } 195bb0ec6b3SJim Harris 196bb0ec6b3SJim Harris switch (tth->opc) { 197bb0ec6b3SJim Harris case NVME_OPC_WRITE: 198bb0ec6b3SJim Harris nvme_ns_cmd_write(tth->ns, tth->buf, tth->idx * 2048, 199bb0ec6b3SJim Harris tth->size/nvme_ns_get_sector_size(tth->ns), 200bb0ec6b3SJim Harris nvme_ns_io_test_cb, tth); 201bb0ec6b3SJim Harris break; 202bb0ec6b3SJim Harris case NVME_OPC_READ: 203bb0ec6b3SJim Harris nvme_ns_cmd_read(tth->ns, tth->buf, tth->idx * 2048, 204bb0ec6b3SJim Harris tth->size/nvme_ns_get_sector_size(tth->ns), 205bb0ec6b3SJim Harris nvme_ns_io_test_cb, tth); 206bb0ec6b3SJim Harris break; 207bb0ec6b3SJim Harris default: 208bb0ec6b3SJim Harris break; 209bb0ec6b3SJim Harris } 210bb0ec6b3SJim Harris } 211bb0ec6b3SJim Harris 212bb0ec6b3SJim Harris static void 213bb0ec6b3SJim Harris nvme_ns_io_test(void *arg) 214bb0ec6b3SJim Harris { 215bb0ec6b3SJim Harris struct nvme_io_test_internal *io_test = arg; 216bb0ec6b3SJim Harris struct nvme_io_test_thread *tth; 217bb0ec6b3SJim Harris struct nvme_completion cpl; 218bb0ec6b3SJim Harris int error; 219bb0ec6b3SJim Harris 220bb0ec6b3SJim Harris tth = malloc(sizeof(*tth), M_NVME, M_NOWAIT | M_ZERO); 221bb0ec6b3SJim Harris tth->ns = io_test->ns; 222bb0ec6b3SJim Harris tth->opc = io_test->opc; 223bb0ec6b3SJim Harris memcpy(&tth->start, &io_test->start, sizeof(tth->start)); 224bb0ec6b3SJim Harris tth->buf = malloc(io_test->size, M_NVME, M_NOWAIT); 225bb0ec6b3SJim Harris tth->size = io_test->size; 226bb0ec6b3SJim Harris tth->time = io_test->time; 227bb0ec6b3SJim Harris tth->idx = atomic_fetchadd_int(&io_test->td_idx, 1); 228bb0ec6b3SJim Harris 229bb0ec6b3SJim Harris memset(&cpl, 0, sizeof(cpl)); 230bb0ec6b3SJim Harris 231bb0ec6b3SJim Harris nvme_ns_io_test_cb(tth, &cpl); 232bb0ec6b3SJim Harris 233bb0ec6b3SJim Harris error = tsleep(tth, 0, "test_wait", tth->time*hz*2); 234bb0ec6b3SJim Harris 235bb0ec6b3SJim Harris if (error) 236bb0ec6b3SJim Harris printf("%s: error = %d\n", __func__, error); 237bb0ec6b3SJim Harris 238bb0ec6b3SJim Harris io_test->io_completed[tth->idx] = tth->io_completed; 239bb0ec6b3SJim Harris wakeup_one(io_test); 240bb0ec6b3SJim Harris 241bb0ec6b3SJim Harris free(tth->buf, M_NVME); 242bb0ec6b3SJim Harris free(tth, M_NVME); 243bb0ec6b3SJim Harris 244bb0ec6b3SJim Harris atomic_subtract_int(&io_test->td_active, 1); 245bb0ec6b3SJim Harris mb(); 246bb0ec6b3SJim Harris 247bb0ec6b3SJim Harris #if __FreeBSD_version >= 800004 248bb0ec6b3SJim Harris kthread_exit(); 249bb0ec6b3SJim Harris #else 250bb0ec6b3SJim Harris kthread_exit(0); 251bb0ec6b3SJim Harris #endif 252bb0ec6b3SJim Harris } 253bb0ec6b3SJim Harris 254bb0ec6b3SJim Harris void 255bb0ec6b3SJim Harris nvme_ns_test(struct nvme_namespace *ns, u_long cmd, caddr_t arg) 256bb0ec6b3SJim Harris { 257bb0ec6b3SJim Harris struct nvme_io_test *io_test; 258bb0ec6b3SJim Harris struct nvme_io_test_internal *io_test_internal; 259bb0ec6b3SJim Harris void (*fn)(void *); 260bb0ec6b3SJim Harris int i; 261bb0ec6b3SJim Harris 262bb0ec6b3SJim Harris io_test = (struct nvme_io_test *)arg; 263bb0ec6b3SJim Harris 264bb0ec6b3SJim Harris if ((io_test->opc != NVME_OPC_READ) && 265bb0ec6b3SJim Harris (io_test->opc != NVME_OPC_WRITE)) 266bb0ec6b3SJim Harris return; 267bb0ec6b3SJim Harris 268bb0ec6b3SJim Harris if (io_test->size % nvme_ns_get_sector_size(ns)) 269bb0ec6b3SJim Harris return; 270bb0ec6b3SJim Harris 271bb0ec6b3SJim Harris io_test_internal = malloc(sizeof(*io_test_internal), M_NVME, 272bb0ec6b3SJim Harris M_NOWAIT | M_ZERO); 273bb0ec6b3SJim Harris io_test_internal->opc = io_test->opc; 274bb0ec6b3SJim Harris io_test_internal->ns = ns; 275bb0ec6b3SJim Harris io_test_internal->td_active = io_test->num_threads; 276bb0ec6b3SJim Harris io_test_internal->time = io_test->time; 277bb0ec6b3SJim Harris io_test_internal->size = io_test->size; 278bb0ec6b3SJim Harris io_test_internal->flags = io_test->flags; 279bb0ec6b3SJim Harris 280bb0ec6b3SJim Harris if (cmd == NVME_IO_TEST) 281bb0ec6b3SJim Harris fn = nvme_ns_io_test; 282bb0ec6b3SJim Harris else 283bb0ec6b3SJim Harris fn = nvme_ns_bio_test; 284bb0ec6b3SJim Harris 285bb0ec6b3SJim Harris getmicrouptime(&io_test_internal->start); 286bb0ec6b3SJim Harris 287bb0ec6b3SJim Harris for (i = 0; i < io_test->num_threads; i++) 288bb0ec6b3SJim Harris #if __FreeBSD_version >= 800004 289bb0ec6b3SJim Harris kthread_add(fn, io_test_internal, 29043e35466SJim Harris NULL, NULL, 0, 0, "nvme_io_test[%d]", i); 291bb0ec6b3SJim Harris #else 292bb0ec6b3SJim Harris kthread_create(fn, io_test_internal, 29343e35466SJim Harris NULL, 0, 0, "nvme_io_test[%d]", i); 294bb0ec6b3SJim Harris #endif 295bb0ec6b3SJim Harris 296bb0ec6b3SJim Harris tsleep(io_test_internal, 0, "nvme_test", io_test->time * 2 * hz); 297bb0ec6b3SJim Harris 298bb0ec6b3SJim Harris while (io_test_internal->td_active > 0) 299bb0ec6b3SJim Harris DELAY(10); 300bb0ec6b3SJim Harris 301bb0ec6b3SJim Harris memcpy(io_test->io_completed, io_test_internal->io_completed, 302bb0ec6b3SJim Harris sizeof(io_test->io_completed)); 303bb0ec6b3SJim Harris 304bb0ec6b3SJim Harris free(io_test_internal, M_NVME); 305bb0ec6b3SJim Harris } 306