1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2012-2013 Intel Corporation 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/bio.h> 34 #include <sys/conf.h> 35 #include <sys/fcntl.h> 36 #include <sys/kthread.h> 37 #include <sys/module.h> 38 #include <sys/proc.h> 39 #include <sys/syscallsubr.h> 40 #include <sys/sysctl.h> 41 #include <sys/sysproto.h> 42 #include <sys/systm.h> 43 #include <sys/unistd.h> 44 45 #include <geom/geom.h> 46 47 #include "nvme_private.h" 48 49 struct nvme_io_test_thread { 50 51 uint32_t idx; 52 struct nvme_namespace *ns; 53 enum nvme_nvm_opcode opc; 54 struct timeval start; 55 void *buf; 56 uint32_t size; 57 uint32_t time; 58 uint64_t io_completed; 59 }; 60 61 struct nvme_io_test_internal { 62 63 struct nvme_namespace *ns; 64 enum nvme_nvm_opcode opc; 65 struct timeval start; 66 uint32_t time; 67 uint32_t size; 68 uint32_t td_active; 69 uint32_t td_idx; 70 uint32_t flags; 71 uint64_t io_completed[NVME_TEST_MAX_THREADS]; 72 }; 73 74 static void 75 nvme_ns_bio_test_cb(struct bio *bio) 76 { 77 struct mtx *mtx; 78 79 mtx = mtx_pool_find(mtxpool_sleep, bio); 80 mtx_lock(mtx); 81 wakeup(bio); 82 mtx_unlock(mtx); 83 } 84 85 static void 86 nvme_ns_bio_test(void *arg) 87 { 88 struct nvme_io_test_internal *io_test = arg; 89 struct cdevsw *csw; 90 struct mtx *mtx; 91 struct bio *bio; 92 struct cdev *dev; 93 void *buf; 94 struct timeval t; 95 uint64_t io_completed = 0, offset; 96 uint32_t idx; 97 int ref; 98 99 buf = malloc(io_test->size, M_NVME, M_WAITOK); 100 idx = atomic_fetchadd_int(&io_test->td_idx, 1); 101 dev = io_test->ns->cdev; 102 103 offset = idx * 2048 * nvme_ns_get_sector_size(io_test->ns); 104 105 while (1) { 106 107 bio = g_alloc_bio(); 108 109 memset(bio, 0, sizeof(*bio)); 110 bio->bio_cmd = (io_test->opc == NVME_OPC_READ) ? 111 BIO_READ : BIO_WRITE; 112 bio->bio_done = nvme_ns_bio_test_cb; 113 bio->bio_dev = dev; 114 bio->bio_offset = offset; 115 bio->bio_data = buf; 116 bio->bio_bcount = io_test->size; 117 118 if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) { 119 csw = dev_refthread(dev, &ref); 120 } else 121 csw = dev->si_devsw; 122 123 mtx = mtx_pool_find(mtxpool_sleep, bio); 124 mtx_lock(mtx); 125 (*csw->d_strategy)(bio); 126 msleep(bio, mtx, PRIBIO, "biotestwait", 0); 127 mtx_unlock(mtx); 128 129 if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) { 130 dev_relthread(dev, ref); 131 } 132 133 if ((bio->bio_flags & BIO_ERROR) || (bio->bio_resid > 0)) 134 break; 135 136 g_destroy_bio(bio); 137 138 io_completed++; 139 140 getmicrouptime(&t); 141 timevalsub(&t, &io_test->start); 142 143 if (t.tv_sec >= io_test->time) 144 break; 145 146 offset += io_test->size; 147 if ((offset + io_test->size) > nvme_ns_get_size(io_test->ns)) 148 offset = 0; 149 } 150 151 io_test->io_completed[idx] = io_completed; 152 wakeup_one(io_test); 153 154 free(buf, M_NVME); 155 156 atomic_subtract_int(&io_test->td_active, 1); 157 mb(); 158 159 kthread_exit(); 160 } 161 162 static void 163 nvme_ns_io_test_cb(void *arg, const struct nvme_completion *cpl) 164 { 165 struct nvme_io_test_thread *tth = arg; 166 struct timeval t; 167 168 tth->io_completed++; 169 170 if (nvme_completion_is_error(cpl)) { 171 printf("%s: error occurred\n", __func__); 172 wakeup_one(tth); 173 return; 174 } 175 176 getmicrouptime(&t); 177 timevalsub(&t, &tth->start); 178 179 if (t.tv_sec >= tth->time) { 180 wakeup_one(tth); 181 return; 182 } 183 184 switch (tth->opc) { 185 case NVME_OPC_WRITE: 186 nvme_ns_cmd_write(tth->ns, tth->buf, tth->idx * 2048, 187 tth->size/nvme_ns_get_sector_size(tth->ns), 188 nvme_ns_io_test_cb, tth); 189 break; 190 case NVME_OPC_READ: 191 nvme_ns_cmd_read(tth->ns, tth->buf, tth->idx * 2048, 192 tth->size/nvme_ns_get_sector_size(tth->ns), 193 nvme_ns_io_test_cb, tth); 194 break; 195 default: 196 break; 197 } 198 } 199 200 static void 201 nvme_ns_io_test(void *arg) 202 { 203 struct nvme_io_test_internal *io_test = arg; 204 struct nvme_io_test_thread *tth; 205 struct nvme_completion cpl; 206 int error; 207 208 tth = malloc(sizeof(*tth), M_NVME, M_WAITOK | M_ZERO); 209 tth->ns = io_test->ns; 210 tth->opc = io_test->opc; 211 memcpy(&tth->start, &io_test->start, sizeof(tth->start)); 212 tth->buf = malloc(io_test->size, M_NVME, M_WAITOK); 213 tth->size = io_test->size; 214 tth->time = io_test->time; 215 tth->idx = atomic_fetchadd_int(&io_test->td_idx, 1); 216 217 memset(&cpl, 0, sizeof(cpl)); 218 219 nvme_ns_io_test_cb(tth, &cpl); 220 221 error = tsleep(tth, 0, "test_wait", tth->time*hz*2); 222 223 if (error) 224 printf("%s: error = %d\n", __func__, error); 225 226 io_test->io_completed[tth->idx] = tth->io_completed; 227 wakeup_one(io_test); 228 229 free(tth->buf, M_NVME); 230 free(tth, M_NVME); 231 232 atomic_subtract_int(&io_test->td_active, 1); 233 mb(); 234 235 kthread_exit(); 236 } 237 238 void 239 nvme_ns_test(struct nvme_namespace *ns, u_long cmd, caddr_t arg) 240 { 241 struct nvme_io_test *io_test; 242 struct nvme_io_test_internal *io_test_internal; 243 void (*fn)(void *); 244 int i; 245 246 io_test = (struct nvme_io_test *)arg; 247 248 if ((io_test->opc != NVME_OPC_READ) && 249 (io_test->opc != NVME_OPC_WRITE)) 250 return; 251 252 if (io_test->size % nvme_ns_get_sector_size(ns)) 253 return; 254 255 io_test_internal = malloc(sizeof(*io_test_internal), M_NVME, 256 M_WAITOK | M_ZERO); 257 io_test_internal->opc = io_test->opc; 258 io_test_internal->ns = ns; 259 io_test_internal->td_active = io_test->num_threads; 260 io_test_internal->time = io_test->time; 261 io_test_internal->size = io_test->size; 262 io_test_internal->flags = io_test->flags; 263 264 if (cmd == NVME_IO_TEST) 265 fn = nvme_ns_io_test; 266 else 267 fn = nvme_ns_bio_test; 268 269 getmicrouptime(&io_test_internal->start); 270 271 for (i = 0; i < io_test->num_threads; i++) 272 kthread_add(fn, io_test_internal, 273 NULL, NULL, 0, 0, "nvme_io_test[%d]", i); 274 275 tsleep(io_test_internal, 0, "nvme_test", io_test->time * 2 * hz); 276 277 while (io_test_internal->td_active > 0) 278 DELAY(10); 279 280 memcpy(io_test->io_completed, io_test_internal->io_completed, 281 sizeof(io_test->io_completed)); 282 283 free(io_test_internal, M_NVME); 284 } 285