1bb0ec6b3SJim Harris /*-
24d846d26SWarner Losh * SPDX-License-Identifier: BSD-2-Clause
3718cf2ccSPedro F. Giffuni *
4e9efbc13SJim Harris * Copyright (C) 2012-2013 Intel Corporation
5bb0ec6b3SJim Harris * All rights reserved.
6bb0ec6b3SJim Harris *
7bb0ec6b3SJim Harris * Redistribution and use in source and binary forms, with or without
8bb0ec6b3SJim Harris * modification, are permitted provided that the following conditions
9bb0ec6b3SJim Harris * are met:
10bb0ec6b3SJim Harris * 1. Redistributions of source code must retain the above copyright
11bb0ec6b3SJim Harris * notice, this list of conditions and the following disclaimer.
12bb0ec6b3SJim Harris * 2. Redistributions in binary form must reproduce the above copyright
13bb0ec6b3SJim Harris * notice, this list of conditions and the following disclaimer in the
14bb0ec6b3SJim Harris * documentation and/or other materials provided with the distribution.
15bb0ec6b3SJim Harris *
16bb0ec6b3SJim Harris * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17bb0ec6b3SJim Harris * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18bb0ec6b3SJim Harris * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19bb0ec6b3SJim Harris * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20bb0ec6b3SJim Harris * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21bb0ec6b3SJim Harris * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22bb0ec6b3SJim Harris * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23bb0ec6b3SJim Harris * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24bb0ec6b3SJim Harris * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25bb0ec6b3SJim Harris * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26bb0ec6b3SJim Harris * SUCH DAMAGE.
27bb0ec6b3SJim Harris */
28bb0ec6b3SJim Harris
29bb0ec6b3SJim Harris #include <sys/param.h>
30bb0ec6b3SJim Harris #include <sys/bio.h>
31bb0ec6b3SJim Harris #include <sys/conf.h>
32bb0ec6b3SJim Harris #include <sys/fcntl.h>
33bb0ec6b3SJim Harris #include <sys/kthread.h>
346bb132baSBrooks Davis #include <sys/malloc.h>
35bb0ec6b3SJim Harris #include <sys/module.h>
36bb0ec6b3SJim Harris #include <sys/proc.h>
37bb0ec6b3SJim Harris #include <sys/syscallsubr.h>
38bb0ec6b3SJim Harris #include <sys/sysctl.h>
39bb0ec6b3SJim Harris #include <sys/sysproto.h>
40bb0ec6b3SJim Harris #include <sys/systm.h>
41bb0ec6b3SJim Harris #include <sys/unistd.h>
42bb0ec6b3SJim Harris
43bb0ec6b3SJim Harris #include <geom/geom.h>
44bb0ec6b3SJim Harris
45bb0ec6b3SJim Harris #include "nvme_private.h"
46bb0ec6b3SJim Harris
47bb0ec6b3SJim Harris struct nvme_io_test_thread {
48bb0ec6b3SJim Harris uint32_t idx;
49bb0ec6b3SJim Harris struct nvme_namespace *ns;
50bb0ec6b3SJim Harris enum nvme_nvm_opcode opc;
51bb0ec6b3SJim Harris struct timeval start;
52bb0ec6b3SJim Harris void *buf;
53bb0ec6b3SJim Harris uint32_t size;
54bb0ec6b3SJim Harris uint32_t time;
55992db80fSJim Harris uint64_t io_completed;
56bb0ec6b3SJim Harris };
57bb0ec6b3SJim Harris
58bb0ec6b3SJim Harris struct nvme_io_test_internal {
59bb0ec6b3SJim Harris struct nvme_namespace *ns;
60bb0ec6b3SJim Harris enum nvme_nvm_opcode opc;
61bb0ec6b3SJim Harris struct timeval start;
62bb0ec6b3SJim Harris uint32_t time;
63bb0ec6b3SJim Harris uint32_t size;
64bb0ec6b3SJim Harris uint32_t td_active;
65bb0ec6b3SJim Harris uint32_t td_idx;
66bb0ec6b3SJim Harris uint32_t flags;
67992db80fSJim Harris uint64_t io_completed[NVME_TEST_MAX_THREADS];
68bb0ec6b3SJim Harris };
69bb0ec6b3SJim Harris
70bb0ec6b3SJim Harris static void
nvme_ns_bio_test_cb(struct bio * bio)71bb0ec6b3SJim Harris nvme_ns_bio_test_cb(struct bio *bio)
72bb0ec6b3SJim Harris {
73bb0ec6b3SJim Harris struct mtx *mtx;
74bb0ec6b3SJim Harris
75bb0ec6b3SJim Harris mtx = mtx_pool_find(mtxpool_sleep, bio);
76bb0ec6b3SJim Harris mtx_lock(mtx);
77bb0ec6b3SJim Harris wakeup(bio);
78bb0ec6b3SJim Harris mtx_unlock(mtx);
79bb0ec6b3SJim Harris }
80bb0ec6b3SJim Harris
81bb0ec6b3SJim Harris static void
nvme_ns_bio_test(void * arg)82bb0ec6b3SJim Harris nvme_ns_bio_test(void *arg)
83bb0ec6b3SJim Harris {
84bb0ec6b3SJim Harris struct nvme_io_test_internal *io_test = arg;
85bb0ec6b3SJim Harris struct cdevsw *csw;
86bb0ec6b3SJim Harris struct mtx *mtx;
87bb0ec6b3SJim Harris struct bio *bio;
88bb0ec6b3SJim Harris struct cdev *dev;
89bb0ec6b3SJim Harris void *buf;
90bb0ec6b3SJim Harris struct timeval t;
91992db80fSJim Harris uint64_t io_completed = 0, offset;
92992db80fSJim Harris uint32_t idx;
93bb0ec6b3SJim Harris int ref;
94bb0ec6b3SJim Harris
95237d2019SJim Harris buf = malloc(io_test->size, M_NVME, M_WAITOK);
96bb0ec6b3SJim Harris idx = atomic_fetchadd_int(&io_test->td_idx, 1);
97bb0ec6b3SJim Harris dev = io_test->ns->cdev;
98bb0ec6b3SJim Harris
994053f8acSDavid Bright offset = idx * 2048ULL * nvme_ns_get_sector_size(io_test->ns);
100bb0ec6b3SJim Harris
101bb0ec6b3SJim Harris while (1) {
102bb0ec6b3SJim Harris bio = g_alloc_bio();
103bb0ec6b3SJim Harris
104bb0ec6b3SJim Harris memset(bio, 0, sizeof(*bio));
105bb0ec6b3SJim Harris bio->bio_cmd = (io_test->opc == NVME_OPC_READ) ?
106bb0ec6b3SJim Harris BIO_READ : BIO_WRITE;
107bb0ec6b3SJim Harris bio->bio_done = nvme_ns_bio_test_cb;
108bb0ec6b3SJim Harris bio->bio_dev = dev;
109bb0ec6b3SJim Harris bio->bio_offset = offset;
110bb0ec6b3SJim Harris bio->bio_data = buf;
111bb0ec6b3SJim Harris bio->bio_bcount = io_test->size;
112bb0ec6b3SJim Harris
113bb0ec6b3SJim Harris if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) {
114bb0ec6b3SJim Harris csw = dev_refthread(dev, &ref);
115bb0ec6b3SJim Harris } else
116bb0ec6b3SJim Harris csw = dev->si_devsw;
117bb0ec6b3SJim Harris
1184053f8acSDavid Bright if (csw == NULL)
1194053f8acSDavid Bright panic("Unable to retrieve device switch");
120bb0ec6b3SJim Harris mtx = mtx_pool_find(mtxpool_sleep, bio);
121bb0ec6b3SJim Harris mtx_lock(mtx);
122bb0ec6b3SJim Harris (*csw->d_strategy)(bio);
123bb0ec6b3SJim Harris msleep(bio, mtx, PRIBIO, "biotestwait", 0);
124bb0ec6b3SJim Harris mtx_unlock(mtx);
125bb0ec6b3SJim Harris
126bb0ec6b3SJim Harris if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) {
127bb0ec6b3SJim Harris dev_relthread(dev, ref);
128bb0ec6b3SJim Harris }
129bb0ec6b3SJim Harris
130bb0ec6b3SJim Harris if ((bio->bio_flags & BIO_ERROR) || (bio->bio_resid > 0))
131bb0ec6b3SJim Harris break;
132bb0ec6b3SJim Harris
133bb0ec6b3SJim Harris g_destroy_bio(bio);
134bb0ec6b3SJim Harris
135bb0ec6b3SJim Harris io_completed++;
136bb0ec6b3SJim Harris
137bb0ec6b3SJim Harris getmicrouptime(&t);
138bb0ec6b3SJim Harris timevalsub(&t, &io_test->start);
139bb0ec6b3SJim Harris
140bb0ec6b3SJim Harris if (t.tv_sec >= io_test->time)
141bb0ec6b3SJim Harris break;
142bb0ec6b3SJim Harris
143bb0ec6b3SJim Harris offset += io_test->size;
144bb0ec6b3SJim Harris if ((offset + io_test->size) > nvme_ns_get_size(io_test->ns))
145bb0ec6b3SJim Harris offset = 0;
146bb0ec6b3SJim Harris }
147bb0ec6b3SJim Harris
148bb0ec6b3SJim Harris io_test->io_completed[idx] = io_completed;
149bb0ec6b3SJim Harris wakeup_one(io_test);
150bb0ec6b3SJim Harris
151bb0ec6b3SJim Harris free(buf, M_NVME);
152bb0ec6b3SJim Harris
153bb0ec6b3SJim Harris atomic_subtract_int(&io_test->td_active, 1);
154bb0ec6b3SJim Harris mb();
155bb0ec6b3SJim Harris
156bb0ec6b3SJim Harris kthread_exit();
157bb0ec6b3SJim Harris }
158bb0ec6b3SJim Harris
159bb0ec6b3SJim Harris static void
nvme_ns_io_test_cb(void * arg,const struct nvme_completion * cpl)160cf81529cSJim Harris nvme_ns_io_test_cb(void *arg, const struct nvme_completion *cpl)
161bb0ec6b3SJim Harris {
162bb0ec6b3SJim Harris struct nvme_io_test_thread *tth = arg;
163bb0ec6b3SJim Harris struct timeval t;
164bb0ec6b3SJim Harris
165bb0ec6b3SJim Harris tth->io_completed++;
166bb0ec6b3SJim Harris
167cf81529cSJim Harris if (nvme_completion_is_error(cpl)) {
168bb0ec6b3SJim Harris printf("%s: error occurred\n", __func__);
169bb0ec6b3SJim Harris wakeup_one(tth);
170bb0ec6b3SJim Harris return;
171bb0ec6b3SJim Harris }
172bb0ec6b3SJim Harris
173bb0ec6b3SJim Harris getmicrouptime(&t);
174bb0ec6b3SJim Harris timevalsub(&t, &tth->start);
175bb0ec6b3SJim Harris
176bb0ec6b3SJim Harris if (t.tv_sec >= tth->time) {
177bb0ec6b3SJim Harris wakeup_one(tth);
178bb0ec6b3SJim Harris return;
179bb0ec6b3SJim Harris }
180bb0ec6b3SJim Harris
181bb0ec6b3SJim Harris switch (tth->opc) {
182bb0ec6b3SJim Harris case NVME_OPC_WRITE:
183bb0ec6b3SJim Harris nvme_ns_cmd_write(tth->ns, tth->buf, tth->idx * 2048,
184bb0ec6b3SJim Harris tth->size/nvme_ns_get_sector_size(tth->ns),
185bb0ec6b3SJim Harris nvme_ns_io_test_cb, tth);
186bb0ec6b3SJim Harris break;
187bb0ec6b3SJim Harris case NVME_OPC_READ:
188bb0ec6b3SJim Harris nvme_ns_cmd_read(tth->ns, tth->buf, tth->idx * 2048,
189bb0ec6b3SJim Harris tth->size/nvme_ns_get_sector_size(tth->ns),
190bb0ec6b3SJim Harris nvme_ns_io_test_cb, tth);
191bb0ec6b3SJim Harris break;
192bb0ec6b3SJim Harris default:
193bb0ec6b3SJim Harris break;
194bb0ec6b3SJim Harris }
195bb0ec6b3SJim Harris }
196bb0ec6b3SJim Harris
197bb0ec6b3SJim Harris static void
nvme_ns_io_test(void * arg)198bb0ec6b3SJim Harris nvme_ns_io_test(void *arg)
199bb0ec6b3SJim Harris {
200bb0ec6b3SJim Harris struct nvme_io_test_internal *io_test = arg;
201bb0ec6b3SJim Harris struct nvme_io_test_thread *tth;
202bb0ec6b3SJim Harris struct nvme_completion cpl;
203bb0ec6b3SJim Harris int error;
204bb0ec6b3SJim Harris
205237d2019SJim Harris tth = malloc(sizeof(*tth), M_NVME, M_WAITOK | M_ZERO);
206bb0ec6b3SJim Harris tth->ns = io_test->ns;
207bb0ec6b3SJim Harris tth->opc = io_test->opc;
208bb0ec6b3SJim Harris memcpy(&tth->start, &io_test->start, sizeof(tth->start));
209237d2019SJim Harris tth->buf = malloc(io_test->size, M_NVME, M_WAITOK);
210bb0ec6b3SJim Harris tth->size = io_test->size;
211bb0ec6b3SJim Harris tth->time = io_test->time;
212bb0ec6b3SJim Harris tth->idx = atomic_fetchadd_int(&io_test->td_idx, 1);
213bb0ec6b3SJim Harris
214bb0ec6b3SJim Harris memset(&cpl, 0, sizeof(cpl));
215bb0ec6b3SJim Harris
216bb0ec6b3SJim Harris nvme_ns_io_test_cb(tth, &cpl);
217bb0ec6b3SJim Harris
218bb0ec6b3SJim Harris error = tsleep(tth, 0, "test_wait", tth->time*hz*2);
219bb0ec6b3SJim Harris
220bb0ec6b3SJim Harris if (error)
221bb0ec6b3SJim Harris printf("%s: error = %d\n", __func__, error);
222bb0ec6b3SJim Harris
223bb0ec6b3SJim Harris io_test->io_completed[tth->idx] = tth->io_completed;
224bb0ec6b3SJim Harris wakeup_one(io_test);
225bb0ec6b3SJim Harris
226bb0ec6b3SJim Harris free(tth->buf, M_NVME);
227bb0ec6b3SJim Harris free(tth, M_NVME);
228bb0ec6b3SJim Harris
229bb0ec6b3SJim Harris atomic_subtract_int(&io_test->td_active, 1);
230bb0ec6b3SJim Harris mb();
231bb0ec6b3SJim Harris
232bb0ec6b3SJim Harris kthread_exit();
233bb0ec6b3SJim Harris }
234bb0ec6b3SJim Harris
235bb0ec6b3SJim Harris void
nvme_ns_test(struct nvme_namespace * ns,u_long cmd,caddr_t arg)236bb0ec6b3SJim Harris nvme_ns_test(struct nvme_namespace *ns, u_long cmd, caddr_t arg)
237bb0ec6b3SJim Harris {
238bb0ec6b3SJim Harris struct nvme_io_test *io_test;
239bb0ec6b3SJim Harris struct nvme_io_test_internal *io_test_internal;
240bb0ec6b3SJim Harris void (*fn)(void *);
241bb0ec6b3SJim Harris int i;
242bb0ec6b3SJim Harris
243bb0ec6b3SJim Harris io_test = (struct nvme_io_test *)arg;
244bb0ec6b3SJim Harris
245bb0ec6b3SJim Harris if ((io_test->opc != NVME_OPC_READ) &&
246bb0ec6b3SJim Harris (io_test->opc != NVME_OPC_WRITE))
247bb0ec6b3SJim Harris return;
248bb0ec6b3SJim Harris
249bb0ec6b3SJim Harris if (io_test->size % nvme_ns_get_sector_size(ns))
250bb0ec6b3SJim Harris return;
251bb0ec6b3SJim Harris
252bb0ec6b3SJim Harris io_test_internal = malloc(sizeof(*io_test_internal), M_NVME,
253237d2019SJim Harris M_WAITOK | M_ZERO);
254bb0ec6b3SJim Harris io_test_internal->opc = io_test->opc;
255bb0ec6b3SJim Harris io_test_internal->ns = ns;
256bb0ec6b3SJim Harris io_test_internal->td_active = io_test->num_threads;
257bb0ec6b3SJim Harris io_test_internal->time = io_test->time;
258bb0ec6b3SJim Harris io_test_internal->size = io_test->size;
259bb0ec6b3SJim Harris io_test_internal->flags = io_test->flags;
260bb0ec6b3SJim Harris
261bb0ec6b3SJim Harris if (cmd == NVME_IO_TEST)
262bb0ec6b3SJim Harris fn = nvme_ns_io_test;
263bb0ec6b3SJim Harris else
264bb0ec6b3SJim Harris fn = nvme_ns_bio_test;
265bb0ec6b3SJim Harris
266bb0ec6b3SJim Harris getmicrouptime(&io_test_internal->start);
267bb0ec6b3SJim Harris
268bb0ec6b3SJim Harris for (i = 0; i < io_test->num_threads; i++)
269bb0ec6b3SJim Harris kthread_add(fn, io_test_internal,
27043e35466SJim Harris NULL, NULL, 0, 0, "nvme_io_test[%d]", i);
271bb0ec6b3SJim Harris
272bb0ec6b3SJim Harris tsleep(io_test_internal, 0, "nvme_test", io_test->time * 2 * hz);
273bb0ec6b3SJim Harris
274bb0ec6b3SJim Harris while (io_test_internal->td_active > 0)
275bb0ec6b3SJim Harris DELAY(10);
276bb0ec6b3SJim Harris
277bb0ec6b3SJim Harris memcpy(io_test->io_completed, io_test_internal->io_completed,
278bb0ec6b3SJim Harris sizeof(io_test->io_completed));
279bb0ec6b3SJim Harris
280bb0ec6b3SJim Harris free(io_test_internal, M_NVME);
281bb0ec6b3SJim Harris }
282