1 /* $NetBSD: nvme.c,v 1.67 2022/09/13 10:14:20 riastradh Exp $ */
2 /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */
3
4 /*
5 * Copyright (c) 2014 David Gwynne <dlg@openbsd.org>
6 *
7 * Permission to use, copy, modify, and distribute this software for any
8 * purpose with or without fee is hereby granted, provided that the above
9 * copyright notice and this permission notice appear in all copies.
10 *
11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18 */
19
20 #include <sys/cdefs.h>
21 __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.67 2022/09/13 10:14:20 riastradh Exp $");
22
23 #include <sys/param.h>
24 #include <sys/systm.h>
25 #include <sys/kernel.h>
26 #include <sys/atomic.h>
27 #include <sys/bus.h>
28 #include <sys/buf.h>
29 #include <sys/conf.h>
30 #include <sys/device.h>
31 #include <sys/kmem.h>
32 #include <sys/once.h>
33 #include <sys/proc.h>
34 #include <sys/queue.h>
35 #include <sys/mutex.h>
36
37 #include <uvm/uvm_extern.h>
38
39 #include <dev/ic/nvmereg.h>
40 #include <dev/ic/nvmevar.h>
41 #include <dev/ic/nvmeio.h>
42
43 #include "ioconf.h"
44 #include "locators.h"
45
46 #define B4_CHK_RDY_DELAY_MS 2300 /* workaround controller bug */
47
48 int nvme_adminq_size = 32;
49 int nvme_ioq_size = 1024;
50
51 static int nvme_print(void *, const char *);
52
53 static int nvme_ready(struct nvme_softc *, uint32_t);
54 static int nvme_enable(struct nvme_softc *, u_int);
55 static int nvme_disable(struct nvme_softc *);
56 static int nvme_shutdown(struct nvme_softc *);
57
58 uint32_t nvme_op_sq_enter(struct nvme_softc *,
59 struct nvme_queue *, struct nvme_ccb *);
60 void nvme_op_sq_leave(struct nvme_softc *,
61 struct nvme_queue *, struct nvme_ccb *);
62 uint32_t nvme_op_sq_enter_locked(struct nvme_softc *,
63 struct nvme_queue *, struct nvme_ccb *);
64 void nvme_op_sq_leave_locked(struct nvme_softc *,
65 struct nvme_queue *, struct nvme_ccb *);
66
67 void nvme_op_cq_done(struct nvme_softc *,
68 struct nvme_queue *, struct nvme_ccb *);
69
70 static const struct nvme_ops nvme_ops = {
71 .op_sq_enter = nvme_op_sq_enter,
72 .op_sq_leave = nvme_op_sq_leave,
73 .op_sq_enter_locked = nvme_op_sq_enter_locked,
74 .op_sq_leave_locked = nvme_op_sq_leave_locked,
75
76 .op_cq_done = nvme_op_cq_done,
77 };
78
79 #ifdef NVME_DEBUG
80 static void nvme_dumpregs(struct nvme_softc *);
81 #endif
82 static int nvme_identify(struct nvme_softc *, u_int);
83 static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *,
84 void *);
85
86 static int nvme_ccbs_alloc(struct nvme_queue *, uint16_t);
87 static void nvme_ccbs_free(struct nvme_queue *);
88
89 static struct nvme_ccb *
90 nvme_ccb_get(struct nvme_queue *, bool);
91 static struct nvme_ccb *
92 nvme_ccb_get_bio(struct nvme_softc *, struct buf *,
93 struct nvme_queue **);
94 static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *);
95
96 static int nvme_poll(struct nvme_softc *, struct nvme_queue *,
97 struct nvme_ccb *, void (*)(struct nvme_queue *,
98 struct nvme_ccb *, void *), int);
99 static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *);
100 static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *,
101 struct nvme_cqe *);
102 static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *);
103 static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *,
104 struct nvme_cqe *);
105
106 static struct nvme_queue *
107 nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int);
108 static int nvme_q_create(struct nvme_softc *, struct nvme_queue *);
109 static void nvme_q_reset(struct nvme_softc *, struct nvme_queue *);
110 static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *);
111 static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *,
112 struct nvme_ccb *, void (*)(struct nvme_queue *,
113 struct nvme_ccb *, void *));
114 static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q);
115 static void nvme_q_free(struct nvme_softc *, struct nvme_queue *);
116 static void nvme_q_wait_complete(struct nvme_softc *, struct nvme_queue *,
117 bool (*)(void *), void *);
118
119 static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *,
120 void *);
121 static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *,
122 struct nvme_cqe *);
123 static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *,
124 void *);
125 static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *,
126 struct nvme_cqe *);
127 static void nvme_getcache_fill(struct nvme_queue *, struct nvme_ccb *,
128 void *);
129 static void nvme_getcache_done(struct nvme_queue *, struct nvme_ccb *,
130 struct nvme_cqe *);
131
132 static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *,
133 void *);
134 static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *,
135 struct nvme_cqe *);
136 static int nvme_command_passthrough(struct nvme_softc *,
137 struct nvme_pt_command *, uint32_t, struct lwp *, bool);
138
139 static int nvme_set_number_of_queues(struct nvme_softc *, u_int, u_int *,
140 u_int *);
141
142 #define NVME_TIMO_QOP 5 /* queue create and delete timeout */
143 #define NVME_TIMO_IDENT 10 /* probe identify timeout */
144 #define NVME_TIMO_PT -1 /* passthrough cmd timeout */
145 #define NVME_TIMO_SY 60 /* sync cache timeout */
146
147 /*
148 * Some controllers, at least Apple NVMe, always require split
149 * transfers, so don't use bus_space_{read,write}_8() on LP64.
150 */
151 uint64_t
nvme_read8(struct nvme_softc * sc,bus_size_t r)152 nvme_read8(struct nvme_softc *sc, bus_size_t r)
153 {
154 uint64_t v;
155 uint32_t *a = (uint32_t *)&v;
156
157 #if _BYTE_ORDER == _LITTLE_ENDIAN
158 a[0] = nvme_read4(sc, r);
159 a[1] = nvme_read4(sc, r + 4);
160 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */
161 a[1] = nvme_read4(sc, r);
162 a[0] = nvme_read4(sc, r + 4);
163 #endif
164
165 return v;
166 }
167
168 void
nvme_write8(struct nvme_softc * sc,bus_size_t r,uint64_t v)169 nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v)
170 {
171 uint32_t *a = (uint32_t *)&v;
172
173 #if _BYTE_ORDER == _LITTLE_ENDIAN
174 nvme_write4(sc, r, a[0]);
175 nvme_write4(sc, r + 4, a[1]);
176 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */
177 nvme_write4(sc, r, a[1]);
178 nvme_write4(sc, r + 4, a[0]);
179 #endif
180 }
181
182 #ifdef NVME_DEBUG
183 static __used void
nvme_dumpregs(struct nvme_softc * sc)184 nvme_dumpregs(struct nvme_softc *sc)
185 {
186 uint64_t r8;
187 uint32_t r4;
188
189 #define DEVNAME(_sc) device_xname((_sc)->sc_dev)
190 r8 = nvme_read8(sc, NVME_CAP);
191 printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP));
192 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc),
193 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8)));
194 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc),
195 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8)));
196 printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8));
197 printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8));
198 printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8));
199 printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8));
200 printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8));
201 printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8));
202 printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8));
203
204 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS));
205
206 r4 = nvme_read4(sc, NVME_CC);
207 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4);
208 printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4),
209 (1 << NVME_CC_IOCQES_R(r4)));
210 printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4),
211 (1 << NVME_CC_IOSQES_R(r4)));
212 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4));
213 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4));
214 printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4),
215 (1 << NVME_CC_MPS_R(r4)));
216 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4));
217 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0);
218
219 r4 = nvme_read4(sc, NVME_CSTS);
220 printf("%s: csts 0x%08x\n", DEVNAME(sc), r4);
221 printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY);
222 printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS);
223 printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK);
224
225 r4 = nvme_read4(sc, NVME_AQA);
226 printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4);
227 printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4));
228 printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4));
229
230 printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ));
231 printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ));
232 #undef DEVNAME
233 }
234 #endif /* NVME_DEBUG */
235
236 static int
nvme_ready(struct nvme_softc * sc,uint32_t rdy)237 nvme_ready(struct nvme_softc *sc, uint32_t rdy)
238 {
239 u_int i = 0;
240
241 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) {
242 if (i++ > sc->sc_rdy_to)
243 return ENXIO;
244
245 delay(1000);
246 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ);
247 }
248
249 return 0;
250 }
251
252 static int
nvme_enable(struct nvme_softc * sc,u_int mps)253 nvme_enable(struct nvme_softc *sc, u_int mps)
254 {
255 uint32_t cc, csts;
256 int error;
257
258 cc = nvme_read4(sc, NVME_CC);
259 csts = nvme_read4(sc, NVME_CSTS);
260
261 /*
262 * See note in nvme_disable. Short circuit if we're already enabled.
263 */
264 if (ISSET(cc, NVME_CC_EN)) {
265 if (ISSET(csts, NVME_CSTS_RDY))
266 return 0;
267
268 goto waitready;
269 } else {
270 /* EN == 0 already wait for RDY == 0 or fail */
271 error = nvme_ready(sc, 0);
272 if (error)
273 return error;
274 }
275
276 if (sc->sc_ops->op_enable != NULL)
277 sc->sc_ops->op_enable(sc);
278
279 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem));
280 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE);
281 delay(5000);
282 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem));
283 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE);
284 delay(5000);
285
286 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) |
287 NVME_AQA_ASQS(sc->sc_admin_q->q_entries));
288 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE);
289 delay(5000);
290
291 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK |
292 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK);
293 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1));
294 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE));
295 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM));
296 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR));
297 SET(cc, NVME_CC_MPS(mps));
298 SET(cc, NVME_CC_EN);
299
300 nvme_write4(sc, NVME_CC, cc);
301 nvme_barrier(sc, 0, sc->sc_ios,
302 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
303
304 waitready:
305 return nvme_ready(sc, NVME_CSTS_RDY);
306 }
307
308 static int
nvme_disable(struct nvme_softc * sc)309 nvme_disable(struct nvme_softc *sc)
310 {
311 uint32_t cc, csts;
312 int error;
313
314 cc = nvme_read4(sc, NVME_CC);
315 csts = nvme_read4(sc, NVME_CSTS);
316
317 /*
318 * Per 3.1.5 in NVME 1.3 spec, transitioning CC.EN from 0 to 1
319 * when CSTS.RDY is 1 or transitioning CC.EN from 1 to 0 when
320 * CSTS.RDY is 0 "has undefined results" So make sure that CSTS.RDY
321 * isn't the desired value. Short circuit if we're already disabled.
322 */
323 if (ISSET(cc, NVME_CC_EN)) {
324 if (!ISSET(csts, NVME_CSTS_RDY)) {
325 /* EN == 1, wait for RDY == 1 or fail */
326 error = nvme_ready(sc, NVME_CSTS_RDY);
327 if (error)
328 return error;
329 }
330 } else {
331 /* EN == 0 already wait for RDY == 0 */
332 if (!ISSET(csts, NVME_CSTS_RDY))
333 return 0;
334
335 goto waitready;
336 }
337
338 CLR(cc, NVME_CC_EN);
339 nvme_write4(sc, NVME_CC, cc);
340 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ);
341
342 /*
343 * Some drives have issues with accessing the mmio after we disable,
344 * so delay for a bit after we write the bit to cope with these issues.
345 */
346 if (ISSET(sc->sc_quirks, NVME_QUIRK_DELAY_B4_CHK_RDY))
347 delay(B4_CHK_RDY_DELAY_MS);
348
349 waitready:
350 return nvme_ready(sc, 0);
351 }
352
353 int
nvme_attach(struct nvme_softc * sc)354 nvme_attach(struct nvme_softc *sc)
355 {
356 uint64_t cap;
357 uint32_t reg;
358 u_int mps = PAGE_SHIFT;
359 u_int ncq, nsq;
360 uint16_t adminq_entries = nvme_adminq_size;
361 uint16_t ioq_entries = nvme_ioq_size;
362 int i;
363
364 if (sc->sc_ops == NULL)
365 sc->sc_ops = &nvme_ops;
366
367 reg = nvme_read4(sc, NVME_VS);
368 if (reg == 0xffffffff) {
369 aprint_error_dev(sc->sc_dev, "invalid mapping\n");
370 return 1;
371 }
372
373 if (NVME_VS_TER(reg) == 0)
374 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d\n", NVME_VS_MJR(reg),
375 NVME_VS_MNR(reg));
376 else
377 aprint_normal_dev(sc->sc_dev, "NVMe %d.%d.%d\n", NVME_VS_MJR(reg),
378 NVME_VS_MNR(reg), NVME_VS_TER(reg));
379
380 cap = nvme_read8(sc, NVME_CAP);
381 sc->sc_dstrd = NVME_CAP_DSTRD(cap);
382 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) {
383 aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u "
384 "is greater than CPU page size %u\n",
385 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT);
386 return 1;
387 }
388 if (NVME_CAP_MPSMAX(cap) < mps)
389 mps = NVME_CAP_MPSMAX(cap);
390 if (ioq_entries > NVME_CAP_MQES(cap))
391 ioq_entries = NVME_CAP_MQES(cap);
392
393 /* set initial values to be used for admin queue during probe */
394 sc->sc_rdy_to = NVME_CAP_TO(cap);
395 sc->sc_mps = 1 << mps;
396 sc->sc_mdts = MAXPHYS;
397 sc->sc_max_sgl = btoc(round_page(sc->sc_mdts));
398
399 if (nvme_disable(sc) != 0) {
400 aprint_error_dev(sc->sc_dev, "unable to disable controller\n");
401 return 1;
402 }
403
404 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries,
405 sc->sc_dstrd);
406 if (sc->sc_admin_q == NULL) {
407 aprint_error_dev(sc->sc_dev,
408 "unable to allocate admin queue\n");
409 return 1;
410 }
411 if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q))
412 goto free_admin_q;
413
414 if (nvme_enable(sc, mps) != 0) {
415 aprint_error_dev(sc->sc_dev, "unable to enable controller\n");
416 goto disestablish_admin_q;
417 }
418
419 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) {
420 aprint_error_dev(sc->sc_dev, "unable to identify controller\n");
421 goto disable;
422 }
423 if (sc->sc_nn == 0) {
424 aprint_error_dev(sc->sc_dev, "namespace not found\n");
425 goto disable;
426 }
427
428 /* we know how big things are now */
429 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps;
430
431 /* reallocate ccbs of admin queue with new max sgl. */
432 nvme_ccbs_free(sc->sc_admin_q);
433 nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries);
434
435 if (sc->sc_use_mq) {
436 /* Limit the number of queues to the number allocated in HW */
437 if (nvme_set_number_of_queues(sc, sc->sc_nq, &ncq, &nsq) != 0) {
438 aprint_error_dev(sc->sc_dev,
439 "unable to get number of queues\n");
440 goto disable;
441 }
442 if (sc->sc_nq > ncq)
443 sc->sc_nq = ncq;
444 if (sc->sc_nq > nsq)
445 sc->sc_nq = nsq;
446 }
447
448 sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP);
449 for (i = 0; i < sc->sc_nq; i++) {
450 sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries,
451 sc->sc_dstrd);
452 if (sc->sc_q[i] == NULL) {
453 aprint_error_dev(sc->sc_dev,
454 "unable to allocate io queue\n");
455 goto free_q;
456 }
457 if (nvme_q_create(sc, sc->sc_q[i]) != 0) {
458 aprint_error_dev(sc->sc_dev,
459 "unable to create io queue\n");
460 nvme_q_free(sc, sc->sc_q[i]);
461 goto free_q;
462 }
463 }
464
465 if (!sc->sc_use_mq)
466 nvme_write4(sc, NVME_INTMC, 1);
467
468 /* probe subdevices */
469 sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn,
470 KM_SLEEP);
471 nvme_rescan(sc->sc_dev, NULL, NULL);
472
473 return 0;
474
475 free_q:
476 while (--i >= 0) {
477 nvme_q_delete(sc, sc->sc_q[i]);
478 nvme_q_free(sc, sc->sc_q[i]);
479 }
480 disable:
481 nvme_disable(sc);
482 disestablish_admin_q:
483 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q);
484 free_admin_q:
485 nvme_q_free(sc, sc->sc_admin_q);
486
487 return 1;
488 }
489
490 int
nvme_rescan(device_t self,const char * ifattr,const int * locs)491 nvme_rescan(device_t self, const char *ifattr, const int *locs)
492 {
493 struct nvme_softc *sc = device_private(self);
494 struct nvme_attach_args naa;
495 struct nvm_namespace_format *f;
496 struct nvme_namespace *ns;
497 uint64_t cap;
498 int ioq_entries = nvme_ioq_size;
499 int i, mlocs[NVMECF_NLOCS];
500 int error;
501
502 cap = nvme_read8(sc, NVME_CAP);
503 if (ioq_entries > NVME_CAP_MQES(cap))
504 ioq_entries = NVME_CAP_MQES(cap);
505
506 for (i = 1; i <= sc->sc_nn; i++) {
507 if (sc->sc_namespaces[i - 1].dev)
508 continue;
509
510 /* identify to check for availability */
511 error = nvme_ns_identify(sc, i);
512 if (error) {
513 aprint_error_dev(self, "couldn't identify namespace #%d\n", i);
514 continue;
515 }
516
517 ns = nvme_ns_get(sc, i);
518 KASSERT(ns);
519
520 f = &ns->ident->lbaf[NVME_ID_NS_FLBAS(ns->ident->flbas)];
521
522 /*
523 * NVME1.0e 6.11 Identify command
524 *
525 * LBADS values smaller than 9 are not supported, a value
526 * of zero means that the format is not used.
527 */
528 if (f->lbads < 9) {
529 if (f->lbads > 0)
530 aprint_error_dev(self,
531 "unsupported logical data size %u\n", f->lbads);
532 continue;
533 }
534
535 mlocs[NVMECF_NSID] = i;
536
537 memset(&naa, 0, sizeof(naa));
538 naa.naa_nsid = i;
539 naa.naa_qentries = (ioq_entries - 1) * sc->sc_nq;
540 naa.naa_maxphys = sc->sc_mdts;
541 naa.naa_typename = sc->sc_modelname;
542 sc->sc_namespaces[i - 1].dev =
543 config_found(sc->sc_dev, &naa, nvme_print,
544 CFARGS(.submatch = config_stdsubmatch,
545 .locators = mlocs));
546 }
547 return 0;
548 }
549
550 static int
nvme_print(void * aux,const char * pnp)551 nvme_print(void *aux, const char *pnp)
552 {
553 struct nvme_attach_args *naa = aux;
554
555 if (pnp)
556 aprint_normal("ld at %s", pnp);
557
558 if (naa->naa_nsid > 0)
559 aprint_normal(" nsid %d", naa->naa_nsid);
560
561 return UNCONF;
562 }
563
564 int
nvme_detach(struct nvme_softc * sc,int flags)565 nvme_detach(struct nvme_softc *sc, int flags)
566 {
567 int i, error;
568
569 error = config_detach_children(sc->sc_dev, flags);
570 if (error)
571 return error;
572
573 error = nvme_shutdown(sc);
574 if (error)
575 return error;
576
577 /* from now on we are committed to detach, following will never fail */
578 sc->sc_intr_disestablish(sc, NVME_ADMIN_Q);
579 for (i = 0; i < sc->sc_nq; i++)
580 nvme_q_free(sc, sc->sc_q[i]);
581 kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq);
582 nvme_q_free(sc, sc->sc_admin_q);
583
584 return 0;
585 }
586
587 int
nvme_suspend(struct nvme_softc * sc)588 nvme_suspend(struct nvme_softc *sc)
589 {
590
591 return nvme_shutdown(sc);
592 }
593
594 int
nvme_resume(struct nvme_softc * sc)595 nvme_resume(struct nvme_softc *sc)
596 {
597 int i, error;
598
599 error = nvme_disable(sc);
600 if (error) {
601 device_printf(sc->sc_dev, "unable to disable controller\n");
602 return error;
603 }
604
605 nvme_q_reset(sc, sc->sc_admin_q);
606
607 error = nvme_enable(sc, ffs(sc->sc_mps) - 1);
608 if (error) {
609 device_printf(sc->sc_dev, "unable to enable controller\n");
610 return error;
611 }
612
613 for (i = 0; i < sc->sc_nq; i++) {
614 nvme_q_reset(sc, sc->sc_q[i]);
615 if (nvme_q_create(sc, sc->sc_q[i]) != 0) {
616 error = EIO;
617 device_printf(sc->sc_dev, "unable to create io q %d"
618 "\n", i);
619 goto disable;
620 }
621 }
622
623 nvme_write4(sc, NVME_INTMC, 1);
624
625 return 0;
626
627 disable:
628 (void)nvme_disable(sc);
629
630 return error;
631 }
632
633 static int
nvme_shutdown(struct nvme_softc * sc)634 nvme_shutdown(struct nvme_softc *sc)
635 {
636 uint32_t cc, csts;
637 bool disabled = false;
638 int i;
639
640 if (!sc->sc_use_mq)
641 nvme_write4(sc, NVME_INTMS, 1);
642
643 for (i = 0; i < sc->sc_nq; i++) {
644 if (nvme_q_delete(sc, sc->sc_q[i]) != 0) {
645 aprint_error_dev(sc->sc_dev,
646 "unable to delete io queue %d, disabling\n", i + 1);
647 disabled = true;
648 }
649 }
650 if (disabled)
651 goto disable;
652
653 cc = nvme_read4(sc, NVME_CC);
654 CLR(cc, NVME_CC_SHN_MASK);
655 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL));
656 nvme_write4(sc, NVME_CC, cc);
657
658 for (i = 0; i < 4000; i++) {
659 nvme_barrier(sc, 0, sc->sc_ios,
660 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
661 csts = nvme_read4(sc, NVME_CSTS);
662 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE)
663 return 0;
664
665 delay(1000);
666 }
667
668 aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n");
669
670 disable:
671 nvme_disable(sc);
672 return 0;
673 }
674
675 void
nvme_childdet(device_t self,device_t child)676 nvme_childdet(device_t self, device_t child)
677 {
678 struct nvme_softc *sc = device_private(self);
679 int i;
680
681 for (i = 0; i < sc->sc_nn; i++) {
682 if (sc->sc_namespaces[i].dev == child) {
683 /* Already freed ns->ident. */
684 sc->sc_namespaces[i].dev = NULL;
685 break;
686 }
687 }
688 }
689
690 int
nvme_ns_identify(struct nvme_softc * sc,uint16_t nsid)691 nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid)
692 {
693 struct nvme_sqe sqe;
694 struct nvm_identify_namespace *identify;
695 struct nvme_dmamem *mem;
696 struct nvme_ccb *ccb;
697 struct nvme_namespace *ns;
698 int rv;
699
700 KASSERT(nsid > 0);
701
702 ns = nvme_ns_get(sc, nsid);
703 KASSERT(ns);
704
705 if (ns->ident != NULL)
706 return 0;
707
708 ccb = nvme_ccb_get(sc->sc_admin_q, false);
709 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */
710
711 mem = nvme_dmamem_alloc(sc, sizeof(*identify));
712 if (mem == NULL) {
713 nvme_ccb_put(sc->sc_admin_q, ccb);
714 return ENOMEM;
715 }
716
717 memset(&sqe, 0, sizeof(sqe));
718 sqe.opcode = NVM_ADMIN_IDENTIFY;
719 htolem32(&sqe.nsid, nsid);
720 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem));
721 htolem32(&sqe.cdw10, 0);
722
723 ccb->ccb_done = nvme_empty_done;
724 ccb->ccb_cookie = &sqe;
725
726 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD);
727 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT);
728 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD);
729
730 nvme_ccb_put(sc->sc_admin_q, ccb);
731
732 if (rv != 0) {
733 rv = EIO;
734 goto done;
735 }
736
737 /* commit */
738
739 identify = kmem_zalloc(sizeof(*identify), KM_SLEEP);
740 *identify = *((volatile struct nvm_identify_namespace *)NVME_DMA_KVA(mem));
741
742 /* Convert data to host endian */
743 nvme_identify_namespace_swapbytes(identify);
744
745 ns->ident = identify;
746
747 done:
748 nvme_dmamem_free(sc, mem);
749
750 return rv;
751 }
752
753 int
nvme_ns_dobio(struct nvme_softc * sc,uint16_t nsid,void * cookie,struct buf * bp,void * data,size_t datasize,int secsize,daddr_t blkno,int flags,nvme_nnc_done nnc_done)754 nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie,
755 struct buf *bp, void *data, size_t datasize,
756 int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done)
757 {
758 struct nvme_queue *q;
759 struct nvme_ccb *ccb;
760 bus_dmamap_t dmap;
761 int i, error;
762
763 ccb = nvme_ccb_get_bio(sc, bp, &q);
764 if (ccb == NULL)
765 return EAGAIN;
766
767 ccb->ccb_done = nvme_ns_io_done;
768 ccb->ccb_cookie = cookie;
769
770 /* namespace context */
771 ccb->nnc_nsid = nsid;
772 ccb->nnc_flags = flags;
773 ccb->nnc_buf = bp;
774 ccb->nnc_datasize = datasize;
775 ccb->nnc_secsize = secsize;
776 ccb->nnc_blkno = blkno;
777 ccb->nnc_done = nnc_done;
778
779 dmap = ccb->ccb_dmamap;
780 error = bus_dmamap_load(sc->sc_dmat, dmap, data,
781 datasize, NULL,
782 (ISSET(flags, NVME_NS_CTX_F_POLL) ?
783 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) |
784 (ISSET(flags, NVME_NS_CTX_F_READ) ?
785 BUS_DMA_READ : BUS_DMA_WRITE));
786 if (error) {
787 nvme_ccb_put(q, ccb);
788 return error;
789 }
790
791 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
792 ISSET(flags, NVME_NS_CTX_F_READ) ?
793 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
794
795 if (dmap->dm_nsegs > 2) {
796 for (i = 1; i < dmap->dm_nsegs; i++) {
797 htolem64(&ccb->ccb_prpl[i - 1],
798 dmap->dm_segs[i].ds_addr);
799 }
800 bus_dmamap_sync(sc->sc_dmat,
801 NVME_DMA_MAP(q->q_ccb_prpls),
802 ccb->ccb_prpl_off,
803 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1),
804 BUS_DMASYNC_PREWRITE);
805 }
806
807 if (ISSET(flags, NVME_NS_CTX_F_POLL)) {
808 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0)
809 return EIO;
810 return 0;
811 }
812
813 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill);
814 return 0;
815 }
816
817 static void
nvme_ns_io_fill(struct nvme_queue * q,struct nvme_ccb * ccb,void * slot)818 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
819 {
820 struct nvme_sqe_io *sqe = slot;
821 bus_dmamap_t dmap = ccb->ccb_dmamap;
822
823 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ?
824 NVM_CMD_READ : NVM_CMD_WRITE;
825 htolem32(&sqe->nsid, ccb->nnc_nsid);
826
827 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr);
828 switch (dmap->dm_nsegs) {
829 case 1:
830 break;
831 case 2:
832 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr);
833 break;
834 default:
835 /* the prp list is already set up and synced */
836 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva);
837 break;
838 }
839
840 htolem64(&sqe->slba, ccb->nnc_blkno);
841
842 if (ISSET(ccb->nnc_flags, NVME_NS_CTX_F_FUA))
843 htolem16(&sqe->ioflags, NVM_SQE_IO_FUA);
844
845 /* guaranteed by upper layers, but check just in case */
846 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0);
847 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1);
848 }
849
850 static void
nvme_ns_io_done(struct nvme_queue * q,struct nvme_ccb * ccb,struct nvme_cqe * cqe)851 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb,
852 struct nvme_cqe *cqe)
853 {
854 struct nvme_softc *sc = q->q_sc;
855 bus_dmamap_t dmap = ccb->ccb_dmamap;
856 void *nnc_cookie = ccb->ccb_cookie;
857 nvme_nnc_done nnc_done = ccb->nnc_done;
858 struct buf *bp = ccb->nnc_buf;
859
860 if (dmap->dm_nsegs > 2) {
861 bus_dmamap_sync(sc->sc_dmat,
862 NVME_DMA_MAP(q->q_ccb_prpls),
863 ccb->ccb_prpl_off,
864 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1),
865 BUS_DMASYNC_POSTWRITE);
866 }
867
868 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
869 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ?
870 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
871
872 bus_dmamap_unload(sc->sc_dmat, dmap);
873 nvme_ccb_put(q, ccb);
874
875 nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0));
876 }
877
878 /*
879 * If there is no volatile write cache, it makes no sense to issue
880 * flush commands or query for the status.
881 */
882 static bool
nvme_has_volatile_write_cache(struct nvme_softc * sc)883 nvme_has_volatile_write_cache(struct nvme_softc *sc)
884 {
885 /* sc_identify is filled during attachment */
886 return ((sc->sc_identify.vwc & NVME_ID_CTRLR_VWC_PRESENT) != 0);
887 }
888
889 static bool
nvme_ns_sync_finished(void * cookie)890 nvme_ns_sync_finished(void *cookie)
891 {
892 int *result = cookie;
893
894 return (*result != 0);
895 }
896
897 int
nvme_ns_sync(struct nvme_softc * sc,uint16_t nsid,int flags)898 nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, int flags)
899 {
900 struct nvme_queue *q = nvme_get_q(sc);
901 struct nvme_ccb *ccb;
902 int result = 0;
903
904 if (!nvme_has_volatile_write_cache(sc)) {
905 /* cache not present, no value in trying to flush it */
906 return 0;
907 }
908
909 ccb = nvme_ccb_get(q, true);
910 KASSERT(ccb != NULL);
911
912 ccb->ccb_done = nvme_ns_sync_done;
913 ccb->ccb_cookie = &result;
914
915 /* namespace context */
916 ccb->nnc_nsid = nsid;
917 ccb->nnc_flags = flags;
918 ccb->nnc_done = NULL;
919
920 if (ISSET(flags, NVME_NS_CTX_F_POLL)) {
921 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0)
922 return EIO;
923 return 0;
924 }
925
926 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill);
927
928 /* wait for completion */
929 nvme_q_wait_complete(sc, q, nvme_ns_sync_finished, &result);
930 KASSERT(result != 0);
931
932 return (result > 0) ? 0 : EIO;
933 }
934
935 static void
nvme_ns_sync_fill(struct nvme_queue * q,struct nvme_ccb * ccb,void * slot)936 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
937 {
938 struct nvme_sqe *sqe = slot;
939
940 sqe->opcode = NVM_CMD_FLUSH;
941 htolem32(&sqe->nsid, ccb->nnc_nsid);
942 }
943
944 static void
nvme_ns_sync_done(struct nvme_queue * q,struct nvme_ccb * ccb,struct nvme_cqe * cqe)945 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb,
946 struct nvme_cqe *cqe)
947 {
948 int *result = ccb->ccb_cookie;
949 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags));
950
951 if (status == NVME_CQE_SC_SUCCESS)
952 *result = 1;
953 else
954 *result = -1;
955
956 nvme_ccb_put(q, ccb);
957 }
958
959 static bool
nvme_getcache_finished(void * xc)960 nvme_getcache_finished(void *xc)
961 {
962 int *addr = xc;
963
964 return (*addr != 0);
965 }
966
967 /*
968 * Get status of volatile write cache. Always asynchronous.
969 */
970 int
nvme_admin_getcache(struct nvme_softc * sc,int * addr)971 nvme_admin_getcache(struct nvme_softc *sc, int *addr)
972 {
973 struct nvme_ccb *ccb;
974 struct nvme_queue *q = sc->sc_admin_q;
975 int result = 0, error;
976
977 if (!nvme_has_volatile_write_cache(sc)) {
978 /* cache simply not present */
979 *addr = 0;
980 return 0;
981 }
982
983 ccb = nvme_ccb_get(q, true);
984 KASSERT(ccb != NULL);
985
986 ccb->ccb_done = nvme_getcache_done;
987 ccb->ccb_cookie = &result;
988
989 /* namespace context */
990 ccb->nnc_flags = 0;
991 ccb->nnc_done = NULL;
992
993 nvme_q_submit(sc, q, ccb, nvme_getcache_fill);
994
995 /* wait for completion */
996 nvme_q_wait_complete(sc, q, nvme_getcache_finished, &result);
997 KASSERT(result != 0);
998
999 if (result > 0) {
1000 *addr = result;
1001 error = 0;
1002 } else
1003 error = EINVAL;
1004
1005 return error;
1006 }
1007
1008 static void
nvme_getcache_fill(struct nvme_queue * q,struct nvme_ccb * ccb,void * slot)1009 nvme_getcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
1010 {
1011 struct nvme_sqe *sqe = slot;
1012
1013 sqe->opcode = NVM_ADMIN_GET_FEATURES;
1014 htolem32(&sqe->cdw10, NVM_FEATURE_VOLATILE_WRITE_CACHE);
1015 htolem32(&sqe->cdw11, NVM_VOLATILE_WRITE_CACHE_WCE);
1016 }
1017
1018 static void
nvme_getcache_done(struct nvme_queue * q,struct nvme_ccb * ccb,struct nvme_cqe * cqe)1019 nvme_getcache_done(struct nvme_queue *q, struct nvme_ccb *ccb,
1020 struct nvme_cqe *cqe)
1021 {
1022 int *addr = ccb->ccb_cookie;
1023 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags));
1024 uint32_t cdw0 = lemtoh32(&cqe->cdw0);
1025 int result;
1026
1027 if (status == NVME_CQE_SC_SUCCESS) {
1028 result = 0;
1029
1030 /*
1031 * DPO not supported, Dataset Management (DSM) field doesn't
1032 * specify the same semantics. FUA is always supported.
1033 */
1034 result = DKCACHE_FUA;
1035
1036 if (cdw0 & NVM_VOLATILE_WRITE_CACHE_WCE)
1037 result |= DKCACHE_WRITE;
1038
1039 /*
1040 * If volatile write cache is present, the flag shall also be
1041 * settable.
1042 */
1043 result |= DKCACHE_WCHANGE;
1044
1045 /*
1046 * ONCS field indicates whether the optional SAVE is also
1047 * supported for Set Features. According to spec v1.3,
1048 * Volatile Write Cache however doesn't support persistency
1049 * across power cycle/reset.
1050 */
1051
1052 } else {
1053 result = -1;
1054 }
1055
1056 *addr = result;
1057
1058 nvme_ccb_put(q, ccb);
1059 }
1060
1061 struct nvme_setcache_state {
1062 int dkcache;
1063 int result;
1064 };
1065
1066 static bool
nvme_setcache_finished(void * xc)1067 nvme_setcache_finished(void *xc)
1068 {
1069 struct nvme_setcache_state *st = xc;
1070
1071 return (st->result != 0);
1072 }
1073
1074 static void
nvme_setcache_fill(struct nvme_queue * q,struct nvme_ccb * ccb,void * slot)1075 nvme_setcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
1076 {
1077 struct nvme_sqe *sqe = slot;
1078 struct nvme_setcache_state *st = ccb->ccb_cookie;
1079
1080 sqe->opcode = NVM_ADMIN_SET_FEATURES;
1081 htolem32(&sqe->cdw10, NVM_FEATURE_VOLATILE_WRITE_CACHE);
1082 if (st->dkcache & DKCACHE_WRITE)
1083 htolem32(&sqe->cdw11, NVM_VOLATILE_WRITE_CACHE_WCE);
1084 }
1085
1086 static void
nvme_setcache_done(struct nvme_queue * q,struct nvme_ccb * ccb,struct nvme_cqe * cqe)1087 nvme_setcache_done(struct nvme_queue *q, struct nvme_ccb *ccb,
1088 struct nvme_cqe *cqe)
1089 {
1090 struct nvme_setcache_state *st = ccb->ccb_cookie;
1091 uint16_t status = NVME_CQE_SC(lemtoh16(&cqe->flags));
1092
1093 if (status == NVME_CQE_SC_SUCCESS) {
1094 st->result = 1;
1095 } else {
1096 st->result = -1;
1097 }
1098
1099 nvme_ccb_put(q, ccb);
1100 }
1101
1102 /*
1103 * Set status of volatile write cache. Always asynchronous.
1104 */
1105 int
nvme_admin_setcache(struct nvme_softc * sc,int dkcache)1106 nvme_admin_setcache(struct nvme_softc *sc, int dkcache)
1107 {
1108 struct nvme_ccb *ccb;
1109 struct nvme_queue *q = sc->sc_admin_q;
1110 int error;
1111 struct nvme_setcache_state st;
1112
1113 if (!nvme_has_volatile_write_cache(sc)) {
1114 /* cache simply not present */
1115 return EOPNOTSUPP;
1116 }
1117
1118 if (dkcache & ~(DKCACHE_WRITE)) {
1119 /* unsupported parameters */
1120 return EOPNOTSUPP;
1121 }
1122
1123 ccb = nvme_ccb_get(q, true);
1124 KASSERT(ccb != NULL);
1125
1126 memset(&st, 0, sizeof(st));
1127 st.dkcache = dkcache;
1128
1129 ccb->ccb_done = nvme_setcache_done;
1130 ccb->ccb_cookie = &st;
1131
1132 /* namespace context */
1133 ccb->nnc_flags = 0;
1134 ccb->nnc_done = NULL;
1135
1136 nvme_q_submit(sc, q, ccb, nvme_setcache_fill);
1137
1138 /* wait for completion */
1139 nvme_q_wait_complete(sc, q, nvme_setcache_finished, &st);
1140 KASSERT(st.result != 0);
1141
1142 if (st.result > 0)
1143 error = 0;
1144 else
1145 error = EINVAL;
1146
1147 return error;
1148 }
1149
1150 void
nvme_ns_free(struct nvme_softc * sc,uint16_t nsid)1151 nvme_ns_free(struct nvme_softc *sc, uint16_t nsid)
1152 {
1153 struct nvme_namespace *ns;
1154 struct nvm_identify_namespace *identify;
1155
1156 ns = nvme_ns_get(sc, nsid);
1157 KASSERT(ns);
1158
1159 identify = ns->ident;
1160 ns->ident = NULL;
1161 if (identify != NULL)
1162 kmem_free(identify, sizeof(*identify));
1163 }
1164
1165 struct nvme_pt_state {
1166 struct nvme_pt_command *pt;
1167 bool finished;
1168 };
1169
1170 static void
nvme_pt_fill(struct nvme_queue * q,struct nvme_ccb * ccb,void * slot)1171 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
1172 {
1173 struct nvme_softc *sc = q->q_sc;
1174 struct nvme_sqe *sqe = slot;
1175 struct nvme_pt_state *state = ccb->ccb_cookie;
1176 struct nvme_pt_command *pt = state->pt;
1177 bus_dmamap_t dmap = ccb->ccb_dmamap;
1178 int i;
1179
1180 sqe->opcode = pt->cmd.opcode;
1181 htolem32(&sqe->nsid, pt->cmd.nsid);
1182
1183 if (pt->buf != NULL && pt->len > 0) {
1184 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr);
1185 switch (dmap->dm_nsegs) {
1186 case 1:
1187 break;
1188 case 2:
1189 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr);
1190 break;
1191 default:
1192 for (i = 1; i < dmap->dm_nsegs; i++) {
1193 htolem64(&ccb->ccb_prpl[i - 1],
1194 dmap->dm_segs[i].ds_addr);
1195 }
1196 bus_dmamap_sync(sc->sc_dmat,
1197 NVME_DMA_MAP(q->q_ccb_prpls),
1198 ccb->ccb_prpl_off,
1199 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1),
1200 BUS_DMASYNC_PREWRITE);
1201 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva);
1202 break;
1203 }
1204 }
1205
1206 htolem32(&sqe->cdw10, pt->cmd.cdw10);
1207 htolem32(&sqe->cdw11, pt->cmd.cdw11);
1208 htolem32(&sqe->cdw12, pt->cmd.cdw12);
1209 htolem32(&sqe->cdw13, pt->cmd.cdw13);
1210 htolem32(&sqe->cdw14, pt->cmd.cdw14);
1211 htolem32(&sqe->cdw15, pt->cmd.cdw15);
1212 }
1213
1214 static void
nvme_pt_done(struct nvme_queue * q,struct nvme_ccb * ccb,struct nvme_cqe * cqe)1215 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe)
1216 {
1217 struct nvme_softc *sc = q->q_sc;
1218 struct nvme_pt_state *state = ccb->ccb_cookie;
1219 struct nvme_pt_command *pt = state->pt;
1220 bus_dmamap_t dmap = ccb->ccb_dmamap;
1221
1222 if (pt->buf != NULL && pt->len > 0) {
1223 if (dmap->dm_nsegs > 2) {
1224 bus_dmamap_sync(sc->sc_dmat,
1225 NVME_DMA_MAP(q->q_ccb_prpls),
1226 ccb->ccb_prpl_off,
1227 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1),
1228 BUS_DMASYNC_POSTWRITE);
1229 }
1230
1231 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1232 pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
1233 bus_dmamap_unload(sc->sc_dmat, dmap);
1234 }
1235
1236 pt->cpl.cdw0 = lemtoh32(&cqe->cdw0);
1237 pt->cpl.flags = lemtoh16(&cqe->flags) & ~NVME_CQE_PHASE;
1238
1239 state->finished = true;
1240
1241 nvme_ccb_put(q, ccb);
1242 }
1243
1244 static bool
nvme_pt_finished(void * cookie)1245 nvme_pt_finished(void *cookie)
1246 {
1247 struct nvme_pt_state *state = cookie;
1248
1249 return state->finished;
1250 }
1251
1252 static int
nvme_command_passthrough(struct nvme_softc * sc,struct nvme_pt_command * pt,uint32_t nsid,struct lwp * l,bool is_adminq)1253 nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt,
1254 uint32_t nsid, struct lwp *l, bool is_adminq)
1255 {
1256 struct nvme_queue *q;
1257 struct nvme_ccb *ccb;
1258 void *buf = NULL;
1259 struct nvme_pt_state state;
1260 int error;
1261
1262 /* limit command size to maximum data transfer size */
1263 if ((pt->buf == NULL && pt->len > 0) ||
1264 (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts)))
1265 return EINVAL;
1266
1267 q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc);
1268 ccb = nvme_ccb_get(q, true);
1269 KASSERT(ccb != NULL);
1270
1271 if (pt->buf != NULL) {
1272 KASSERT(pt->len > 0);
1273 buf = kmem_alloc(pt->len, KM_SLEEP);
1274 if (!pt->is_read) {
1275 error = copyin(pt->buf, buf, pt->len);
1276 if (error)
1277 goto kmem_free;
1278 }
1279 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf,
1280 pt->len, NULL,
1281 BUS_DMA_WAITOK |
1282 (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE));
1283 if (error)
1284 goto kmem_free;
1285 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap,
1286 0, ccb->ccb_dmamap->dm_mapsize,
1287 pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
1288 }
1289
1290 memset(&state, 0, sizeof(state));
1291 state.pt = pt;
1292 state.finished = false;
1293
1294 ccb->ccb_done = nvme_pt_done;
1295 ccb->ccb_cookie = &state;
1296
1297 pt->cmd.nsid = nsid;
1298
1299 nvme_q_submit(sc, q, ccb, nvme_pt_fill);
1300
1301 /* wait for completion */
1302 nvme_q_wait_complete(sc, q, nvme_pt_finished, &state);
1303 KASSERT(state.finished);
1304
1305 error = 0;
1306
1307 if (buf != NULL) {
1308 if (error == 0 && pt->is_read)
1309 error = copyout(buf, pt->buf, pt->len);
1310 kmem_free:
1311 kmem_free(buf, pt->len);
1312 }
1313
1314 return error;
1315 }
1316
1317 uint32_t
nvme_op_sq_enter(struct nvme_softc * sc,struct nvme_queue * q,struct nvme_ccb * ccb)1318 nvme_op_sq_enter(struct nvme_softc *sc,
1319 struct nvme_queue *q, struct nvme_ccb *ccb)
1320 {
1321 mutex_enter(&q->q_sq_mtx);
1322
1323 return nvme_op_sq_enter_locked(sc, q, ccb);
1324 }
1325
1326 uint32_t
nvme_op_sq_enter_locked(struct nvme_softc * sc,struct nvme_queue * q,struct nvme_ccb * ccb)1327 nvme_op_sq_enter_locked(struct nvme_softc *sc,
1328 struct nvme_queue *q, struct nvme_ccb *ccb)
1329 {
1330 return q->q_sq_tail;
1331 }
1332
1333 void
nvme_op_sq_leave_locked(struct nvme_softc * sc,struct nvme_queue * q,struct nvme_ccb * ccb)1334 nvme_op_sq_leave_locked(struct nvme_softc *sc,
1335 struct nvme_queue *q, struct nvme_ccb *ccb)
1336 {
1337 uint32_t tail;
1338
1339 tail = ++q->q_sq_tail;
1340 if (tail >= q->q_entries)
1341 tail = 0;
1342 q->q_sq_tail = tail;
1343 nvme_write4(sc, q->q_sqtdbl, tail);
1344 }
1345
1346 void
nvme_op_sq_leave(struct nvme_softc * sc,struct nvme_queue * q,struct nvme_ccb * ccb)1347 nvme_op_sq_leave(struct nvme_softc *sc,
1348 struct nvme_queue *q, struct nvme_ccb *ccb)
1349 {
1350 nvme_op_sq_leave_locked(sc, q, ccb);
1351
1352 mutex_exit(&q->q_sq_mtx);
1353 }
1354
1355 static void
nvme_q_submit(struct nvme_softc * sc,struct nvme_queue * q,struct nvme_ccb * ccb,void (* fill)(struct nvme_queue *,struct nvme_ccb *,void *))1356 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb,
1357 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *))
1358 {
1359 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem);
1360 uint32_t tail;
1361
1362 tail = sc->sc_ops->op_sq_enter(sc, q, ccb);
1363
1364 sqe += tail;
1365
1366 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem),
1367 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE);
1368 memset(sqe, 0, sizeof(*sqe));
1369 (*fill)(q, ccb, sqe);
1370 htolem16(&sqe->cid, ccb->ccb_id);
1371 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem),
1372 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE);
1373
1374 sc->sc_ops->op_sq_leave(sc, q, ccb);
1375 }
1376
1377 struct nvme_poll_state {
1378 struct nvme_sqe s;
1379 struct nvme_cqe c;
1380 void *cookie;
1381 void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *);
1382 };
1383
1384 static int
nvme_poll(struct nvme_softc * sc,struct nvme_queue * q,struct nvme_ccb * ccb,void (* fill)(struct nvme_queue *,struct nvme_ccb *,void *),int timo_sec)1385 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb,
1386 void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec)
1387 {
1388 struct nvme_poll_state state;
1389 uint16_t flags;
1390 int step = 10;
1391 int maxloop = timo_sec * 1000000 / step;
1392 int error = 0;
1393
1394 memset(&state, 0, sizeof(state));
1395 (*fill)(q, ccb, &state.s);
1396
1397 state.done = ccb->ccb_done;
1398 state.cookie = ccb->ccb_cookie;
1399
1400 ccb->ccb_done = nvme_poll_done;
1401 ccb->ccb_cookie = &state;
1402
1403 nvme_q_submit(sc, q, ccb, nvme_poll_fill);
1404 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) {
1405 if (nvme_q_complete(sc, q) == 0)
1406 delay(step);
1407
1408 if (timo_sec >= 0 && --maxloop <= 0) {
1409 error = ETIMEDOUT;
1410 break;
1411 }
1412 }
1413
1414 if (error == 0) {
1415 flags = lemtoh16(&state.c.flags);
1416 return flags & ~NVME_CQE_PHASE;
1417 } else {
1418 /*
1419 * If it succeds later, it would hit ccb which will have been
1420 * already reused for something else. Not good. Cross
1421 * fingers and hope for best. XXX do controller reset?
1422 */
1423 aprint_error_dev(sc->sc_dev, "polled command timed out\n");
1424
1425 /* Invoke the callback to clean state anyway */
1426 struct nvme_cqe cqe;
1427 memset(&cqe, 0, sizeof(cqe));
1428 ccb->ccb_done(q, ccb, &cqe);
1429
1430 return 1;
1431 }
1432 }
1433
1434 static void
nvme_poll_fill(struct nvme_queue * q,struct nvme_ccb * ccb,void * slot)1435 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
1436 {
1437 struct nvme_sqe *sqe = slot;
1438 struct nvme_poll_state *state = ccb->ccb_cookie;
1439
1440 *sqe = state->s;
1441 }
1442
1443 static void
nvme_poll_done(struct nvme_queue * q,struct nvme_ccb * ccb,struct nvme_cqe * cqe)1444 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb,
1445 struct nvme_cqe *cqe)
1446 {
1447 struct nvme_poll_state *state = ccb->ccb_cookie;
1448
1449 state->c = *cqe;
1450 SET(state->c.flags, htole16(NVME_CQE_PHASE));
1451
1452 ccb->ccb_cookie = state->cookie;
1453 state->done(q, ccb, &state->c);
1454 }
1455
1456 static void
nvme_sqe_fill(struct nvme_queue * q,struct nvme_ccb * ccb,void * slot)1457 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
1458 {
1459 struct nvme_sqe *src = ccb->ccb_cookie;
1460 struct nvme_sqe *dst = slot;
1461
1462 *dst = *src;
1463 }
1464
1465 static void
nvme_empty_done(struct nvme_queue * q,struct nvme_ccb * ccb,struct nvme_cqe * cqe)1466 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb,
1467 struct nvme_cqe *cqe)
1468 {
1469 }
1470
1471 void
nvme_op_cq_done(struct nvme_softc * sc,struct nvme_queue * q,struct nvme_ccb * ccb)1472 nvme_op_cq_done(struct nvme_softc *sc,
1473 struct nvme_queue *q, struct nvme_ccb *ccb)
1474 {
1475 /* nop */
1476 }
1477
1478 static int
nvme_q_complete(struct nvme_softc * sc,struct nvme_queue * q)1479 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q)
1480 {
1481 struct nvme_ccb *ccb;
1482 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe;
1483 uint16_t flags;
1484 int rv = 0;
1485
1486 mutex_enter(&q->q_cq_mtx);
1487
1488 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD);
1489 for (;;) {
1490 cqe = &ring[q->q_cq_head];
1491 flags = lemtoh16(&cqe->flags);
1492 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase)
1493 break;
1494
1495 /*
1496 * Make sure we have read the flags _before_ we read
1497 * the cid. Otherwise the CPU might speculatively read
1498 * the cid before the entry has been assigned to our
1499 * phase.
1500 */
1501 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD);
1502
1503 ccb = &q->q_ccbs[lemtoh16(&cqe->cid)];
1504
1505 if (++q->q_cq_head >= q->q_entries) {
1506 q->q_cq_head = 0;
1507 q->q_cq_phase ^= NVME_CQE_PHASE;
1508 }
1509
1510 #ifdef DEBUG
1511 /*
1512 * If we get spurious completion notification, something
1513 * is seriously hosed up. Very likely DMA to some random
1514 * memory place happened, so just bail out.
1515 */
1516 if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) {
1517 panic("%s: invalid ccb detected",
1518 device_xname(sc->sc_dev));
1519 /* NOTREACHED */
1520 }
1521 #endif
1522
1523 rv++;
1524
1525 sc->sc_ops->op_cq_done(sc, q, ccb);
1526
1527 /*
1528 * Unlock the mutex before calling the ccb_done callback
1529 * and re-lock afterwards. The callback triggers lddone()
1530 * which schedules another i/o, and also calls nvme_ccb_put().
1531 * Unlock/relock avoids possibility of deadlock.
1532 */
1533 mutex_exit(&q->q_cq_mtx);
1534 ccb->ccb_done(q, ccb, cqe);
1535 mutex_enter(&q->q_cq_mtx);
1536 }
1537 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD);
1538
1539 if (rv)
1540 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head);
1541
1542 mutex_exit(&q->q_cq_mtx);
1543
1544 return rv;
1545 }
1546
1547 static void
nvme_q_wait_complete(struct nvme_softc * sc,struct nvme_queue * q,bool (* finished)(void *),void * cookie)1548 nvme_q_wait_complete(struct nvme_softc *sc,
1549 struct nvme_queue *q, bool (*finished)(void *), void *cookie)
1550 {
1551 mutex_enter(&q->q_ccb_mtx);
1552 if (finished(cookie))
1553 goto out;
1554
1555 for(;;) {
1556 q->q_ccb_waiting = true;
1557 cv_wait(&q->q_ccb_wait, &q->q_ccb_mtx);
1558
1559 if (finished(cookie))
1560 break;
1561 }
1562
1563 out:
1564 mutex_exit(&q->q_ccb_mtx);
1565 }
1566
1567 static int
nvme_identify(struct nvme_softc * sc,u_int mps)1568 nvme_identify(struct nvme_softc *sc, u_int mps)
1569 {
1570 char sn[41], mn[81], fr[17];
1571 struct nvm_identify_controller *identify;
1572 struct nvme_dmamem *mem;
1573 struct nvme_ccb *ccb;
1574 u_int mdts;
1575 int rv = 1;
1576
1577 ccb = nvme_ccb_get(sc->sc_admin_q, false);
1578 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */
1579
1580 mem = nvme_dmamem_alloc(sc, sizeof(*identify));
1581 if (mem == NULL)
1582 return 1;
1583
1584 ccb->ccb_done = nvme_empty_done;
1585 ccb->ccb_cookie = mem;
1586
1587 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD);
1588 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify,
1589 NVME_TIMO_IDENT);
1590 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD);
1591
1592 nvme_ccb_put(sc->sc_admin_q, ccb);
1593
1594 if (rv != 0)
1595 goto done;
1596
1597 identify = NVME_DMA_KVA(mem);
1598 sc->sc_identify = *identify;
1599 identify = NULL;
1600
1601 /* Convert data to host endian */
1602 nvme_identify_controller_swapbytes(&sc->sc_identify);
1603
1604 strnvisx(sn, sizeof(sn), (const char *)sc->sc_identify.sn,
1605 sizeof(sc->sc_identify.sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL);
1606 strnvisx(mn, sizeof(mn), (const char *)sc->sc_identify.mn,
1607 sizeof(sc->sc_identify.mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL);
1608 strnvisx(fr, sizeof(fr), (const char *)sc->sc_identify.fr,
1609 sizeof(sc->sc_identify.fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL);
1610 aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr,
1611 sn);
1612
1613 strlcpy(sc->sc_modelname, mn, sizeof(sc->sc_modelname));
1614
1615 if (sc->sc_identify.mdts > 0) {
1616 mdts = (1 << sc->sc_identify.mdts) * (1 << mps);
1617 if (mdts < sc->sc_mdts)
1618 sc->sc_mdts = mdts;
1619 }
1620
1621 sc->sc_nn = sc->sc_identify.nn;
1622
1623 done:
1624 nvme_dmamem_free(sc, mem);
1625
1626 return rv;
1627 }
1628
1629 static int
nvme_q_create(struct nvme_softc * sc,struct nvme_queue * q)1630 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q)
1631 {
1632 struct nvme_sqe_q sqe;
1633 struct nvme_ccb *ccb;
1634 int rv;
1635
1636 if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0)
1637 return 1;
1638
1639 ccb = nvme_ccb_get(sc->sc_admin_q, false);
1640 KASSERT(ccb != NULL);
1641
1642 ccb->ccb_done = nvme_empty_done;
1643 ccb->ccb_cookie = &sqe;
1644
1645 memset(&sqe, 0, sizeof(sqe));
1646 sqe.opcode = NVM_ADMIN_ADD_IOCQ;
1647 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem));
1648 htolem16(&sqe.qsize, q->q_entries - 1);
1649 htolem16(&sqe.qid, q->q_id);
1650 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC;
1651 if (sc->sc_use_mq)
1652 htolem16(&sqe.cqid, q->q_id); /* qid == vector */
1653
1654 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP);
1655 if (rv != 0)
1656 goto fail;
1657
1658 ccb->ccb_done = nvme_empty_done;
1659 ccb->ccb_cookie = &sqe;
1660
1661 memset(&sqe, 0, sizeof(sqe));
1662 sqe.opcode = NVM_ADMIN_ADD_IOSQ;
1663 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem));
1664 htolem16(&sqe.qsize, q->q_entries - 1);
1665 htolem16(&sqe.qid, q->q_id);
1666 htolem16(&sqe.cqid, q->q_id);
1667 sqe.qflags = NVM_SQE_Q_PC;
1668
1669 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP);
1670 if (rv != 0)
1671 goto fail;
1672
1673 nvme_ccb_put(sc->sc_admin_q, ccb);
1674 return 0;
1675
1676 fail:
1677 if (sc->sc_use_mq)
1678 sc->sc_intr_disestablish(sc, q->q_id);
1679
1680 nvme_ccb_put(sc->sc_admin_q, ccb);
1681 return rv;
1682 }
1683
1684 static int
nvme_q_delete(struct nvme_softc * sc,struct nvme_queue * q)1685 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q)
1686 {
1687 struct nvme_sqe_q sqe;
1688 struct nvme_ccb *ccb;
1689 int rv;
1690
1691 ccb = nvme_ccb_get(sc->sc_admin_q, false);
1692 KASSERT(ccb != NULL);
1693
1694 ccb->ccb_done = nvme_empty_done;
1695 ccb->ccb_cookie = &sqe;
1696
1697 memset(&sqe, 0, sizeof(sqe));
1698 sqe.opcode = NVM_ADMIN_DEL_IOSQ;
1699 htolem16(&sqe.qid, q->q_id);
1700
1701 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP);
1702 if (rv != 0)
1703 goto fail;
1704
1705 ccb->ccb_done = nvme_empty_done;
1706 ccb->ccb_cookie = &sqe;
1707
1708 memset(&sqe, 0, sizeof(sqe));
1709 sqe.opcode = NVM_ADMIN_DEL_IOCQ;
1710 htolem16(&sqe.qid, q->q_id);
1711
1712 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP);
1713 if (rv != 0)
1714 goto fail;
1715
1716 fail:
1717 nvme_ccb_put(sc->sc_admin_q, ccb);
1718
1719 if (rv == 0 && sc->sc_use_mq) {
1720 if (sc->sc_intr_disestablish(sc, q->q_id))
1721 rv = 1;
1722 }
1723
1724 return rv;
1725 }
1726
1727 static void
nvme_fill_identify(struct nvme_queue * q,struct nvme_ccb * ccb,void * slot)1728 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
1729 {
1730 struct nvme_sqe *sqe = slot;
1731 struct nvme_dmamem *mem = ccb->ccb_cookie;
1732
1733 sqe->opcode = NVM_ADMIN_IDENTIFY;
1734 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem));
1735 htolem32(&sqe->cdw10, 1);
1736 }
1737
1738 static int
nvme_set_number_of_queues(struct nvme_softc * sc,u_int nq,u_int * ncqa,u_int * nsqa)1739 nvme_set_number_of_queues(struct nvme_softc *sc, u_int nq, u_int *ncqa,
1740 u_int *nsqa)
1741 {
1742 struct nvme_pt_state state;
1743 struct nvme_pt_command pt;
1744 struct nvme_ccb *ccb;
1745 int rv;
1746
1747 ccb = nvme_ccb_get(sc->sc_admin_q, false);
1748 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */
1749
1750 memset(&pt, 0, sizeof(pt));
1751 pt.cmd.opcode = NVM_ADMIN_SET_FEATURES;
1752 pt.cmd.cdw10 = NVM_FEATURE_NUMBER_OF_QUEUES;
1753 pt.cmd.cdw11 = ((nq - 1) << 16) | (nq - 1);
1754
1755 memset(&state, 0, sizeof(state));
1756 state.pt = &pt;
1757 state.finished = false;
1758
1759 ccb->ccb_done = nvme_pt_done;
1760 ccb->ccb_cookie = &state;
1761
1762 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_pt_fill, NVME_TIMO_QOP);
1763
1764 if (rv != 0) {
1765 *ncqa = *nsqa = 0;
1766 return EIO;
1767 }
1768
1769 *ncqa = (pt.cpl.cdw0 >> 16) + 1;
1770 *nsqa = (pt.cpl.cdw0 & 0xffff) + 1;
1771
1772 return 0;
1773 }
1774
1775 static int
nvme_ccbs_alloc(struct nvme_queue * q,uint16_t nccbs)1776 nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs)
1777 {
1778 struct nvme_softc *sc = q->q_sc;
1779 struct nvme_ccb *ccb;
1780 bus_addr_t off;
1781 uint64_t *prpl;
1782 u_int i;
1783
1784 mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO);
1785 cv_init(&q->q_ccb_wait, "nvmeqw");
1786 q->q_ccb_waiting = false;
1787 SIMPLEQ_INIT(&q->q_ccb_list);
1788
1789 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP);
1790
1791 q->q_nccbs = nccbs;
1792 q->q_ccb_prpls = nvme_dmamem_alloc(sc,
1793 sizeof(*prpl) * sc->sc_max_sgl * nccbs);
1794
1795 prpl = NVME_DMA_KVA(q->q_ccb_prpls);
1796 off = 0;
1797
1798 for (i = 0; i < nccbs; i++) {
1799 ccb = &q->q_ccbs[i];
1800
1801 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts,
1802 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */,
1803 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
1804 &ccb->ccb_dmamap) != 0)
1805 goto free_maps;
1806
1807 ccb->ccb_id = i;
1808 ccb->ccb_prpl = prpl;
1809 ccb->ccb_prpl_off = off;
1810 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off;
1811
1812 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry);
1813
1814 prpl += sc->sc_max_sgl;
1815 off += sizeof(*prpl) * sc->sc_max_sgl;
1816 }
1817
1818 return 0;
1819
1820 free_maps:
1821 nvme_ccbs_free(q);
1822 return 1;
1823 }
1824
1825 static struct nvme_ccb *
nvme_ccb_get(struct nvme_queue * q,bool wait)1826 nvme_ccb_get(struct nvme_queue *q, bool wait)
1827 {
1828 struct nvme_ccb *ccb = NULL;
1829
1830 mutex_enter(&q->q_ccb_mtx);
1831 again:
1832 ccb = SIMPLEQ_FIRST(&q->q_ccb_list);
1833 if (ccb != NULL) {
1834 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry);
1835 #ifdef DEBUG
1836 ccb->ccb_cookie = NULL;
1837 #endif
1838 } else {
1839 if (__predict_false(wait)) {
1840 q->q_ccb_waiting = true;
1841 cv_wait(&q->q_ccb_wait, &q->q_ccb_mtx);
1842 goto again;
1843 }
1844 }
1845 mutex_exit(&q->q_ccb_mtx);
1846
1847 return ccb;
1848 }
1849
1850 static struct nvme_ccb *
nvme_ccb_get_bio(struct nvme_softc * sc,struct buf * bp,struct nvme_queue ** selq)1851 nvme_ccb_get_bio(struct nvme_softc *sc, struct buf *bp,
1852 struct nvme_queue **selq)
1853 {
1854 u_int cpuindex = cpu_index((bp && bp->b_ci) ? bp->b_ci : curcpu());
1855
1856 /*
1857 * Find a queue with available ccbs, preferring the originating
1858 * CPU's queue.
1859 */
1860
1861 for (u_int qoff = 0; qoff < sc->sc_nq; qoff++) {
1862 struct nvme_queue *q = sc->sc_q[(cpuindex + qoff) % sc->sc_nq];
1863 struct nvme_ccb *ccb;
1864
1865 mutex_enter(&q->q_ccb_mtx);
1866 ccb = SIMPLEQ_FIRST(&q->q_ccb_list);
1867 if (ccb != NULL) {
1868 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry);
1869 #ifdef DEBUG
1870 ccb->ccb_cookie = NULL;
1871 #endif
1872 }
1873 mutex_exit(&q->q_ccb_mtx);
1874
1875 if (ccb != NULL) {
1876 *selq = q;
1877 return ccb;
1878 }
1879 }
1880
1881 return NULL;
1882 }
1883
1884 static void
nvme_ccb_put(struct nvme_queue * q,struct nvme_ccb * ccb)1885 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb)
1886 {
1887
1888 mutex_enter(&q->q_ccb_mtx);
1889 #ifdef DEBUG
1890 ccb->ccb_cookie = (void *)NVME_CCB_FREE;
1891 #endif
1892 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry);
1893
1894 /* It's unlikely there are any waiters, it's not used for regular I/O */
1895 if (__predict_false(q->q_ccb_waiting)) {
1896 q->q_ccb_waiting = false;
1897 cv_broadcast(&q->q_ccb_wait);
1898 }
1899
1900 mutex_exit(&q->q_ccb_mtx);
1901 }
1902
1903 static void
nvme_ccbs_free(struct nvme_queue * q)1904 nvme_ccbs_free(struct nvme_queue *q)
1905 {
1906 struct nvme_softc *sc = q->q_sc;
1907 struct nvme_ccb *ccb;
1908
1909 mutex_enter(&q->q_ccb_mtx);
1910 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) {
1911 SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry);
1912 /*
1913 * bus_dmamap_destroy() may call vm_map_lock() and rw_enter()
1914 * internally. don't hold spin mutex
1915 */
1916 mutex_exit(&q->q_ccb_mtx);
1917 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
1918 mutex_enter(&q->q_ccb_mtx);
1919 }
1920 mutex_exit(&q->q_ccb_mtx);
1921
1922 nvme_dmamem_free(sc, q->q_ccb_prpls);
1923 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs);
1924 q->q_ccbs = NULL;
1925 cv_destroy(&q->q_ccb_wait);
1926 mutex_destroy(&q->q_ccb_mtx);
1927 }
1928
1929 static struct nvme_queue *
nvme_q_alloc(struct nvme_softc * sc,uint16_t id,u_int entries,u_int dstrd)1930 nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd)
1931 {
1932 struct nvme_queue *q;
1933
1934 q = kmem_alloc(sizeof(*q), KM_SLEEP);
1935 q->q_sc = sc;
1936 q->q_sq_dmamem = nvme_dmamem_alloc(sc,
1937 sizeof(struct nvme_sqe) * entries);
1938 if (q->q_sq_dmamem == NULL)
1939 goto free;
1940
1941 q->q_cq_dmamem = nvme_dmamem_alloc(sc,
1942 sizeof(struct nvme_cqe) * entries);
1943 if (q->q_cq_dmamem == NULL)
1944 goto free_sq;
1945
1946 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem));
1947 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem));
1948
1949 mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO);
1950 mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO);
1951 q->q_sqtdbl = NVME_SQTDBL(id, dstrd);
1952 q->q_cqhdbl = NVME_CQHDBL(id, dstrd);
1953 q->q_id = id;
1954 q->q_entries = entries;
1955 q->q_sq_tail = 0;
1956 q->q_cq_head = 0;
1957 q->q_cq_phase = NVME_CQE_PHASE;
1958
1959 if (sc->sc_ops->op_q_alloc != NULL) {
1960 if (sc->sc_ops->op_q_alloc(sc, q) != 0)
1961 goto free_cq;
1962 }
1963
1964 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE);
1965 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD);
1966
1967 /*
1968 * Due to definition of full and empty queue (queue is empty
1969 * when head == tail, full when tail is one less then head),
1970 * we can actually only have (entries - 1) in-flight commands.
1971 */
1972 if (nvme_ccbs_alloc(q, entries - 1) != 0) {
1973 aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n");
1974 goto free_cq;
1975 }
1976
1977 return q;
1978
1979 free_cq:
1980 nvme_dmamem_free(sc, q->q_cq_dmamem);
1981 free_sq:
1982 nvme_dmamem_free(sc, q->q_sq_dmamem);
1983 free:
1984 kmem_free(q, sizeof(*q));
1985
1986 return NULL;
1987 }
1988
1989 static void
nvme_q_reset(struct nvme_softc * sc,struct nvme_queue * q)1990 nvme_q_reset(struct nvme_softc *sc, struct nvme_queue *q)
1991 {
1992
1993 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem));
1994 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem));
1995
1996 q->q_sq_tail = 0;
1997 q->q_cq_head = 0;
1998 q->q_cq_phase = NVME_CQE_PHASE;
1999
2000 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE);
2001 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD);
2002 }
2003
2004 static void
nvme_q_free(struct nvme_softc * sc,struct nvme_queue * q)2005 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q)
2006 {
2007 nvme_ccbs_free(q);
2008 mutex_destroy(&q->q_sq_mtx);
2009 mutex_destroy(&q->q_cq_mtx);
2010 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD);
2011 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE);
2012
2013 if (sc->sc_ops->op_q_alloc != NULL)
2014 sc->sc_ops->op_q_free(sc, q);
2015
2016 nvme_dmamem_free(sc, q->q_cq_dmamem);
2017 nvme_dmamem_free(sc, q->q_sq_dmamem);
2018 kmem_free(q, sizeof(*q));
2019 }
2020
2021 int
nvme_intr(void * xsc)2022 nvme_intr(void *xsc)
2023 {
2024 struct nvme_softc *sc = xsc;
2025
2026 /*
2027 * INTx is level triggered, controller deasserts the interrupt only
2028 * when we advance command queue head via write to the doorbell.
2029 * Tell the controller to block the interrupts while we process
2030 * the queue(s).
2031 */
2032 nvme_write4(sc, NVME_INTMS, 1);
2033
2034 softint_schedule(sc->sc_softih[0]);
2035
2036 /* don't know, might not have been for us */
2037 return 1;
2038 }
2039
2040 void
nvme_softintr_intx(void * xq)2041 nvme_softintr_intx(void *xq)
2042 {
2043 struct nvme_queue *q = xq;
2044 struct nvme_softc *sc = q->q_sc;
2045
2046 nvme_q_complete(sc, sc->sc_admin_q);
2047 if (sc->sc_q != NULL)
2048 nvme_q_complete(sc, sc->sc_q[0]);
2049
2050 /*
2051 * Processing done, tell controller to issue interrupts again. There
2052 * is no race, as NVMe spec requires the controller to maintain state,
2053 * and assert the interrupt whenever there are unacknowledged
2054 * completion queue entries.
2055 */
2056 nvme_write4(sc, NVME_INTMC, 1);
2057 }
2058
2059 int
nvme_intr_msi(void * xq)2060 nvme_intr_msi(void *xq)
2061 {
2062 struct nvme_queue *q = xq;
2063
2064 KASSERT(q);
2065 KASSERT(q->q_sc);
2066 KASSERT(q->q_sc->sc_softih);
2067 KASSERT(q->q_sc->sc_softih[q->q_id]);
2068
2069 /*
2070 * MSI/MSI-X are edge triggered, so can handover processing to softint
2071 * without masking the interrupt.
2072 */
2073 softint_schedule(q->q_sc->sc_softih[q->q_id]);
2074
2075 return 1;
2076 }
2077
2078 void
nvme_softintr_msi(void * xq)2079 nvme_softintr_msi(void *xq)
2080 {
2081 struct nvme_queue *q = xq;
2082 struct nvme_softc *sc = q->q_sc;
2083
2084 nvme_q_complete(sc, q);
2085 }
2086
2087 struct nvme_dmamem *
nvme_dmamem_alloc(struct nvme_softc * sc,size_t size)2088 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size)
2089 {
2090 struct nvme_dmamem *ndm;
2091 int nsegs;
2092
2093 ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP);
2094 if (ndm == NULL)
2095 return NULL;
2096
2097 ndm->ndm_size = size;
2098
2099 if (bus_dmamap_create(sc->sc_dmat, size, btoc(round_page(size)), size, 0,
2100 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0)
2101 goto ndmfree;
2102
2103 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg,
2104 1, &nsegs, BUS_DMA_WAITOK) != 0)
2105 goto destroy;
2106
2107 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size,
2108 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0)
2109 goto free;
2110
2111 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size,
2112 NULL, BUS_DMA_WAITOK) != 0)
2113 goto unmap;
2114
2115 memset(ndm->ndm_kva, 0, size);
2116 bus_dmamap_sync(sc->sc_dmat, ndm->ndm_map, 0, size, BUS_DMASYNC_PREREAD);
2117
2118 return ndm;
2119
2120 unmap:
2121 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size);
2122 free:
2123 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1);
2124 destroy:
2125 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map);
2126 ndmfree:
2127 kmem_free(ndm, sizeof(*ndm));
2128 return NULL;
2129 }
2130
2131 void
nvme_dmamem_sync(struct nvme_softc * sc,struct nvme_dmamem * mem,int ops)2132 nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops)
2133 {
2134 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem),
2135 0, NVME_DMA_LEN(mem), ops);
2136 }
2137
2138 void
nvme_dmamem_free(struct nvme_softc * sc,struct nvme_dmamem * ndm)2139 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm)
2140 {
2141 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map);
2142 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size);
2143 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1);
2144 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map);
2145 kmem_free(ndm, sizeof(*ndm));
2146 }
2147
2148 /*
2149 * ioctl
2150 */
2151
2152 dev_type_open(nvmeopen);
2153 dev_type_close(nvmeclose);
2154 dev_type_ioctl(nvmeioctl);
2155
2156 const struct cdevsw nvme_cdevsw = {
2157 .d_open = nvmeopen,
2158 .d_close = nvmeclose,
2159 .d_read = noread,
2160 .d_write = nowrite,
2161 .d_ioctl = nvmeioctl,
2162 .d_stop = nostop,
2163 .d_tty = notty,
2164 .d_poll = nopoll,
2165 .d_mmap = nommap,
2166 .d_kqfilter = nokqfilter,
2167 .d_discard = nodiscard,
2168 .d_flag = D_OTHER,
2169 };
2170
2171 /*
2172 * Accept an open operation on the control device.
2173 */
2174 int
nvmeopen(dev_t dev,int flag,int mode,struct lwp * l)2175 nvmeopen(dev_t dev, int flag, int mode, struct lwp *l)
2176 {
2177 struct nvme_softc *sc;
2178 int unit = minor(dev) / 0x10000;
2179 int nsid = minor(dev) & 0xffff;
2180 int nsidx;
2181
2182 if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL)
2183 return ENXIO;
2184 if ((sc->sc_flags & NVME_F_ATTACHED) == 0)
2185 return ENXIO;
2186
2187 if (nsid == 0) {
2188 /* controller */
2189 if (ISSET(sc->sc_flags, NVME_F_OPEN))
2190 return EBUSY;
2191 SET(sc->sc_flags, NVME_F_OPEN);
2192 } else {
2193 /* namespace */
2194 nsidx = nsid - 1;
2195 if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL)
2196 return ENXIO;
2197 if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN))
2198 return EBUSY;
2199 SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN);
2200 }
2201 return 0;
2202 }
2203
2204 /*
2205 * Accept the last close on the control device.
2206 */
2207 int
nvmeclose(dev_t dev,int flag,int mode,struct lwp * l)2208 nvmeclose(dev_t dev, int flag, int mode, struct lwp *l)
2209 {
2210 struct nvme_softc *sc;
2211 int unit = minor(dev) / 0x10000;
2212 int nsid = minor(dev) & 0xffff;
2213 int nsidx;
2214
2215 sc = device_lookup_private(&nvme_cd, unit);
2216 if (sc == NULL)
2217 return ENXIO;
2218
2219 if (nsid == 0) {
2220 /* controller */
2221 CLR(sc->sc_flags, NVME_F_OPEN);
2222 } else {
2223 /* namespace */
2224 nsidx = nsid - 1;
2225 if (nsidx >= sc->sc_nn)
2226 return ENXIO;
2227 CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN);
2228 }
2229
2230 return 0;
2231 }
2232
2233 /*
2234 * Handle control operations.
2235 */
2236 int
nvmeioctl(dev_t dev,u_long cmd,void * data,int flag,struct lwp * l)2237 nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l)
2238 {
2239 struct nvme_softc *sc;
2240 int unit = minor(dev) / 0x10000;
2241 int nsid = minor(dev) & 0xffff;
2242 struct nvme_pt_command *pt;
2243
2244 sc = device_lookup_private(&nvme_cd, unit);
2245 if (sc == NULL)
2246 return ENXIO;
2247
2248 switch (cmd) {
2249 case NVME_PASSTHROUGH_CMD:
2250 pt = data;
2251 return nvme_command_passthrough(sc, data,
2252 nsid == 0 ? pt->cmd.nsid : (uint32_t)nsid, l, nsid == 0);
2253 }
2254
2255 return ENOTTY;
2256 }
2257