1 /* $NetBSD: ld_virtio.c,v 1.32 2023/03/23 03:55:11 yamaguchi Exp $ */
2
3 /*
4 * Copyright (c) 2010 Minoura Makoto.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include <sys/cdefs.h>
29 __KERNEL_RCSID(0, "$NetBSD: ld_virtio.c,v 1.32 2023/03/23 03:55:11 yamaguchi Exp $");
30
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/kernel.h>
34 #include <sys/buf.h>
35 #include <sys/bufq.h>
36 #include <sys/bus.h>
37 #include <sys/device.h>
38 #include <sys/disk.h>
39 #include <sys/mutex.h>
40 #include <sys/module.h>
41
42 #include <dev/ldvar.h>
43 #include <dev/pci/virtioreg.h>
44 #include <dev/pci/virtiovar.h>
45
46 #include "ioconf.h"
47
48 /*
49 * ld_virtioreg:
50 */
51 /* Configuration registers */
52 #define VIRTIO_BLK_CONFIG_CAPACITY 0 /* 64bit */
53 #define VIRTIO_BLK_CONFIG_SIZE_MAX 8 /* 32bit */
54 #define VIRTIO_BLK_CONFIG_SEG_MAX 12 /* 32bit */
55 #define VIRTIO_BLK_CONFIG_GEOMETRY_C 16 /* 16bit */
56 #define VIRTIO_BLK_CONFIG_GEOMETRY_H 18 /* 8bit */
57 #define VIRTIO_BLK_CONFIG_GEOMETRY_S 19 /* 8bit */
58 #define VIRTIO_BLK_CONFIG_BLK_SIZE 20 /* 32bit */
59 #define VIRTIO_BLK_CONFIG_WRITEBACK 32 /* 8bit */
60
61 /* Feature bits */
62 #define VIRTIO_BLK_F_BARRIER (1<<0)
63 #define VIRTIO_BLK_F_SIZE_MAX (1<<1)
64 #define VIRTIO_BLK_F_SEG_MAX (1<<2)
65 #define VIRTIO_BLK_F_GEOMETRY (1<<4)
66 #define VIRTIO_BLK_F_RO (1<<5)
67 #define VIRTIO_BLK_F_BLK_SIZE (1<<6)
68 #define VIRTIO_BLK_F_SCSI (1<<7)
69 #define VIRTIO_BLK_F_FLUSH (1<<9)
70 #define VIRTIO_BLK_F_TOPOLOGY (1<<10)
71 #define VIRTIO_BLK_F_CONFIG_WCE (1<<11)
72
73 /*
74 * Each block request uses at least two segments - one for the header
75 * and one for the status.
76 */
77 #define VIRTIO_BLK_MIN_SEGMENTS 2
78
79 #define VIRTIO_BLK_FLAG_BITS \
80 VIRTIO_COMMON_FLAG_BITS \
81 "b\x0b" "CONFIG_WCE\0" \
82 "b\x0a" "TOPOLOGY\0" \
83 "b\x09" "FLUSH\0" \
84 "b\x07" "SCSI\0" \
85 "b\x06" "BLK_SIZE\0" \
86 "b\x05" "RO\0" \
87 "b\x04" "GEOMETRY\0" \
88 "b\x02" "SEG_MAX\0" \
89 "b\x01" "SIZE_MAX\0" \
90 "b\x00" "BARRIER\0"
91
92 /* Command */
93 #define VIRTIO_BLK_T_IN 0
94 #define VIRTIO_BLK_T_OUT 1
95 #define VIRTIO_BLK_T_FLUSH 4
96 #define VIRTIO_BLK_T_BARRIER 0x80000000
97
98 /* Sector */
99 #define VIRTIO_BLK_BSIZE 512
100
101 /* Status */
102 #define VIRTIO_BLK_S_OK 0
103 #define VIRTIO_BLK_S_IOERR 1
104 #define VIRTIO_BLK_S_UNSUPP 2
105
106 /* Request header structure */
107 struct virtio_blk_req_hdr {
108 uint32_t type; /* VIRTIO_BLK_T_* */
109 uint32_t ioprio;
110 uint64_t sector;
111 } __packed;
112 /* payload and 1 byte status follows */
113
114
115 /*
116 * ld_virtiovar:
117 */
118 struct virtio_blk_req {
119 struct virtio_blk_req_hdr vr_hdr;
120 uint8_t vr_status;
121 struct buf *vr_bp;
122 #define DUMMY_VR_BP ((void *)1)
123 bus_dmamap_t vr_cmdsts;
124 bus_dmamap_t vr_payload;
125 };
126
127 struct ld_virtio_softc {
128 struct ld_softc sc_ld;
129 device_t sc_dev;
130
131 struct virtio_softc *sc_virtio;
132 struct virtqueue sc_vq;
133
134 struct virtio_blk_req *sc_reqs;
135 bus_dma_segment_t sc_reqs_seg;
136
137 int sc_readonly;
138
139 enum {
140 SYNC_FREE, SYNC_BUSY, SYNC_DONE
141 } sc_sync_use;
142 kcondvar_t sc_sync_wait;
143 kmutex_t sc_sync_wait_lock;
144 uint8_t sc_sync_status;
145 };
146
147 static int ld_virtio_match(device_t, cfdata_t, void *);
148 static void ld_virtio_attach(device_t, device_t, void *);
149 static int ld_virtio_detach(device_t, int);
150
151 CFATTACH_DECL_NEW(ld_virtio, sizeof(struct ld_virtio_softc),
152 ld_virtio_match, ld_virtio_attach, ld_virtio_detach, NULL);
153
154 static int
ld_virtio_match(device_t parent,cfdata_t match,void * aux)155 ld_virtio_match(device_t parent, cfdata_t match, void *aux)
156 {
157 struct virtio_attach_args *va = aux;
158
159 if (va->sc_childdevid == VIRTIO_DEVICE_ID_BLOCK)
160 return 1;
161
162 return 0;
163 }
164
165 static int ld_virtio_vq_done(struct virtqueue *);
166 static int ld_virtio_dump(struct ld_softc *, void *, int, int);
167 static int ld_virtio_start(struct ld_softc *, struct buf *);
168 static int ld_virtio_ioctl(struct ld_softc *, u_long, void *, int32_t, bool);
169
170 static int
ld_virtio_alloc_reqs(struct ld_virtio_softc * sc,int qsize)171 ld_virtio_alloc_reqs(struct ld_virtio_softc *sc, int qsize)
172 {
173 int allocsize, r, rsegs, i;
174 struct ld_softc *ld = &sc->sc_ld;
175 void *vaddr;
176
177 allocsize = sizeof(struct virtio_blk_req) * qsize;
178 r = bus_dmamem_alloc(virtio_dmat(sc->sc_virtio), allocsize, 0, 0,
179 &sc->sc_reqs_seg, 1, &rsegs, BUS_DMA_WAITOK);
180 if (r != 0) {
181 aprint_error_dev(sc->sc_dev,
182 "DMA memory allocation failed, size %d, "
183 "error code %d\n", allocsize, r);
184 goto err_none;
185 }
186 r = bus_dmamem_map(virtio_dmat(sc->sc_virtio),
187 &sc->sc_reqs_seg, 1, allocsize,
188 &vaddr, BUS_DMA_WAITOK);
189 if (r != 0) {
190 aprint_error_dev(sc->sc_dev,
191 "DMA memory map failed, "
192 "error code %d\n", r);
193 goto err_dmamem_alloc;
194 }
195 sc->sc_reqs = vaddr;
196 memset(vaddr, 0, allocsize);
197 for (i = 0; i < qsize; i++) {
198 struct virtio_blk_req *vr = &sc->sc_reqs[i];
199 r = bus_dmamap_create(virtio_dmat(sc->sc_virtio),
200 offsetof(struct virtio_blk_req, vr_bp),
201 1,
202 offsetof(struct virtio_blk_req, vr_bp),
203 0,
204 BUS_DMA_WAITOK|BUS_DMA_ALLOCNOW,
205 &vr->vr_cmdsts);
206 if (r != 0) {
207 aprint_error_dev(sc->sc_dev,
208 "command dmamap creation failed, "
209 "error code %d\n", r);
210 goto err_reqs;
211 }
212 r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), vr->vr_cmdsts,
213 &vr->vr_hdr,
214 offsetof(struct virtio_blk_req, vr_bp),
215 NULL, BUS_DMA_WAITOK);
216 if (r != 0) {
217 aprint_error_dev(sc->sc_dev,
218 "command dmamap load failed, "
219 "error code %d\n", r);
220 goto err_reqs;
221 }
222 r = bus_dmamap_create(virtio_dmat(sc->sc_virtio),
223 ld->sc_maxxfer,
224 (ld->sc_maxxfer / NBPG) +
225 VIRTIO_BLK_MIN_SEGMENTS,
226 ld->sc_maxxfer,
227 0,
228 BUS_DMA_WAITOK|BUS_DMA_ALLOCNOW,
229 &vr->vr_payload);
230 if (r != 0) {
231 aprint_error_dev(sc->sc_dev,
232 "payload dmamap creation failed, "
233 "error code %d\n", r);
234 goto err_reqs;
235 }
236 }
237 return 0;
238
239 err_reqs:
240 for (i = 0; i < qsize; i++) {
241 struct virtio_blk_req *vr = &sc->sc_reqs[i];
242 if (vr->vr_cmdsts) {
243 bus_dmamap_destroy(virtio_dmat(sc->sc_virtio),
244 vr->vr_cmdsts);
245 vr->vr_cmdsts = 0;
246 }
247 if (vr->vr_payload) {
248 bus_dmamap_destroy(virtio_dmat(sc->sc_virtio),
249 vr->vr_payload);
250 vr->vr_payload = 0;
251 }
252 }
253 bus_dmamem_unmap(virtio_dmat(sc->sc_virtio), sc->sc_reqs, allocsize);
254 err_dmamem_alloc:
255 bus_dmamem_free(virtio_dmat(sc->sc_virtio), &sc->sc_reqs_seg, 1);
256 err_none:
257 return -1;
258 }
259
260 static void
ld_virtio_attach(device_t parent,device_t self,void * aux)261 ld_virtio_attach(device_t parent, device_t self, void *aux)
262 {
263 struct ld_virtio_softc *sc = device_private(self);
264 struct ld_softc *ld = &sc->sc_ld;
265 struct virtio_softc *vsc = device_private(parent);
266 uint64_t features;
267 int qsize, maxxfersize, maxnsegs;
268
269 if (virtio_child(vsc) != NULL) {
270 aprint_normal(": child already attached for %s; "
271 "something wrong...\n", device_xname(parent));
272 return;
273 }
274
275 sc->sc_dev = self;
276 sc->sc_virtio = vsc;
277
278 virtio_child_attach_start(vsc, self, IPL_BIO,
279 (VIRTIO_BLK_F_SIZE_MAX | VIRTIO_BLK_F_SEG_MAX |
280 VIRTIO_BLK_F_GEOMETRY | VIRTIO_BLK_F_RO | VIRTIO_BLK_F_BLK_SIZE |
281 VIRTIO_BLK_F_FLUSH | VIRTIO_BLK_F_CONFIG_WCE),
282 VIRTIO_BLK_FLAG_BITS);
283
284 features = virtio_features(vsc);
285 if (features == 0)
286 goto err;
287
288 if (features & VIRTIO_BLK_F_RO)
289 sc->sc_readonly = 1;
290 else
291 sc->sc_readonly = 0;
292
293 if (features & VIRTIO_BLK_F_BLK_SIZE) {
294 ld->sc_secsize = virtio_read_device_config_4(vsc,
295 VIRTIO_BLK_CONFIG_BLK_SIZE);
296 } else
297 ld->sc_secsize = VIRTIO_BLK_BSIZE;
298
299 /* At least genfs_io assumes maxxfer == MAXPHYS. */
300 if (features & VIRTIO_BLK_F_SIZE_MAX) {
301 maxxfersize = virtio_read_device_config_4(vsc,
302 VIRTIO_BLK_CONFIG_SIZE_MAX);
303 if (maxxfersize < MAXPHYS) {
304 aprint_error_dev(sc->sc_dev,
305 "Too small SIZE_MAX %dK minimum is %dK\n",
306 maxxfersize / 1024, MAXPHYS / 1024);
307 // goto err;
308 maxxfersize = MAXPHYS;
309 } else if (maxxfersize > MAXPHYS) {
310 aprint_normal_dev(sc->sc_dev,
311 "Clip SEG_MAX from %dK to %dK\n",
312 maxxfersize / 1024,
313 MAXPHYS / 1024);
314 maxxfersize = MAXPHYS;
315 }
316 } else
317 maxxfersize = MAXPHYS;
318
319 if (features & VIRTIO_BLK_F_SEG_MAX) {
320 maxnsegs = virtio_read_device_config_4(vsc,
321 VIRTIO_BLK_CONFIG_SEG_MAX);
322 if (maxnsegs < VIRTIO_BLK_MIN_SEGMENTS) {
323 aprint_error_dev(sc->sc_dev,
324 "Too small SEG_MAX %d minimum is %d\n",
325 maxnsegs, VIRTIO_BLK_MIN_SEGMENTS);
326 maxnsegs = maxxfersize / NBPG;
327 // goto err;
328 }
329 } else
330 maxnsegs = maxxfersize / NBPG;
331
332 /* 2 for the minimum size */
333 maxnsegs += VIRTIO_BLK_MIN_SEGMENTS;
334
335 virtio_init_vq_vqdone(vsc, &sc->sc_vq, 0,
336 ld_virtio_vq_done);
337
338 if (virtio_alloc_vq(vsc, &sc->sc_vq, maxxfersize, maxnsegs,
339 "I/O request") != 0) {
340 goto err;
341 }
342 qsize = sc->sc_vq.vq_num;
343
344 if (virtio_child_attach_finish(vsc, &sc->sc_vq, 1,
345 NULL, VIRTIO_F_INTR_MSIX) != 0)
346 goto err;
347
348 ld->sc_dv = self;
349 ld->sc_secperunit = virtio_read_device_config_8(vsc,
350 VIRTIO_BLK_CONFIG_CAPACITY) / (ld->sc_secsize / VIRTIO_BLK_BSIZE);
351 ld->sc_maxxfer = maxxfersize;
352 if (features & VIRTIO_BLK_F_GEOMETRY) {
353 ld->sc_ncylinders = virtio_read_device_config_2(vsc,
354 VIRTIO_BLK_CONFIG_GEOMETRY_C);
355 ld->sc_nheads = virtio_read_device_config_1(vsc,
356 VIRTIO_BLK_CONFIG_GEOMETRY_H);
357 ld->sc_nsectors = virtio_read_device_config_1(vsc,
358 VIRTIO_BLK_CONFIG_GEOMETRY_S);
359 }
360 ld->sc_maxqueuecnt = qsize - 1; /* reserve slot for dumps, flushes */
361
362 if (ld_virtio_alloc_reqs(sc, qsize) < 0)
363 goto err;
364
365 cv_init(&sc->sc_sync_wait, "vblksync");
366 mutex_init(&sc->sc_sync_wait_lock, MUTEX_DEFAULT, IPL_BIO);
367 sc->sc_sync_use = SYNC_FREE;
368
369 ld->sc_dump = ld_virtio_dump;
370 ld->sc_start = ld_virtio_start;
371 ld->sc_ioctl = ld_virtio_ioctl;
372
373 ld->sc_flags = LDF_ENABLED | LDF_MPSAFE;
374 ldattach(ld, BUFQ_DISK_DEFAULT_STRAT);
375
376 return;
377
378 err:
379 virtio_child_attach_failed(vsc);
380 return;
381 }
382
383 static int
ld_virtio_start(struct ld_softc * ld,struct buf * bp)384 ld_virtio_start(struct ld_softc *ld, struct buf *bp)
385 {
386 /* splbio */
387 struct ld_virtio_softc *sc = device_private(ld->sc_dv);
388 struct virtio_softc *vsc = sc->sc_virtio;
389 struct virtqueue *vq = &sc->sc_vq;
390 struct virtio_blk_req *vr;
391 int r;
392 int isread = (bp->b_flags & B_READ);
393 int slot;
394
395 if (sc->sc_readonly && !isread)
396 return EIO;
397
398 r = virtio_enqueue_prep(vsc, vq, &slot);
399 if (r != 0)
400 return r;
401
402 vr = &sc->sc_reqs[slot];
403 KASSERT(vr->vr_bp == NULL);
404
405 r = bus_dmamap_load(virtio_dmat(vsc), vr->vr_payload,
406 bp->b_data, bp->b_bcount, NULL,
407 ((isread?BUS_DMA_READ:BUS_DMA_WRITE)
408 |BUS_DMA_NOWAIT));
409 if (r != 0) {
410 aprint_error_dev(sc->sc_dev,
411 "payload dmamap failed, error code %d\n", r);
412 virtio_enqueue_abort(vsc, vq, slot);
413 return r;
414 }
415
416 r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs +
417 VIRTIO_BLK_MIN_SEGMENTS);
418 if (r != 0) {
419 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload);
420 return r;
421 }
422
423 vr->vr_bp = bp;
424 vr->vr_hdr.type = virtio_rw32(vsc,
425 isread ? VIRTIO_BLK_T_IN : VIRTIO_BLK_T_OUT);
426 vr->vr_hdr.ioprio = virtio_rw32(vsc, 0);
427 vr->vr_hdr.sector = virtio_rw64(vsc,
428 bp->b_rawblkno * sc->sc_ld.sc_secsize /
429 VIRTIO_BLK_BSIZE);
430
431 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts,
432 0, sizeof(struct virtio_blk_req_hdr),
433 BUS_DMASYNC_PREWRITE);
434 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload,
435 0, bp->b_bcount,
436 isread?BUS_DMASYNC_PREREAD:BUS_DMASYNC_PREWRITE);
437 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts,
438 offsetof(struct virtio_blk_req, vr_status),
439 sizeof(uint8_t),
440 BUS_DMASYNC_PREREAD);
441
442 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts,
443 0, sizeof(struct virtio_blk_req_hdr),
444 true);
445 virtio_enqueue(vsc, vq, slot, vr->vr_payload, !isread);
446 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts,
447 offsetof(struct virtio_blk_req, vr_status),
448 sizeof(uint8_t),
449 false);
450 virtio_enqueue_commit(vsc, vq, slot, true);
451
452 return 0;
453 }
454
455 static void
ld_virtio_vq_done1(struct ld_virtio_softc * sc,struct virtio_softc * vsc,struct virtqueue * vq,int slot)456 ld_virtio_vq_done1(struct ld_virtio_softc *sc, struct virtio_softc *vsc,
457 struct virtqueue *vq, int slot)
458 {
459 struct virtio_blk_req *vr = &sc->sc_reqs[slot];
460 struct buf *bp = vr->vr_bp;
461
462 vr->vr_bp = NULL;
463
464 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts,
465 0, sizeof(struct virtio_blk_req_hdr),
466 BUS_DMASYNC_POSTWRITE);
467 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts,
468 sizeof(struct virtio_blk_req_hdr), sizeof(uint8_t),
469 BUS_DMASYNC_POSTREAD);
470 if (bp == DUMMY_VR_BP) {
471 mutex_enter(&sc->sc_sync_wait_lock);
472 sc->sc_sync_status = vr->vr_status;
473 sc->sc_sync_use = SYNC_DONE;
474 cv_broadcast(&sc->sc_sync_wait);
475 mutex_exit(&sc->sc_sync_wait_lock);
476 virtio_dequeue_commit(vsc, vq, slot);
477 return;
478 }
479 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload,
480 0, bp->b_bcount,
481 (bp->b_flags & B_READ)?BUS_DMASYNC_POSTREAD
482 :BUS_DMASYNC_POSTWRITE);
483 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload);
484
485 if (vr->vr_status != VIRTIO_BLK_S_OK) {
486 bp->b_error = EIO;
487 bp->b_resid = bp->b_bcount;
488 } else {
489 bp->b_error = 0;
490 bp->b_resid = 0;
491 }
492
493 virtio_dequeue_commit(vsc, vq, slot);
494
495 lddone(&sc->sc_ld, bp);
496 }
497
498 static int
ld_virtio_vq_done(struct virtqueue * vq)499 ld_virtio_vq_done(struct virtqueue *vq)
500 {
501 struct virtio_softc *vsc = vq->vq_owner;
502 struct ld_virtio_softc *sc = device_private(virtio_child(vsc));
503 int r = 0;
504 int slot;
505
506 again:
507 if (virtio_dequeue(vsc, vq, &slot, NULL))
508 return r;
509 r = 1;
510
511 ld_virtio_vq_done1(sc, vsc, vq, slot);
512 goto again;
513 }
514
515 static int
ld_virtio_dump(struct ld_softc * ld,void * data,int blkno,int blkcnt)516 ld_virtio_dump(struct ld_softc *ld, void *data, int blkno, int blkcnt)
517 {
518 struct ld_virtio_softc *sc = device_private(ld->sc_dv);
519 struct virtio_softc *vsc = sc->sc_virtio;
520 struct virtqueue *vq = &sc->sc_vq;
521 struct virtio_blk_req *vr;
522 int slot, r;
523
524 if (sc->sc_readonly)
525 return EIO;
526
527 r = virtio_enqueue_prep(vsc, vq, &slot);
528 if (r != 0) {
529 if (r == EAGAIN) { /* no free slot; dequeue first */
530 delay(100);
531 ld_virtio_vq_done(vq);
532 r = virtio_enqueue_prep(vsc, vq, &slot);
533 if (r != 0)
534 return r;
535 }
536 return r;
537 }
538 vr = &sc->sc_reqs[slot];
539 r = bus_dmamap_load(virtio_dmat(vsc), vr->vr_payload,
540 data, blkcnt*ld->sc_secsize, NULL,
541 BUS_DMA_WRITE|BUS_DMA_NOWAIT);
542 if (r != 0)
543 return r;
544
545 r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs +
546 VIRTIO_BLK_MIN_SEGMENTS);
547 if (r != 0) {
548 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload);
549 return r;
550 }
551
552 vr->vr_bp = (void*)0xdeadbeef;
553 vr->vr_hdr.type = virtio_rw32(vsc, VIRTIO_BLK_T_OUT);
554 vr->vr_hdr.ioprio = virtio_rw32(vsc, 0);
555 vr->vr_hdr.sector = virtio_rw64(vsc,
556 (daddr_t) blkno * ld->sc_secsize /
557 VIRTIO_BLK_BSIZE);
558
559 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts,
560 0, sizeof(struct virtio_blk_req_hdr),
561 BUS_DMASYNC_PREWRITE);
562 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload,
563 0, blkcnt*ld->sc_secsize,
564 BUS_DMASYNC_PREWRITE);
565 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts,
566 offsetof(struct virtio_blk_req, vr_status),
567 sizeof(uint8_t),
568 BUS_DMASYNC_PREREAD);
569
570 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts,
571 0, sizeof(struct virtio_blk_req_hdr),
572 true);
573 virtio_enqueue(vsc, vq, slot, vr->vr_payload, true);
574 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts,
575 offsetof(struct virtio_blk_req, vr_status),
576 sizeof(uint8_t),
577 false);
578 virtio_enqueue_commit(vsc, vq, slot, true);
579
580 for ( ; ; ) {
581 int dslot;
582
583 r = virtio_dequeue(vsc, vq, &dslot, NULL);
584 if (r != 0)
585 continue;
586 if (dslot != slot) {
587 ld_virtio_vq_done1(sc, vsc, vq, dslot);
588 continue;
589 } else
590 break;
591 }
592
593 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts,
594 0, sizeof(struct virtio_blk_req_hdr),
595 BUS_DMASYNC_POSTWRITE);
596 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload,
597 0, blkcnt*ld->sc_secsize,
598 BUS_DMASYNC_POSTWRITE);
599 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts,
600 offsetof(struct virtio_blk_req, vr_status),
601 sizeof(uint8_t),
602 BUS_DMASYNC_POSTREAD);
603 if (vr->vr_status == VIRTIO_BLK_S_OK)
604 r = 0;
605 else
606 r = EIO;
607 virtio_dequeue_commit(vsc, vq, slot);
608
609 return r;
610 }
611
612 static int
ld_virtio_detach(device_t self,int flags)613 ld_virtio_detach(device_t self, int flags)
614 {
615 struct ld_virtio_softc *sc = device_private(self);
616 struct ld_softc *ld = &sc->sc_ld;
617 bus_dma_tag_t dmat = virtio_dmat(sc->sc_virtio);
618 int r, i, qsize;
619
620 qsize = sc->sc_vq.vq_num;
621 r = ldbegindetach(ld, flags);
622 if (r != 0)
623 return r;
624 virtio_reset(sc->sc_virtio);
625 virtio_free_vq(sc->sc_virtio, &sc->sc_vq);
626
627 for (i = 0; i < qsize; i++) {
628 bus_dmamap_destroy(dmat,
629 sc->sc_reqs[i].vr_cmdsts);
630 bus_dmamap_destroy(dmat,
631 sc->sc_reqs[i].vr_payload);
632 }
633 bus_dmamem_unmap(dmat, sc->sc_reqs,
634 sizeof(struct virtio_blk_req) * qsize);
635 bus_dmamem_free(dmat, &sc->sc_reqs_seg, 1);
636
637 ldenddetach(ld);
638
639 cv_destroy(&sc->sc_sync_wait);
640 mutex_destroy(&sc->sc_sync_wait_lock);
641
642 virtio_child_detach(sc->sc_virtio);
643
644 return 0;
645 }
646
647 static int
ld_virtio_flush(struct ld_softc * ld,bool poll)648 ld_virtio_flush(struct ld_softc *ld, bool poll)
649 {
650 struct ld_virtio_softc * const sc = device_private(ld->sc_dv);
651 struct virtio_softc * const vsc = sc->sc_virtio;
652 const uint64_t features = virtio_features(vsc);
653 struct virtqueue *vq = &sc->sc_vq;
654 struct virtio_blk_req *vr;
655 int slot;
656 int r;
657
658 if ((features & VIRTIO_BLK_F_FLUSH) == 0)
659 return 0;
660
661 mutex_enter(&sc->sc_sync_wait_lock);
662 while (sc->sc_sync_use != SYNC_FREE) {
663 if (poll) {
664 mutex_exit(&sc->sc_sync_wait_lock);
665 ld_virtio_vq_done(vq);
666 mutex_enter(&sc->sc_sync_wait_lock);
667 continue;
668 }
669 cv_wait(&sc->sc_sync_wait, &sc->sc_sync_wait_lock);
670 }
671 sc->sc_sync_use = SYNC_BUSY;
672 mutex_exit(&sc->sc_sync_wait_lock);
673
674 r = virtio_enqueue_prep(vsc, vq, &slot);
675 if (r != 0) {
676 return r;
677 }
678
679 vr = &sc->sc_reqs[slot];
680 KASSERT(vr->vr_bp == NULL);
681
682 r = virtio_enqueue_reserve(vsc, vq, slot, VIRTIO_BLK_MIN_SEGMENTS);
683 if (r != 0) {
684 return r;
685 }
686
687 vr->vr_bp = DUMMY_VR_BP;
688 vr->vr_hdr.type = virtio_rw32(vsc, VIRTIO_BLK_T_FLUSH);
689 vr->vr_hdr.ioprio = virtio_rw32(vsc, 0);
690 vr->vr_hdr.sector = virtio_rw64(vsc, 0);
691
692 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts,
693 0, sizeof(struct virtio_blk_req_hdr),
694 BUS_DMASYNC_PREWRITE);
695 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts,
696 offsetof(struct virtio_blk_req, vr_status),
697 sizeof(uint8_t),
698 BUS_DMASYNC_PREREAD);
699
700 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts,
701 0, sizeof(struct virtio_blk_req_hdr),
702 true);
703 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts,
704 offsetof(struct virtio_blk_req, vr_status),
705 sizeof(uint8_t),
706 false);
707 virtio_enqueue_commit(vsc, vq, slot, true);
708
709 mutex_enter(&sc->sc_sync_wait_lock);
710 while (sc->sc_sync_use != SYNC_DONE) {
711 if (poll) {
712 mutex_exit(&sc->sc_sync_wait_lock);
713 ld_virtio_vq_done(vq);
714 mutex_enter(&sc->sc_sync_wait_lock);
715 continue;
716 }
717 cv_wait(&sc->sc_sync_wait, &sc->sc_sync_wait_lock);
718 }
719
720 if (sc->sc_sync_status == VIRTIO_BLK_S_OK)
721 r = 0;
722 else
723 r = EIO;
724
725 sc->sc_sync_use = SYNC_FREE;
726 cv_broadcast(&sc->sc_sync_wait);
727 mutex_exit(&sc->sc_sync_wait_lock);
728
729 return r;
730 }
731
732 static int
ld_virtio_getcache(struct ld_softc * ld,int * bitsp)733 ld_virtio_getcache(struct ld_softc *ld, int *bitsp)
734 {
735 struct ld_virtio_softc * const sc = device_private(ld->sc_dv);
736 struct virtio_softc * const vsc = sc->sc_virtio;
737 const uint64_t features = virtio_features(vsc);
738
739 *bitsp = DKCACHE_READ;
740 if ((features & VIRTIO_BLK_F_CONFIG_WCE) != 0)
741 *bitsp |= DKCACHE_WCHANGE;
742 if (virtio_read_device_config_1(vsc,
743 VIRTIO_BLK_CONFIG_WRITEBACK) != 0x00)
744 *bitsp |= DKCACHE_WRITE;
745
746 return 0;
747 }
748
749 static int
ld_virtio_setcache(struct ld_softc * ld,int bits)750 ld_virtio_setcache(struct ld_softc *ld, int bits)
751 {
752 struct ld_virtio_softc * const sc = device_private(ld->sc_dv);
753 struct virtio_softc * const vsc = sc->sc_virtio;
754 const uint8_t wce = (bits & DKCACHE_WRITE) ? 0x01 : 0x00;
755
756 virtio_write_device_config_1(vsc,
757 VIRTIO_BLK_CONFIG_WRITEBACK, wce);
758 if (virtio_read_device_config_1(vsc,
759 VIRTIO_BLK_CONFIG_WRITEBACK) != wce)
760 return EIO;
761
762 return 0;
763 }
764
765 static int
ld_virtio_ioctl(struct ld_softc * ld,u_long cmd,void * addr,int32_t flag,bool poll)766 ld_virtio_ioctl(struct ld_softc *ld, u_long cmd, void *addr, int32_t flag, bool poll)
767 {
768 int error;
769
770 switch (cmd) {
771 case DIOCCACHESYNC:
772 error = ld_virtio_flush(ld, poll);
773 break;
774
775 case DIOCGCACHE:
776 error = ld_virtio_getcache(ld, (int *)addr);
777 break;
778
779 case DIOCSCACHE:
780 error = ld_virtio_setcache(ld, *(int *)addr);
781 break;
782
783 default:
784 error = EPASSTHROUGH;
785 break;
786 }
787
788 return error;
789 }
790
791 MODULE(MODULE_CLASS_DRIVER, ld_virtio, "ld,virtio");
792
793 #ifdef _MODULE
794 /*
795 * XXX Don't allow ioconf.c to redefine the "struct cfdriver ld_cd"
796 * XXX it will be defined in the common-code module
797 */
798 #undef CFDRIVER_DECL
799 #define CFDRIVER_DECL(name, class, attr)
800 #include "ioconf.c"
801 #endif
802
803 static int
ld_virtio_modcmd(modcmd_t cmd,void * opaque)804 ld_virtio_modcmd(modcmd_t cmd, void *opaque)
805 {
806 #ifdef _MODULE
807 /*
808 * We ignore the cfdriver_vec[] that ioconf provides, since
809 * the cfdrivers are attached already.
810 */
811 static struct cfdriver * const no_cfdriver_vec[] = { NULL };
812 #endif
813 int error = 0;
814
815 #ifdef _MODULE
816 switch (cmd) {
817 case MODULE_CMD_INIT:
818 error = config_init_component(no_cfdriver_vec,
819 cfattach_ioconf_ld_virtio, cfdata_ioconf_ld_virtio);
820 break;
821 case MODULE_CMD_FINI:
822 error = config_fini_component(no_cfdriver_vec,
823 cfattach_ioconf_ld_virtio, cfdata_ioconf_ld_virtio);
824 break;
825 default:
826 error = ENOTTY;
827 break;
828 }
829 #endif
830
831 return error;
832 }
833