1 /*-
2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com>
3 * All rights reserved.
4 *
5 * This software was developed by SRI International and the University of
6 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
7 * ("CTSRD"), as part of the DARPA CRASH research programme.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 /*
32 * BERI virtio block backend driver
33 */
34
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/bus.h>
38 #include <sys/kernel.h>
39 #include <sys/module.h>
40 #include <sys/rman.h>
41 #include <sys/conf.h>
42 #include <sys/stat.h>
43 #include <sys/endian.h>
44 #include <sys/disk.h>
45 #include <sys/vnode.h>
46 #include <sys/fcntl.h>
47 #include <sys/kthread.h>
48 #include <sys/buf.h>
49 #include <sys/mdioctl.h>
50 #include <sys/namei.h>
51
52 #include <machine/bus.h>
53 #include <machine/fdt.h>
54 #include <machine/cpu.h>
55 #include <machine/intr.h>
56
57 #include <dev/fdt/fdt_common.h>
58 #include <dev/ofw/openfirm.h>
59 #include <dev/ofw/ofw_bus.h>
60 #include <dev/ofw/ofw_bus_subr.h>
61
62 #include <dev/beri/virtio/virtio.h>
63 #include <dev/beri/virtio/virtio_mmio_platform.h>
64 #include <dev/altera/pio/pio.h>
65 #include <dev/virtio/mmio/virtio_mmio.h>
66 #include <dev/virtio/block/virtio_blk.h>
67 #include <dev/virtio/virtio_ids.h>
68 #include <dev/virtio/virtio_config.h>
69 #include <dev/virtio/virtio_ring.h>
70
71 #include "pio_if.h"
72
73 #define DPRINTF(fmt, ...)
74
75 /* We use indirect descriptors */
76 #define NUM_DESCS 1
77 #define NUM_QUEUES 1
78
79 #define VTBLK_BLK_ID_BYTES 20
80 #define VTBLK_MAXSEGS 256
81
82 struct beri_vtblk_softc {
83 struct resource *res[1];
84 bus_space_tag_t bst;
85 bus_space_handle_t bsh;
86 struct cdev *cdev;
87 device_t dev;
88 int opened;
89 device_t pio_recv;
90 device_t pio_send;
91 struct vqueue_info vs_queues[NUM_QUEUES];
92 char ident[VTBLK_BLK_ID_BYTES];
93 struct ucred *cred;
94 struct vnode *vnode;
95 struct thread *vtblk_ktd;
96 struct sx sc_mtx;
97 int beri_mem_offset;
98 struct md_ioctl *mdio;
99 struct virtio_blk_config *cfg;
100 };
101
102 static struct resource_spec beri_spec[] = {
103 { SYS_RES_MEMORY, 0, RF_ACTIVE },
104 { -1, 0 }
105 };
106
107 static int
vtblk_rdwr(struct beri_vtblk_softc * sc,struct iovec * iov,int cnt,int offset,int operation,int iolen)108 vtblk_rdwr(struct beri_vtblk_softc *sc, struct iovec *iov,
109 int cnt, int offset, int operation, int iolen)
110 {
111 struct vnode *vp;
112 struct mount *mp;
113 struct uio auio;
114 int error;
115
116 bzero(&auio, sizeof(auio));
117
118 vp = sc->vnode;
119
120 KASSERT(vp != NULL, ("file not opened"));
121
122 auio.uio_iov = iov;
123 auio.uio_iovcnt = cnt;
124 auio.uio_offset = offset;
125 auio.uio_segflg = UIO_SYSSPACE;
126 auio.uio_rw = operation;
127 auio.uio_resid = iolen;
128 auio.uio_td = curthread;
129
130 if (operation == 0) {
131 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
132 error = VOP_READ(vp, &auio, IO_DIRECT, sc->cred);
133 VOP_UNLOCK(vp);
134 } else {
135 (void) vn_start_write(vp, &mp, V_WAIT);
136 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
137 error = VOP_WRITE(vp, &auio, IO_SYNC, sc->cred);
138 VOP_UNLOCK(vp);
139 vn_finished_write(mp);
140 }
141
142 return (error);
143 }
144
145 static void
vtblk_proc(struct beri_vtblk_softc * sc,struct vqueue_info * vq)146 vtblk_proc(struct beri_vtblk_softc *sc, struct vqueue_info *vq)
147 {
148 struct iovec iov[VTBLK_MAXSEGS + 2];
149 uint16_t flags[VTBLK_MAXSEGS + 2];
150 struct virtio_blk_outhdr *vbh;
151 struct iovec *tiov;
152 uint8_t *status;
153 off_t offset;
154 int iolen;
155 int type;
156 int i, n;
157 int err;
158
159 n = vq_getchain(sc->beri_mem_offset, vq, iov,
160 VTBLK_MAXSEGS + 2, flags);
161 KASSERT(n >= 2 && n <= VTBLK_MAXSEGS + 2,
162 ("wrong n value %d", n));
163
164 tiov = getcopy(iov, n);
165 vbh = iov[0].iov_base;
166
167 status = iov[n-1].iov_base;
168 KASSERT(iov[n-1].iov_len == 1,
169 ("iov_len == %d", iov[n-1].iov_len));
170
171 type = be32toh(vbh->type) & ~VIRTIO_BLK_T_BARRIER;
172 offset = be64toh(vbh->sector) * DEV_BSIZE;
173
174 iolen = 0;
175 for (i = 1; i < (n-1); i++) {
176 iolen += iov[i].iov_len;
177 }
178
179 switch (type) {
180 case VIRTIO_BLK_T_OUT:
181 case VIRTIO_BLK_T_IN:
182 err = vtblk_rdwr(sc, tiov + 1, i - 1,
183 offset, type, iolen);
184 break;
185 case VIRTIO_BLK_T_GET_ID:
186 /* Assume a single buffer */
187 strncpy(iov[1].iov_base, sc->ident,
188 MIN(iov[1].iov_len, sizeof(sc->ident)));
189 err = 0;
190 break;
191 case VIRTIO_BLK_T_FLUSH:
192 /* Possible? */
193 default:
194 err = -ENOSYS;
195 break;
196 }
197
198 if (err < 0) {
199 if (err == -ENOSYS) {
200 *status = VIRTIO_BLK_S_UNSUPP;
201 } else
202 *status = VIRTIO_BLK_S_IOERR;
203 } else
204 *status = VIRTIO_BLK_S_OK;
205
206 free(tiov, M_DEVBUF);
207 vq_relchain(vq, iov, n, 1);
208 }
209
210 static int
close_file(struct beri_vtblk_softc * sc,struct thread * td)211 close_file(struct beri_vtblk_softc *sc, struct thread *td)
212 {
213 int error;
214
215 if (sc->vnode != NULL) {
216 vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY);
217 sc->vnode->v_vflag &= ~VV_MD;
218 VOP_UNLOCK(sc->vnode);
219 error = vn_close(sc->vnode, (FREAD|FWRITE),
220 sc->cred, td);
221 if (error != 0)
222 return (error);
223 sc->vnode = NULL;
224 }
225
226 if (sc->cred != NULL)
227 crfree(sc->cred);
228
229 return (0);
230 }
231
232 static int
open_file(struct beri_vtblk_softc * sc,struct thread * td)233 open_file(struct beri_vtblk_softc *sc, struct thread *td)
234 {
235 struct nameidata nd;
236 struct vattr vattr;
237 int error;
238 int flags;
239
240 flags = (FREAD | FWRITE);
241 NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, sc->mdio->md_file);
242 error = vn_open(&nd, &flags, 0, NULL);
243 if (error != 0)
244 return (error);
245 NDFREE_PNBUF(&nd);
246
247 if (nd.ni_vp->v_type != VREG) {
248 return (EINVAL);
249 }
250
251 error = VOP_GETATTR(nd.ni_vp, &vattr, td->td_ucred);
252 if (error != 0)
253 return (error);
254
255 if (VOP_ISLOCKED(nd.ni_vp) != LK_EXCLUSIVE) {
256 vn_lock(nd.ni_vp, LK_UPGRADE | LK_RETRY);
257 if (VN_IS_DOOMED(nd.ni_vp)) {
258 return (1);
259 }
260 }
261 nd.ni_vp->v_vflag |= VV_MD;
262 VOP_UNLOCK(nd.ni_vp);
263
264 sc->vnode = nd.ni_vp;
265 sc->cred = crhold(td->td_ucred);
266
267 return (0);
268 }
269
270 static int
vtblk_notify(struct beri_vtblk_softc * sc)271 vtblk_notify(struct beri_vtblk_softc *sc)
272 {
273 struct vqueue_info *vq;
274 int queue;
275 int reg;
276
277 vq = &sc->vs_queues[0];
278 if (!vq_ring_ready(vq))
279 return (0);
280
281 if (!sc->opened)
282 return (0);
283
284 reg = READ2(sc, VIRTIO_MMIO_QUEUE_NOTIFY);
285 queue = be16toh(reg);
286
287 KASSERT(queue == 0, ("we support single queue only"));
288
289 /* Process new descriptors */
290 vq = &sc->vs_queues[queue];
291 vq->vq_save_used = be16toh(vq->vq_used->idx);
292 while (vq_has_descs(vq))
293 vtblk_proc(sc, vq);
294
295 /* Interrupt the other side */
296 if ((be16toh(vq->vq_avail->flags) & VRING_AVAIL_F_NO_INTERRUPT) == 0) {
297 reg = htobe32(VIRTIO_MMIO_INT_VRING);
298 WRITE4(sc, VIRTIO_MMIO_INTERRUPT_STATUS, reg);
299 PIO_SET(sc->pio_send, Q_INTR, 1);
300 }
301
302 return (0);
303 }
304
305 static int
vq_init(struct beri_vtblk_softc * sc)306 vq_init(struct beri_vtblk_softc *sc)
307 {
308 struct vqueue_info *vq;
309 uint8_t *base;
310 int size;
311 int reg;
312 int pfn;
313
314 vq = &sc->vs_queues[0];
315 vq->vq_qsize = NUM_DESCS;
316
317 reg = READ4(sc, VIRTIO_MMIO_QUEUE_PFN);
318 pfn = be32toh(reg);
319 vq->vq_pfn = pfn;
320
321 size = vring_size(vq->vq_qsize, VRING_ALIGN);
322 base = paddr_map(sc->beri_mem_offset,
323 (pfn << PAGE_SHIFT), size);
324
325 /* First pages are descriptors */
326 vq->vq_desc = (struct vring_desc *)base;
327 base += vq->vq_qsize * sizeof(struct vring_desc);
328
329 /* Then avail ring */
330 vq->vq_avail = (struct vring_avail *)base;
331 base += (2 + vq->vq_qsize + 1) * sizeof(uint16_t);
332
333 /* Then it's rounded up to the next page */
334 base = (uint8_t *)roundup2((uintptr_t)base, VRING_ALIGN);
335
336 /* And the last pages are the used ring */
337 vq->vq_used = (struct vring_used *)base;
338
339 /* Mark queue as allocated, and start at 0 when we use it. */
340 vq->vq_flags = VQ_ALLOC;
341 vq->vq_last_avail = 0;
342
343 return (0);
344 }
345
346 static void
vtblk_thread(void * arg)347 vtblk_thread(void *arg)
348 {
349 struct beri_vtblk_softc *sc;
350 int err;
351
352 sc = arg;
353
354 sx_xlock(&sc->sc_mtx);
355 for (;;) {
356 err = msleep(sc, &sc->sc_mtx, PCATCH | PZERO, "prd", hz);
357 vtblk_notify(sc);
358 }
359 sx_xunlock(&sc->sc_mtx);
360
361 kthread_exit();
362 }
363
364 static int
backend_info(struct beri_vtblk_softc * sc)365 backend_info(struct beri_vtblk_softc *sc)
366 {
367 struct virtio_blk_config *cfg;
368 uint32_t *s;
369 int reg;
370 int i;
371
372 /* Specify that we provide block device */
373 reg = htobe32(VIRTIO_ID_BLOCK);
374 WRITE4(sc, VIRTIO_MMIO_DEVICE_ID, reg);
375
376 /* Queue size */
377 reg = htobe32(NUM_DESCS);
378 WRITE4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX, reg);
379
380 /* Our features */
381 reg = htobe32(VIRTIO_RING_F_INDIRECT_DESC
382 | VIRTIO_BLK_F_BLK_SIZE
383 | VIRTIO_BLK_F_SEG_MAX);
384 WRITE4(sc, VIRTIO_MMIO_HOST_FEATURES, reg);
385
386 cfg = sc->cfg;
387 cfg->capacity = htobe64(sc->mdio->md_mediasize / DEV_BSIZE);
388 cfg->size_max = 0; /* not negotiated */
389 cfg->seg_max = htobe32(VTBLK_MAXSEGS);
390 cfg->blk_size = htobe32(DEV_BSIZE);
391
392 s = (uint32_t *)cfg;
393
394 for (i = 0; i < sizeof(struct virtio_blk_config); i+=4) {
395 WRITE4(sc, VIRTIO_MMIO_CONFIG + i, *s);
396 s+=1;
397 }
398
399 strncpy(sc->ident, "Virtio block backend", sizeof(sc->ident));
400
401 return (0);
402 }
403
404 static void
vtblk_intr(void * arg)405 vtblk_intr(void *arg)
406 {
407 struct beri_vtblk_softc *sc;
408 int pending;
409 int reg;
410
411 sc = arg;
412
413 reg = PIO_READ(sc->pio_recv);
414
415 /* Ack */
416 PIO_SET(sc->pio_recv, reg, 0);
417
418 pending = htobe32(reg);
419
420 if (pending & Q_PFN) {
421 vq_init(sc);
422 }
423
424 if (pending & Q_NOTIFY) {
425 wakeup(sc);
426 }
427 }
428
429 static int
beri_ioctl(struct cdev * dev,u_long cmd,caddr_t addr,int flags,struct thread * td)430 beri_ioctl(struct cdev *dev, u_long cmd, caddr_t addr,
431 int flags, struct thread *td)
432 {
433 struct beri_vtblk_softc *sc;
434 int err;
435
436 sc = dev->si_drv1;
437
438 switch (cmd) {
439 case MDIOCATTACH:
440 /* take file as argument */
441 if (sc->vnode != NULL) {
442 /* Already opened */
443 return (1);
444 }
445 sc->mdio = (struct md_ioctl *)addr;
446 backend_info(sc);
447 DPRINTF("opening file, td 0x%08x\n", (int)td);
448 err = open_file(sc, td);
449 if (err)
450 return (err);
451 PIO_SETUP_IRQ(sc->pio_recv, vtblk_intr, sc);
452 sc->opened = 1;
453 break;
454 case MDIOCDETACH:
455 if (sc->vnode == NULL) {
456 /* File not opened */
457 return (1);
458 }
459 sc->opened = 0;
460 DPRINTF("closing file, td 0x%08x\n", (int)td);
461 err = close_file(sc, td);
462 if (err)
463 return (err);
464 PIO_TEARDOWN_IRQ(sc->pio_recv);
465 break;
466 default:
467 break;
468 }
469
470 return (0);
471 }
472
473 static struct cdevsw beri_cdevsw = {
474 .d_version = D_VERSION,
475 .d_ioctl = beri_ioctl,
476 .d_name = "virtio block backend",
477 };
478
479 static int
beri_vtblk_probe(device_t dev)480 beri_vtblk_probe(device_t dev)
481 {
482
483 if (!ofw_bus_status_okay(dev))
484 return (ENXIO);
485
486 if (!ofw_bus_is_compatible(dev, "sri-cambridge,beri-vtblk"))
487 return (ENXIO);
488
489 device_set_desc(dev, "SRI-Cambridge BERI block");
490 return (BUS_PROBE_DEFAULT);
491 }
492
493 static int
beri_vtblk_attach(device_t dev)494 beri_vtblk_attach(device_t dev)
495 {
496 struct beri_vtblk_softc *sc;
497 int error;
498
499 sc = device_get_softc(dev);
500 sc->dev = dev;
501
502 if (bus_alloc_resources(dev, beri_spec, sc->res)) {
503 device_printf(dev, "could not allocate resources\n");
504 return (ENXIO);
505 }
506
507 /* Memory interface */
508 sc->bst = rman_get_bustag(sc->res[0]);
509 sc->bsh = rman_get_bushandle(sc->res[0]);
510
511 sc->cfg = malloc(sizeof(struct virtio_blk_config),
512 M_DEVBUF, M_NOWAIT|M_ZERO);
513
514 sx_init(&sc->sc_mtx, device_get_nameunit(sc->dev));
515
516 error = kthread_add(vtblk_thread, sc, NULL, &sc->vtblk_ktd,
517 0, 0, "beri_virtio_block");
518 if (error) {
519 device_printf(dev, "cannot create kthread\n");
520 return (ENXIO);
521 }
522
523 if (setup_offset(dev, &sc->beri_mem_offset) != 0)
524 return (ENXIO);
525 if (setup_pio(dev, "pio-send", &sc->pio_send) != 0)
526 return (ENXIO);
527 if (setup_pio(dev, "pio-recv", &sc->pio_recv) != 0)
528 return (ENXIO);
529
530 sc->cdev = make_dev(&beri_cdevsw, 0, UID_ROOT, GID_WHEEL,
531 S_IRWXU, "beri_vtblk");
532 if (sc->cdev == NULL) {
533 device_printf(dev, "Failed to create character device.\n");
534 return (ENXIO);
535 }
536
537 sc->cdev->si_drv1 = sc;
538 return (0);
539 }
540
541 static device_method_t beri_vtblk_methods[] = {
542 DEVMETHOD(device_probe, beri_vtblk_probe),
543 DEVMETHOD(device_attach, beri_vtblk_attach),
544 { 0, 0 }
545 };
546
547 static driver_t beri_vtblk_driver = {
548 "beri_vtblk",
549 beri_vtblk_methods,
550 sizeof(struct beri_vtblk_softc),
551 };
552
553 DRIVER_MODULE(beri_vtblk, simplebus, beri_vtblk_driver, 0, 0);
554