1 /*- 2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * This software was developed by SRI International and the University of 6 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) 7 * ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /* 32 * BERI virtio block backend driver 33 */ 34 35 #include <sys/cdefs.h> 36 __FBSDID("$FreeBSD$"); 37 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/bus.h> 41 #include <sys/kernel.h> 42 #include <sys/module.h> 43 #include <sys/rman.h> 44 #include <sys/conf.h> 45 #include <sys/stat.h> 46 #include <sys/endian.h> 47 #include <sys/disk.h> 48 #include <sys/vnode.h> 49 #include <sys/fcntl.h> 50 #include <sys/kthread.h> 51 #include <sys/buf.h> 52 #include <sys/mdioctl.h> 53 #include <sys/namei.h> 54 55 #include <machine/bus.h> 56 #include <machine/fdt.h> 57 #include <machine/cpu.h> 58 #include <machine/intr.h> 59 60 #include <dev/fdt/fdt_common.h> 61 #include <dev/ofw/openfirm.h> 62 #include <dev/ofw/ofw_bus.h> 63 #include <dev/ofw/ofw_bus_subr.h> 64 65 #include <dev/beri/virtio/virtio.h> 66 #include <dev/beri/virtio/virtio_mmio_platform.h> 67 #include <dev/altera/pio/pio.h> 68 #include <dev/virtio/mmio/virtio_mmio.h> 69 #include <dev/virtio/block/virtio_blk.h> 70 #include <dev/virtio/virtio_ids.h> 71 #include <dev/virtio/virtio_config.h> 72 #include <dev/virtio/virtio_ring.h> 73 74 #include "pio_if.h" 75 76 #define DPRINTF(fmt, ...) 77 78 /* We use indirect descriptors */ 79 #define NUM_DESCS 1 80 #define NUM_QUEUES 1 81 82 #define VTBLK_BLK_ID_BYTES 20 83 #define VTBLK_MAXSEGS 256 84 85 struct beri_vtblk_softc { 86 struct resource *res[1]; 87 bus_space_tag_t bst; 88 bus_space_handle_t bsh; 89 struct cdev *cdev; 90 device_t dev; 91 int opened; 92 device_t pio_recv; 93 device_t pio_send; 94 struct vqueue_info vs_queues[NUM_QUEUES]; 95 char ident[VTBLK_BLK_ID_BYTES]; 96 struct ucred *cred; 97 struct vnode *vnode; 98 struct thread *vtblk_ktd; 99 struct sx sc_mtx; 100 int beri_mem_offset; 101 struct md_ioctl *mdio; 102 struct virtio_blk_config *cfg; 103 }; 104 105 static struct resource_spec beri_spec[] = { 106 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 107 { -1, 0 } 108 }; 109 110 static int 111 vtblk_rdwr(struct beri_vtblk_softc *sc, struct iovec *iov, 112 int cnt, int offset, int operation, int iolen) 113 { 114 struct vnode *vp; 115 struct mount *mp; 116 struct uio auio; 117 int error; 118 119 bzero(&auio, sizeof(auio)); 120 121 vp = sc->vnode; 122 123 KASSERT(vp != NULL, ("file not opened")); 124 125 auio.uio_iov = iov; 126 auio.uio_iovcnt = cnt; 127 auio.uio_offset = offset; 128 auio.uio_segflg = UIO_SYSSPACE; 129 auio.uio_rw = operation; 130 auio.uio_resid = iolen; 131 auio.uio_td = curthread; 132 133 if (operation == 0) { 134 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 135 error = VOP_READ(vp, &auio, IO_DIRECT, sc->cred); 136 VOP_UNLOCK(vp, 0); 137 } else { 138 (void) vn_start_write(vp, &mp, V_WAIT); 139 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 140 error = VOP_WRITE(vp, &auio, IO_SYNC, sc->cred); 141 VOP_UNLOCK(vp, 0); 142 vn_finished_write(mp); 143 } 144 145 return (error); 146 } 147 148 static void 149 vtblk_proc(struct beri_vtblk_softc *sc, struct vqueue_info *vq) 150 { 151 struct iovec iov[VTBLK_MAXSEGS + 2]; 152 uint16_t flags[VTBLK_MAXSEGS + 2]; 153 struct virtio_blk_outhdr *vbh; 154 struct iovec *tiov; 155 uint8_t *status; 156 off_t offset; 157 int iolen; 158 int type; 159 int i, n; 160 int err; 161 162 n = vq_getchain(sc->beri_mem_offset, vq, iov, 163 VTBLK_MAXSEGS + 2, flags); 164 KASSERT(n >= 2 && n <= VTBLK_MAXSEGS + 2, 165 ("wrong n value %d", n)); 166 167 tiov = getcopy(iov, n); 168 vbh = iov[0].iov_base; 169 170 status = iov[n-1].iov_base; 171 KASSERT(iov[n-1].iov_len == 1, 172 ("iov_len == %d", iov[n-1].iov_len)); 173 174 type = be32toh(vbh->type) & ~VIRTIO_BLK_T_BARRIER; 175 offset = be64toh(vbh->sector) * DEV_BSIZE; 176 177 iolen = 0; 178 for (i = 1; i < (n-1); i++) { 179 iolen += iov[i].iov_len; 180 } 181 182 switch (type) { 183 case VIRTIO_BLK_T_OUT: 184 case VIRTIO_BLK_T_IN: 185 err = vtblk_rdwr(sc, tiov + 1, i - 1, 186 offset, type, iolen); 187 break; 188 case VIRTIO_BLK_T_GET_ID: 189 /* Assume a single buffer */ 190 strlcpy(iov[1].iov_base, sc->ident, 191 MIN(iov[1].iov_len, sizeof(sc->ident))); 192 err = 0; 193 break; 194 case VIRTIO_BLK_T_FLUSH: 195 /* Possible? */ 196 default: 197 err = -ENOSYS; 198 break; 199 } 200 201 if (err < 0) { 202 if (err == -ENOSYS) { 203 *status = VIRTIO_BLK_S_UNSUPP; 204 } else 205 *status = VIRTIO_BLK_S_IOERR; 206 } else 207 *status = VIRTIO_BLK_S_OK; 208 209 free(tiov, M_DEVBUF); 210 vq_relchain(vq, iov, n, 1); 211 } 212 213 static int 214 close_file(struct beri_vtblk_softc *sc, struct thread *td) 215 { 216 int error; 217 218 if (sc->vnode != NULL) { 219 vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY); 220 sc->vnode->v_vflag &= ~VV_MD; 221 VOP_UNLOCK(sc->vnode, 0); 222 error = vn_close(sc->vnode, (FREAD|FWRITE), 223 sc->cred, td); 224 if (error != 0) 225 return (error); 226 sc->vnode = NULL; 227 } 228 229 if (sc->cred != NULL) 230 crfree(sc->cred); 231 232 return (0); 233 } 234 235 static int 236 open_file(struct beri_vtblk_softc *sc, struct thread *td) 237 { 238 struct nameidata nd; 239 struct vattr vattr; 240 int error; 241 int flags; 242 243 flags = (FREAD | FWRITE); 244 NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, 245 sc->mdio->md_file, td); 246 error = vn_open(&nd, &flags, 0, NULL); 247 if (error != 0) 248 return (error); 249 NDFREE(&nd, NDF_ONLY_PNBUF); 250 251 if (nd.ni_vp->v_type != VREG) { 252 return (EINVAL); 253 } 254 255 error = VOP_GETATTR(nd.ni_vp, &vattr, td->td_ucred); 256 if (error != 0) 257 return (error); 258 259 if (VOP_ISLOCKED(nd.ni_vp) != LK_EXCLUSIVE) { 260 vn_lock(nd.ni_vp, LK_UPGRADE | LK_RETRY); 261 if (nd.ni_vp->v_iflag & VI_DOOMED) { 262 return (1); 263 } 264 } 265 nd.ni_vp->v_vflag |= VV_MD; 266 VOP_UNLOCK(nd.ni_vp, 0); 267 268 sc->vnode = nd.ni_vp; 269 sc->cred = crhold(td->td_ucred); 270 271 return (0); 272 } 273 274 static int 275 vtblk_notify(struct beri_vtblk_softc *sc) 276 { 277 struct vqueue_info *vq; 278 int queue; 279 int reg; 280 281 vq = &sc->vs_queues[0]; 282 if (!vq_ring_ready(vq)) 283 return (0); 284 285 if (!sc->opened) 286 return (0); 287 288 reg = READ2(sc, VIRTIO_MMIO_QUEUE_NOTIFY); 289 queue = be16toh(reg); 290 291 KASSERT(queue == 0, ("we support single queue only")); 292 293 /* Process new descriptors */ 294 vq = &sc->vs_queues[queue]; 295 vq->vq_save_used = be16toh(vq->vq_used->idx); 296 while (vq_has_descs(vq)) 297 vtblk_proc(sc, vq); 298 299 /* Interrupt the other side */ 300 if ((be16toh(vq->vq_avail->flags) & VRING_AVAIL_F_NO_INTERRUPT) == 0) { 301 reg = htobe32(VIRTIO_MMIO_INT_VRING); 302 WRITE4(sc, VIRTIO_MMIO_INTERRUPT_STATUS, reg); 303 PIO_SET(sc->pio_send, Q_INTR, 1); 304 } 305 306 return (0); 307 } 308 309 static int 310 vq_init(struct beri_vtblk_softc *sc) 311 { 312 struct vqueue_info *vq; 313 uint8_t *base; 314 int size; 315 int reg; 316 int pfn; 317 318 vq = &sc->vs_queues[0]; 319 vq->vq_qsize = NUM_DESCS; 320 321 reg = READ4(sc, VIRTIO_MMIO_QUEUE_PFN); 322 pfn = be32toh(reg); 323 vq->vq_pfn = pfn; 324 325 size = vring_size(vq->vq_qsize, VRING_ALIGN); 326 base = paddr_map(sc->beri_mem_offset, 327 (pfn << PAGE_SHIFT), size); 328 329 /* First pages are descriptors */ 330 vq->vq_desc = (struct vring_desc *)base; 331 base += vq->vq_qsize * sizeof(struct vring_desc); 332 333 /* Then avail ring */ 334 vq->vq_avail = (struct vring_avail *)base; 335 base += (2 + vq->vq_qsize + 1) * sizeof(uint16_t); 336 337 /* Then it's rounded up to the next page */ 338 base = (uint8_t *)roundup2((uintptr_t)base, VRING_ALIGN); 339 340 /* And the last pages are the used ring */ 341 vq->vq_used = (struct vring_used *)base; 342 343 /* Mark queue as allocated, and start at 0 when we use it. */ 344 vq->vq_flags = VQ_ALLOC; 345 vq->vq_last_avail = 0; 346 347 return (0); 348 } 349 350 351 static void 352 vtblk_thread(void *arg) 353 { 354 struct beri_vtblk_softc *sc; 355 int err; 356 357 sc = arg; 358 359 sx_xlock(&sc->sc_mtx); 360 for (;;) { 361 err = msleep(sc, &sc->sc_mtx, PCATCH | PZERO, "prd", hz); 362 vtblk_notify(sc); 363 } 364 sx_xunlock(&sc->sc_mtx); 365 366 kthread_exit(); 367 } 368 369 static int 370 backend_info(struct beri_vtblk_softc *sc) 371 { 372 struct virtio_blk_config *cfg; 373 uint32_t *s; 374 int reg; 375 int i; 376 377 /* Specify that we provide block device */ 378 reg = htobe32(VIRTIO_ID_BLOCK); 379 WRITE4(sc, VIRTIO_MMIO_DEVICE_ID, reg); 380 381 /* Queue size */ 382 reg = htobe32(NUM_DESCS); 383 WRITE4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX, reg); 384 385 /* Our features */ 386 reg = htobe32(VIRTIO_RING_F_INDIRECT_DESC 387 | VIRTIO_BLK_F_BLK_SIZE 388 | VIRTIO_BLK_F_SEG_MAX); 389 WRITE4(sc, VIRTIO_MMIO_HOST_FEATURES, reg); 390 391 cfg = sc->cfg; 392 cfg->capacity = htobe64(sc->mdio->md_mediasize / DEV_BSIZE); 393 cfg->size_max = 0; /* not negotiated */ 394 cfg->seg_max = htobe32(VTBLK_MAXSEGS); 395 cfg->blk_size = htobe32(DEV_BSIZE); 396 397 s = (uint32_t *)cfg; 398 399 for (i = 0; i < sizeof(struct virtio_blk_config); i+=4) { 400 WRITE4(sc, VIRTIO_MMIO_CONFIG + i, *s); 401 s+=1; 402 } 403 404 sprintf(sc->ident, "Virtio block backend"); 405 406 return (0); 407 } 408 409 static void 410 vtblk_intr(void *arg) 411 { 412 struct beri_vtblk_softc *sc; 413 int pending; 414 int reg; 415 416 sc = arg; 417 418 reg = PIO_READ(sc->pio_recv); 419 420 /* Ack */ 421 PIO_SET(sc->pio_recv, reg, 0); 422 423 pending = htobe32(reg); 424 425 if (pending & Q_PFN) { 426 vq_init(sc); 427 } 428 429 if (pending & Q_NOTIFY) { 430 wakeup(sc); 431 } 432 } 433 434 static int 435 beri_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, 436 int flags, struct thread *td) 437 { 438 struct beri_vtblk_softc *sc; 439 int err; 440 441 sc = dev->si_drv1; 442 443 switch (cmd) { 444 case MDIOCATTACH: 445 /* take file as argument */ 446 if (sc->vnode != NULL) { 447 /* Already opened */ 448 return (1); 449 } 450 sc->mdio = (struct md_ioctl *)addr; 451 backend_info(sc); 452 DPRINTF("opening file, td 0x%08x\n", (int)td); 453 err = open_file(sc, td); 454 if (err) 455 return (err); 456 PIO_SETUP_IRQ(sc->pio_recv, vtblk_intr, sc); 457 sc->opened = 1; 458 break; 459 case MDIOCDETACH: 460 if (sc->vnode == NULL) { 461 /* File not opened */ 462 return (1); 463 } 464 sc->opened = 0; 465 DPRINTF("closing file, td 0x%08x\n", (int)td); 466 err = close_file(sc, td); 467 if (err) 468 return (err); 469 PIO_TEARDOWN_IRQ(sc->pio_recv); 470 break; 471 default: 472 break; 473 } 474 475 return (0); 476 } 477 478 static struct cdevsw beri_cdevsw = { 479 .d_version = D_VERSION, 480 .d_ioctl = beri_ioctl, 481 .d_name = "virtio block backend", 482 }; 483 484 static int 485 beri_vtblk_probe(device_t dev) 486 { 487 488 if (!ofw_bus_status_okay(dev)) 489 return (ENXIO); 490 491 if (!ofw_bus_is_compatible(dev, "sri-cambridge,beri-vtblk")) 492 return (ENXIO); 493 494 device_set_desc(dev, "SRI-Cambridge BERI block"); 495 return (BUS_PROBE_DEFAULT); 496 } 497 498 static int 499 beri_vtblk_attach(device_t dev) 500 { 501 struct beri_vtblk_softc *sc; 502 int error; 503 504 sc = device_get_softc(dev); 505 sc->dev = dev; 506 507 if (bus_alloc_resources(dev, beri_spec, sc->res)) { 508 device_printf(dev, "could not allocate resources\n"); 509 return (ENXIO); 510 } 511 512 /* Memory interface */ 513 sc->bst = rman_get_bustag(sc->res[0]); 514 sc->bsh = rman_get_bushandle(sc->res[0]); 515 516 sc->cfg = malloc(sizeof(struct virtio_blk_config), 517 M_DEVBUF, M_NOWAIT|M_ZERO); 518 519 sx_init(&sc->sc_mtx, device_get_nameunit(sc->dev)); 520 521 error = kthread_add(vtblk_thread, sc, NULL, &sc->vtblk_ktd, 522 0, 0, "beri_virtio_block"); 523 if (error) { 524 device_printf(dev, "cannot create kthread\n"); 525 return (ENXIO); 526 } 527 528 if (setup_offset(dev, &sc->beri_mem_offset) != 0) 529 return (ENXIO); 530 if (setup_pio(dev, "pio-send", &sc->pio_send) != 0) 531 return (ENXIO); 532 if (setup_pio(dev, "pio-recv", &sc->pio_recv) != 0) 533 return (ENXIO); 534 535 sc->cdev = make_dev(&beri_cdevsw, 0, UID_ROOT, GID_WHEEL, 536 S_IRWXU, "beri_vtblk"); 537 if (sc->cdev == NULL) { 538 device_printf(dev, "Failed to create character device.\n"); 539 return (ENXIO); 540 } 541 542 sc->cdev->si_drv1 = sc; 543 return (0); 544 } 545 546 static device_method_t beri_vtblk_methods[] = { 547 DEVMETHOD(device_probe, beri_vtblk_probe), 548 DEVMETHOD(device_attach, beri_vtblk_attach), 549 { 0, 0 } 550 }; 551 552 static driver_t beri_vtblk_driver = { 553 "beri_vtblk", 554 beri_vtblk_methods, 555 sizeof(struct beri_vtblk_softc), 556 }; 557 558 static devclass_t beri_vtblk_devclass; 559 560 DRIVER_MODULE(beri_vtblk, simplebus, beri_vtblk_driver, 561 beri_vtblk_devclass, 0, 0); 562