1 /*- 2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * This software was developed by SRI International and the University of 6 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) 7 * ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /* 32 * BERI virtio block backend driver 33 */ 34 35 #include <sys/cdefs.h> 36 __FBSDID("$FreeBSD$"); 37 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/bus.h> 41 #include <sys/kernel.h> 42 #include <sys/module.h> 43 #include <sys/rman.h> 44 #include <sys/conf.h> 45 #include <sys/stat.h> 46 #include <sys/endian.h> 47 #include <sys/disk.h> 48 #include <sys/vnode.h> 49 #include <sys/fcntl.h> 50 #include <sys/kthread.h> 51 #include <sys/buf.h> 52 #include <sys/mdioctl.h> 53 #include <sys/namei.h> 54 55 #include <machine/bus.h> 56 #include <machine/fdt.h> 57 #include <machine/cpu.h> 58 #include <machine/intr.h> 59 60 #include <dev/fdt/fdt_common.h> 61 #include <dev/ofw/openfirm.h> 62 #include <dev/ofw/ofw_bus.h> 63 #include <dev/ofw/ofw_bus_subr.h> 64 65 #include <dev/beri/virtio/virtio.h> 66 #include <dev/beri/virtio/virtio_mmio_platform.h> 67 #include <dev/altera/pio/pio.h> 68 #include <dev/virtio/mmio/virtio_mmio.h> 69 #include <dev/virtio/block/virtio_blk.h> 70 #include <dev/virtio/virtio_ids.h> 71 #include <dev/virtio/virtio_config.h> 72 #include <dev/virtio/virtio_ring.h> 73 74 #include "pio_if.h" 75 76 #define DPRINTF(fmt, ...) 77 78 /* We use indirect descriptors */ 79 #define NUM_DESCS 1 80 #define NUM_QUEUES 1 81 82 #define VTBLK_BLK_ID_BYTES 20 83 #define VTBLK_MAXSEGS 256 84 85 struct beri_vtblk_softc { 86 struct resource *res[1]; 87 bus_space_tag_t bst; 88 bus_space_handle_t bsh; 89 struct cdev *cdev; 90 device_t dev; 91 int opened; 92 device_t pio_recv; 93 device_t pio_send; 94 struct vqueue_info vs_queues[NUM_QUEUES]; 95 char ident[VTBLK_BLK_ID_BYTES]; 96 struct ucred *cred; 97 struct vnode *vnode; 98 struct thread *vtblk_ktd; 99 struct sx sc_mtx; 100 int beri_mem_offset; 101 struct md_ioctl *mdio; 102 struct virtio_blk_config *cfg; 103 }; 104 105 static struct resource_spec beri_spec[] = { 106 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 107 { -1, 0 } 108 }; 109 110 static int 111 vtblk_rdwr(struct beri_vtblk_softc *sc, struct iovec *iov, 112 int cnt, int offset, int operation, int iolen) 113 { 114 struct vnode *vp; 115 struct mount *mp; 116 struct uio auio; 117 int error; 118 119 bzero(&auio, sizeof(auio)); 120 121 vp = sc->vnode; 122 123 KASSERT(vp != NULL, ("file not opened")); 124 125 auio.uio_iov = iov; 126 auio.uio_iovcnt = cnt; 127 auio.uio_offset = offset; 128 auio.uio_segflg = UIO_SYSSPACE; 129 auio.uio_rw = operation; 130 auio.uio_resid = iolen; 131 auio.uio_td = curthread; 132 133 if (operation == 0) { 134 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 135 error = VOP_READ(vp, &auio, IO_DIRECT, sc->cred); 136 VOP_UNLOCK(vp); 137 } else { 138 (void) vn_start_write(vp, &mp, V_WAIT); 139 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 140 error = VOP_WRITE(vp, &auio, IO_SYNC, sc->cred); 141 VOP_UNLOCK(vp); 142 vn_finished_write(mp); 143 } 144 145 return (error); 146 } 147 148 static void 149 vtblk_proc(struct beri_vtblk_softc *sc, struct vqueue_info *vq) 150 { 151 struct iovec iov[VTBLK_MAXSEGS + 2]; 152 uint16_t flags[VTBLK_MAXSEGS + 2]; 153 struct virtio_blk_outhdr *vbh; 154 struct iovec *tiov; 155 uint8_t *status; 156 off_t offset; 157 int iolen; 158 int type; 159 int i, n; 160 int err; 161 162 n = vq_getchain(sc->beri_mem_offset, vq, iov, 163 VTBLK_MAXSEGS + 2, flags); 164 KASSERT(n >= 2 && n <= VTBLK_MAXSEGS + 2, 165 ("wrong n value %d", n)); 166 167 tiov = getcopy(iov, n); 168 vbh = iov[0].iov_base; 169 170 status = iov[n-1].iov_base; 171 KASSERT(iov[n-1].iov_len == 1, 172 ("iov_len == %d", iov[n-1].iov_len)); 173 174 type = be32toh(vbh->type) & ~VIRTIO_BLK_T_BARRIER; 175 offset = be64toh(vbh->sector) * DEV_BSIZE; 176 177 iolen = 0; 178 for (i = 1; i < (n-1); i++) { 179 iolen += iov[i].iov_len; 180 } 181 182 switch (type) { 183 case VIRTIO_BLK_T_OUT: 184 case VIRTIO_BLK_T_IN: 185 err = vtblk_rdwr(sc, tiov + 1, i - 1, 186 offset, type, iolen); 187 break; 188 case VIRTIO_BLK_T_GET_ID: 189 /* Assume a single buffer */ 190 strncpy(iov[1].iov_base, sc->ident, 191 MIN(iov[1].iov_len, sizeof(sc->ident))); 192 err = 0; 193 break; 194 case VIRTIO_BLK_T_FLUSH: 195 /* Possible? */ 196 default: 197 err = -ENOSYS; 198 break; 199 } 200 201 if (err < 0) { 202 if (err == -ENOSYS) { 203 *status = VIRTIO_BLK_S_UNSUPP; 204 } else 205 *status = VIRTIO_BLK_S_IOERR; 206 } else 207 *status = VIRTIO_BLK_S_OK; 208 209 free(tiov, M_DEVBUF); 210 vq_relchain(vq, iov, n, 1); 211 } 212 213 static int 214 close_file(struct beri_vtblk_softc *sc, struct thread *td) 215 { 216 int error; 217 218 if (sc->vnode != NULL) { 219 vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY); 220 sc->vnode->v_vflag &= ~VV_MD; 221 VOP_UNLOCK(sc->vnode); 222 error = vn_close(sc->vnode, (FREAD|FWRITE), 223 sc->cred, td); 224 if (error != 0) 225 return (error); 226 sc->vnode = NULL; 227 } 228 229 if (sc->cred != NULL) 230 crfree(sc->cred); 231 232 return (0); 233 } 234 235 static int 236 open_file(struct beri_vtblk_softc *sc, struct thread *td) 237 { 238 struct nameidata nd; 239 struct vattr vattr; 240 int error; 241 int flags; 242 243 flags = (FREAD | FWRITE); 244 NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, sc->mdio->md_file); 245 error = vn_open(&nd, &flags, 0, NULL); 246 if (error != 0) 247 return (error); 248 NDFREE_PNBUF(&nd); 249 250 if (nd.ni_vp->v_type != VREG) { 251 return (EINVAL); 252 } 253 254 error = VOP_GETATTR(nd.ni_vp, &vattr, td->td_ucred); 255 if (error != 0) 256 return (error); 257 258 if (VOP_ISLOCKED(nd.ni_vp) != LK_EXCLUSIVE) { 259 vn_lock(nd.ni_vp, LK_UPGRADE | LK_RETRY); 260 if (VN_IS_DOOMED(nd.ni_vp)) { 261 return (1); 262 } 263 } 264 nd.ni_vp->v_vflag |= VV_MD; 265 VOP_UNLOCK(nd.ni_vp); 266 267 sc->vnode = nd.ni_vp; 268 sc->cred = crhold(td->td_ucred); 269 270 return (0); 271 } 272 273 static int 274 vtblk_notify(struct beri_vtblk_softc *sc) 275 { 276 struct vqueue_info *vq; 277 int queue; 278 int reg; 279 280 vq = &sc->vs_queues[0]; 281 if (!vq_ring_ready(vq)) 282 return (0); 283 284 if (!sc->opened) 285 return (0); 286 287 reg = READ2(sc, VIRTIO_MMIO_QUEUE_NOTIFY); 288 queue = be16toh(reg); 289 290 KASSERT(queue == 0, ("we support single queue only")); 291 292 /* Process new descriptors */ 293 vq = &sc->vs_queues[queue]; 294 vq->vq_save_used = be16toh(vq->vq_used->idx); 295 while (vq_has_descs(vq)) 296 vtblk_proc(sc, vq); 297 298 /* Interrupt the other side */ 299 if ((be16toh(vq->vq_avail->flags) & VRING_AVAIL_F_NO_INTERRUPT) == 0) { 300 reg = htobe32(VIRTIO_MMIO_INT_VRING); 301 WRITE4(sc, VIRTIO_MMIO_INTERRUPT_STATUS, reg); 302 PIO_SET(sc->pio_send, Q_INTR, 1); 303 } 304 305 return (0); 306 } 307 308 static int 309 vq_init(struct beri_vtblk_softc *sc) 310 { 311 struct vqueue_info *vq; 312 uint8_t *base; 313 int size; 314 int reg; 315 int pfn; 316 317 vq = &sc->vs_queues[0]; 318 vq->vq_qsize = NUM_DESCS; 319 320 reg = READ4(sc, VIRTIO_MMIO_QUEUE_PFN); 321 pfn = be32toh(reg); 322 vq->vq_pfn = pfn; 323 324 size = vring_size(vq->vq_qsize, VRING_ALIGN); 325 base = paddr_map(sc->beri_mem_offset, 326 (pfn << PAGE_SHIFT), size); 327 328 /* First pages are descriptors */ 329 vq->vq_desc = (struct vring_desc *)base; 330 base += vq->vq_qsize * sizeof(struct vring_desc); 331 332 /* Then avail ring */ 333 vq->vq_avail = (struct vring_avail *)base; 334 base += (2 + vq->vq_qsize + 1) * sizeof(uint16_t); 335 336 /* Then it's rounded up to the next page */ 337 base = (uint8_t *)roundup2((uintptr_t)base, VRING_ALIGN); 338 339 /* And the last pages are the used ring */ 340 vq->vq_used = (struct vring_used *)base; 341 342 /* Mark queue as allocated, and start at 0 when we use it. */ 343 vq->vq_flags = VQ_ALLOC; 344 vq->vq_last_avail = 0; 345 346 return (0); 347 } 348 349 static void 350 vtblk_thread(void *arg) 351 { 352 struct beri_vtblk_softc *sc; 353 int err; 354 355 sc = arg; 356 357 sx_xlock(&sc->sc_mtx); 358 for (;;) { 359 err = msleep(sc, &sc->sc_mtx, PCATCH | PZERO, "prd", hz); 360 vtblk_notify(sc); 361 } 362 sx_xunlock(&sc->sc_mtx); 363 364 kthread_exit(); 365 } 366 367 static int 368 backend_info(struct beri_vtblk_softc *sc) 369 { 370 struct virtio_blk_config *cfg; 371 uint32_t *s; 372 int reg; 373 int i; 374 375 /* Specify that we provide block device */ 376 reg = htobe32(VIRTIO_ID_BLOCK); 377 WRITE4(sc, VIRTIO_MMIO_DEVICE_ID, reg); 378 379 /* Queue size */ 380 reg = htobe32(NUM_DESCS); 381 WRITE4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX, reg); 382 383 /* Our features */ 384 reg = htobe32(VIRTIO_RING_F_INDIRECT_DESC 385 | VIRTIO_BLK_F_BLK_SIZE 386 | VIRTIO_BLK_F_SEG_MAX); 387 WRITE4(sc, VIRTIO_MMIO_HOST_FEATURES, reg); 388 389 cfg = sc->cfg; 390 cfg->capacity = htobe64(sc->mdio->md_mediasize / DEV_BSIZE); 391 cfg->size_max = 0; /* not negotiated */ 392 cfg->seg_max = htobe32(VTBLK_MAXSEGS); 393 cfg->blk_size = htobe32(DEV_BSIZE); 394 395 s = (uint32_t *)cfg; 396 397 for (i = 0; i < sizeof(struct virtio_blk_config); i+=4) { 398 WRITE4(sc, VIRTIO_MMIO_CONFIG + i, *s); 399 s+=1; 400 } 401 402 strncpy(sc->ident, "Virtio block backend", sizeof(sc->ident)); 403 404 return (0); 405 } 406 407 static void 408 vtblk_intr(void *arg) 409 { 410 struct beri_vtblk_softc *sc; 411 int pending; 412 int reg; 413 414 sc = arg; 415 416 reg = PIO_READ(sc->pio_recv); 417 418 /* Ack */ 419 PIO_SET(sc->pio_recv, reg, 0); 420 421 pending = htobe32(reg); 422 423 if (pending & Q_PFN) { 424 vq_init(sc); 425 } 426 427 if (pending & Q_NOTIFY) { 428 wakeup(sc); 429 } 430 } 431 432 static int 433 beri_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, 434 int flags, struct thread *td) 435 { 436 struct beri_vtblk_softc *sc; 437 int err; 438 439 sc = dev->si_drv1; 440 441 switch (cmd) { 442 case MDIOCATTACH: 443 /* take file as argument */ 444 if (sc->vnode != NULL) { 445 /* Already opened */ 446 return (1); 447 } 448 sc->mdio = (struct md_ioctl *)addr; 449 backend_info(sc); 450 DPRINTF("opening file, td 0x%08x\n", (int)td); 451 err = open_file(sc, td); 452 if (err) 453 return (err); 454 PIO_SETUP_IRQ(sc->pio_recv, vtblk_intr, sc); 455 sc->opened = 1; 456 break; 457 case MDIOCDETACH: 458 if (sc->vnode == NULL) { 459 /* File not opened */ 460 return (1); 461 } 462 sc->opened = 0; 463 DPRINTF("closing file, td 0x%08x\n", (int)td); 464 err = close_file(sc, td); 465 if (err) 466 return (err); 467 PIO_TEARDOWN_IRQ(sc->pio_recv); 468 break; 469 default: 470 break; 471 } 472 473 return (0); 474 } 475 476 static struct cdevsw beri_cdevsw = { 477 .d_version = D_VERSION, 478 .d_ioctl = beri_ioctl, 479 .d_name = "virtio block backend", 480 }; 481 482 static int 483 beri_vtblk_probe(device_t dev) 484 { 485 486 if (!ofw_bus_status_okay(dev)) 487 return (ENXIO); 488 489 if (!ofw_bus_is_compatible(dev, "sri-cambridge,beri-vtblk")) 490 return (ENXIO); 491 492 device_set_desc(dev, "SRI-Cambridge BERI block"); 493 return (BUS_PROBE_DEFAULT); 494 } 495 496 static int 497 beri_vtblk_attach(device_t dev) 498 { 499 struct beri_vtblk_softc *sc; 500 int error; 501 502 sc = device_get_softc(dev); 503 sc->dev = dev; 504 505 if (bus_alloc_resources(dev, beri_spec, sc->res)) { 506 device_printf(dev, "could not allocate resources\n"); 507 return (ENXIO); 508 } 509 510 /* Memory interface */ 511 sc->bst = rman_get_bustag(sc->res[0]); 512 sc->bsh = rman_get_bushandle(sc->res[0]); 513 514 sc->cfg = malloc(sizeof(struct virtio_blk_config), 515 M_DEVBUF, M_NOWAIT|M_ZERO); 516 517 sx_init(&sc->sc_mtx, device_get_nameunit(sc->dev)); 518 519 error = kthread_add(vtblk_thread, sc, NULL, &sc->vtblk_ktd, 520 0, 0, "beri_virtio_block"); 521 if (error) { 522 device_printf(dev, "cannot create kthread\n"); 523 return (ENXIO); 524 } 525 526 if (setup_offset(dev, &sc->beri_mem_offset) != 0) 527 return (ENXIO); 528 if (setup_pio(dev, "pio-send", &sc->pio_send) != 0) 529 return (ENXIO); 530 if (setup_pio(dev, "pio-recv", &sc->pio_recv) != 0) 531 return (ENXIO); 532 533 sc->cdev = make_dev(&beri_cdevsw, 0, UID_ROOT, GID_WHEEL, 534 S_IRWXU, "beri_vtblk"); 535 if (sc->cdev == NULL) { 536 device_printf(dev, "Failed to create character device.\n"); 537 return (ENXIO); 538 } 539 540 sc->cdev->si_drv1 = sc; 541 return (0); 542 } 543 544 static device_method_t beri_vtblk_methods[] = { 545 DEVMETHOD(device_probe, beri_vtblk_probe), 546 DEVMETHOD(device_attach, beri_vtblk_attach), 547 { 0, 0 } 548 }; 549 550 static driver_t beri_vtblk_driver = { 551 "beri_vtblk", 552 beri_vtblk_methods, 553 sizeof(struct beri_vtblk_softc), 554 }; 555 556 DRIVER_MODULE(beri_vtblk, simplebus, beri_vtblk_driver, 0, 0); 557