1 /*- 2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * This software was developed by SRI International and the University of 6 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) 7 * ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /* 32 * BERI Virtio Networking Frontend 33 */ 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/bus.h> 38 #include <sys/kernel.h> 39 #include <sys/module.h> 40 #include <sys/malloc.h> 41 #include <sys/rman.h> 42 #include <sys/timeet.h> 43 #include <sys/timetc.h> 44 #include <sys/endian.h> 45 #include <sys/lock.h> 46 #include <sys/mbuf.h> 47 #include <sys/mutex.h> 48 #include <sys/socket.h> 49 #include <sys/sockio.h> 50 #include <sys/sysctl.h> 51 #include <sys/mdioctl.h> 52 #include <sys/conf.h> 53 #include <sys/stat.h> 54 #include <sys/uio.h> 55 56 #include <dev/fdt/fdt_common.h> 57 #include <dev/ofw/openfirm.h> 58 #include <dev/ofw/ofw_bus.h> 59 #include <dev/ofw/ofw_bus_subr.h> 60 61 #include <net/bpf.h> 62 #include <net/if.h> 63 #include <net/ethernet.h> 64 #include <net/if_dl.h> 65 #include <net/if_media.h> 66 #include <net/if_types.h> 67 #include <net/if_var.h> 68 #include <net/if_vlan_var.h> 69 70 #include <netinet/in.h> 71 #include <netinet/udp.h> 72 #include <netinet/tcp.h> 73 74 #include <machine/bus.h> 75 #include <machine/fdt.h> 76 #include <machine/cpu.h> 77 #include <machine/intr.h> 78 79 #include <dev/beri/virtio/virtio.h> 80 #include <dev/beri/virtio/virtio_mmio_platform.h> 81 82 #include <dev/altera/pio/pio.h> 83 84 #include <dev/virtio/mmio/virtio_mmio.h> 85 #include <dev/virtio/network/virtio_net.h> 86 #include <dev/virtio/virtio_ids.h> 87 #include <dev/virtio/virtio_config.h> 88 #include <dev/virtio/virtio_ring.h> 89 90 #include "pio_if.h" 91 92 #define DPRINTF(fmt, args...) printf(fmt, ##args) 93 94 #define READ4(_sc, _reg) \ 95 bus_read_4((_sc)->res[0], _reg) 96 #define WRITE4(_sc, _reg, _val) \ 97 bus_write_4((_sc)->res[0], _reg, _val) 98 99 #define VTBE_LOCK(sc) mtx_lock(&(sc)->mtx) 100 #define VTBE_UNLOCK(sc) mtx_unlock(&(sc)->mtx) 101 #define VTBE_ASSERT_LOCKED(sc) mtx_assert(&(sc)->mtx, MA_OWNED); 102 #define VTBE_ASSERT_UNLOCKED(sc) mtx_assert(&(sc)->mtx, MA_NOTOWNED); 103 104 /* 105 * Driver data and defines. 106 */ 107 #define DESC_COUNT 256 108 109 struct vtbe_softc { 110 struct resource *res[2]; 111 bus_space_tag_t bst; 112 bus_space_handle_t bsh; 113 device_t dev; 114 if_t ifp; 115 int if_flags; 116 struct mtx mtx; 117 boolean_t is_attached; 118 119 int beri_mem_offset; 120 device_t pio_send; 121 device_t pio_recv; 122 int opened; 123 124 struct vqueue_info vs_queues[2]; 125 int vs_curq; 126 int hdrsize; 127 }; 128 129 static struct resource_spec vtbe_spec[] = { 130 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 131 { -1, 0 } 132 }; 133 134 static void vtbe_txfinish_locked(struct vtbe_softc *sc); 135 static void vtbe_rxfinish_locked(struct vtbe_softc *sc); 136 static void vtbe_stop_locked(struct vtbe_softc *sc); 137 static int pio_enable_irq(struct vtbe_softc *sc, int enable); 138 139 static void 140 vtbe_txstart_locked(struct vtbe_softc *sc) 141 { 142 struct iovec iov[DESC_COUNT]; 143 struct virtio_net_hdr *vnh; 144 struct vqueue_info *vq; 145 struct iovec *tiov; 146 if_t ifp; 147 struct mbuf *m; 148 struct uio uio; 149 int enqueued; 150 int iolen; 151 int error; 152 int reg; 153 int len; 154 int n; 155 156 VTBE_ASSERT_LOCKED(sc); 157 158 /* RX queue */ 159 vq = &sc->vs_queues[0]; 160 if (!vq_has_descs(vq)) { 161 return; 162 } 163 164 ifp = sc->ifp; 165 if (if_getdrvflags(ifp) & IFF_DRV_OACTIVE) { 166 return; 167 } 168 169 enqueued = 0; 170 171 if (!vq_ring_ready(vq)) 172 return; 173 174 vq->vq_save_used = be16toh(vq->vq_used->idx); 175 176 for (;;) { 177 if (!vq_has_descs(vq)) { 178 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0); 179 break; 180 } 181 182 m = if_dequeue(ifp); 183 if (m == NULL) { 184 break; 185 } 186 187 n = vq_getchain(sc->beri_mem_offset, vq, iov, 188 DESC_COUNT, NULL); 189 KASSERT(n == 2, 190 ("Unexpected amount of descriptors (%d)", n)); 191 192 tiov = getcopy(iov, n); 193 vnh = iov[0].iov_base; 194 memset(vnh, 0, sc->hdrsize); 195 196 len = iov[1].iov_len; 197 uio.uio_resid = len; 198 uio.uio_iov = &tiov[1]; 199 uio.uio_segflg = UIO_SYSSPACE; 200 uio.uio_iovcnt = 1; 201 uio.uio_offset = 0; 202 uio.uio_rw = UIO_READ; 203 204 error = m_mbuftouio(&uio, m, 0); 205 if (error) 206 panic("m_mbuftouio failed\n"); 207 208 iolen = (len - uio.uio_resid + sc->hdrsize); 209 210 free(tiov, M_DEVBUF); 211 vq_relchain(vq, iov, n, iolen); 212 213 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); 214 215 BPF_MTAP(ifp, m); 216 m_freem(m); 217 218 ++enqueued; 219 } 220 221 if (enqueued != 0) { 222 reg = htobe32(VIRTIO_MMIO_INT_VRING); 223 WRITE4(sc, VIRTIO_MMIO_INTERRUPT_STATUS, reg); 224 225 PIO_SET(sc->pio_send, Q_INTR, 1); 226 } 227 } 228 229 static void 230 vtbe_txstart(if_t ifp) 231 { 232 struct vtbe_softc *sc = if_getsoftc(ifp); 233 234 VTBE_LOCK(sc); 235 vtbe_txstart_locked(sc); 236 VTBE_UNLOCK(sc); 237 } 238 239 static void 240 vtbe_stop_locked(struct vtbe_softc *sc) 241 { 242 if_t ifp; 243 244 VTBE_ASSERT_LOCKED(sc); 245 246 ifp = sc->ifp; 247 if_setdrvflagbits(ifp, 0, (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)); 248 } 249 250 static void 251 vtbe_init_locked(struct vtbe_softc *sc) 252 { 253 if_t ifp = sc->ifp; 254 255 VTBE_ASSERT_LOCKED(sc); 256 257 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 258 return; 259 260 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0); 261 } 262 263 static void 264 vtbe_init(void *if_softc) 265 { 266 struct vtbe_softc *sc = if_softc; 267 268 VTBE_LOCK(sc); 269 vtbe_init_locked(sc); 270 VTBE_UNLOCK(sc); 271 } 272 273 static int 274 vtbe_ioctl(if_t ifp, u_long cmd, caddr_t data) 275 { 276 struct ifmediareq *ifmr; 277 struct vtbe_softc *sc; 278 struct ifreq *ifr; 279 int mask, error; 280 281 sc = if_getsoftc(ifp); 282 ifr = (struct ifreq *)data; 283 284 error = 0; 285 switch (cmd) { 286 case SIOCSIFFLAGS: 287 VTBE_LOCK(sc); 288 if (if_getflags(ifp) & IFF_UP) { 289 pio_enable_irq(sc, 1); 290 291 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { 292 vtbe_init_locked(sc); 293 } 294 } else { 295 pio_enable_irq(sc, 0); 296 297 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 298 vtbe_stop_locked(sc); 299 } 300 } 301 sc->if_flags = if_getflags(ifp); 302 VTBE_UNLOCK(sc); 303 break; 304 case SIOCADDMULTI: 305 case SIOCDELMULTI: 306 break; 307 case SIOCSIFMEDIA: 308 case SIOCGIFMEDIA: 309 ifmr = (struct ifmediareq *)data; 310 ifmr->ifm_count = 1; 311 ifmr->ifm_status = (IFM_AVALID | IFM_ACTIVE); 312 ifmr->ifm_active = (IFM_ETHER | IFM_10G_T | IFM_FDX); 313 ifmr->ifm_current = ifmr->ifm_active; 314 break; 315 case SIOCSIFCAP: 316 mask = if_getcapenable(ifp) ^ ifr->ifr_reqcap; 317 if (mask & IFCAP_VLAN_MTU) { 318 if_togglecapenable(ifp, IFCAP_VLAN_MTU); 319 } 320 break; 321 322 case SIOCSIFADDR: 323 pio_enable_irq(sc, 1); 324 default: 325 error = ether_ioctl(ifp, cmd, data); 326 break; 327 } 328 329 return (error); 330 } 331 332 static void 333 vtbe_txfinish_locked(struct vtbe_softc *sc) 334 { 335 if_t ifp; 336 337 VTBE_ASSERT_LOCKED(sc); 338 339 ifp = sc->ifp; 340 } 341 342 static int 343 vq_init(struct vtbe_softc *sc) 344 { 345 struct vqueue_info *vq; 346 uint8_t *base; 347 int size; 348 int reg; 349 int pfn; 350 351 vq = &sc->vs_queues[sc->vs_curq]; 352 vq->vq_qsize = DESC_COUNT; 353 354 reg = READ4(sc, VIRTIO_MMIO_QUEUE_PFN); 355 pfn = be32toh(reg); 356 vq->vq_pfn = pfn; 357 358 size = vring_size(vq->vq_qsize, VRING_ALIGN); 359 base = paddr_map(sc->beri_mem_offset, 360 (pfn << PAGE_SHIFT), size); 361 362 /* First pages are descriptors */ 363 vq->vq_desc = (struct vring_desc *)base; 364 base += vq->vq_qsize * sizeof(struct vring_desc); 365 366 /* Then avail ring */ 367 vq->vq_avail = (struct vring_avail *)base; 368 base += (2 + vq->vq_qsize + 1) * sizeof(uint16_t); 369 370 /* Then it's rounded up to the next page */ 371 base = (uint8_t *)roundup2((uintptr_t)base, VRING_ALIGN); 372 373 /* And the last pages are the used ring */ 374 vq->vq_used = (struct vring_used *)base; 375 376 /* Mark queue as allocated, and start at 0 when we use it. */ 377 vq->vq_flags = VQ_ALLOC; 378 vq->vq_last_avail = 0; 379 380 return (0); 381 } 382 383 static void 384 vtbe_proc_rx(struct vtbe_softc *sc, struct vqueue_info *vq) 385 { 386 struct iovec iov[DESC_COUNT]; 387 struct iovec *tiov; 388 if_t ifp; 389 struct uio uio; 390 struct mbuf *m; 391 int iolen; 392 int i; 393 int n; 394 395 ifp = sc->ifp; 396 397 n = vq_getchain(sc->beri_mem_offset, vq, iov, 398 DESC_COUNT, NULL); 399 400 KASSERT(n >= 1 && n <= DESC_COUNT, 401 ("wrong n %d", n)); 402 403 tiov = getcopy(iov, n); 404 405 iolen = 0; 406 for (i = 1; i < n; i++) { 407 iolen += iov[i].iov_len; 408 } 409 410 uio.uio_resid = iolen; 411 uio.uio_iov = &tiov[1]; 412 uio.uio_segflg = UIO_SYSSPACE; 413 uio.uio_iovcnt = (n - 1); 414 uio.uio_rw = UIO_WRITE; 415 416 if ((m = m_uiotombuf(&uio, M_NOWAIT, 0, ETHER_ALIGN, 417 M_PKTHDR)) == NULL) { 418 if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); 419 goto done; 420 } 421 422 m->m_pkthdr.rcvif = ifp; 423 424 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); 425 426 CURVNET_SET(if_getvnet(ifp)); 427 VTBE_UNLOCK(sc); 428 if_input(ifp, m); 429 VTBE_LOCK(sc); 430 CURVNET_RESTORE(); 431 432 done: 433 free(tiov, M_DEVBUF); 434 vq_relchain(vq, iov, n, iolen + sc->hdrsize); 435 } 436 437 static void 438 vtbe_rxfinish_locked(struct vtbe_softc *sc) 439 { 440 struct vqueue_info *vq; 441 int reg; 442 443 /* TX queue */ 444 vq = &sc->vs_queues[1]; 445 if (!vq_ring_ready(vq)) 446 return; 447 448 /* Process new descriptors */ 449 vq->vq_save_used = be16toh(vq->vq_used->idx); 450 451 while (vq_has_descs(vq)) { 452 vtbe_proc_rx(sc, vq); 453 } 454 455 /* Interrupt the other side */ 456 reg = htobe32(VIRTIO_MMIO_INT_VRING); 457 WRITE4(sc, VIRTIO_MMIO_INTERRUPT_STATUS, reg); 458 459 PIO_SET(sc->pio_send, Q_INTR, 1); 460 } 461 462 static void 463 vtbe_intr(void *arg) 464 { 465 struct vtbe_softc *sc; 466 int pending; 467 uint32_t reg; 468 469 sc = arg; 470 471 VTBE_LOCK(sc); 472 473 reg = PIO_READ(sc->pio_recv); 474 475 /* Ack */ 476 PIO_SET(sc->pio_recv, reg, 0); 477 478 pending = htobe32(reg); 479 if (pending & Q_SEL) { 480 reg = READ4(sc, VIRTIO_MMIO_QUEUE_SEL); 481 sc->vs_curq = be32toh(reg); 482 } 483 484 if (pending & Q_PFN) { 485 vq_init(sc); 486 } 487 488 if (pending & Q_NOTIFY) { 489 /* beri rx / arm tx notify */ 490 vtbe_txfinish_locked(sc); 491 } 492 493 if (pending & Q_NOTIFY1) { 494 vtbe_rxfinish_locked(sc); 495 } 496 497 VTBE_UNLOCK(sc); 498 } 499 500 static int 501 vtbe_get_hwaddr(struct vtbe_softc *sc, uint8_t *hwaddr) 502 { 503 int rnd; 504 505 /* 506 * Generate MAC address, use 'bsd' + random 24 low-order bits. 507 */ 508 509 rnd = arc4random() & 0x00ffffff; 510 511 hwaddr[0] = 'b'; 512 hwaddr[1] = 's'; 513 hwaddr[2] = 'd'; 514 hwaddr[3] = rnd >> 16; 515 hwaddr[4] = rnd >> 8; 516 hwaddr[5] = rnd >> 0; 517 518 return (0); 519 } 520 521 static int 522 pio_enable_irq(struct vtbe_softc *sc, int enable) 523 { 524 525 /* 526 * IRQ lines should be disabled while reprogram FPGA core. 527 */ 528 529 if (enable) { 530 if (sc->opened == 0) { 531 sc->opened = 1; 532 PIO_SETUP_IRQ(sc->pio_recv, vtbe_intr, sc); 533 } 534 } else { 535 if (sc->opened == 1) { 536 PIO_TEARDOWN_IRQ(sc->pio_recv); 537 sc->opened = 0; 538 } 539 } 540 541 return (0); 542 } 543 544 static int 545 vtbe_probe(device_t dev) 546 { 547 548 if (!ofw_bus_status_okay(dev)) 549 return (ENXIO); 550 551 if (!ofw_bus_is_compatible(dev, "sri-cambridge,beri-vtnet")) 552 return (ENXIO); 553 554 device_set_desc(dev, "Virtio BERI Ethernet Controller"); 555 return (BUS_PROBE_DEFAULT); 556 } 557 558 static int 559 vtbe_attach(device_t dev) 560 { 561 uint8_t macaddr[ETHER_ADDR_LEN]; 562 struct vtbe_softc *sc; 563 if_t ifp; 564 int reg; 565 566 sc = device_get_softc(dev); 567 sc->dev = dev; 568 569 sc->hdrsize = sizeof(struct virtio_net_hdr); 570 571 if (bus_alloc_resources(dev, vtbe_spec, sc->res)) { 572 device_printf(dev, "could not allocate resources\n"); 573 return (ENXIO); 574 } 575 576 /* Memory interface */ 577 sc->bst = rman_get_bustag(sc->res[0]); 578 sc->bsh = rman_get_bushandle(sc->res[0]); 579 580 mtx_init(&sc->mtx, device_get_nameunit(sc->dev), 581 MTX_NETWORK_LOCK, MTX_DEF); 582 583 if (setup_offset(dev, &sc->beri_mem_offset) != 0) 584 return (ENXIO); 585 if (setup_pio(dev, "pio-send", &sc->pio_send) != 0) 586 return (ENXIO); 587 if (setup_pio(dev, "pio-recv", &sc->pio_recv) != 0) 588 return (ENXIO); 589 590 /* Setup MMIO */ 591 592 /* Specify that we provide network device */ 593 reg = htobe32(VIRTIO_ID_NETWORK); 594 WRITE4(sc, VIRTIO_MMIO_DEVICE_ID, reg); 595 596 /* The number of desc we support */ 597 reg = htobe32(DESC_COUNT); 598 WRITE4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX, reg); 599 600 /* Our features */ 601 reg = htobe32(VIRTIO_NET_F_MAC | 602 VIRTIO_F_NOTIFY_ON_EMPTY); 603 WRITE4(sc, VIRTIO_MMIO_HOST_FEATURES, reg); 604 605 /* Get MAC */ 606 if (vtbe_get_hwaddr(sc, macaddr)) { 607 device_printf(sc->dev, "can't get mac\n"); 608 return (ENXIO); 609 } 610 611 /* Set up the ethernet interface. */ 612 sc->ifp = ifp = if_alloc(IFT_ETHER); 613 if_setbaudrate(ifp, IF_Gbps(10)); 614 if_setsoftc(ifp, sc); 615 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 616 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | 617 IFF_MULTICAST | IFF_PROMISC); 618 if_setcapabilities(ifp, IFCAP_VLAN_MTU); 619 if_setcapenable(ifp, if_getcapabilities(ifp)); 620 if_setstartfn(ifp, vtbe_txstart); 621 if_setioctlfn(ifp, vtbe_ioctl); 622 if_setinitfn(ifp, vtbe_init); 623 if_setsendqlen(ifp, DESC_COUNT - 1); 624 if_setsendqready(ifp); 625 if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); 626 627 /* All ready to run, attach the ethernet interface. */ 628 ether_ifattach(ifp, macaddr); 629 630 sc->is_attached = true; 631 632 return (0); 633 } 634 635 static device_method_t vtbe_methods[] = { 636 DEVMETHOD(device_probe, vtbe_probe), 637 DEVMETHOD(device_attach, vtbe_attach), 638 { 0, 0 } 639 }; 640 641 static driver_t vtbe_driver = { 642 "vtbe", 643 vtbe_methods, 644 sizeof(struct vtbe_softc), 645 }; 646 647 DRIVER_MODULE(vtbe, simplebus, vtbe_driver, 0, 0); 648 MODULE_DEPEND(vtbe, ether, 1, 1, 1); 649