1 /*- 2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * This software was developed by SRI International and the University of 6 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) 7 * ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /* 32 * BERI Virtio Networking Frontend 33 */ 34 35 #include <sys/cdefs.h> 36 __FBSDID("$FreeBSD$"); 37 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/bus.h> 41 #include <sys/kernel.h> 42 #include <sys/module.h> 43 #include <sys/malloc.h> 44 #include <sys/rman.h> 45 #include <sys/timeet.h> 46 #include <sys/timetc.h> 47 #include <sys/endian.h> 48 #include <sys/lock.h> 49 #include <sys/mbuf.h> 50 #include <sys/mutex.h> 51 #include <sys/socket.h> 52 #include <sys/sockio.h> 53 #include <sys/sysctl.h> 54 #include <sys/mdioctl.h> 55 #include <sys/conf.h> 56 #include <sys/stat.h> 57 #include <sys/uio.h> 58 59 #include <dev/fdt/fdt_common.h> 60 #include <dev/ofw/openfirm.h> 61 #include <dev/ofw/ofw_bus.h> 62 #include <dev/ofw/ofw_bus_subr.h> 63 64 #include <net/bpf.h> 65 #include <net/if.h> 66 #include <net/ethernet.h> 67 #include <net/if_dl.h> 68 #include <net/if_media.h> 69 #include <net/if_types.h> 70 #include <net/if_var.h> 71 #include <net/if_vlan_var.h> 72 73 #include <machine/bus.h> 74 #include <machine/fdt.h> 75 #include <machine/cpu.h> 76 #include <machine/intr.h> 77 78 #include <dev/beri/virtio/virtio.h> 79 #include <dev/beri/virtio/virtio_mmio_platform.h> 80 81 #include <dev/altera/pio/pio.h> 82 83 #include <dev/virtio/mmio/virtio_mmio.h> 84 #include <dev/virtio/network/virtio_net.h> 85 #include <dev/virtio/virtio_ids.h> 86 #include <dev/virtio/virtio_config.h> 87 #include <dev/virtio/virtio_ring.h> 88 89 #include "pio_if.h" 90 91 #define DPRINTF(fmt, args...) printf(fmt, ##args) 92 93 #define READ4(_sc, _reg) \ 94 bus_read_4((_sc)->res[0], _reg) 95 #define WRITE4(_sc, _reg, _val) \ 96 bus_write_4((_sc)->res[0], _reg, _val) 97 98 #define VTBE_LOCK(sc) mtx_lock(&(sc)->mtx) 99 #define VTBE_UNLOCK(sc) mtx_unlock(&(sc)->mtx) 100 #define VTBE_ASSERT_LOCKED(sc) mtx_assert(&(sc)->mtx, MA_OWNED); 101 #define VTBE_ASSERT_UNLOCKED(sc) mtx_assert(&(sc)->mtx, MA_NOTOWNED); 102 103 /* 104 * Driver data and defines. 105 */ 106 #define DESC_COUNT 256 107 108 struct vtbe_softc { 109 struct resource *res[2]; 110 bus_space_tag_t bst; 111 bus_space_handle_t bsh; 112 device_t dev; 113 struct ifnet *ifp; 114 int if_flags; 115 struct mtx mtx; 116 boolean_t is_attached; 117 118 int beri_mem_offset; 119 device_t pio_send; 120 device_t pio_recv; 121 int opened; 122 123 struct vqueue_info vs_queues[2]; 124 int vs_curq; 125 int hdrsize; 126 }; 127 128 static struct resource_spec vtbe_spec[] = { 129 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 130 { -1, 0 } 131 }; 132 133 static void vtbe_txfinish_locked(struct vtbe_softc *sc); 134 static void vtbe_rxfinish_locked(struct vtbe_softc *sc); 135 static void vtbe_stop_locked(struct vtbe_softc *sc); 136 static int pio_enable_irq(struct vtbe_softc *sc, int enable); 137 138 static void 139 vtbe_txstart_locked(struct vtbe_softc *sc) 140 { 141 struct iovec iov[DESC_COUNT]; 142 struct virtio_net_hdr *vnh; 143 struct vqueue_info *vq; 144 struct iovec *tiov; 145 struct ifnet *ifp; 146 struct mbuf *m; 147 struct uio uio; 148 int enqueued; 149 int iolen; 150 int error; 151 int reg; 152 int len; 153 int n; 154 155 VTBE_ASSERT_LOCKED(sc); 156 157 /* RX queue */ 158 vq = &sc->vs_queues[0]; 159 if (!vq_has_descs(vq)) { 160 return; 161 } 162 163 ifp = sc->ifp; 164 if (ifp->if_drv_flags & IFF_DRV_OACTIVE) { 165 return; 166 } 167 168 enqueued = 0; 169 170 if (!vq_ring_ready(vq)) 171 return; 172 173 vq->vq_save_used = be16toh(vq->vq_used->idx); 174 175 for (;;) { 176 if (!vq_has_descs(vq)) { 177 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 178 break; 179 } 180 181 IFQ_DRV_DEQUEUE(&ifp->if_snd, m); 182 if (m == NULL) { 183 break; 184 } 185 186 n = vq_getchain(sc->beri_mem_offset, vq, iov, 187 DESC_COUNT, NULL); 188 KASSERT(n == 2, 189 ("Unexpected amount of descriptors (%d)", n)); 190 191 tiov = getcopy(iov, n); 192 vnh = iov[0].iov_base; 193 memset(vnh, 0, sc->hdrsize); 194 195 len = iov[1].iov_len; 196 uio.uio_resid = len; 197 uio.uio_iov = &tiov[1]; 198 uio.uio_segflg = UIO_SYSSPACE; 199 uio.uio_iovcnt = 1; 200 uio.uio_offset = 0; 201 uio.uio_rw = UIO_READ; 202 203 error = m_mbuftouio(&uio, m, 0); 204 if (error) 205 panic("m_mbuftouio failed\n"); 206 207 iolen = (len - uio.uio_resid + sc->hdrsize); 208 209 free(tiov, M_DEVBUF); 210 vq_relchain(vq, iov, n, iolen); 211 212 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); 213 214 BPF_MTAP(ifp, m); 215 m_freem(m); 216 217 ++enqueued; 218 } 219 220 if (enqueued != 0) { 221 reg = htobe32(VIRTIO_MMIO_INT_VRING); 222 WRITE4(sc, VIRTIO_MMIO_INTERRUPT_STATUS, reg); 223 224 PIO_SET(sc->pio_send, Q_INTR, 1); 225 } 226 } 227 228 static void 229 vtbe_txstart(struct ifnet *ifp) 230 { 231 struct vtbe_softc *sc = ifp->if_softc; 232 233 VTBE_LOCK(sc); 234 vtbe_txstart_locked(sc); 235 VTBE_UNLOCK(sc); 236 } 237 238 static void 239 vtbe_stop_locked(struct vtbe_softc *sc) 240 { 241 struct ifnet *ifp; 242 243 VTBE_ASSERT_LOCKED(sc); 244 245 ifp = sc->ifp; 246 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 247 } 248 249 static void 250 vtbe_init_locked(struct vtbe_softc *sc) 251 { 252 struct ifnet *ifp = sc->ifp; 253 254 VTBE_ASSERT_LOCKED(sc); 255 256 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 257 return; 258 259 ifp->if_drv_flags |= IFF_DRV_RUNNING; 260 } 261 262 static void 263 vtbe_init(void *if_softc) 264 { 265 struct vtbe_softc *sc = if_softc; 266 267 VTBE_LOCK(sc); 268 vtbe_init_locked(sc); 269 VTBE_UNLOCK(sc); 270 } 271 272 static int 273 vtbe_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 274 { 275 struct ifmediareq *ifmr; 276 struct vtbe_softc *sc; 277 struct ifreq *ifr; 278 int mask, error; 279 280 sc = ifp->if_softc; 281 ifr = (struct ifreq *)data; 282 283 error = 0; 284 switch (cmd) { 285 case SIOCSIFFLAGS: 286 VTBE_LOCK(sc); 287 if (ifp->if_flags & IFF_UP) { 288 pio_enable_irq(sc, 1); 289 290 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 291 vtbe_init_locked(sc); 292 } 293 } else { 294 pio_enable_irq(sc, 0); 295 296 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 297 vtbe_stop_locked(sc); 298 } 299 } 300 sc->if_flags = ifp->if_flags; 301 VTBE_UNLOCK(sc); 302 break; 303 case SIOCADDMULTI: 304 case SIOCDELMULTI: 305 break; 306 case SIOCSIFMEDIA: 307 case SIOCGIFMEDIA: 308 ifmr = (struct ifmediareq *)data; 309 ifmr->ifm_count = 1; 310 ifmr->ifm_status = (IFM_AVALID | IFM_ACTIVE); 311 ifmr->ifm_active = (IFM_ETHER | IFM_10G_T | IFM_FDX); 312 ifmr->ifm_current = ifmr->ifm_active; 313 break; 314 case SIOCSIFCAP: 315 mask = ifp->if_capenable ^ ifr->ifr_reqcap; 316 if (mask & IFCAP_VLAN_MTU) { 317 ifp->if_capenable ^= IFCAP_VLAN_MTU; 318 } 319 break; 320 321 case SIOCSIFADDR: 322 pio_enable_irq(sc, 1); 323 default: 324 error = ether_ioctl(ifp, cmd, data); 325 break; 326 } 327 328 return (error); 329 } 330 331 static void 332 vtbe_txfinish_locked(struct vtbe_softc *sc) 333 { 334 struct ifnet *ifp; 335 336 VTBE_ASSERT_LOCKED(sc); 337 338 ifp = sc->ifp; 339 } 340 341 static int 342 vq_init(struct vtbe_softc *sc) 343 { 344 struct vqueue_info *vq; 345 uint8_t *base; 346 int size; 347 int reg; 348 int pfn; 349 350 vq = &sc->vs_queues[sc->vs_curq]; 351 vq->vq_qsize = DESC_COUNT; 352 353 reg = READ4(sc, VIRTIO_MMIO_QUEUE_PFN); 354 pfn = be32toh(reg); 355 vq->vq_pfn = pfn; 356 357 size = vring_size(vq->vq_qsize, VRING_ALIGN); 358 base = paddr_map(sc->beri_mem_offset, 359 (pfn << PAGE_SHIFT), size); 360 361 /* First pages are descriptors */ 362 vq->vq_desc = (struct vring_desc *)base; 363 base += vq->vq_qsize * sizeof(struct vring_desc); 364 365 /* Then avail ring */ 366 vq->vq_avail = (struct vring_avail *)base; 367 base += (2 + vq->vq_qsize + 1) * sizeof(uint16_t); 368 369 /* Then it's rounded up to the next page */ 370 base = (uint8_t *)roundup2((uintptr_t)base, VRING_ALIGN); 371 372 /* And the last pages are the used ring */ 373 vq->vq_used = (struct vring_used *)base; 374 375 /* Mark queue as allocated, and start at 0 when we use it. */ 376 vq->vq_flags = VQ_ALLOC; 377 vq->vq_last_avail = 0; 378 379 return (0); 380 } 381 382 static void 383 vtbe_proc_rx(struct vtbe_softc *sc, struct vqueue_info *vq) 384 { 385 struct iovec iov[DESC_COUNT]; 386 struct iovec *tiov; 387 struct ifnet *ifp; 388 struct uio uio; 389 struct mbuf *m; 390 int iolen; 391 int i; 392 int n; 393 394 ifp = sc->ifp; 395 396 n = vq_getchain(sc->beri_mem_offset, vq, iov, 397 DESC_COUNT, NULL); 398 399 KASSERT(n >= 1 && n <= DESC_COUNT, 400 ("wrong n %d", n)); 401 402 tiov = getcopy(iov, n); 403 404 iolen = 0; 405 for (i = 1; i < n; i++) { 406 iolen += iov[i].iov_len; 407 } 408 409 uio.uio_resid = iolen; 410 uio.uio_iov = &tiov[1]; 411 uio.uio_segflg = UIO_SYSSPACE; 412 uio.uio_iovcnt = (n - 1); 413 uio.uio_rw = UIO_WRITE; 414 415 if ((m = m_uiotombuf(&uio, M_NOWAIT, 0, ETHER_ALIGN, 416 M_PKTHDR)) == NULL) { 417 if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); 418 goto done; 419 } 420 421 m->m_pkthdr.rcvif = ifp; 422 423 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); 424 425 CURVNET_SET(ifp->if_vnet); 426 VTBE_UNLOCK(sc); 427 (*ifp->if_input)(ifp, m); 428 VTBE_LOCK(sc); 429 CURVNET_RESTORE(); 430 431 done: 432 free(tiov, M_DEVBUF); 433 vq_relchain(vq, iov, n, iolen + sc->hdrsize); 434 } 435 436 static void 437 vtbe_rxfinish_locked(struct vtbe_softc *sc) 438 { 439 struct vqueue_info *vq; 440 int reg; 441 442 /* TX queue */ 443 vq = &sc->vs_queues[1]; 444 if (!vq_ring_ready(vq)) 445 return; 446 447 /* Process new descriptors */ 448 vq->vq_save_used = be16toh(vq->vq_used->idx); 449 450 while (vq_has_descs(vq)) { 451 vtbe_proc_rx(sc, vq); 452 } 453 454 /* Interrupt the other side */ 455 reg = htobe32(VIRTIO_MMIO_INT_VRING); 456 WRITE4(sc, VIRTIO_MMIO_INTERRUPT_STATUS, reg); 457 458 PIO_SET(sc->pio_send, Q_INTR, 1); 459 } 460 461 static void 462 vtbe_intr(void *arg) 463 { 464 struct vtbe_softc *sc; 465 int pending; 466 uint32_t reg; 467 468 sc = arg; 469 470 VTBE_LOCK(sc); 471 472 reg = PIO_READ(sc->pio_recv); 473 474 /* Ack */ 475 PIO_SET(sc->pio_recv, reg, 0); 476 477 pending = htobe32(reg); 478 if (pending & Q_SEL) { 479 reg = READ4(sc, VIRTIO_MMIO_QUEUE_SEL); 480 sc->vs_curq = be32toh(reg); 481 } 482 483 if (pending & Q_PFN) { 484 vq_init(sc); 485 } 486 487 if (pending & Q_NOTIFY) { 488 /* beri rx / arm tx notify */ 489 vtbe_txfinish_locked(sc); 490 } 491 492 if (pending & Q_NOTIFY1) { 493 vtbe_rxfinish_locked(sc); 494 } 495 496 VTBE_UNLOCK(sc); 497 } 498 499 static int 500 vtbe_get_hwaddr(struct vtbe_softc *sc, uint8_t *hwaddr) 501 { 502 int rnd; 503 504 /* 505 * Generate MAC address, use 'bsd' + random 24 low-order bits. 506 */ 507 508 rnd = arc4random() & 0x00ffffff; 509 510 hwaddr[0] = 'b'; 511 hwaddr[1] = 's'; 512 hwaddr[2] = 'd'; 513 hwaddr[3] = rnd >> 16; 514 hwaddr[4] = rnd >> 8; 515 hwaddr[5] = rnd >> 0; 516 517 return (0); 518 } 519 520 static int 521 pio_enable_irq(struct vtbe_softc *sc, int enable) 522 { 523 524 /* 525 * IRQ lines should be disabled while reprogram FPGA core. 526 */ 527 528 if (enable) { 529 if (sc->opened == 0) { 530 sc->opened = 1; 531 PIO_SETUP_IRQ(sc->pio_recv, vtbe_intr, sc); 532 } 533 } else { 534 if (sc->opened == 1) { 535 PIO_TEARDOWN_IRQ(sc->pio_recv); 536 sc->opened = 0; 537 } 538 } 539 540 return (0); 541 } 542 543 static int 544 vtbe_probe(device_t dev) 545 { 546 547 if (!ofw_bus_status_okay(dev)) 548 return (ENXIO); 549 550 if (!ofw_bus_is_compatible(dev, "sri-cambridge,beri-vtnet")) 551 return (ENXIO); 552 553 device_set_desc(dev, "Virtio BERI Ethernet Controller"); 554 return (BUS_PROBE_DEFAULT); 555 } 556 557 static int 558 vtbe_attach(device_t dev) 559 { 560 uint8_t macaddr[ETHER_ADDR_LEN]; 561 struct vtbe_softc *sc; 562 struct ifnet *ifp; 563 int reg; 564 565 sc = device_get_softc(dev); 566 sc->dev = dev; 567 568 sc->hdrsize = sizeof(struct virtio_net_hdr); 569 570 if (bus_alloc_resources(dev, vtbe_spec, sc->res)) { 571 device_printf(dev, "could not allocate resources\n"); 572 return (ENXIO); 573 } 574 575 /* Memory interface */ 576 sc->bst = rman_get_bustag(sc->res[0]); 577 sc->bsh = rman_get_bushandle(sc->res[0]); 578 579 mtx_init(&sc->mtx, device_get_nameunit(sc->dev), 580 MTX_NETWORK_LOCK, MTX_DEF); 581 582 if (setup_offset(dev, &sc->beri_mem_offset) != 0) 583 return (ENXIO); 584 if (setup_pio(dev, "pio-send", &sc->pio_send) != 0) 585 return (ENXIO); 586 if (setup_pio(dev, "pio-recv", &sc->pio_recv) != 0) 587 return (ENXIO); 588 589 /* Setup MMIO */ 590 591 /* Specify that we provide network device */ 592 reg = htobe32(VIRTIO_ID_NETWORK); 593 WRITE4(sc, VIRTIO_MMIO_DEVICE_ID, reg); 594 595 /* The number of desc we support */ 596 reg = htobe32(DESC_COUNT); 597 WRITE4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX, reg); 598 599 /* Our features */ 600 reg = htobe32(VIRTIO_NET_F_MAC | 601 VIRTIO_F_NOTIFY_ON_EMPTY); 602 WRITE4(sc, VIRTIO_MMIO_HOST_FEATURES, reg); 603 604 /* Get MAC */ 605 if (vtbe_get_hwaddr(sc, macaddr)) { 606 device_printf(sc->dev, "can't get mac\n"); 607 return (ENXIO); 608 } 609 610 /* Set up the ethernet interface. */ 611 sc->ifp = ifp = if_alloc(IFT_ETHER); 612 ifp->if_baudrate = IF_Gbps(10); 613 ifp->if_softc = sc; 614 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 615 ifp->if_flags = (IFF_BROADCAST | IFF_SIMPLEX | 616 IFF_MULTICAST | IFF_PROMISC); 617 ifp->if_capabilities = IFCAP_VLAN_MTU; 618 ifp->if_capenable = ifp->if_capabilities; 619 ifp->if_start = vtbe_txstart; 620 ifp->if_ioctl = vtbe_ioctl; 621 ifp->if_init = vtbe_init; 622 IFQ_SET_MAXLEN(&ifp->if_snd, DESC_COUNT - 1); 623 ifp->if_snd.ifq_drv_maxlen = DESC_COUNT - 1; 624 IFQ_SET_READY(&ifp->if_snd); 625 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 626 627 /* All ready to run, attach the ethernet interface. */ 628 ether_ifattach(ifp, macaddr); 629 630 sc->is_attached = true; 631 632 return (0); 633 } 634 635 static device_method_t vtbe_methods[] = { 636 DEVMETHOD(device_probe, vtbe_probe), 637 DEVMETHOD(device_attach, vtbe_attach), 638 639 { 0, 0 } 640 }; 641 642 static driver_t vtbe_driver = { 643 "vtbe", 644 vtbe_methods, 645 sizeof(struct vtbe_softc), 646 }; 647 648 static devclass_t vtbe_devclass; 649 650 DRIVER_MODULE(vtbe, simplebus, vtbe_driver, vtbe_devclass, 0, 0); 651 MODULE_DEPEND(vtbe, ether, 1, 1, 1); 652