1 /*- 2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * This software was developed by SRI International and the University of 6 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) 7 * ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /* 32 * BERI Virtio Networking Frontend 33 */ 34 35 #include <sys/cdefs.h> 36 __FBSDID("$FreeBSD$"); 37 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/bus.h> 41 #include <sys/kernel.h> 42 #include <sys/module.h> 43 #include <sys/malloc.h> 44 #include <sys/rman.h> 45 #include <sys/timeet.h> 46 #include <sys/timetc.h> 47 #include <sys/endian.h> 48 #include <sys/lock.h> 49 #include <sys/mbuf.h> 50 #include <sys/mutex.h> 51 #include <sys/socket.h> 52 #include <sys/sockio.h> 53 #include <sys/sysctl.h> 54 #include <sys/mdioctl.h> 55 #include <sys/conf.h> 56 #include <sys/stat.h> 57 #include <sys/uio.h> 58 59 #include <dev/fdt/fdt_common.h> 60 #include <dev/ofw/openfirm.h> 61 #include <dev/ofw/ofw_bus.h> 62 #include <dev/ofw/ofw_bus_subr.h> 63 64 #include <net/bpf.h> 65 #include <net/if.h> 66 #include <net/ethernet.h> 67 #include <net/if_dl.h> 68 #include <net/if_media.h> 69 #include <net/if_types.h> 70 #include <net/if_var.h> 71 #include <net/if_vlan_var.h> 72 73 #include <netinet/in.h> 74 #include <netinet/udp.h> 75 #include <netinet/tcp.h> 76 77 #include <machine/bus.h> 78 #include <machine/fdt.h> 79 #include <machine/cpu.h> 80 #include <machine/intr.h> 81 82 #include <dev/beri/virtio/virtio.h> 83 #include <dev/beri/virtio/virtio_mmio_platform.h> 84 85 #include <dev/altera/pio/pio.h> 86 87 #include <dev/virtio/mmio/virtio_mmio.h> 88 #include <dev/virtio/network/virtio_net.h> 89 #include <dev/virtio/virtio_ids.h> 90 #include <dev/virtio/virtio_config.h> 91 #include <dev/virtio/virtio_ring.h> 92 93 #include "pio_if.h" 94 95 #define DPRINTF(fmt, args...) printf(fmt, ##args) 96 97 #define READ4(_sc, _reg) \ 98 bus_read_4((_sc)->res[0], _reg) 99 #define WRITE4(_sc, _reg, _val) \ 100 bus_write_4((_sc)->res[0], _reg, _val) 101 102 #define VTBE_LOCK(sc) mtx_lock(&(sc)->mtx) 103 #define VTBE_UNLOCK(sc) mtx_unlock(&(sc)->mtx) 104 #define VTBE_ASSERT_LOCKED(sc) mtx_assert(&(sc)->mtx, MA_OWNED); 105 #define VTBE_ASSERT_UNLOCKED(sc) mtx_assert(&(sc)->mtx, MA_NOTOWNED); 106 107 /* 108 * Driver data and defines. 109 */ 110 #define DESC_COUNT 256 111 112 struct vtbe_softc { 113 struct resource *res[2]; 114 bus_space_tag_t bst; 115 bus_space_handle_t bsh; 116 device_t dev; 117 if_t ifp; 118 int if_flags; 119 struct mtx mtx; 120 boolean_t is_attached; 121 122 int beri_mem_offset; 123 device_t pio_send; 124 device_t pio_recv; 125 int opened; 126 127 struct vqueue_info vs_queues[2]; 128 int vs_curq; 129 int hdrsize; 130 }; 131 132 static struct resource_spec vtbe_spec[] = { 133 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 134 { -1, 0 } 135 }; 136 137 static void vtbe_txfinish_locked(struct vtbe_softc *sc); 138 static void vtbe_rxfinish_locked(struct vtbe_softc *sc); 139 static void vtbe_stop_locked(struct vtbe_softc *sc); 140 static int pio_enable_irq(struct vtbe_softc *sc, int enable); 141 142 static void 143 vtbe_txstart_locked(struct vtbe_softc *sc) 144 { 145 struct iovec iov[DESC_COUNT]; 146 struct virtio_net_hdr *vnh; 147 struct vqueue_info *vq; 148 struct iovec *tiov; 149 if_t ifp; 150 struct mbuf *m; 151 struct uio uio; 152 int enqueued; 153 int iolen; 154 int error; 155 int reg; 156 int len; 157 int n; 158 159 VTBE_ASSERT_LOCKED(sc); 160 161 /* RX queue */ 162 vq = &sc->vs_queues[0]; 163 if (!vq_has_descs(vq)) { 164 return; 165 } 166 167 ifp = sc->ifp; 168 if (if_getdrvflags(ifp) & IFF_DRV_OACTIVE) { 169 return; 170 } 171 172 enqueued = 0; 173 174 if (!vq_ring_ready(vq)) 175 return; 176 177 vq->vq_save_used = be16toh(vq->vq_used->idx); 178 179 for (;;) { 180 if (!vq_has_descs(vq)) { 181 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0); 182 break; 183 } 184 185 m = if_dequeue(ifp); 186 if (m == NULL) { 187 break; 188 } 189 190 n = vq_getchain(sc->beri_mem_offset, vq, iov, 191 DESC_COUNT, NULL); 192 KASSERT(n == 2, 193 ("Unexpected amount of descriptors (%d)", n)); 194 195 tiov = getcopy(iov, n); 196 vnh = iov[0].iov_base; 197 memset(vnh, 0, sc->hdrsize); 198 199 len = iov[1].iov_len; 200 uio.uio_resid = len; 201 uio.uio_iov = &tiov[1]; 202 uio.uio_segflg = UIO_SYSSPACE; 203 uio.uio_iovcnt = 1; 204 uio.uio_offset = 0; 205 uio.uio_rw = UIO_READ; 206 207 error = m_mbuftouio(&uio, m, 0); 208 if (error) 209 panic("m_mbuftouio failed\n"); 210 211 iolen = (len - uio.uio_resid + sc->hdrsize); 212 213 free(tiov, M_DEVBUF); 214 vq_relchain(vq, iov, n, iolen); 215 216 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); 217 218 BPF_MTAP(ifp, m); 219 m_freem(m); 220 221 ++enqueued; 222 } 223 224 if (enqueued != 0) { 225 reg = htobe32(VIRTIO_MMIO_INT_VRING); 226 WRITE4(sc, VIRTIO_MMIO_INTERRUPT_STATUS, reg); 227 228 PIO_SET(sc->pio_send, Q_INTR, 1); 229 } 230 } 231 232 static void 233 vtbe_txstart(if_t ifp) 234 { 235 struct vtbe_softc *sc = if_getsoftc(ifp); 236 237 VTBE_LOCK(sc); 238 vtbe_txstart_locked(sc); 239 VTBE_UNLOCK(sc); 240 } 241 242 static void 243 vtbe_stop_locked(struct vtbe_softc *sc) 244 { 245 if_t ifp; 246 247 VTBE_ASSERT_LOCKED(sc); 248 249 ifp = sc->ifp; 250 if_setdrvflagbits(ifp, 0, (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)); 251 } 252 253 static void 254 vtbe_init_locked(struct vtbe_softc *sc) 255 { 256 if_t ifp = sc->ifp; 257 258 VTBE_ASSERT_LOCKED(sc); 259 260 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 261 return; 262 263 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0); 264 } 265 266 static void 267 vtbe_init(void *if_softc) 268 { 269 struct vtbe_softc *sc = if_softc; 270 271 VTBE_LOCK(sc); 272 vtbe_init_locked(sc); 273 VTBE_UNLOCK(sc); 274 } 275 276 static int 277 vtbe_ioctl(if_t ifp, u_long cmd, caddr_t data) 278 { 279 struct ifmediareq *ifmr; 280 struct vtbe_softc *sc; 281 struct ifreq *ifr; 282 int mask, error; 283 284 sc = if_getsoftc(ifp); 285 ifr = (struct ifreq *)data; 286 287 error = 0; 288 switch (cmd) { 289 case SIOCSIFFLAGS: 290 VTBE_LOCK(sc); 291 if (if_getflags(ifp) & IFF_UP) { 292 pio_enable_irq(sc, 1); 293 294 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { 295 vtbe_init_locked(sc); 296 } 297 } else { 298 pio_enable_irq(sc, 0); 299 300 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 301 vtbe_stop_locked(sc); 302 } 303 } 304 sc->if_flags = if_getflags(ifp); 305 VTBE_UNLOCK(sc); 306 break; 307 case SIOCADDMULTI: 308 case SIOCDELMULTI: 309 break; 310 case SIOCSIFMEDIA: 311 case SIOCGIFMEDIA: 312 ifmr = (struct ifmediareq *)data; 313 ifmr->ifm_count = 1; 314 ifmr->ifm_status = (IFM_AVALID | IFM_ACTIVE); 315 ifmr->ifm_active = (IFM_ETHER | IFM_10G_T | IFM_FDX); 316 ifmr->ifm_current = ifmr->ifm_active; 317 break; 318 case SIOCSIFCAP: 319 mask = if_getcapenable(ifp) ^ ifr->ifr_reqcap; 320 if (mask & IFCAP_VLAN_MTU) { 321 if_togglecapenable(ifp, IFCAP_VLAN_MTU); 322 } 323 break; 324 325 case SIOCSIFADDR: 326 pio_enable_irq(sc, 1); 327 default: 328 error = ether_ioctl(ifp, cmd, data); 329 break; 330 } 331 332 return (error); 333 } 334 335 static void 336 vtbe_txfinish_locked(struct vtbe_softc *sc) 337 { 338 if_t ifp; 339 340 VTBE_ASSERT_LOCKED(sc); 341 342 ifp = sc->ifp; 343 } 344 345 static int 346 vq_init(struct vtbe_softc *sc) 347 { 348 struct vqueue_info *vq; 349 uint8_t *base; 350 int size; 351 int reg; 352 int pfn; 353 354 vq = &sc->vs_queues[sc->vs_curq]; 355 vq->vq_qsize = DESC_COUNT; 356 357 reg = READ4(sc, VIRTIO_MMIO_QUEUE_PFN); 358 pfn = be32toh(reg); 359 vq->vq_pfn = pfn; 360 361 size = vring_size(vq->vq_qsize, VRING_ALIGN); 362 base = paddr_map(sc->beri_mem_offset, 363 (pfn << PAGE_SHIFT), size); 364 365 /* First pages are descriptors */ 366 vq->vq_desc = (struct vring_desc *)base; 367 base += vq->vq_qsize * sizeof(struct vring_desc); 368 369 /* Then avail ring */ 370 vq->vq_avail = (struct vring_avail *)base; 371 base += (2 + vq->vq_qsize + 1) * sizeof(uint16_t); 372 373 /* Then it's rounded up to the next page */ 374 base = (uint8_t *)roundup2((uintptr_t)base, VRING_ALIGN); 375 376 /* And the last pages are the used ring */ 377 vq->vq_used = (struct vring_used *)base; 378 379 /* Mark queue as allocated, and start at 0 when we use it. */ 380 vq->vq_flags = VQ_ALLOC; 381 vq->vq_last_avail = 0; 382 383 return (0); 384 } 385 386 static void 387 vtbe_proc_rx(struct vtbe_softc *sc, struct vqueue_info *vq) 388 { 389 struct iovec iov[DESC_COUNT]; 390 struct iovec *tiov; 391 if_t ifp; 392 struct uio uio; 393 struct mbuf *m; 394 int iolen; 395 int i; 396 int n; 397 398 ifp = sc->ifp; 399 400 n = vq_getchain(sc->beri_mem_offset, vq, iov, 401 DESC_COUNT, NULL); 402 403 KASSERT(n >= 1 && n <= DESC_COUNT, 404 ("wrong n %d", n)); 405 406 tiov = getcopy(iov, n); 407 408 iolen = 0; 409 for (i = 1; i < n; i++) { 410 iolen += iov[i].iov_len; 411 } 412 413 uio.uio_resid = iolen; 414 uio.uio_iov = &tiov[1]; 415 uio.uio_segflg = UIO_SYSSPACE; 416 uio.uio_iovcnt = (n - 1); 417 uio.uio_rw = UIO_WRITE; 418 419 if ((m = m_uiotombuf(&uio, M_NOWAIT, 0, ETHER_ALIGN, 420 M_PKTHDR)) == NULL) { 421 if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); 422 goto done; 423 } 424 425 m->m_pkthdr.rcvif = ifp; 426 427 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); 428 429 CURVNET_SET(if_getvnet(ifp)); 430 VTBE_UNLOCK(sc); 431 if_input(ifp, m); 432 VTBE_LOCK(sc); 433 CURVNET_RESTORE(); 434 435 done: 436 free(tiov, M_DEVBUF); 437 vq_relchain(vq, iov, n, iolen + sc->hdrsize); 438 } 439 440 static void 441 vtbe_rxfinish_locked(struct vtbe_softc *sc) 442 { 443 struct vqueue_info *vq; 444 int reg; 445 446 /* TX queue */ 447 vq = &sc->vs_queues[1]; 448 if (!vq_ring_ready(vq)) 449 return; 450 451 /* Process new descriptors */ 452 vq->vq_save_used = be16toh(vq->vq_used->idx); 453 454 while (vq_has_descs(vq)) { 455 vtbe_proc_rx(sc, vq); 456 } 457 458 /* Interrupt the other side */ 459 reg = htobe32(VIRTIO_MMIO_INT_VRING); 460 WRITE4(sc, VIRTIO_MMIO_INTERRUPT_STATUS, reg); 461 462 PIO_SET(sc->pio_send, Q_INTR, 1); 463 } 464 465 static void 466 vtbe_intr(void *arg) 467 { 468 struct vtbe_softc *sc; 469 int pending; 470 uint32_t reg; 471 472 sc = arg; 473 474 VTBE_LOCK(sc); 475 476 reg = PIO_READ(sc->pio_recv); 477 478 /* Ack */ 479 PIO_SET(sc->pio_recv, reg, 0); 480 481 pending = htobe32(reg); 482 if (pending & Q_SEL) { 483 reg = READ4(sc, VIRTIO_MMIO_QUEUE_SEL); 484 sc->vs_curq = be32toh(reg); 485 } 486 487 if (pending & Q_PFN) { 488 vq_init(sc); 489 } 490 491 if (pending & Q_NOTIFY) { 492 /* beri rx / arm tx notify */ 493 vtbe_txfinish_locked(sc); 494 } 495 496 if (pending & Q_NOTIFY1) { 497 vtbe_rxfinish_locked(sc); 498 } 499 500 VTBE_UNLOCK(sc); 501 } 502 503 static int 504 vtbe_get_hwaddr(struct vtbe_softc *sc, uint8_t *hwaddr) 505 { 506 int rnd; 507 508 /* 509 * Generate MAC address, use 'bsd' + random 24 low-order bits. 510 */ 511 512 rnd = arc4random() & 0x00ffffff; 513 514 hwaddr[0] = 'b'; 515 hwaddr[1] = 's'; 516 hwaddr[2] = 'd'; 517 hwaddr[3] = rnd >> 16; 518 hwaddr[4] = rnd >> 8; 519 hwaddr[5] = rnd >> 0; 520 521 return (0); 522 } 523 524 static int 525 pio_enable_irq(struct vtbe_softc *sc, int enable) 526 { 527 528 /* 529 * IRQ lines should be disabled while reprogram FPGA core. 530 */ 531 532 if (enable) { 533 if (sc->opened == 0) { 534 sc->opened = 1; 535 PIO_SETUP_IRQ(sc->pio_recv, vtbe_intr, sc); 536 } 537 } else { 538 if (sc->opened == 1) { 539 PIO_TEARDOWN_IRQ(sc->pio_recv); 540 sc->opened = 0; 541 } 542 } 543 544 return (0); 545 } 546 547 static int 548 vtbe_probe(device_t dev) 549 { 550 551 if (!ofw_bus_status_okay(dev)) 552 return (ENXIO); 553 554 if (!ofw_bus_is_compatible(dev, "sri-cambridge,beri-vtnet")) 555 return (ENXIO); 556 557 device_set_desc(dev, "Virtio BERI Ethernet Controller"); 558 return (BUS_PROBE_DEFAULT); 559 } 560 561 static int 562 vtbe_attach(device_t dev) 563 { 564 uint8_t macaddr[ETHER_ADDR_LEN]; 565 struct vtbe_softc *sc; 566 if_t ifp; 567 int reg; 568 569 sc = device_get_softc(dev); 570 sc->dev = dev; 571 572 sc->hdrsize = sizeof(struct virtio_net_hdr); 573 574 if (bus_alloc_resources(dev, vtbe_spec, sc->res)) { 575 device_printf(dev, "could not allocate resources\n"); 576 return (ENXIO); 577 } 578 579 /* Memory interface */ 580 sc->bst = rman_get_bustag(sc->res[0]); 581 sc->bsh = rman_get_bushandle(sc->res[0]); 582 583 mtx_init(&sc->mtx, device_get_nameunit(sc->dev), 584 MTX_NETWORK_LOCK, MTX_DEF); 585 586 if (setup_offset(dev, &sc->beri_mem_offset) != 0) 587 return (ENXIO); 588 if (setup_pio(dev, "pio-send", &sc->pio_send) != 0) 589 return (ENXIO); 590 if (setup_pio(dev, "pio-recv", &sc->pio_recv) != 0) 591 return (ENXIO); 592 593 /* Setup MMIO */ 594 595 /* Specify that we provide network device */ 596 reg = htobe32(VIRTIO_ID_NETWORK); 597 WRITE4(sc, VIRTIO_MMIO_DEVICE_ID, reg); 598 599 /* The number of desc we support */ 600 reg = htobe32(DESC_COUNT); 601 WRITE4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX, reg); 602 603 /* Our features */ 604 reg = htobe32(VIRTIO_NET_F_MAC | 605 VIRTIO_F_NOTIFY_ON_EMPTY); 606 WRITE4(sc, VIRTIO_MMIO_HOST_FEATURES, reg); 607 608 /* Get MAC */ 609 if (vtbe_get_hwaddr(sc, macaddr)) { 610 device_printf(sc->dev, "can't get mac\n"); 611 return (ENXIO); 612 } 613 614 /* Set up the ethernet interface. */ 615 sc->ifp = ifp = if_alloc(IFT_ETHER); 616 if_setbaudrate(ifp, IF_Gbps(10)); 617 if_setsoftc(ifp, sc); 618 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 619 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | 620 IFF_MULTICAST | IFF_PROMISC); 621 if_setcapabilities(ifp, IFCAP_VLAN_MTU); 622 if_setcapenable(ifp, if_getcapabilities(ifp)); 623 if_setstartfn(ifp, vtbe_txstart); 624 if_setioctlfn(ifp, vtbe_ioctl); 625 if_setinitfn(ifp, vtbe_init); 626 if_setsendqlen(ifp, DESC_COUNT - 1); 627 if_setsendqready(ifp); 628 if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); 629 630 /* All ready to run, attach the ethernet interface. */ 631 ether_ifattach(ifp, macaddr); 632 633 sc->is_attached = true; 634 635 return (0); 636 } 637 638 static device_method_t vtbe_methods[] = { 639 DEVMETHOD(device_probe, vtbe_probe), 640 DEVMETHOD(device_attach, vtbe_attach), 641 { 0, 0 } 642 }; 643 644 static driver_t vtbe_driver = { 645 "vtbe", 646 vtbe_methods, 647 sizeof(struct vtbe_softc), 648 }; 649 650 DRIVER_MODULE(vtbe, simplebus, vtbe_driver, 0, 0); 651 MODULE_DEPEND(vtbe, ether, 1, 1, 1); 652