1e47937d1SCarl Delsey /*- 29a5325c2SAlexander Motin * Copyright (c) 2016 Alexander Motin <mav@FreeBSD.org> 3e47937d1SCarl Delsey * Copyright (C) 2013 Intel Corporation 4763fa8aeSConrad Meyer * Copyright (C) 2015 EMC Corporation 5e47937d1SCarl Delsey * All rights reserved. 6e47937d1SCarl Delsey * 7e47937d1SCarl Delsey * Redistribution and use in source and binary forms, with or without 8e47937d1SCarl Delsey * modification, are permitted provided that the following conditions 9e47937d1SCarl Delsey * are met: 10e47937d1SCarl Delsey * 1. Redistributions of source code must retain the above copyright 11e47937d1SCarl Delsey * notice, this list of conditions and the following disclaimer. 12e47937d1SCarl Delsey * 2. Redistributions in binary form must reproduce the above copyright 13e47937d1SCarl Delsey * notice, this list of conditions and the following disclaimer in the 14e47937d1SCarl Delsey * documentation and/or other materials provided with the distribution. 15e47937d1SCarl Delsey * 16e47937d1SCarl Delsey * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17e47937d1SCarl Delsey * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18e47937d1SCarl Delsey * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19e47937d1SCarl Delsey * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20e47937d1SCarl Delsey * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21e47937d1SCarl Delsey * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22e47937d1SCarl Delsey * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23e47937d1SCarl Delsey * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24e47937d1SCarl Delsey * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25e47937d1SCarl Delsey * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26e47937d1SCarl Delsey * SUCH DAMAGE. 27e47937d1SCarl Delsey */ 28e47937d1SCarl Delsey 299a5325c2SAlexander Motin /* 309a5325c2SAlexander Motin * The Non-Transparent Bridge (NTB) is a device that allows you to connect 319a5325c2SAlexander Motin * two or more systems using a PCI-e links, providing remote memory access. 329a5325c2SAlexander Motin * 339a5325c2SAlexander Motin * This module contains a driver for simulated Ethernet device, using 349a5325c2SAlexander Motin * underlying NTB Transport device. 359a5325c2SAlexander Motin * 369a5325c2SAlexander Motin * NOTE: Much of the code in this module is shared with Linux. Any patches may 379a5325c2SAlexander Motin * be picked up and redistributed in Linux with a dual GPL/BSD license. 389a5325c2SAlexander Motin */ 399a5325c2SAlexander Motin 40e47937d1SCarl Delsey #include <sys/cdefs.h> 41e47937d1SCarl Delsey __FBSDID("$FreeBSD$"); 42e47937d1SCarl Delsey 43e47937d1SCarl Delsey #include <sys/param.h> 44e47937d1SCarl Delsey #include <sys/kernel.h> 45e47937d1SCarl Delsey #include <sys/systm.h> 468795de77SAlexander Motin #include <sys/buf_ring.h> 47e47937d1SCarl Delsey #include <sys/bus.h> 48737bc501SConrad Meyer #include <sys/limits.h> 49e47937d1SCarl Delsey #include <sys/module.h> 50e47937d1SCarl Delsey #include <sys/socket.h> 51e47937d1SCarl Delsey #include <sys/sockio.h> 528795de77SAlexander Motin #include <sys/sysctl.h> 538795de77SAlexander Motin #include <sys/taskqueue.h> 54b67ddac2SConrad Meyer 55e47937d1SCarl Delsey #include <net/if.h> 56e47937d1SCarl Delsey #include <net/if_media.h> 57e47937d1SCarl Delsey #include <net/if_types.h> 588795de77SAlexander Motin #include <net/if_media.h> 59e47937d1SCarl Delsey #include <net/if_var.h> 60e47937d1SCarl Delsey #include <net/bpf.h> 61e47937d1SCarl Delsey #include <net/ethernet.h> 62b67ddac2SConrad Meyer 63e47937d1SCarl Delsey #include <machine/bus.h> 64e47937d1SCarl Delsey 659a5325c2SAlexander Motin #include "../ntb_transport.h" 66e47937d1SCarl Delsey 67e47937d1SCarl Delsey #define KTR_NTB KTR_SPARE3 688795de77SAlexander Motin #define NTB_MEDIATYPE (IFM_ETHER | IFM_AUTO | IFM_FDX) 69e47937d1SCarl Delsey 70c266ab38SAlexander Motin #define NTB_CSUM_FEATURES (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP) 71c266ab38SAlexander Motin #define NTB_CSUM_FEATURES6 (CSUM_TCP_IPV6 | CSUM_UDP_IPV6 | CSUM_SCTP_IPV6) 72c266ab38SAlexander Motin #define NTB_CSUM_SET (CSUM_DATA_VALID | CSUM_DATA_VALID_IPV6 | \ 73c266ab38SAlexander Motin CSUM_PSEUDO_HDR | \ 74c266ab38SAlexander Motin CSUM_IP_CHECKED | CSUM_IP_VALID | \ 75c266ab38SAlexander Motin CSUM_SCTP_VALID) 76c266ab38SAlexander Motin 778795de77SAlexander Motin static SYSCTL_NODE(_hw, OID_AUTO, if_ntb, CTLFLAG_RW, 0, "if_ntb"); 788795de77SAlexander Motin 796bd57d14SAlexander Motin static unsigned g_if_ntb_num_queues = UINT_MAX; 808795de77SAlexander Motin SYSCTL_UINT(_hw_if_ntb, OID_AUTO, num_queues, CTLFLAG_RWTUN, 818795de77SAlexander Motin &g_if_ntb_num_queues, 0, "Number of queues per interface"); 828795de77SAlexander Motin 838795de77SAlexander Motin struct ntb_net_queue { 848795de77SAlexander Motin struct ntb_net_ctx *sc; 858795de77SAlexander Motin if_t ifp; 869a5325c2SAlexander Motin struct ntb_transport_qp *qp; 878795de77SAlexander Motin struct buf_ring *br; 888795de77SAlexander Motin struct task tx_task; 898795de77SAlexander Motin struct taskqueue *tx_tq; 90e47937d1SCarl Delsey struct mtx tx_lock; 919a5325c2SAlexander Motin struct callout queue_full; 92e47937d1SCarl Delsey }; 93e47937d1SCarl Delsey 948795de77SAlexander Motin struct ntb_net_ctx { 958795de77SAlexander Motin if_t ifp; 968795de77SAlexander Motin struct ifmedia media; 978795de77SAlexander Motin u_char eaddr[ETHER_ADDR_LEN]; 988795de77SAlexander Motin int num_queues; 998795de77SAlexander Motin struct ntb_net_queue *queues; 1008795de77SAlexander Motin int mtu; 1018795de77SAlexander Motin }; 1028795de77SAlexander Motin 1039a5325c2SAlexander Motin static int ntb_net_probe(device_t dev); 1049a5325c2SAlexander Motin static int ntb_net_attach(device_t dev); 1059a5325c2SAlexander Motin static int ntb_net_detach(device_t dev); 106e47937d1SCarl Delsey static void ntb_net_init(void *arg); 1078795de77SAlexander Motin static int ntb_ifmedia_upd(struct ifnet *); 1088795de77SAlexander Motin static void ntb_ifmedia_sts(struct ifnet *, struct ifmediareq *); 1098795de77SAlexander Motin static int ntb_ioctl(if_t ifp, u_long command, caddr_t data); 1108795de77SAlexander Motin static int ntb_transmit(if_t ifp, struct mbuf *m); 111e47937d1SCarl Delsey static void ntb_net_tx_handler(struct ntb_transport_qp *qp, void *qp_data, 112e47937d1SCarl Delsey void *data, int len); 113e47937d1SCarl Delsey static void ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data, 114e47937d1SCarl Delsey void *data, int len); 115531c7b99SConrad Meyer static void ntb_net_event_handler(void *data, enum ntb_link_event status); 1168795de77SAlexander Motin static void ntb_handle_tx(void *arg, int pending); 117e47937d1SCarl Delsey static void ntb_qp_full(void *arg); 1188795de77SAlexander Motin static void ntb_qflush(if_t ifp); 119e47937d1SCarl Delsey static void create_random_local_eui48(u_char *eaddr); 120e47937d1SCarl Delsey 121e47937d1SCarl Delsey static int 1229a5325c2SAlexander Motin ntb_net_probe(device_t dev) 123e47937d1SCarl Delsey { 1249a5325c2SAlexander Motin 1259a5325c2SAlexander Motin device_set_desc(dev, "NTB Network Interface"); 1269a5325c2SAlexander Motin return (0); 1279a5325c2SAlexander Motin } 1289a5325c2SAlexander Motin 1299a5325c2SAlexander Motin static int 1309a5325c2SAlexander Motin ntb_net_attach(device_t dev) 1319a5325c2SAlexander Motin { 1329a5325c2SAlexander Motin struct ntb_net_ctx *sc = device_get_softc(dev); 1338795de77SAlexander Motin struct ntb_net_queue *q; 1348795de77SAlexander Motin if_t ifp; 135e47937d1SCarl Delsey struct ntb_queue_handlers handlers = { ntb_net_rx_handler, 136e47937d1SCarl Delsey ntb_net_tx_handler, ntb_net_event_handler }; 1378795de77SAlexander Motin int i; 138e47937d1SCarl Delsey 1398795de77SAlexander Motin ifp = sc->ifp = if_gethandle(IFT_ETHER); 140e47937d1SCarl Delsey if (ifp == NULL) { 14195a3f7fbSConrad Meyer printf("ntb: Cannot allocate ifnet structure\n"); 142e47937d1SCarl Delsey return (ENOMEM); 143e47937d1SCarl Delsey } 1449a5325c2SAlexander Motin if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1458795de77SAlexander Motin if_setdev(ifp, dev); 14698bdb1ceSConrad Meyer 1476bd57d14SAlexander Motin sc->num_queues = min(g_if_ntb_num_queues, 1486bd57d14SAlexander Motin ntb_transport_queue_count(dev)); 1498795de77SAlexander Motin sc->queues = malloc(sc->num_queues * sizeof(struct ntb_net_queue), 1508795de77SAlexander Motin M_DEVBUF, M_WAITOK | M_ZERO); 1518795de77SAlexander Motin sc->mtu = INT_MAX; 1528795de77SAlexander Motin for (i = 0; i < sc->num_queues; i++) { 1538795de77SAlexander Motin q = &sc->queues[i]; 1548795de77SAlexander Motin q->sc = sc; 1558795de77SAlexander Motin q->ifp = ifp; 1566bd57d14SAlexander Motin q->qp = ntb_transport_create_queue(dev, i, &handlers, q); 1578795de77SAlexander Motin if (q->qp == NULL) 1588795de77SAlexander Motin break; 1598795de77SAlexander Motin sc->mtu = imin(sc->mtu, ntb_transport_max_size(q->qp)); 1608795de77SAlexander Motin mtx_init(&q->tx_lock, "ntb tx", NULL, MTX_DEF); 1618795de77SAlexander Motin q->br = buf_ring_alloc(4096, M_DEVBUF, M_WAITOK, &q->tx_lock); 1628795de77SAlexander Motin TASK_INIT(&q->tx_task, 0, ntb_handle_tx, q); 1638795de77SAlexander Motin q->tx_tq = taskqueue_create_fast("ntb_txq", M_NOWAIT, 1648795de77SAlexander Motin taskqueue_thread_enqueue, &q->tx_tq); 1658795de77SAlexander Motin taskqueue_start_threads(&q->tx_tq, 1, PI_NET, "%s txq%d", 1668795de77SAlexander Motin device_get_nameunit(dev), i); 1678795de77SAlexander Motin callout_init(&q->queue_full, 1); 1688795de77SAlexander Motin } 1698795de77SAlexander Motin sc->num_queues = i; 1706bd57d14SAlexander Motin device_printf(dev, "%d queue(s)\n", sc->num_queues); 171e47937d1SCarl Delsey 1728795de77SAlexander Motin if_setinitfn(ifp, ntb_net_init); 1738795de77SAlexander Motin if_setsoftc(ifp, sc); 1748795de77SAlexander Motin if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); 1758795de77SAlexander Motin if_setioctlfn(ifp, ntb_ioctl); 1768795de77SAlexander Motin if_settransmitfn(ifp, ntb_transmit); 1778795de77SAlexander Motin if_setqflushfn(ifp, ntb_qflush); 1789a5325c2SAlexander Motin create_random_local_eui48(sc->eaddr); 1799a5325c2SAlexander Motin ether_ifattach(ifp, sc->eaddr); 1808795de77SAlexander Motin if_setcapabilities(ifp, IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 | 1818795de77SAlexander Motin IFCAP_JUMBO_MTU | IFCAP_LINKSTATE); 182c266ab38SAlexander Motin if_setcapenable(ifp, IFCAP_JUMBO_MTU | IFCAP_LINKSTATE); 1838795de77SAlexander Motin if_setmtu(ifp, sc->mtu - ETHER_HDR_LEN); 184e47937d1SCarl Delsey 1858795de77SAlexander Motin ifmedia_init(&sc->media, IFM_IMASK, ntb_ifmedia_upd, 1868795de77SAlexander Motin ntb_ifmedia_sts); 1878795de77SAlexander Motin ifmedia_add(&sc->media, NTB_MEDIATYPE, 0, NULL); 1888795de77SAlexander Motin ifmedia_set(&sc->media, NTB_MEDIATYPE); 1898795de77SAlexander Motin 1908795de77SAlexander Motin for (i = 0; i < sc->num_queues; i++) 1918795de77SAlexander Motin ntb_transport_link_up(sc->queues[i].qp); 192e47937d1SCarl Delsey return (0); 193e47937d1SCarl Delsey } 194e47937d1SCarl Delsey 195e47937d1SCarl Delsey static int 1969a5325c2SAlexander Motin ntb_net_detach(device_t dev) 197e47937d1SCarl Delsey { 1989a5325c2SAlexander Motin struct ntb_net_ctx *sc = device_get_softc(dev); 1998795de77SAlexander Motin struct ntb_net_queue *q; 2008795de77SAlexander Motin int i; 201e47937d1SCarl Delsey 2028795de77SAlexander Motin for (i = 0; i < sc->num_queues; i++) 2038795de77SAlexander Motin ntb_transport_link_down(sc->queues[i].qp); 2049a5325c2SAlexander Motin ether_ifdetach(sc->ifp); 2059a5325c2SAlexander Motin if_free(sc->ifp); 2068795de77SAlexander Motin ifmedia_removeall(&sc->media); 2078795de77SAlexander Motin for (i = 0; i < sc->num_queues; i++) { 2088795de77SAlexander Motin q = &sc->queues[i]; 2098795de77SAlexander Motin ntb_transport_free_queue(q->qp); 2108795de77SAlexander Motin buf_ring_free(q->br, M_DEVBUF); 2118795de77SAlexander Motin callout_drain(&q->queue_full); 2128795de77SAlexander Motin taskqueue_drain_all(q->tx_tq); 2138795de77SAlexander Motin mtx_destroy(&q->tx_lock); 214538779c1SCarl Delsey } 2158795de77SAlexander Motin free(sc->queues, M_DEVBUF); 216e47937d1SCarl Delsey return (0); 217e47937d1SCarl Delsey } 218e47937d1SCarl Delsey 219e47937d1SCarl Delsey /* Network device interface */ 220e47937d1SCarl Delsey 221e47937d1SCarl Delsey static void 222e47937d1SCarl Delsey ntb_net_init(void *arg) 223e47937d1SCarl Delsey { 2249a5325c2SAlexander Motin struct ntb_net_ctx *sc = arg; 2258795de77SAlexander Motin if_t ifp = sc->ifp; 226e47937d1SCarl Delsey 2278795de77SAlexander Motin if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 2288795de77SAlexander Motin if_link_state_change(ifp, ntb_transport_link_query(sc->queues[0].qp) ? 2298795de77SAlexander Motin LINK_STATE_UP : LINK_STATE_DOWN); 230e47937d1SCarl Delsey } 231e47937d1SCarl Delsey 232e47937d1SCarl Delsey static int 2338795de77SAlexander Motin ntb_ioctl(if_t ifp, u_long command, caddr_t data) 234e47937d1SCarl Delsey { 2358795de77SAlexander Motin struct ntb_net_ctx *sc = if_getsoftc(ifp); 236e47937d1SCarl Delsey struct ifreq *ifr = (struct ifreq *)data; 237e47937d1SCarl Delsey int error = 0; 238e47937d1SCarl Delsey 239e47937d1SCarl Delsey switch (command) { 240e47937d1SCarl Delsey case SIOCSIFMTU: 241e47937d1SCarl Delsey { 2428795de77SAlexander Motin if (ifr->ifr_mtu > sc->mtu - ETHER_HDR_LEN) { 243e47937d1SCarl Delsey error = EINVAL; 244e47937d1SCarl Delsey break; 245e47937d1SCarl Delsey } 246e47937d1SCarl Delsey 2478795de77SAlexander Motin if_setmtu(ifp, ifr->ifr_mtu); 248e47937d1SCarl Delsey break; 249e47937d1SCarl Delsey } 2508795de77SAlexander Motin 2518795de77SAlexander Motin case SIOCSIFMEDIA: 2528795de77SAlexander Motin case SIOCGIFMEDIA: 2538795de77SAlexander Motin error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 2548795de77SAlexander Motin break; 2558795de77SAlexander Motin 256c266ab38SAlexander Motin case SIOCSIFCAP: 257c266ab38SAlexander Motin if (ifr->ifr_reqcap & IFCAP_RXCSUM) 258c266ab38SAlexander Motin if_setcapenablebit(ifp, IFCAP_RXCSUM, 0); 259c266ab38SAlexander Motin else 260c266ab38SAlexander Motin if_setcapenablebit(ifp, 0, IFCAP_RXCSUM); 261c266ab38SAlexander Motin if (ifr->ifr_reqcap & IFCAP_TXCSUM) { 262c266ab38SAlexander Motin if_setcapenablebit(ifp, IFCAP_TXCSUM, 0); 263c266ab38SAlexander Motin if_sethwassistbits(ifp, NTB_CSUM_FEATURES, 0); 264c266ab38SAlexander Motin } else { 265c266ab38SAlexander Motin if_setcapenablebit(ifp, 0, IFCAP_TXCSUM); 266c266ab38SAlexander Motin if_sethwassistbits(ifp, 0, NTB_CSUM_FEATURES); 267c266ab38SAlexander Motin } 268c266ab38SAlexander Motin if (ifr->ifr_reqcap & IFCAP_RXCSUM_IPV6) 269c266ab38SAlexander Motin if_setcapenablebit(ifp, IFCAP_RXCSUM_IPV6, 0); 270c266ab38SAlexander Motin else 271c266ab38SAlexander Motin if_setcapenablebit(ifp, 0, IFCAP_RXCSUM_IPV6); 272c266ab38SAlexander Motin if (ifr->ifr_reqcap & IFCAP_TXCSUM_IPV6) { 273c266ab38SAlexander Motin if_setcapenablebit(ifp, IFCAP_TXCSUM_IPV6, 0); 274c266ab38SAlexander Motin if_sethwassistbits(ifp, NTB_CSUM_FEATURES6, 0); 275c266ab38SAlexander Motin } else { 276c266ab38SAlexander Motin if_setcapenablebit(ifp, 0, IFCAP_TXCSUM_IPV6); 277c266ab38SAlexander Motin if_sethwassistbits(ifp, 0, NTB_CSUM_FEATURES6); 278c266ab38SAlexander Motin } 279c266ab38SAlexander Motin break; 280c266ab38SAlexander Motin 281e47937d1SCarl Delsey default: 282e47937d1SCarl Delsey error = ether_ioctl(ifp, command, data); 283e47937d1SCarl Delsey break; 284e47937d1SCarl Delsey } 285e47937d1SCarl Delsey 286e47937d1SCarl Delsey return (error); 287e47937d1SCarl Delsey } 288e47937d1SCarl Delsey 2898795de77SAlexander Motin static int 2908795de77SAlexander Motin ntb_ifmedia_upd(struct ifnet *ifp) 2918795de77SAlexander Motin { 2928795de77SAlexander Motin struct ntb_net_ctx *sc = if_getsoftc(ifp); 2938795de77SAlexander Motin struct ifmedia *ifm = &sc->media; 2948795de77SAlexander Motin 2958795de77SAlexander Motin if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 2968795de77SAlexander Motin return (EINVAL); 2978795de77SAlexander Motin 2988795de77SAlexander Motin return (0); 2998795de77SAlexander Motin } 300e47937d1SCarl Delsey 301e47937d1SCarl Delsey static void 3028795de77SAlexander Motin ntb_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr) 303e47937d1SCarl Delsey { 3048795de77SAlexander Motin struct ntb_net_ctx *sc = if_getsoftc(ifp); 305e47937d1SCarl Delsey 3068795de77SAlexander Motin ifmr->ifm_status = IFM_AVALID; 3078795de77SAlexander Motin ifmr->ifm_active = NTB_MEDIATYPE; 3088795de77SAlexander Motin if (ntb_transport_link_query(sc->queues[0].qp)) 3098795de77SAlexander Motin ifmr->ifm_status |= IFM_ACTIVE; 3108795de77SAlexander Motin } 3118795de77SAlexander Motin 3128795de77SAlexander Motin static void 3138795de77SAlexander Motin ntb_transmit_locked(struct ntb_net_queue *q) 3148795de77SAlexander Motin { 3158795de77SAlexander Motin if_t ifp = q->ifp; 3168795de77SAlexander Motin struct mbuf *m; 3178795de77SAlexander Motin int rc, len; 3188795de77SAlexander Motin short mflags; 3198795de77SAlexander Motin 3208795de77SAlexander Motin CTR0(KTR_NTB, "TX: ntb_transmit_locked"); 3218795de77SAlexander Motin while ((m = drbr_peek(ifp, q->br)) != NULL) { 3228795de77SAlexander Motin CTR1(KTR_NTB, "TX: start mbuf %p", m); 3238795de77SAlexander Motin if_etherbpfmtap(ifp, m); 3248795de77SAlexander Motin len = m->m_pkthdr.len; 3258795de77SAlexander Motin mflags = m->m_flags; 3268795de77SAlexander Motin rc = ntb_transport_tx_enqueue(q->qp, m, m, len); 327e47937d1SCarl Delsey if (rc != 0) { 3288795de77SAlexander Motin CTR2(KTR_NTB, "TX: could not tx mbuf %p: %d", m, rc); 329e47937d1SCarl Delsey if (rc == EAGAIN) { 3308795de77SAlexander Motin drbr_putback(ifp, q->br, m); 3318795de77SAlexander Motin callout_reset_sbt(&q->queue_full, 3328795de77SAlexander Motin SBT_1MS / 4, SBT_1MS / 4, 3338795de77SAlexander Motin ntb_qp_full, q, 0); 3348795de77SAlexander Motin } else { 3358795de77SAlexander Motin m_freem(m); 3368795de77SAlexander Motin drbr_advance(ifp, q->br); 3378795de77SAlexander Motin if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 338e47937d1SCarl Delsey } 339e47937d1SCarl Delsey break; 340e47937d1SCarl Delsey } 3418795de77SAlexander Motin drbr_advance(ifp, q->br); 3428795de77SAlexander Motin if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); 3438795de77SAlexander Motin if_inc_counter(ifp, IFCOUNTER_OBYTES, len); 3448795de77SAlexander Motin if (mflags & M_MCAST) 3458795de77SAlexander Motin if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); 346e47937d1SCarl Delsey } 3478795de77SAlexander Motin } 3488795de77SAlexander Motin 3498795de77SAlexander Motin static int 3508795de77SAlexander Motin ntb_transmit(if_t ifp, struct mbuf *m) 3518795de77SAlexander Motin { 3528795de77SAlexander Motin struct ntb_net_ctx *sc = if_getsoftc(ifp); 3538795de77SAlexander Motin struct ntb_net_queue *q; 3548795de77SAlexander Motin int error, i; 3558795de77SAlexander Motin 3568795de77SAlexander Motin CTR0(KTR_NTB, "TX: ntb_transmit"); 3578795de77SAlexander Motin if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 3588795de77SAlexander Motin i = m->m_pkthdr.flowid % sc->num_queues; 3598795de77SAlexander Motin else 3608795de77SAlexander Motin i = curcpu % sc->num_queues; 3618795de77SAlexander Motin q = &sc->queues[i]; 3628795de77SAlexander Motin 3638795de77SAlexander Motin error = drbr_enqueue(ifp, q->br, m); 3648795de77SAlexander Motin if (error) 3658795de77SAlexander Motin return (error); 3668795de77SAlexander Motin 3678795de77SAlexander Motin if (mtx_trylock(&q->tx_lock)) { 3688795de77SAlexander Motin ntb_transmit_locked(q); 3698795de77SAlexander Motin mtx_unlock(&q->tx_lock); 3708795de77SAlexander Motin } else 3718795de77SAlexander Motin taskqueue_enqueue(q->tx_tq, &q->tx_task); 3728795de77SAlexander Motin return (0); 3738795de77SAlexander Motin } 3748795de77SAlexander Motin 3758795de77SAlexander Motin static void 3768795de77SAlexander Motin ntb_handle_tx(void *arg, int pending) 3778795de77SAlexander Motin { 3788795de77SAlexander Motin struct ntb_net_queue *q = arg; 3798795de77SAlexander Motin 3808795de77SAlexander Motin mtx_lock(&q->tx_lock); 3818795de77SAlexander Motin ntb_transmit_locked(q); 3828795de77SAlexander Motin mtx_unlock(&q->tx_lock); 3838795de77SAlexander Motin } 3848795de77SAlexander Motin 3858795de77SAlexander Motin static void 3868795de77SAlexander Motin ntb_qp_full(void *arg) 3878795de77SAlexander Motin { 3888795de77SAlexander Motin struct ntb_net_queue *q = arg; 3898795de77SAlexander Motin 3908795de77SAlexander Motin CTR0(KTR_NTB, "TX: qp_full callout"); 3918795de77SAlexander Motin if (ntb_transport_tx_free_entry(q->qp) > 0) 3928795de77SAlexander Motin taskqueue_enqueue(q->tx_tq, &q->tx_task); 3938795de77SAlexander Motin else 3948795de77SAlexander Motin callout_schedule_sbt(&q->queue_full, 3958795de77SAlexander Motin SBT_1MS / 4, SBT_1MS / 4, 0); 3968795de77SAlexander Motin } 3978795de77SAlexander Motin 3988795de77SAlexander Motin static void 3998795de77SAlexander Motin ntb_qflush(if_t ifp) 4008795de77SAlexander Motin { 4018795de77SAlexander Motin struct ntb_net_ctx *sc = if_getsoftc(ifp); 4028795de77SAlexander Motin struct ntb_net_queue *q; 4038795de77SAlexander Motin struct mbuf *m; 4048795de77SAlexander Motin int i; 4058795de77SAlexander Motin 4068795de77SAlexander Motin for (i = 0; i < sc->num_queues; i++) { 4078795de77SAlexander Motin q = &sc->queues[i]; 4088795de77SAlexander Motin mtx_lock(&q->tx_lock); 4098795de77SAlexander Motin while ((m = buf_ring_dequeue_sc(q->br)) != NULL) 4108795de77SAlexander Motin m_freem(m); 4118795de77SAlexander Motin mtx_unlock(&q->tx_lock); 4128795de77SAlexander Motin } 4138795de77SAlexander Motin if_qflush(ifp); 414e47937d1SCarl Delsey } 415e47937d1SCarl Delsey 416e47937d1SCarl Delsey /* Network Device Callbacks */ 417e47937d1SCarl Delsey static void 418e47937d1SCarl Delsey ntb_net_tx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data, 419e47937d1SCarl Delsey int len) 420e47937d1SCarl Delsey { 421e47937d1SCarl Delsey 422e47937d1SCarl Delsey m_freem(data); 423e47937d1SCarl Delsey CTR1(KTR_NTB, "TX: tx_handler freeing mbuf %p", data); 424e47937d1SCarl Delsey } 425e47937d1SCarl Delsey 426e47937d1SCarl Delsey static void 427e47937d1SCarl Delsey ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data, 428e47937d1SCarl Delsey int len) 429e47937d1SCarl Delsey { 4308795de77SAlexander Motin struct ntb_net_queue *q = qp_data; 4318795de77SAlexander Motin struct ntb_net_ctx *sc = q->sc; 432e47937d1SCarl Delsey struct mbuf *m = data; 4338795de77SAlexander Motin if_t ifp = q->ifp; 434c266ab38SAlexander Motin uint16_t proto; 435e47937d1SCarl Delsey 436b42e3ec6SAlexander Motin CTR1(KTR_NTB, "RX: rx handler (%d)", len); 437b42e3ec6SAlexander Motin if (len < 0) { 438b42e3ec6SAlexander Motin if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); 439b42e3ec6SAlexander Motin return; 440b42e3ec6SAlexander Motin } 441b42e3ec6SAlexander Motin 4428795de77SAlexander Motin m->m_pkthdr.rcvif = ifp; 4438795de77SAlexander Motin if (sc->num_queues > 1) { 4448795de77SAlexander Motin m->m_pkthdr.flowid = q - sc->queues; 4458795de77SAlexander Motin M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 4468795de77SAlexander Motin } 447c266ab38SAlexander Motin if (if_getcapenable(ifp) & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) { 448c266ab38SAlexander Motin m_copydata(m, 12, 2, (void *)&proto); 449c266ab38SAlexander Motin switch (ntohs(proto)) { 450c266ab38SAlexander Motin case ETHERTYPE_IP: 451c266ab38SAlexander Motin if (if_getcapenable(ifp) & IFCAP_RXCSUM) { 452c266ab38SAlexander Motin m->m_pkthdr.csum_data = 0xffff; 453c266ab38SAlexander Motin m->m_pkthdr.csum_flags = NTB_CSUM_SET; 454c266ab38SAlexander Motin } 455c266ab38SAlexander Motin break; 456c266ab38SAlexander Motin case ETHERTYPE_IPV6: 457c266ab38SAlexander Motin if (if_getcapenable(ifp) & IFCAP_RXCSUM_IPV6) { 458c266ab38SAlexander Motin m->m_pkthdr.csum_data = 0xffff; 459c266ab38SAlexander Motin m->m_pkthdr.csum_flags = NTB_CSUM_SET; 460c266ab38SAlexander Motin } 461c266ab38SAlexander Motin break; 462c266ab38SAlexander Motin } 4638795de77SAlexander Motin } 4648795de77SAlexander Motin if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); 4658795de77SAlexander Motin if_input(ifp, m); 466e47937d1SCarl Delsey } 467e47937d1SCarl Delsey 468e47937d1SCarl Delsey static void 469531c7b99SConrad Meyer ntb_net_event_handler(void *data, enum ntb_link_event status) 470e47937d1SCarl Delsey { 4718795de77SAlexander Motin struct ntb_net_queue *q = data; 4728795de77SAlexander Motin int new_state; 473531c7b99SConrad Meyer 474531c7b99SConrad Meyer switch (status) { 475531c7b99SConrad Meyer case NTB_LINK_DOWN: 4768795de77SAlexander Motin new_state = LINK_STATE_DOWN; 477531c7b99SConrad Meyer break; 478531c7b99SConrad Meyer case NTB_LINK_UP: 4798795de77SAlexander Motin new_state = LINK_STATE_UP; 480531c7b99SConrad Meyer break; 481531c7b99SConrad Meyer default: 4828795de77SAlexander Motin new_state = LINK_STATE_UNKNOWN; 4838795de77SAlexander Motin break; 484531c7b99SConrad Meyer } 4858795de77SAlexander Motin if_link_state_change(q->ifp, new_state); 486e47937d1SCarl Delsey } 487e47937d1SCarl Delsey 488e47937d1SCarl Delsey /* Helper functions */ 489e47937d1SCarl Delsey /* TODO: This too should really be part of the kernel */ 490e47937d1SCarl Delsey #define EUI48_MULTICAST 1 << 0 491e47937d1SCarl Delsey #define EUI48_LOCALLY_ADMINISTERED 1 << 1 492e47937d1SCarl Delsey static void 493e47937d1SCarl Delsey create_random_local_eui48(u_char *eaddr) 494e47937d1SCarl Delsey { 495e47937d1SCarl Delsey static uint8_t counter = 0; 496e47937d1SCarl Delsey uint32_t seed = ticks; 497e47937d1SCarl Delsey 498e47937d1SCarl Delsey eaddr[0] = EUI48_LOCALLY_ADMINISTERED; 499e47937d1SCarl Delsey memcpy(&eaddr[1], &seed, sizeof(uint32_t)); 500e47937d1SCarl Delsey eaddr[5] = counter++; 501e47937d1SCarl Delsey } 502e47937d1SCarl Delsey 5039a5325c2SAlexander Motin static device_method_t ntb_net_methods[] = { 5049a5325c2SAlexander Motin /* Device interface */ 5059a5325c2SAlexander Motin DEVMETHOD(device_probe, ntb_net_probe), 5069a5325c2SAlexander Motin DEVMETHOD(device_attach, ntb_net_attach), 5079a5325c2SAlexander Motin DEVMETHOD(device_detach, ntb_net_detach), 5089a5325c2SAlexander Motin DEVMETHOD_END 5099a5325c2SAlexander Motin }; 510e47937d1SCarl Delsey 5119a5325c2SAlexander Motin devclass_t ntb_net_devclass; 5129a5325c2SAlexander Motin static DEFINE_CLASS_0(ntb, ntb_net_driver, ntb_net_methods, 5139a5325c2SAlexander Motin sizeof(struct ntb_net_ctx)); 5149a5325c2SAlexander Motin DRIVER_MODULE(if_ntb, ntb_transport, ntb_net_driver, ntb_net_devclass, 5159a5325c2SAlexander Motin NULL, NULL); 5169a5325c2SAlexander Motin MODULE_DEPEND(if_ntb, ntb_transport, 1, 1, 1); 5179a5325c2SAlexander Motin MODULE_VERSION(if_ntb, 1); 518