198c230c8SBjoern A. Zeeb /*- 2fe267a55SPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3fe267a55SPedro F. Giffuni * 498c230c8SBjoern A. Zeeb * Copyright (c) 2008 The FreeBSD Foundation 5eea3faf7SBjoern A. Zeeb * Copyright (c) 2009-2010 Bjoern A. Zeeb <bz@FreeBSD.org> 698c230c8SBjoern A. Zeeb * All rights reserved. 798c230c8SBjoern A. Zeeb * 898c230c8SBjoern A. Zeeb * This software was developed by CK Software GmbH under sponsorship 998c230c8SBjoern A. Zeeb * from the FreeBSD Foundation. 1098c230c8SBjoern A. Zeeb * 1198c230c8SBjoern A. Zeeb * Redistribution and use in source and binary forms, with or without 1298c230c8SBjoern A. Zeeb * modification, are permitted provided that the following conditions 1398c230c8SBjoern A. Zeeb * are met: 1498c230c8SBjoern A. Zeeb * 1. Redistributions of source code must retain the above copyright 1598c230c8SBjoern A. Zeeb * notice, this list of conditions and the following disclaimer. 1698c230c8SBjoern A. Zeeb * 2. Redistributions in binary form must reproduce the above copyright 1798c230c8SBjoern A. Zeeb * notice, this list of conditions and the following disclaimer in the 1898c230c8SBjoern A. Zeeb * documentation and/or other materials provided with the distribution. 1998c230c8SBjoern A. Zeeb * 2098c230c8SBjoern A. Zeeb * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 2198c230c8SBjoern A. Zeeb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2298c230c8SBjoern A. Zeeb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2398c230c8SBjoern A. Zeeb * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2498c230c8SBjoern A. Zeeb * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2598c230c8SBjoern A. Zeeb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2698c230c8SBjoern A. Zeeb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2798c230c8SBjoern A. Zeeb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2898c230c8SBjoern A. Zeeb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2998c230c8SBjoern A. Zeeb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 3098c230c8SBjoern A. Zeeb * SUCH DAMAGE. 3198c230c8SBjoern A. Zeeb */ 3298c230c8SBjoern A. Zeeb 3398c230c8SBjoern A. Zeeb /* 34d0ea4743SBjoern A. Zeeb * A pair of virtual back-to-back connected ethernet like interfaces 35d0ea4743SBjoern A. Zeeb * (``two interfaces with a virtual cross-over cable''). 36d0ea4743SBjoern A. Zeeb * 3798c230c8SBjoern A. Zeeb * This is mostly intended to be used to provide connectivity between 3898c230c8SBjoern A. Zeeb * different virtual network stack instances. 3998c230c8SBjoern A. Zeeb */ 4098c230c8SBjoern A. Zeeb /* 4198c230c8SBjoern A. Zeeb * Things to re-think once we have more experience: 42d0ea4743SBjoern A. Zeeb * - ifp->if_reassign function once we can test with vimage. Depending on 43530c0060SRobert Watson * how if_vmove() is going to be improved. 44d0ea4743SBjoern A. Zeeb * - Real random etheraddrs that are checked to be uniquish; we would need 45d0ea4743SBjoern A. Zeeb * to re-do them in case we move the interface between network stacks 46d0ea4743SBjoern A. Zeeb * in a private if_reassign function. 47d0ea4743SBjoern A. Zeeb * In case we bridge to a real interface/network or between indepedent 48d0ea4743SBjoern A. Zeeb * epairs on multiple stacks/machines, we may need this. 49d0ea4743SBjoern A. Zeeb * For now let the user handle that case. 5098c230c8SBjoern A. Zeeb */ 5198c230c8SBjoern A. Zeeb 5298c230c8SBjoern A. Zeeb #include <sys/cdefs.h> 5398c230c8SBjoern A. Zeeb __FBSDID("$FreeBSD$"); 5498c230c8SBjoern A. Zeeb 5598c230c8SBjoern A. Zeeb #include <sys/param.h> 5611d41666SLuca Pizzamiglio #include <sys/hash.h> 5711d41666SLuca Pizzamiglio #include <sys/jail.h> 5898c230c8SBjoern A. Zeeb #include <sys/kernel.h> 5911d41666SLuca Pizzamiglio #include <sys/libkern.h> 608ec07310SGleb Smirnoff #include <sys/malloc.h> 6198c230c8SBjoern A. Zeeb #include <sys/mbuf.h> 6298c230c8SBjoern A. Zeeb #include <sys/module.h> 6311d41666SLuca Pizzamiglio #include <sys/proc.h> 6498c230c8SBjoern A. Zeeb #include <sys/refcount.h> 6598c230c8SBjoern A. Zeeb #include <sys/queue.h> 66d0ea4743SBjoern A. Zeeb #include <sys/smp.h> 6798c230c8SBjoern A. Zeeb #include <sys/socket.h> 6898c230c8SBjoern A. Zeeb #include <sys/sockio.h> 6998c230c8SBjoern A. Zeeb #include <sys/sysctl.h> 7098c230c8SBjoern A. Zeeb #include <sys/types.h> 7198c230c8SBjoern A. Zeeb 7298c230c8SBjoern A. Zeeb #include <net/bpf.h> 7398c230c8SBjoern A. Zeeb #include <net/ethernet.h> 7498c230c8SBjoern A. Zeeb #include <net/if.h> 7576039bc8SGleb Smirnoff #include <net/if_var.h> 7698c230c8SBjoern A. Zeeb #include <net/if_clone.h> 772dccdd45SMarko Zec #include <net/if_media.h> 7898c230c8SBjoern A. Zeeb #include <net/if_var.h> 7998c230c8SBjoern A. Zeeb #include <net/if_types.h> 8098c230c8SBjoern A. Zeeb #include <net/netisr.h> 81530c0060SRobert Watson #include <net/vnet.h> 8298c230c8SBjoern A. Zeeb 8398c230c8SBjoern A. Zeeb SYSCTL_DECL(_net_link); 847029da5cSPawel Biernacki static SYSCTL_NODE(_net_link, OID_AUTO, epair, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 857029da5cSPawel Biernacki "epair sysctl"); 86d0ea4743SBjoern A. Zeeb 87d0ea4743SBjoern A. Zeeb #ifdef EPAIR_DEBUG 88d0ea4743SBjoern A. Zeeb static int epair_debug = 0; 892c8b047cSBjoern A. Zeeb SYSCTL_INT(_net_link_epair, OID_AUTO, epair_debug, CTLFLAG_RW, 9098c230c8SBjoern A. Zeeb &epair_debug, 0, "if_epair(4) debugging."); 91d0ea4743SBjoern A. Zeeb #define DPRINTF(fmt, arg...) \ 92d0ea4743SBjoern A. Zeeb if (epair_debug) \ 9398c230c8SBjoern A. Zeeb printf("[%s:%d] " fmt, __func__, __LINE__, ##arg) 9498c230c8SBjoern A. Zeeb #else 9598c230c8SBjoern A. Zeeb #define DPRINTF(fmt, arg...) 9698c230c8SBjoern A. Zeeb #endif 9798c230c8SBjoern A. Zeeb 98d0ea4743SBjoern A. Zeeb static void epair_nh_sintr(struct mbuf *); 99d0ea4743SBjoern A. Zeeb static struct mbuf *epair_nh_m2cpuid(struct mbuf *, uintptr_t, u_int *); 100d0ea4743SBjoern A. Zeeb static void epair_nh_drainedcpu(u_int); 10198c230c8SBjoern A. Zeeb 102d0ea4743SBjoern A. Zeeb static void epair_start_locked(struct ifnet *); 1032dccdd45SMarko Zec static int epair_media_change(struct ifnet *); 1042dccdd45SMarko Zec static void epair_media_status(struct ifnet *, struct ifmediareq *); 10598c230c8SBjoern A. Zeeb 10698c230c8SBjoern A. Zeeb static int epair_clone_match(struct if_clone *, const char *); 10798c230c8SBjoern A. Zeeb static int epair_clone_create(struct if_clone *, char *, size_t, caddr_t); 10898c230c8SBjoern A. Zeeb static int epair_clone_destroy(struct if_clone *, struct ifnet *); 10998c230c8SBjoern A. Zeeb 11042a58907SGleb Smirnoff static const char epairname[] = "epair"; 111804771f5SEugene Grosbein static unsigned int next_index = 0; 11242a58907SGleb Smirnoff 1133c3136b1SHiroki Sato /* Netisr related definitions and sysctl. */ 114d0ea4743SBjoern A. Zeeb static struct netisr_handler epair_nh = { 11542a58907SGleb Smirnoff .nh_name = epairname, 116d0ea4743SBjoern A. Zeeb .nh_proto = NETISR_EPAIR, 117d0ea4743SBjoern A. Zeeb .nh_policy = NETISR_POLICY_CPU, 118d0ea4743SBjoern A. Zeeb .nh_handler = epair_nh_sintr, 119d0ea4743SBjoern A. Zeeb .nh_m2cpuid = epair_nh_m2cpuid, 120d0ea4743SBjoern A. Zeeb .nh_drainedcpu = epair_nh_drainedcpu, 121d0ea4743SBjoern A. Zeeb }; 122d0ea4743SBjoern A. Zeeb 123d0ea4743SBjoern A. Zeeb static int 124d0ea4743SBjoern A. Zeeb sysctl_epair_netisr_maxqlen(SYSCTL_HANDLER_ARGS) 125d0ea4743SBjoern A. Zeeb { 126d0ea4743SBjoern A. Zeeb int error, qlimit; 127d0ea4743SBjoern A. Zeeb 128d0ea4743SBjoern A. Zeeb netisr_getqlimit(&epair_nh, &qlimit); 129d0ea4743SBjoern A. Zeeb error = sysctl_handle_int(oidp, &qlimit, 0, req); 130d0ea4743SBjoern A. Zeeb if (error || !req->newptr) 131d0ea4743SBjoern A. Zeeb return (error); 132d0ea4743SBjoern A. Zeeb if (qlimit < 1) 133d0ea4743SBjoern A. Zeeb return (EINVAL); 134d0ea4743SBjoern A. Zeeb return (netisr_setqlimit(&epair_nh, qlimit)); 135d0ea4743SBjoern A. Zeeb } 1367029da5cSPawel Biernacki SYSCTL_PROC(_net_link_epair, OID_AUTO, netisr_maxqlen, 1377029da5cSPawel Biernacki CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0, 1387029da5cSPawel Biernacki sysctl_epair_netisr_maxqlen, "I", 139d0ea4743SBjoern A. Zeeb "Maximum if_epair(4) netisr \"hw\" queue length"); 140d0ea4743SBjoern A. Zeeb 141d0ea4743SBjoern A. Zeeb struct epair_softc { 142d0ea4743SBjoern A. Zeeb struct ifnet *ifp; /* This ifp. */ 143d0ea4743SBjoern A. Zeeb struct ifnet *oifp; /* other ifp of pair. */ 1442dccdd45SMarko Zec struct ifmedia media; /* Media config (fake). */ 145d0ea4743SBjoern A. Zeeb u_int refcount; /* # of mbufs in flight. */ 146d0ea4743SBjoern A. Zeeb u_int cpuid; /* CPU ID assigned upon creation. */ 147d0ea4743SBjoern A. Zeeb void (*if_qflush)(struct ifnet *); 148d0ea4743SBjoern A. Zeeb /* Original if_qflush routine. */ 149d0ea4743SBjoern A. Zeeb }; 150d0ea4743SBjoern A. Zeeb 151d0ea4743SBjoern A. Zeeb /* 152d0ea4743SBjoern A. Zeeb * Per-CPU list of ifps with data in the ifq that needs to be flushed 153d0ea4743SBjoern A. Zeeb * to the netisr ``hw'' queue before we allow any further direct queuing 154d0ea4743SBjoern A. Zeeb * to the ``hw'' queue. 155d0ea4743SBjoern A. Zeeb */ 156d0ea4743SBjoern A. Zeeb struct epair_ifp_drain { 157d0ea4743SBjoern A. Zeeb STAILQ_ENTRY(epair_ifp_drain) ifp_next; 158d0ea4743SBjoern A. Zeeb struct ifnet *ifp; 159d0ea4743SBjoern A. Zeeb }; 160d0ea4743SBjoern A. Zeeb STAILQ_HEAD(eid_list, epair_ifp_drain); 161d0ea4743SBjoern A. Zeeb 162d0ea4743SBjoern A. Zeeb #define EPAIR_LOCK_INIT(dpcpu) mtx_init(&(dpcpu)->if_epair_mtx, \ 163d0ea4743SBjoern A. Zeeb "if_epair", NULL, MTX_DEF) 164d0ea4743SBjoern A. Zeeb #define EPAIR_LOCK_DESTROY(dpcpu) mtx_destroy(&(dpcpu)->if_epair_mtx) 165d0ea4743SBjoern A. Zeeb #define EPAIR_LOCK_ASSERT(dpcpu) mtx_assert(&(dpcpu)->if_epair_mtx, \ 166d0ea4743SBjoern A. Zeeb MA_OWNED) 167d0ea4743SBjoern A. Zeeb #define EPAIR_LOCK(dpcpu) mtx_lock(&(dpcpu)->if_epair_mtx) 168d0ea4743SBjoern A. Zeeb #define EPAIR_UNLOCK(dpcpu) mtx_unlock(&(dpcpu)->if_epair_mtx) 169d0ea4743SBjoern A. Zeeb 170d0ea4743SBjoern A. Zeeb #ifdef INVARIANTS 171d0ea4743SBjoern A. Zeeb #define EPAIR_REFCOUNT_INIT(r, v) refcount_init((r), (v)) 172d0ea4743SBjoern A. Zeeb #define EPAIR_REFCOUNT_AQUIRE(r) refcount_acquire((r)) 173d0ea4743SBjoern A. Zeeb #define EPAIR_REFCOUNT_RELEASE(r) refcount_release((r)) 174d0ea4743SBjoern A. Zeeb #define EPAIR_REFCOUNT_ASSERT(a, p) KASSERT(a, p) 175d0ea4743SBjoern A. Zeeb #else 176d0ea4743SBjoern A. Zeeb #define EPAIR_REFCOUNT_INIT(r, v) 177d0ea4743SBjoern A. Zeeb #define EPAIR_REFCOUNT_AQUIRE(r) 178d0ea4743SBjoern A. Zeeb #define EPAIR_REFCOUNT_RELEASE(r) 179d0ea4743SBjoern A. Zeeb #define EPAIR_REFCOUNT_ASSERT(a, p) 180d0ea4743SBjoern A. Zeeb #endif 181d0ea4743SBjoern A. Zeeb 18242a58907SGleb Smirnoff static MALLOC_DEFINE(M_EPAIR, epairname, 183d0ea4743SBjoern A. Zeeb "Pair of virtual cross-over connected Ethernet-like interfaces"); 18498c230c8SBjoern A. Zeeb 1855f901c92SAndrew Turner VNET_DEFINE_STATIC(struct if_clone *, epair_cloner); 1863c3136b1SHiroki Sato #define V_epair_cloner VNET(epair_cloner) 18798c230c8SBjoern A. Zeeb 188d0ea4743SBjoern A. Zeeb /* 189d0ea4743SBjoern A. Zeeb * DPCPU area and functions. 190d0ea4743SBjoern A. Zeeb */ 191d0ea4743SBjoern A. Zeeb struct epair_dpcpu { 192d0ea4743SBjoern A. Zeeb struct mtx if_epair_mtx; /* Per-CPU locking. */ 193d0ea4743SBjoern A. Zeeb int epair_drv_flags; /* Per-CPU ``hw'' drv flags. */ 194d0ea4743SBjoern A. Zeeb struct eid_list epair_ifp_drain_list; /* Per-CPU list of ifps with 195d0ea4743SBjoern A. Zeeb * data in the ifq. */ 196d0ea4743SBjoern A. Zeeb }; 197d0ea4743SBjoern A. Zeeb DPCPU_DEFINE(struct epair_dpcpu, epair_dpcpu); 198d0ea4743SBjoern A. Zeeb 199d0ea4743SBjoern A. Zeeb static void 20062d2dcafSKristof Provost epair_clear_mbuf(struct mbuf *m) 20162d2dcafSKristof Provost { 20262d2dcafSKristof Provost /* Remove any CSUM_SND_TAG as ether_input will barf. */ 20362d2dcafSKristof Provost if (m->m_pkthdr.csum_flags & CSUM_SND_TAG) { 20462d2dcafSKristof Provost m_snd_tag_rele(m->m_pkthdr.snd_tag); 20562d2dcafSKristof Provost m->m_pkthdr.snd_tag = NULL; 20662d2dcafSKristof Provost m->m_pkthdr.csum_flags &= ~CSUM_SND_TAG; 20762d2dcafSKristof Provost } 20862d2dcafSKristof Provost 20962d2dcafSKristof Provost m_tag_delete_nonpersistent(m); 21062d2dcafSKristof Provost } 21162d2dcafSKristof Provost 21262d2dcafSKristof Provost static void 213d0ea4743SBjoern A. Zeeb epair_dpcpu_init(void) 214d0ea4743SBjoern A. Zeeb { 215d0ea4743SBjoern A. Zeeb struct epair_dpcpu *epair_dpcpu; 216d0ea4743SBjoern A. Zeeb struct eid_list *s; 217d0ea4743SBjoern A. Zeeb u_int cpuid; 218d0ea4743SBjoern A. Zeeb 2193aa6d94eSJohn Baldwin CPU_FOREACH(cpuid) { 220d0ea4743SBjoern A. Zeeb epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); 221d0ea4743SBjoern A. Zeeb 222d0ea4743SBjoern A. Zeeb /* Initialize per-cpu lock. */ 223d0ea4743SBjoern A. Zeeb EPAIR_LOCK_INIT(epair_dpcpu); 224d0ea4743SBjoern A. Zeeb 225d0ea4743SBjoern A. Zeeb /* Driver flags are per-cpu as are our netisr "hw" queues. */ 226d0ea4743SBjoern A. Zeeb epair_dpcpu->epair_drv_flags = 0; 227d0ea4743SBjoern A. Zeeb 228d0ea4743SBjoern A. Zeeb /* 229d0ea4743SBjoern A. Zeeb * Initialize per-cpu drain list. 230d0ea4743SBjoern A. Zeeb * Manually do what STAILQ_HEAD_INITIALIZER would do. 231d0ea4743SBjoern A. Zeeb */ 232d0ea4743SBjoern A. Zeeb s = &epair_dpcpu->epair_ifp_drain_list; 233d0ea4743SBjoern A. Zeeb s->stqh_first = NULL; 234d0ea4743SBjoern A. Zeeb s->stqh_last = &s->stqh_first; 235d0ea4743SBjoern A. Zeeb } 236d0ea4743SBjoern A. Zeeb } 237d0ea4743SBjoern A. Zeeb 238d0ea4743SBjoern A. Zeeb static void 239d0ea4743SBjoern A. Zeeb epair_dpcpu_detach(void) 240d0ea4743SBjoern A. Zeeb { 241d0ea4743SBjoern A. Zeeb struct epair_dpcpu *epair_dpcpu; 242d0ea4743SBjoern A. Zeeb u_int cpuid; 243d0ea4743SBjoern A. Zeeb 2443aa6d94eSJohn Baldwin CPU_FOREACH(cpuid) { 245d0ea4743SBjoern A. Zeeb epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); 246d0ea4743SBjoern A. Zeeb 247d0ea4743SBjoern A. Zeeb /* Destroy per-cpu lock. */ 248d0ea4743SBjoern A. Zeeb EPAIR_LOCK_DESTROY(epair_dpcpu); 249d0ea4743SBjoern A. Zeeb } 250d0ea4743SBjoern A. Zeeb } 251d0ea4743SBjoern A. Zeeb 252d0ea4743SBjoern A. Zeeb /* 253d0ea4743SBjoern A. Zeeb * Helper functions. 254d0ea4743SBjoern A. Zeeb */ 255d0ea4743SBjoern A. Zeeb static u_int 256d0ea4743SBjoern A. Zeeb cpuid_from_ifp(struct ifnet *ifp) 257d0ea4743SBjoern A. Zeeb { 258d0ea4743SBjoern A. Zeeb struct epair_softc *sc; 259d0ea4743SBjoern A. Zeeb 260d0ea4743SBjoern A. Zeeb if (ifp == NULL) 261d0ea4743SBjoern A. Zeeb return (0); 262d0ea4743SBjoern A. Zeeb sc = ifp->if_softc; 263d0ea4743SBjoern A. Zeeb 264d0ea4743SBjoern A. Zeeb return (sc->cpuid); 265d0ea4743SBjoern A. Zeeb } 26698c230c8SBjoern A. Zeeb 26798c230c8SBjoern A. Zeeb /* 26898c230c8SBjoern A. Zeeb * Netisr handler functions. 26998c230c8SBjoern A. Zeeb */ 27098c230c8SBjoern A. Zeeb static void 271d0ea4743SBjoern A. Zeeb epair_nh_sintr(struct mbuf *m) 27298c230c8SBjoern A. Zeeb { 27398c230c8SBjoern A. Zeeb struct ifnet *ifp; 27446d0f824SMatt Macy struct epair_softc *sc __unused; 27598c230c8SBjoern A. Zeeb 27698c230c8SBjoern A. Zeeb ifp = m->m_pkthdr.rcvif; 27798c230c8SBjoern A. Zeeb (*ifp->if_input)(ifp, m); 27898c230c8SBjoern A. Zeeb sc = ifp->if_softc; 279d0ea4743SBjoern A. Zeeb EPAIR_REFCOUNT_RELEASE(&sc->refcount); 280eea3faf7SBjoern A. Zeeb EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 281eea3faf7SBjoern A. Zeeb ("%s: ifp=%p sc->refcount not >= 1: %d", 282eea3faf7SBjoern A. Zeeb __func__, ifp, sc->refcount)); 28398c230c8SBjoern A. Zeeb DPRINTF("ifp=%p refcount=%u\n", ifp, sc->refcount); 28498c230c8SBjoern A. Zeeb } 28598c230c8SBjoern A. Zeeb 286d0ea4743SBjoern A. Zeeb static struct mbuf * 287d0ea4743SBjoern A. Zeeb epair_nh_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) 28898c230c8SBjoern A. Zeeb { 289d0ea4743SBjoern A. Zeeb 290d0ea4743SBjoern A. Zeeb *cpuid = cpuid_from_ifp(m->m_pkthdr.rcvif); 291d0ea4743SBjoern A. Zeeb 292d0ea4743SBjoern A. Zeeb return (m); 293d0ea4743SBjoern A. Zeeb } 294d0ea4743SBjoern A. Zeeb 295d0ea4743SBjoern A. Zeeb static void 296d0ea4743SBjoern A. Zeeb epair_nh_drainedcpu(u_int cpuid) 297d0ea4743SBjoern A. Zeeb { 298d0ea4743SBjoern A. Zeeb struct epair_dpcpu *epair_dpcpu; 29998c230c8SBjoern A. Zeeb struct epair_ifp_drain *elm, *tvar; 30098c230c8SBjoern A. Zeeb struct ifnet *ifp; 30198c230c8SBjoern A. Zeeb 302d0ea4743SBjoern A. Zeeb epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); 303d0ea4743SBjoern A. Zeeb EPAIR_LOCK(epair_dpcpu); 30498c230c8SBjoern A. Zeeb /* 30598c230c8SBjoern A. Zeeb * Assume our "hw" queue and possibly ifq will be emptied 30698c230c8SBjoern A. Zeeb * again. In case we will overflow the "hw" queue while 30798c230c8SBjoern A. Zeeb * draining, epair_start_locked will set IFF_DRV_OACTIVE 30898c230c8SBjoern A. Zeeb * again and we will stop and return. 30998c230c8SBjoern A. Zeeb */ 310d0ea4743SBjoern A. Zeeb STAILQ_FOREACH_SAFE(elm, &epair_dpcpu->epair_ifp_drain_list, 311d0ea4743SBjoern A. Zeeb ifp_next, tvar) { 31298c230c8SBjoern A. Zeeb ifp = elm->ifp; 313d0ea4743SBjoern A. Zeeb epair_dpcpu->epair_drv_flags &= ~IFF_DRV_OACTIVE; 31498c230c8SBjoern A. Zeeb ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 31598c230c8SBjoern A. Zeeb epair_start_locked(ifp); 31698c230c8SBjoern A. Zeeb 31798c230c8SBjoern A. Zeeb IFQ_LOCK(&ifp->if_snd); 31898c230c8SBjoern A. Zeeb if (IFQ_IS_EMPTY(&ifp->if_snd)) { 31946d0f824SMatt Macy struct epair_softc *sc __unused; 320eea3faf7SBjoern A. Zeeb 321d0ea4743SBjoern A. Zeeb STAILQ_REMOVE(&epair_dpcpu->epair_ifp_drain_list, 322d0ea4743SBjoern A. Zeeb elm, epair_ifp_drain, ifp_next); 323eea3faf7SBjoern A. Zeeb /* The cached ifp goes off the list. */ 324eea3faf7SBjoern A. Zeeb sc = ifp->if_softc; 325eea3faf7SBjoern A. Zeeb EPAIR_REFCOUNT_RELEASE(&sc->refcount); 326eea3faf7SBjoern A. Zeeb EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 327eea3faf7SBjoern A. Zeeb ("%s: ifp=%p sc->refcount not >= 1: %d", 328eea3faf7SBjoern A. Zeeb __func__, ifp, sc->refcount)); 32998c230c8SBjoern A. Zeeb free(elm, M_EPAIR); 33098c230c8SBjoern A. Zeeb } 33198c230c8SBjoern A. Zeeb IFQ_UNLOCK(&ifp->if_snd); 33298c230c8SBjoern A. Zeeb 33398c230c8SBjoern A. Zeeb if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) != 0) { 33498c230c8SBjoern A. Zeeb /* Our "hw"q overflew again. */ 3352c8b047cSBjoern A. Zeeb epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; 33698c230c8SBjoern A. Zeeb DPRINTF("hw queue length overflow at %u\n", 337d0ea4743SBjoern A. Zeeb epair_nh.nh_qlimit); 33898c230c8SBjoern A. Zeeb break; 33998c230c8SBjoern A. Zeeb } 34098c230c8SBjoern A. Zeeb } 341d0ea4743SBjoern A. Zeeb EPAIR_UNLOCK(epair_dpcpu); 34298c230c8SBjoern A. Zeeb } 34398c230c8SBjoern A. Zeeb 34498c230c8SBjoern A. Zeeb /* 34598c230c8SBjoern A. Zeeb * Network interface (`if') related functions. 34698c230c8SBjoern A. Zeeb */ 347eea3faf7SBjoern A. Zeeb static void 348eea3faf7SBjoern A. Zeeb epair_remove_ifp_from_draining(struct ifnet *ifp) 349eea3faf7SBjoern A. Zeeb { 350eea3faf7SBjoern A. Zeeb struct epair_dpcpu *epair_dpcpu; 351eea3faf7SBjoern A. Zeeb struct epair_ifp_drain *elm, *tvar; 352eea3faf7SBjoern A. Zeeb u_int cpuid; 353eea3faf7SBjoern A. Zeeb 3543aa6d94eSJohn Baldwin CPU_FOREACH(cpuid) { 355eea3faf7SBjoern A. Zeeb epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); 356eea3faf7SBjoern A. Zeeb EPAIR_LOCK(epair_dpcpu); 357eea3faf7SBjoern A. Zeeb STAILQ_FOREACH_SAFE(elm, &epair_dpcpu->epair_ifp_drain_list, 358eea3faf7SBjoern A. Zeeb ifp_next, tvar) { 359eea3faf7SBjoern A. Zeeb if (ifp == elm->ifp) { 36046d0f824SMatt Macy struct epair_softc *sc __unused; 361eea3faf7SBjoern A. Zeeb 362eea3faf7SBjoern A. Zeeb STAILQ_REMOVE( 363eea3faf7SBjoern A. Zeeb &epair_dpcpu->epair_ifp_drain_list, elm, 364eea3faf7SBjoern A. Zeeb epair_ifp_drain, ifp_next); 365eea3faf7SBjoern A. Zeeb /* The cached ifp goes off the list. */ 366eea3faf7SBjoern A. Zeeb sc = ifp->if_softc; 367eea3faf7SBjoern A. Zeeb EPAIR_REFCOUNT_RELEASE(&sc->refcount); 368eea3faf7SBjoern A. Zeeb EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 369eea3faf7SBjoern A. Zeeb ("%s: ifp=%p sc->refcount not >= 1: %d", 370eea3faf7SBjoern A. Zeeb __func__, ifp, sc->refcount)); 371eea3faf7SBjoern A. Zeeb free(elm, M_EPAIR); 372eea3faf7SBjoern A. Zeeb } 373eea3faf7SBjoern A. Zeeb } 374eea3faf7SBjoern A. Zeeb EPAIR_UNLOCK(epair_dpcpu); 375eea3faf7SBjoern A. Zeeb } 376eea3faf7SBjoern A. Zeeb } 377eea3faf7SBjoern A. Zeeb 378d0ea4743SBjoern A. Zeeb static int 379d0ea4743SBjoern A. Zeeb epair_add_ifp_for_draining(struct ifnet *ifp) 380d0ea4743SBjoern A. Zeeb { 381d0ea4743SBjoern A. Zeeb struct epair_dpcpu *epair_dpcpu; 382eea3faf7SBjoern A. Zeeb struct epair_softc *sc; 383d0ea4743SBjoern A. Zeeb struct epair_ifp_drain *elm = NULL; 384d0ea4743SBjoern A. Zeeb 385eea3faf7SBjoern A. Zeeb sc = ifp->if_softc; 386d0ea4743SBjoern A. Zeeb epair_dpcpu = DPCPU_ID_PTR(sc->cpuid, epair_dpcpu); 387eea3faf7SBjoern A. Zeeb EPAIR_LOCK_ASSERT(epair_dpcpu); 388d0ea4743SBjoern A. Zeeb STAILQ_FOREACH(elm, &epair_dpcpu->epair_ifp_drain_list, ifp_next) 389d0ea4743SBjoern A. Zeeb if (elm->ifp == ifp) 390d0ea4743SBjoern A. Zeeb break; 3913c20163aSBjoern A. Zeeb /* If the ifp is there already, return success. */ 392d0ea4743SBjoern A. Zeeb if (elm != NULL) 393d0ea4743SBjoern A. Zeeb return (0); 394d0ea4743SBjoern A. Zeeb 395d0ea4743SBjoern A. Zeeb elm = malloc(sizeof(struct epair_ifp_drain), M_EPAIR, M_NOWAIT|M_ZERO); 396d0ea4743SBjoern A. Zeeb if (elm == NULL) 397d0ea4743SBjoern A. Zeeb return (ENOMEM); 398d0ea4743SBjoern A. Zeeb 399d0ea4743SBjoern A. Zeeb elm->ifp = ifp; 400eea3faf7SBjoern A. Zeeb /* Add a reference for the ifp pointer on the list. */ 401eea3faf7SBjoern A. Zeeb EPAIR_REFCOUNT_AQUIRE(&sc->refcount); 402d0ea4743SBjoern A. Zeeb STAILQ_INSERT_TAIL(&epair_dpcpu->epair_ifp_drain_list, elm, ifp_next); 403d0ea4743SBjoern A. Zeeb 404d0ea4743SBjoern A. Zeeb return (0); 405d0ea4743SBjoern A. Zeeb } 406d0ea4743SBjoern A. Zeeb 40798c230c8SBjoern A. Zeeb static void 40898c230c8SBjoern A. Zeeb epair_start_locked(struct ifnet *ifp) 40998c230c8SBjoern A. Zeeb { 410d0ea4743SBjoern A. Zeeb struct epair_dpcpu *epair_dpcpu; 41198c230c8SBjoern A. Zeeb struct mbuf *m; 41298c230c8SBjoern A. Zeeb struct epair_softc *sc; 41398c230c8SBjoern A. Zeeb struct ifnet *oifp; 41498c230c8SBjoern A. Zeeb int error; 41598c230c8SBjoern A. Zeeb 41698c230c8SBjoern A. Zeeb DPRINTF("ifp=%p\n", ifp); 417d0ea4743SBjoern A. Zeeb sc = ifp->if_softc; 418d0ea4743SBjoern A. Zeeb epair_dpcpu = DPCPU_ID_PTR(sc->cpuid, epair_dpcpu); 419d0ea4743SBjoern A. Zeeb EPAIR_LOCK_ASSERT(epair_dpcpu); 42098c230c8SBjoern A. Zeeb 42198c230c8SBjoern A. Zeeb if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 42298c230c8SBjoern A. Zeeb return; 42398c230c8SBjoern A. Zeeb if ((ifp->if_flags & IFF_UP) == 0) 42498c230c8SBjoern A. Zeeb return; 42598c230c8SBjoern A. Zeeb 42698c230c8SBjoern A. Zeeb /* 42759f35a82SPatrick Kelsey * We get packets here from ether_output via if_handoff() 42875580d58SPatrick Kelsey * and need to put them into the input queue of the oifp 42998c230c8SBjoern A. Zeeb * and call oifp->if_input() via netisr/epair_sintr(). 43098c230c8SBjoern A. Zeeb */ 43198c230c8SBjoern A. Zeeb oifp = sc->oifp; 43298c230c8SBjoern A. Zeeb sc = oifp->if_softc; 43398c230c8SBjoern A. Zeeb for (;;) { 43498c230c8SBjoern A. Zeeb IFQ_DEQUEUE(&ifp->if_snd, m); 43598c230c8SBjoern A. Zeeb if (m == NULL) 43698c230c8SBjoern A. Zeeb break; 43798c230c8SBjoern A. Zeeb BPF_MTAP(ifp, m); 43898c230c8SBjoern A. Zeeb 43998c230c8SBjoern A. Zeeb /* 44098c230c8SBjoern A. Zeeb * In case the outgoing interface is not usable, 44198c230c8SBjoern A. Zeeb * drop the packet. 44298c230c8SBjoern A. Zeeb */ 44398c230c8SBjoern A. Zeeb if ((oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 44498c230c8SBjoern A. Zeeb (oifp->if_flags & IFF_UP) ==0) { 4453751dddbSGleb Smirnoff if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 44698c230c8SBjoern A. Zeeb m_freem(m); 44798c230c8SBjoern A. Zeeb continue; 44898c230c8SBjoern A. Zeeb } 44998c230c8SBjoern A. Zeeb DPRINTF("packet %s -> %s\n", ifp->if_xname, oifp->if_xname); 45098c230c8SBjoern A. Zeeb 45162d2dcafSKristof Provost epair_clear_mbuf(m); 45262d2dcafSKristof Provost 45398c230c8SBjoern A. Zeeb /* 45498c230c8SBjoern A. Zeeb * Add a reference so the interface cannot go while the 45598c230c8SBjoern A. Zeeb * packet is in transit as we rely on rcvif to stay valid. 45698c230c8SBjoern A. Zeeb */ 457d0ea4743SBjoern A. Zeeb EPAIR_REFCOUNT_AQUIRE(&sc->refcount); 45898c230c8SBjoern A. Zeeb m->m_pkthdr.rcvif = oifp; 45998c230c8SBjoern A. Zeeb CURVNET_SET_QUIET(oifp->if_vnet); 46098c230c8SBjoern A. Zeeb error = netisr_queue(NETISR_EPAIR, m); 46198c230c8SBjoern A. Zeeb CURVNET_RESTORE(); 46298c230c8SBjoern A. Zeeb if (!error) { 4633751dddbSGleb Smirnoff if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); 46498c230c8SBjoern A. Zeeb /* Someone else received the packet. */ 4653751dddbSGleb Smirnoff if_inc_counter(oifp, IFCOUNTER_IPACKETS, 1); 46698c230c8SBjoern A. Zeeb } else { 467eea3faf7SBjoern A. Zeeb /* The packet was freed already. */ 468d0ea4743SBjoern A. Zeeb epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; 46998c230c8SBjoern A. Zeeb ifp->if_drv_flags |= IFF_DRV_OACTIVE; 470eea3faf7SBjoern A. Zeeb (void) epair_add_ifp_for_draining(ifp); 4713751dddbSGleb Smirnoff if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 472d0ea4743SBjoern A. Zeeb EPAIR_REFCOUNT_RELEASE(&sc->refcount); 473eea3faf7SBjoern A. Zeeb EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 474eea3faf7SBjoern A. Zeeb ("%s: ifp=%p sc->refcount not >= 1: %d", 475eea3faf7SBjoern A. Zeeb __func__, oifp, sc->refcount)); 47698c230c8SBjoern A. Zeeb } 47798c230c8SBjoern A. Zeeb } 47898c230c8SBjoern A. Zeeb } 47998c230c8SBjoern A. Zeeb 48098c230c8SBjoern A. Zeeb static void 48198c230c8SBjoern A. Zeeb epair_start(struct ifnet *ifp) 48298c230c8SBjoern A. Zeeb { 483d0ea4743SBjoern A. Zeeb struct epair_dpcpu *epair_dpcpu; 48498c230c8SBjoern A. Zeeb 485d0ea4743SBjoern A. Zeeb epair_dpcpu = DPCPU_ID_PTR(cpuid_from_ifp(ifp), epair_dpcpu); 486d0ea4743SBjoern A. Zeeb EPAIR_LOCK(epair_dpcpu); 48798c230c8SBjoern A. Zeeb epair_start_locked(ifp); 488d0ea4743SBjoern A. Zeeb EPAIR_UNLOCK(epair_dpcpu); 48998c230c8SBjoern A. Zeeb } 49098c230c8SBjoern A. Zeeb 49198c230c8SBjoern A. Zeeb static int 49298c230c8SBjoern A. Zeeb epair_transmit_locked(struct ifnet *ifp, struct mbuf *m) 49398c230c8SBjoern A. Zeeb { 494d0ea4743SBjoern A. Zeeb struct epair_dpcpu *epair_dpcpu; 49598c230c8SBjoern A. Zeeb struct epair_softc *sc; 49698c230c8SBjoern A. Zeeb struct ifnet *oifp; 49798c230c8SBjoern A. Zeeb int error, len; 49898c230c8SBjoern A. Zeeb short mflags; 49998c230c8SBjoern A. Zeeb 50098c230c8SBjoern A. Zeeb DPRINTF("ifp=%p m=%p\n", ifp, m); 501d0ea4743SBjoern A. Zeeb sc = ifp->if_softc; 502d0ea4743SBjoern A. Zeeb epair_dpcpu = DPCPU_ID_PTR(sc->cpuid, epair_dpcpu); 503d0ea4743SBjoern A. Zeeb EPAIR_LOCK_ASSERT(epair_dpcpu); 50498c230c8SBjoern A. Zeeb 50598c230c8SBjoern A. Zeeb if (m == NULL) 50698c230c8SBjoern A. Zeeb return (0); 50798c230c8SBjoern A. Zeeb 50898c230c8SBjoern A. Zeeb /* 50998c230c8SBjoern A. Zeeb * We are not going to use the interface en/dequeue mechanism 51098c230c8SBjoern A. Zeeb * on the TX side. We are called from ether_output_frame() 51198c230c8SBjoern A. Zeeb * and will put the packet into the incoming queue of the 51298c230c8SBjoern A. Zeeb * other interface of our pair via the netsir. 51398c230c8SBjoern A. Zeeb */ 51498c230c8SBjoern A. Zeeb if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 51598c230c8SBjoern A. Zeeb m_freem(m); 51698c230c8SBjoern A. Zeeb return (ENXIO); 51798c230c8SBjoern A. Zeeb } 51898c230c8SBjoern A. Zeeb if ((ifp->if_flags & IFF_UP) == 0) { 51998c230c8SBjoern A. Zeeb m_freem(m); 52098c230c8SBjoern A. Zeeb return (ENETDOWN); 52198c230c8SBjoern A. Zeeb } 52298c230c8SBjoern A. Zeeb 52398c230c8SBjoern A. Zeeb BPF_MTAP(ifp, m); 52498c230c8SBjoern A. Zeeb 52598c230c8SBjoern A. Zeeb /* 52698c230c8SBjoern A. Zeeb * In case the outgoing interface is not usable, 52798c230c8SBjoern A. Zeeb * drop the packet. 52898c230c8SBjoern A. Zeeb */ 52998c230c8SBjoern A. Zeeb oifp = sc->oifp; 53098c230c8SBjoern A. Zeeb if ((oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 53198c230c8SBjoern A. Zeeb (oifp->if_flags & IFF_UP) ==0) { 5323751dddbSGleb Smirnoff if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 53398c230c8SBjoern A. Zeeb m_freem(m); 53498c230c8SBjoern A. Zeeb return (0); 53598c230c8SBjoern A. Zeeb } 53698c230c8SBjoern A. Zeeb len = m->m_pkthdr.len; 53798c230c8SBjoern A. Zeeb mflags = m->m_flags; 53898c230c8SBjoern A. Zeeb DPRINTF("packet %s -> %s\n", ifp->if_xname, oifp->if_xname); 53998c230c8SBjoern A. Zeeb 54098c230c8SBjoern A. Zeeb #ifdef ALTQ 541a4641f4eSPedro F. Giffuni /* Support ALTQ via the classic if_start() path. */ 54298c230c8SBjoern A. Zeeb IF_LOCK(&ifp->if_snd); 54398c230c8SBjoern A. Zeeb if (ALTQ_IS_ENABLED(&ifp->if_snd)) { 54498c230c8SBjoern A. Zeeb ALTQ_ENQUEUE(&ifp->if_snd, m, NULL, error); 54598c230c8SBjoern A. Zeeb if (error) 5463751dddbSGleb Smirnoff if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); 54798c230c8SBjoern A. Zeeb IF_UNLOCK(&ifp->if_snd); 54898c230c8SBjoern A. Zeeb if (!error) { 5493751dddbSGleb Smirnoff if_inc_counter(ifp, IFCOUNTER_OBYTES, len); 55098c230c8SBjoern A. Zeeb if (mflags & (M_BCAST|M_MCAST)) 5513751dddbSGleb Smirnoff if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); 55298c230c8SBjoern A. Zeeb 55398c230c8SBjoern A. Zeeb if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) 55498c230c8SBjoern A. Zeeb epair_start_locked(ifp); 55598c230c8SBjoern A. Zeeb else 556d0ea4743SBjoern A. Zeeb (void)epair_add_ifp_for_draining(ifp); 55798c230c8SBjoern A. Zeeb } 55898c230c8SBjoern A. Zeeb return (error); 55998c230c8SBjoern A. Zeeb } 56098c230c8SBjoern A. Zeeb IF_UNLOCK(&ifp->if_snd); 56198c230c8SBjoern A. Zeeb #endif 56298c230c8SBjoern A. Zeeb 563d0ea4743SBjoern A. Zeeb if ((epair_dpcpu->epair_drv_flags & IFF_DRV_OACTIVE) != 0) { 56498c230c8SBjoern A. Zeeb /* 56598c230c8SBjoern A. Zeeb * Our hardware queue is full, try to fall back 56698c230c8SBjoern A. Zeeb * queuing to the ifq but do not call ifp->if_start. 56798c230c8SBjoern A. Zeeb * Either we are lucky or the packet is gone. 56898c230c8SBjoern A. Zeeb */ 56998c230c8SBjoern A. Zeeb IFQ_ENQUEUE(&ifp->if_snd, m, error); 57098c230c8SBjoern A. Zeeb if (!error) 571d0ea4743SBjoern A. Zeeb (void)epair_add_ifp_for_draining(ifp); 57298c230c8SBjoern A. Zeeb return (error); 57398c230c8SBjoern A. Zeeb } 57462d2dcafSKristof Provost 57562d2dcafSKristof Provost epair_clear_mbuf(m); 57662d2dcafSKristof Provost 57798c230c8SBjoern A. Zeeb sc = oifp->if_softc; 57898c230c8SBjoern A. Zeeb /* 57998c230c8SBjoern A. Zeeb * Add a reference so the interface cannot go while the 58098c230c8SBjoern A. Zeeb * packet is in transit as we rely on rcvif to stay valid. 58198c230c8SBjoern A. Zeeb */ 582d0ea4743SBjoern A. Zeeb EPAIR_REFCOUNT_AQUIRE(&sc->refcount); 58398c230c8SBjoern A. Zeeb m->m_pkthdr.rcvif = oifp; 58498c230c8SBjoern A. Zeeb CURVNET_SET_QUIET(oifp->if_vnet); 58598c230c8SBjoern A. Zeeb error = netisr_queue(NETISR_EPAIR, m); 58698c230c8SBjoern A. Zeeb CURVNET_RESTORE(); 58798c230c8SBjoern A. Zeeb if (!error) { 5883751dddbSGleb Smirnoff if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); 58998c230c8SBjoern A. Zeeb /* 59098c230c8SBjoern A. Zeeb * IFQ_HANDOFF_ADJ/ip_handoff() update statistics, 59198c230c8SBjoern A. Zeeb * but as we bypass all this we have to duplicate 59298c230c8SBjoern A. Zeeb * the logic another time. 59398c230c8SBjoern A. Zeeb */ 5943751dddbSGleb Smirnoff if_inc_counter(ifp, IFCOUNTER_OBYTES, len); 59598c230c8SBjoern A. Zeeb if (mflags & (M_BCAST|M_MCAST)) 5963751dddbSGleb Smirnoff if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); 59798c230c8SBjoern A. Zeeb /* Someone else received the packet. */ 5983751dddbSGleb Smirnoff if_inc_counter(oifp, IFCOUNTER_IPACKETS, 1); 59998c230c8SBjoern A. Zeeb } else { 60098c230c8SBjoern A. Zeeb /* The packet was freed already. */ 601d0ea4743SBjoern A. Zeeb epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; 60298c230c8SBjoern A. Zeeb ifp->if_drv_flags |= IFF_DRV_OACTIVE; 6033751dddbSGleb Smirnoff if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 604eea3faf7SBjoern A. Zeeb EPAIR_REFCOUNT_RELEASE(&sc->refcount); 605eea3faf7SBjoern A. Zeeb EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 606eea3faf7SBjoern A. Zeeb ("%s: ifp=%p sc->refcount not >= 1: %d", 607eea3faf7SBjoern A. Zeeb __func__, oifp, sc->refcount)); 60898c230c8SBjoern A. Zeeb } 60998c230c8SBjoern A. Zeeb 61098c230c8SBjoern A. Zeeb return (error); 61198c230c8SBjoern A. Zeeb } 61298c230c8SBjoern A. Zeeb 61398c230c8SBjoern A. Zeeb static int 61498c230c8SBjoern A. Zeeb epair_transmit(struct ifnet *ifp, struct mbuf *m) 61598c230c8SBjoern A. Zeeb { 616d0ea4743SBjoern A. Zeeb struct epair_dpcpu *epair_dpcpu; 61798c230c8SBjoern A. Zeeb int error; 61898c230c8SBjoern A. Zeeb 619d0ea4743SBjoern A. Zeeb epair_dpcpu = DPCPU_ID_PTR(cpuid_from_ifp(ifp), epair_dpcpu); 620d0ea4743SBjoern A. Zeeb EPAIR_LOCK(epair_dpcpu); 62198c230c8SBjoern A. Zeeb error = epair_transmit_locked(ifp, m); 622d0ea4743SBjoern A. Zeeb EPAIR_UNLOCK(epair_dpcpu); 62398c230c8SBjoern A. Zeeb return (error); 62498c230c8SBjoern A. Zeeb } 62598c230c8SBjoern A. Zeeb 62698c230c8SBjoern A. Zeeb static void 62798c230c8SBjoern A. Zeeb epair_qflush(struct ifnet *ifp) 62898c230c8SBjoern A. Zeeb { 62998c230c8SBjoern A. Zeeb struct epair_softc *sc; 63098c230c8SBjoern A. Zeeb 63198c230c8SBjoern A. Zeeb sc = ifp->if_softc; 6327f883a9bSKristof Provost KASSERT(sc != NULL, ("%s: ifp=%p, epair_softc gone? sc=%p\n", 6337f883a9bSKristof Provost __func__, ifp, sc)); 63498c230c8SBjoern A. Zeeb /* 635eea3faf7SBjoern A. Zeeb * Remove this ifp from all backpointer lists. The interface will not 636eea3faf7SBjoern A. Zeeb * usable for flushing anyway nor should it have anything to flush 637eea3faf7SBjoern A. Zeeb * after if_qflush(). 63898c230c8SBjoern A. Zeeb */ 639eea3faf7SBjoern A. Zeeb epair_remove_ifp_from_draining(ifp); 640eea3faf7SBjoern A. Zeeb 64198c230c8SBjoern A. Zeeb if (sc->if_qflush) 64298c230c8SBjoern A. Zeeb sc->if_qflush(ifp); 64398c230c8SBjoern A. Zeeb } 64498c230c8SBjoern A. Zeeb 64598c230c8SBjoern A. Zeeb static int 6462dccdd45SMarko Zec epair_media_change(struct ifnet *ifp __unused) 6472dccdd45SMarko Zec { 6482dccdd45SMarko Zec 6492dccdd45SMarko Zec /* Do nothing. */ 6502dccdd45SMarko Zec return (0); 6512dccdd45SMarko Zec } 6522dccdd45SMarko Zec 6532dccdd45SMarko Zec static void 6542dccdd45SMarko Zec epair_media_status(struct ifnet *ifp __unused, struct ifmediareq *imr) 6552dccdd45SMarko Zec { 6562dccdd45SMarko Zec 6572dccdd45SMarko Zec imr->ifm_status = IFM_AVALID | IFM_ACTIVE; 6582dccdd45SMarko Zec imr->ifm_active = IFM_ETHER | IFM_10G_T | IFM_FDX; 6592dccdd45SMarko Zec } 6602dccdd45SMarko Zec 6612dccdd45SMarko Zec static int 66298c230c8SBjoern A. Zeeb epair_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 66398c230c8SBjoern A. Zeeb { 6642dccdd45SMarko Zec struct epair_softc *sc; 66598c230c8SBjoern A. Zeeb struct ifreq *ifr; 66698c230c8SBjoern A. Zeeb int error; 66798c230c8SBjoern A. Zeeb 66898c230c8SBjoern A. Zeeb ifr = (struct ifreq *)data; 66998c230c8SBjoern A. Zeeb switch (cmd) { 67098c230c8SBjoern A. Zeeb case SIOCSIFFLAGS: 67198c230c8SBjoern A. Zeeb case SIOCADDMULTI: 67298c230c8SBjoern A. Zeeb case SIOCDELMULTI: 67398c230c8SBjoern A. Zeeb error = 0; 67498c230c8SBjoern A. Zeeb break; 67598c230c8SBjoern A. Zeeb 6762dccdd45SMarko Zec case SIOCSIFMEDIA: 6772dccdd45SMarko Zec case SIOCGIFMEDIA: 6782dccdd45SMarko Zec sc = ifp->if_softc; 6792dccdd45SMarko Zec error = ifmedia_ioctl(ifp, ifr, &sc->media, cmd); 6802dccdd45SMarko Zec break; 6812dccdd45SMarko Zec 682d0ea4743SBjoern A. Zeeb case SIOCSIFMTU: 683d0ea4743SBjoern A. Zeeb /* We basically allow all kinds of MTUs. */ 684d0ea4743SBjoern A. Zeeb ifp->if_mtu = ifr->ifr_mtu; 685d0ea4743SBjoern A. Zeeb error = 0; 686d0ea4743SBjoern A. Zeeb break; 687d0ea4743SBjoern A. Zeeb 68898c230c8SBjoern A. Zeeb default: 68998c230c8SBjoern A. Zeeb /* Let the common ethernet handler process this. */ 69098c230c8SBjoern A. Zeeb error = ether_ioctl(ifp, cmd, data); 69198c230c8SBjoern A. Zeeb break; 69298c230c8SBjoern A. Zeeb } 69398c230c8SBjoern A. Zeeb 69498c230c8SBjoern A. Zeeb return (error); 69598c230c8SBjoern A. Zeeb } 69698c230c8SBjoern A. Zeeb 69798c230c8SBjoern A. Zeeb static void 69898c230c8SBjoern A. Zeeb epair_init(void *dummy __unused) 69998c230c8SBjoern A. Zeeb { 70098c230c8SBjoern A. Zeeb } 70198c230c8SBjoern A. Zeeb 70298c230c8SBjoern A. Zeeb /* 70398c230c8SBjoern A. Zeeb * Interface cloning functions. 70498c230c8SBjoern A. Zeeb * We use our private ones so that we can create/destroy our secondary 70598c230c8SBjoern A. Zeeb * device along with the primary one. 70698c230c8SBjoern A. Zeeb */ 70798c230c8SBjoern A. Zeeb static int 70898c230c8SBjoern A. Zeeb epair_clone_match(struct if_clone *ifc, const char *name) 70998c230c8SBjoern A. Zeeb { 71098c230c8SBjoern A. Zeeb const char *cp; 71198c230c8SBjoern A. Zeeb 71298c230c8SBjoern A. Zeeb DPRINTF("name='%s'\n", name); 71398c230c8SBjoern A. Zeeb 71498c230c8SBjoern A. Zeeb /* 71598c230c8SBjoern A. Zeeb * Our base name is epair. 71698c230c8SBjoern A. Zeeb * Our interfaces will be named epair<n>[ab]. 71798c230c8SBjoern A. Zeeb * So accept anything of the following list: 71898c230c8SBjoern A. Zeeb * - epair 71998c230c8SBjoern A. Zeeb * - epair<n> 72098c230c8SBjoern A. Zeeb * but not the epair<n>[ab] versions. 72198c230c8SBjoern A. Zeeb */ 72242a58907SGleb Smirnoff if (strncmp(epairname, name, sizeof(epairname)-1) != 0) 72398c230c8SBjoern A. Zeeb return (0); 72498c230c8SBjoern A. Zeeb 72542a58907SGleb Smirnoff for (cp = name + sizeof(epairname) - 1; *cp != '\0'; cp++) { 72698c230c8SBjoern A. Zeeb if (*cp < '0' || *cp > '9') 72798c230c8SBjoern A. Zeeb return (0); 72898c230c8SBjoern A. Zeeb } 72998c230c8SBjoern A. Zeeb 73098c230c8SBjoern A. Zeeb return (1); 73198c230c8SBjoern A. Zeeb } 73298c230c8SBjoern A. Zeeb 733b02fd8b7SKristof Provost static void 734b02fd8b7SKristof Provost epair_clone_add(struct if_clone *ifc, struct epair_softc *scb) 735b02fd8b7SKristof Provost { 736b02fd8b7SKristof Provost struct ifnet *ifp; 737b02fd8b7SKristof Provost uint8_t eaddr[ETHER_ADDR_LEN]; /* 00:00:00:00:00:00 */ 738b02fd8b7SKristof Provost 739b02fd8b7SKristof Provost ifp = scb->ifp; 740b02fd8b7SKristof Provost /* Copy epairNa etheraddr and change the last byte. */ 741b02fd8b7SKristof Provost memcpy(eaddr, scb->oifp->if_hw_addr, ETHER_ADDR_LEN); 742b02fd8b7SKristof Provost eaddr[5] = 0x0b; 743b02fd8b7SKristof Provost ether_ifattach(ifp, eaddr); 744b02fd8b7SKristof Provost 745b02fd8b7SKristof Provost if_clone_addif(ifc, ifp); 746b02fd8b7SKristof Provost } 747b02fd8b7SKristof Provost 74898c230c8SBjoern A. Zeeb static int 74998c230c8SBjoern A. Zeeb epair_clone_create(struct if_clone *ifc, char *name, size_t len, caddr_t params) 75098c230c8SBjoern A. Zeeb { 75198c230c8SBjoern A. Zeeb struct epair_softc *sca, *scb; 75298c230c8SBjoern A. Zeeb struct ifnet *ifp; 75398c230c8SBjoern A. Zeeb char *dp; 75498c230c8SBjoern A. Zeeb int error, unit, wildcard; 75511d41666SLuca Pizzamiglio uint64_t hostid; 75611d41666SLuca Pizzamiglio uint32_t key[3]; 75711d41666SLuca Pizzamiglio uint32_t hash; 75898c230c8SBjoern A. Zeeb uint8_t eaddr[ETHER_ADDR_LEN]; /* 00:00:00:00:00:00 */ 75998c230c8SBjoern A. Zeeb 76098c230c8SBjoern A. Zeeb /* Try to see if a special unit was requested. */ 76198c230c8SBjoern A. Zeeb error = ifc_name2unit(name, &unit); 76298c230c8SBjoern A. Zeeb if (error != 0) 76398c230c8SBjoern A. Zeeb return (error); 76498c230c8SBjoern A. Zeeb wildcard = (unit < 0); 76598c230c8SBjoern A. Zeeb 76698c230c8SBjoern A. Zeeb error = ifc_alloc_unit(ifc, &unit); 76798c230c8SBjoern A. Zeeb if (error != 0) 76898c230c8SBjoern A. Zeeb return (error); 76998c230c8SBjoern A. Zeeb 77098c230c8SBjoern A. Zeeb /* 77198c230c8SBjoern A. Zeeb * If no unit had been given, we need to adjust the ifName. 77298c230c8SBjoern A. Zeeb * Also make sure there is space for our extra [ab] suffix. 77398c230c8SBjoern A. Zeeb */ 77498c230c8SBjoern A. Zeeb for (dp = name; *dp != '\0'; dp++); 77598c230c8SBjoern A. Zeeb if (wildcard) { 77698c230c8SBjoern A. Zeeb error = snprintf(dp, len - (dp - name), "%d", unit); 77798c230c8SBjoern A. Zeeb if (error > len - (dp - name) - 1) { 77898c230c8SBjoern A. Zeeb /* ifName too long. */ 77998c230c8SBjoern A. Zeeb ifc_free_unit(ifc, unit); 78098c230c8SBjoern A. Zeeb return (ENOSPC); 78198c230c8SBjoern A. Zeeb } 78298c230c8SBjoern A. Zeeb dp += error; 78398c230c8SBjoern A. Zeeb } 78498c230c8SBjoern A. Zeeb if (len - (dp - name) - 1 < 1) { 78598c230c8SBjoern A. Zeeb /* No space left for our [ab] suffix. */ 78698c230c8SBjoern A. Zeeb ifc_free_unit(ifc, unit); 78798c230c8SBjoern A. Zeeb return (ENOSPC); 78898c230c8SBjoern A. Zeeb } 7893c3136b1SHiroki Sato *dp = 'b'; 79098c230c8SBjoern A. Zeeb /* Must not change dp so we can replace 'a' by 'b' later. */ 79198c230c8SBjoern A. Zeeb *(dp+1) = '\0'; 79298c230c8SBjoern A. Zeeb 7933c3136b1SHiroki Sato /* Check if 'a' and 'b' interfaces already exist. */ 7943c3136b1SHiroki Sato if (ifunit(name) != NULL) 7953c3136b1SHiroki Sato return (EEXIST); 7963c3136b1SHiroki Sato *dp = 'a'; 7973c3136b1SHiroki Sato if (ifunit(name) != NULL) 7983c3136b1SHiroki Sato return (EEXIST); 7993c3136b1SHiroki Sato 80098c230c8SBjoern A. Zeeb /* Allocate memory for both [ab] interfaces */ 80198c230c8SBjoern A. Zeeb sca = malloc(sizeof(struct epair_softc), M_EPAIR, M_WAITOK | M_ZERO); 802d0ea4743SBjoern A. Zeeb EPAIR_REFCOUNT_INIT(&sca->refcount, 1); 80398c230c8SBjoern A. Zeeb sca->ifp = if_alloc(IFT_ETHER); 80498c230c8SBjoern A. Zeeb if (sca->ifp == NULL) { 80598c230c8SBjoern A. Zeeb free(sca, M_EPAIR); 80698c230c8SBjoern A. Zeeb ifc_free_unit(ifc, unit); 80798c230c8SBjoern A. Zeeb return (ENOSPC); 80898c230c8SBjoern A. Zeeb } 80998c230c8SBjoern A. Zeeb 81098c230c8SBjoern A. Zeeb scb = malloc(sizeof(struct epair_softc), M_EPAIR, M_WAITOK | M_ZERO); 811d0ea4743SBjoern A. Zeeb EPAIR_REFCOUNT_INIT(&scb->refcount, 1); 81298c230c8SBjoern A. Zeeb scb->ifp = if_alloc(IFT_ETHER); 81398c230c8SBjoern A. Zeeb if (scb->ifp == NULL) { 81498c230c8SBjoern A. Zeeb free(scb, M_EPAIR); 81598c230c8SBjoern A. Zeeb if_free(sca->ifp); 81698c230c8SBjoern A. Zeeb free(sca, M_EPAIR); 81798c230c8SBjoern A. Zeeb ifc_free_unit(ifc, unit); 81898c230c8SBjoern A. Zeeb return (ENOSPC); 81998c230c8SBjoern A. Zeeb } 82098c230c8SBjoern A. Zeeb 82198c230c8SBjoern A. Zeeb /* 82298c230c8SBjoern A. Zeeb * Cross-reference the interfaces so we will be able to free both. 82398c230c8SBjoern A. Zeeb */ 82498c230c8SBjoern A. Zeeb sca->oifp = scb->ifp; 82598c230c8SBjoern A. Zeeb scb->oifp = sca->ifp; 82698c230c8SBjoern A. Zeeb 827d0ea4743SBjoern A. Zeeb /* 828d0ea4743SBjoern A. Zeeb * Calculate the cpuid for netisr queueing based on the 829d0ea4743SBjoern A. Zeeb * ifIndex of the interfaces. As long as we cannot configure 830d0ea4743SBjoern A. Zeeb * this or use cpuset information easily we cannot guarantee 831d0ea4743SBjoern A. Zeeb * cache locality but we can at least allow parallelism. 832d0ea4743SBjoern A. Zeeb */ 833d0ea4743SBjoern A. Zeeb sca->cpuid = 834fdf95c0bSAndrey V. Elsukov netisr_get_cpuid(sca->ifp->if_index); 835d0ea4743SBjoern A. Zeeb scb->cpuid = 836fdf95c0bSAndrey V. Elsukov netisr_get_cpuid(scb->ifp->if_index); 837d0ea4743SBjoern A. Zeeb 83818e199adSHiroki Sato /* Initialise pseudo media types. */ 83918e199adSHiroki Sato ifmedia_init(&sca->media, 0, epair_media_change, epair_media_status); 84018e199adSHiroki Sato ifmedia_add(&sca->media, IFM_ETHER | IFM_10G_T, 0, NULL); 84118e199adSHiroki Sato ifmedia_set(&sca->media, IFM_ETHER | IFM_10G_T); 84218e199adSHiroki Sato ifmedia_init(&scb->media, 0, epair_media_change, epair_media_status); 84318e199adSHiroki Sato ifmedia_add(&scb->media, IFM_ETHER | IFM_10G_T, 0, NULL); 84418e199adSHiroki Sato ifmedia_set(&scb->media, IFM_ETHER | IFM_10G_T); 84518e199adSHiroki Sato 84698c230c8SBjoern A. Zeeb /* Finish initialization of interface <n>a. */ 84798c230c8SBjoern A. Zeeb ifp = sca->ifp; 84898c230c8SBjoern A. Zeeb ifp->if_softc = sca; 84998c230c8SBjoern A. Zeeb strlcpy(ifp->if_xname, name, IFNAMSIZ); 85042a58907SGleb Smirnoff ifp->if_dname = epairname; 85198c230c8SBjoern A. Zeeb ifp->if_dunit = unit; 85298c230c8SBjoern A. Zeeb ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 8539f8cab7fSMarko Zec ifp->if_capabilities = IFCAP_VLAN_MTU; 8549f8cab7fSMarko Zec ifp->if_capenable = IFCAP_VLAN_MTU; 85598c230c8SBjoern A. Zeeb ifp->if_start = epair_start; 85698c230c8SBjoern A. Zeeb ifp->if_ioctl = epair_ioctl; 85798c230c8SBjoern A. Zeeb ifp->if_init = epair_init; 8584e950412SErmal Luçi if_setsendqlen(ifp, ifqmaxlen); 8594e950412SErmal Luçi if_setsendqready(ifp); 86011d41666SLuca Pizzamiglio 86111d41666SLuca Pizzamiglio /* 86211d41666SLuca Pizzamiglio * Calculate the etheraddr hashing the hostid and the 863804771f5SEugene Grosbein * interface index. The result would be hopefully unique. 864804771f5SEugene Grosbein * Note that the "a" component of an epair instance may get moved 865804771f5SEugene Grosbein * to a different VNET after creation. In that case its index 866804771f5SEugene Grosbein * will be freed and the index can get reused by new epair instance. 867804771f5SEugene Grosbein * Make sure we do not create same etheraddr again. 86811d41666SLuca Pizzamiglio */ 86911d41666SLuca Pizzamiglio getcredhostid(curthread->td_ucred, (unsigned long *)&hostid); 87011d41666SLuca Pizzamiglio if (hostid == 0) 87111d41666SLuca Pizzamiglio arc4rand(&hostid, sizeof(hostid), 0); 872804771f5SEugene Grosbein 873804771f5SEugene Grosbein if (ifp->if_index > next_index) 874804771f5SEugene Grosbein next_index = ifp->if_index; 875804771f5SEugene Grosbein else 876804771f5SEugene Grosbein next_index++; 877804771f5SEugene Grosbein 878804771f5SEugene Grosbein key[0] = (uint32_t)next_index; 87911d41666SLuca Pizzamiglio key[1] = (uint32_t)(hostid & 0xffffffff); 88011d41666SLuca Pizzamiglio key[2] = (uint32_t)((hostid >> 32) & 0xfffffffff); 88111d41666SLuca Pizzamiglio hash = jenkins_hash32(key, 3, 0); 88211d41666SLuca Pizzamiglio 88398c230c8SBjoern A. Zeeb eaddr[0] = 0x02; 88411d41666SLuca Pizzamiglio memcpy(&eaddr[1], &hash, 4); 88598c230c8SBjoern A. Zeeb eaddr[5] = 0x0a; 88698c230c8SBjoern A. Zeeb ether_ifattach(ifp, eaddr); 88798c230c8SBjoern A. Zeeb sca->if_qflush = ifp->if_qflush; 88898c230c8SBjoern A. Zeeb ifp->if_qflush = epair_qflush; 88998c230c8SBjoern A. Zeeb ifp->if_transmit = epair_transmit; 890b245f96cSGleb Smirnoff ifp->if_baudrate = IF_Gbps(10); /* arbitrary maximum */ 89198c230c8SBjoern A. Zeeb 89298c230c8SBjoern A. Zeeb /* Swap the name and finish initialization of interface <n>b. */ 89398c230c8SBjoern A. Zeeb *dp = 'b'; 89498c230c8SBjoern A. Zeeb 89598c230c8SBjoern A. Zeeb ifp = scb->ifp; 89698c230c8SBjoern A. Zeeb ifp->if_softc = scb; 89798c230c8SBjoern A. Zeeb strlcpy(ifp->if_xname, name, IFNAMSIZ); 89842a58907SGleb Smirnoff ifp->if_dname = epairname; 89998c230c8SBjoern A. Zeeb ifp->if_dunit = unit; 90098c230c8SBjoern A. Zeeb ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 9019f8cab7fSMarko Zec ifp->if_capabilities = IFCAP_VLAN_MTU; 9029f8cab7fSMarko Zec ifp->if_capenable = IFCAP_VLAN_MTU; 90398c230c8SBjoern A. Zeeb ifp->if_start = epair_start; 90498c230c8SBjoern A. Zeeb ifp->if_ioctl = epair_ioctl; 90598c230c8SBjoern A. Zeeb ifp->if_init = epair_init; 9064e950412SErmal Luçi if_setsendqlen(ifp, ifqmaxlen); 9074e950412SErmal Luçi if_setsendqready(ifp); 90898c230c8SBjoern A. Zeeb /* We need to play some tricks here for the second interface. */ 90942a58907SGleb Smirnoff strlcpy(name, epairname, len); 910b02fd8b7SKristof Provost 911b02fd8b7SKristof Provost /* Correctly set the name for the cloner list. */ 912b02fd8b7SKristof Provost strlcpy(name, scb->ifp->if_xname, len); 913b02fd8b7SKristof Provost epair_clone_add(ifc, scb); 914b02fd8b7SKristof Provost 91598c230c8SBjoern A. Zeeb scb->if_qflush = ifp->if_qflush; 91698c230c8SBjoern A. Zeeb ifp->if_qflush = epair_qflush; 91798c230c8SBjoern A. Zeeb ifp->if_transmit = epair_transmit; 918b245f96cSGleb Smirnoff ifp->if_baudrate = IF_Gbps(10); /* arbitrary maximum */ 91998c230c8SBjoern A. Zeeb 92098c230c8SBjoern A. Zeeb /* 92198c230c8SBjoern A. Zeeb * Restore name to <n>a as the ifp for this will go into the 92298c230c8SBjoern A. Zeeb * cloner list for the initial call. 92398c230c8SBjoern A. Zeeb */ 92498c230c8SBjoern A. Zeeb strlcpy(name, sca->ifp->if_xname, len); 92598c230c8SBjoern A. Zeeb DPRINTF("name='%s/%db' created sca=%p scb=%p\n", name, unit, sca, scb); 92698c230c8SBjoern A. Zeeb 92798c230c8SBjoern A. Zeeb /* Tell the world, that we are ready to rock. */ 92898c230c8SBjoern A. Zeeb sca->ifp->if_drv_flags |= IFF_DRV_RUNNING; 92998c230c8SBjoern A. Zeeb scb->ifp->if_drv_flags |= IFF_DRV_RUNNING; 930c7493539SBjoern A. Zeeb if_link_state_change(sca->ifp, LINK_STATE_UP); 931c7493539SBjoern A. Zeeb if_link_state_change(scb->ifp, LINK_STATE_UP); 93298c230c8SBjoern A. Zeeb 93398c230c8SBjoern A. Zeeb return (0); 93498c230c8SBjoern A. Zeeb } 93598c230c8SBjoern A. Zeeb 93698c230c8SBjoern A. Zeeb static int 93798c230c8SBjoern A. Zeeb epair_clone_destroy(struct if_clone *ifc, struct ifnet *ifp) 93898c230c8SBjoern A. Zeeb { 93998c230c8SBjoern A. Zeeb struct ifnet *oifp; 94098c230c8SBjoern A. Zeeb struct epair_softc *sca, *scb; 94198c230c8SBjoern A. Zeeb int unit, error; 94298c230c8SBjoern A. Zeeb 94398c230c8SBjoern A. Zeeb DPRINTF("ifp=%p\n", ifp); 94498c230c8SBjoern A. Zeeb 94598c230c8SBjoern A. Zeeb /* 94698c230c8SBjoern A. Zeeb * In case we called into if_clone_destroyif() ourselves 94798c230c8SBjoern A. Zeeb * again to remove the second interface, the softc will be 94898c230c8SBjoern A. Zeeb * NULL. In that case so not do anything but return success. 94998c230c8SBjoern A. Zeeb */ 95098c230c8SBjoern A. Zeeb if (ifp->if_softc == NULL) 95198c230c8SBjoern A. Zeeb return (0); 95298c230c8SBjoern A. Zeeb 95398c230c8SBjoern A. Zeeb unit = ifp->if_dunit; 95498c230c8SBjoern A. Zeeb sca = ifp->if_softc; 95598c230c8SBjoern A. Zeeb oifp = sca->oifp; 95698c230c8SBjoern A. Zeeb scb = oifp->if_softc; 95798c230c8SBjoern A. Zeeb 95898c230c8SBjoern A. Zeeb DPRINTF("ifp=%p oifp=%p\n", ifp, oifp); 959c7493539SBjoern A. Zeeb if_link_state_change(ifp, LINK_STATE_DOWN); 960c7493539SBjoern A. Zeeb if_link_state_change(oifp, LINK_STATE_DOWN); 96198c230c8SBjoern A. Zeeb ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 96298c230c8SBjoern A. Zeeb oifp->if_drv_flags &= ~IFF_DRV_RUNNING; 96398c230c8SBjoern A. Zeeb 96498c230c8SBjoern A. Zeeb /* 9657edc3d88SMikolaj Golub * Get rid of our second half. As the other of the two 9667edc3d88SMikolaj Golub * interfaces may reside in a different vnet, we need to 9677edc3d88SMikolaj Golub * switch before freeing them. 96898c230c8SBjoern A. Zeeb */ 9697edc3d88SMikolaj Golub CURVNET_SET_QUIET(oifp->if_vnet); 9707edc3d88SMikolaj Golub ether_ifdetach(oifp); 9717edc3d88SMikolaj Golub /* 9727edc3d88SMikolaj Golub * Wait for all packets to be dispatched to if_input. 9737edc3d88SMikolaj Golub * The numbers can only go down as the interface is 9747edc3d88SMikolaj Golub * detached so there is no need to use atomics. 9757edc3d88SMikolaj Golub */ 9767edc3d88SMikolaj Golub DPRINTF("scb refcnt=%u\n", scb->refcount); 9777edc3d88SMikolaj Golub EPAIR_REFCOUNT_ASSERT(scb->refcount == 1, 9787edc3d88SMikolaj Golub ("%s: ifp=%p scb->refcount!=1: %d", __func__, oifp, scb->refcount)); 97998c230c8SBjoern A. Zeeb oifp->if_softc = NULL; 98098c230c8SBjoern A. Zeeb error = if_clone_destroyif(ifc, oifp); 98198c230c8SBjoern A. Zeeb if (error) 98298c230c8SBjoern A. Zeeb panic("%s: if_clone_destroyif() for our 2nd iface failed: %d", 98398c230c8SBjoern A. Zeeb __func__, error); 98473e39d61SBjoern A. Zeeb if_free(oifp); 9852dccdd45SMarko Zec ifmedia_removeall(&scb->media); 98698c230c8SBjoern A. Zeeb free(scb, M_EPAIR); 9877edc3d88SMikolaj Golub CURVNET_RESTORE(); 9887edc3d88SMikolaj Golub 9897edc3d88SMikolaj Golub ether_ifdetach(ifp); 9907edc3d88SMikolaj Golub /* 9917edc3d88SMikolaj Golub * Wait for all packets to be dispatched to if_input. 9927edc3d88SMikolaj Golub */ 9937edc3d88SMikolaj Golub DPRINTF("sca refcnt=%u\n", sca->refcount); 9947edc3d88SMikolaj Golub EPAIR_REFCOUNT_ASSERT(sca->refcount == 1, 9957edc3d88SMikolaj Golub ("%s: ifp=%p sca->refcount!=1: %d", __func__, ifp, sca->refcount)); 9967edc3d88SMikolaj Golub if_free(ifp); 9977edc3d88SMikolaj Golub ifmedia_removeall(&sca->media); 99898c230c8SBjoern A. Zeeb free(sca, M_EPAIR); 99998c230c8SBjoern A. Zeeb ifc_free_unit(ifc, unit); 100098c230c8SBjoern A. Zeeb 100198c230c8SBjoern A. Zeeb return (0); 100298c230c8SBjoern A. Zeeb } 100398c230c8SBjoern A. Zeeb 10043c3136b1SHiroki Sato static void 10053c3136b1SHiroki Sato vnet_epair_init(const void *unused __unused) 10063c3136b1SHiroki Sato { 10073c3136b1SHiroki Sato 10083c3136b1SHiroki Sato V_epair_cloner = if_clone_advanced(epairname, 0, 10093c3136b1SHiroki Sato epair_clone_match, epair_clone_create, epair_clone_destroy); 1010484149deSBjoern A. Zeeb #ifdef VIMAGE 1011484149deSBjoern A. Zeeb netisr_register_vnet(&epair_nh); 1012484149deSBjoern A. Zeeb #endif 10133c3136b1SHiroki Sato } 101489856f7eSBjoern A. Zeeb VNET_SYSINIT(vnet_epair_init, SI_SUB_PSEUDO, SI_ORDER_ANY, 10153c3136b1SHiroki Sato vnet_epair_init, NULL); 10163c3136b1SHiroki Sato 10173c3136b1SHiroki Sato static void 10183c3136b1SHiroki Sato vnet_epair_uninit(const void *unused __unused) 10193c3136b1SHiroki Sato { 10203c3136b1SHiroki Sato 1021484149deSBjoern A. Zeeb #ifdef VIMAGE 1022484149deSBjoern A. Zeeb netisr_unregister_vnet(&epair_nh); 1023484149deSBjoern A. Zeeb #endif 10243c3136b1SHiroki Sato if_clone_detach(V_epair_cloner); 10253c3136b1SHiroki Sato } 102689856f7eSBjoern A. Zeeb VNET_SYSUNINIT(vnet_epair_uninit, SI_SUB_INIT_IF, SI_ORDER_ANY, 10273c3136b1SHiroki Sato vnet_epair_uninit, NULL); 10283c3136b1SHiroki Sato 102985f330e5SKristof Provost static void 103085f330e5SKristof Provost epair_uninit(const void *unused __unused) 103185f330e5SKristof Provost { 103285f330e5SKristof Provost netisr_unregister(&epair_nh); 103385f330e5SKristof Provost epair_dpcpu_detach(); 103485f330e5SKristof Provost if (bootverbose) 103585f330e5SKristof Provost printf("%s unloaded.\n", epairname); 103685f330e5SKristof Provost } 103785f330e5SKristof Provost SYSUNINIT(epair_uninit, SI_SUB_INIT_IF, SI_ORDER_MIDDLE, 103885f330e5SKristof Provost epair_uninit, NULL); 103985f330e5SKristof Provost 104098c230c8SBjoern A. Zeeb static int 104198c230c8SBjoern A. Zeeb epair_modevent(module_t mod, int type, void *data) 104298c230c8SBjoern A. Zeeb { 1043d0ea4743SBjoern A. Zeeb int qlimit; 104498c230c8SBjoern A. Zeeb 104598c230c8SBjoern A. Zeeb switch (type) { 104698c230c8SBjoern A. Zeeb case MOD_LOAD: 104798c230c8SBjoern A. Zeeb /* For now limit us to one global mutex and one inq. */ 1048d0ea4743SBjoern A. Zeeb epair_dpcpu_init(); 1049d0ea4743SBjoern A. Zeeb epair_nh.nh_qlimit = 42 * ifqmaxlen; /* 42 shall be the number. */ 1050d0ea4743SBjoern A. Zeeb if (TUNABLE_INT_FETCH("net.link.epair.netisr_maxqlen", &qlimit)) 1051d0ea4743SBjoern A. Zeeb epair_nh.nh_qlimit = qlimit; 1052d0ea4743SBjoern A. Zeeb netisr_register(&epair_nh); 105398c230c8SBjoern A. Zeeb if (bootverbose) 105442a58907SGleb Smirnoff printf("%s initialized.\n", epairname); 105598c230c8SBjoern A. Zeeb break; 105698c230c8SBjoern A. Zeeb case MOD_UNLOAD: 105785f330e5SKristof Provost /* Handled in epair_uninit() */ 105898c230c8SBjoern A. Zeeb break; 105998c230c8SBjoern A. Zeeb default: 106098c230c8SBjoern A. Zeeb return (EOPNOTSUPP); 106198c230c8SBjoern A. Zeeb } 106298c230c8SBjoern A. Zeeb return (0); 106398c230c8SBjoern A. Zeeb } 106498c230c8SBjoern A. Zeeb 106598c230c8SBjoern A. Zeeb static moduledata_t epair_mod = { 106698c230c8SBjoern A. Zeeb "if_epair", 106798c230c8SBjoern A. Zeeb epair_modevent, 10689823d527SKevin Lo 0 106998c230c8SBjoern A. Zeeb }; 107098c230c8SBjoern A. Zeeb 107189856f7eSBjoern A. Zeeb DECLARE_MODULE(if_epair, epair_mod, SI_SUB_PSEUDO, SI_ORDER_MIDDLE); 107298c230c8SBjoern A. Zeeb MODULE_VERSION(if_epair, 1); 1073