/*- * SPDX-License-Identifier: ISC * * Copyright (C) 2015-2021 Jason A. Donenfeld . All Rights Reserved. * Copyright (C) 2019-2021 Matt Dunwoodie * Copyright (c) 2019-2020 Rubicon Communications, LLC (Netgate) * Copyright (c) 2021 Kyle Evans * Copyright (c) 2022 The FreeBSD Foundation * Copyright (c) 2023-2024 Aaron LI * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include "opt_inet6.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* so_pru_*() functions */ #include #include /* SIOC* ioctl commands */ #include #include #include #include #include /* ETHERMTU */ #include #include #include /* IFT_WIREGUARD */ #include #include #include #include #include /* struct rtentry */ #include #include #include #include #include #include /* in6_mask2len() */ #include "wg_cookie.h" #include "wg_noise.h" #include "if_wg.h" CTASSERT(WG_KEY_SIZE >= NOISE_PUBLIC_KEY_LEN); CTASSERT(WG_KEY_SIZE >= NOISE_SYMMETRIC_KEY_LEN); #define DEFAULT_MTU (ETHERMTU - 80) #define MAX_MTU (IF_MAXMTU - 80) #ifndef ENOKEY #define ENOKEY ENOENT #endif /* * mbuf flags to clear after in-place encryption/decryption, so that the * mbuf can be reused for re-entering the network stack or delivering to * the remote peer. * * For example, the M_HASH and M_LENCHECKED flags must be cleared for an * inbound packet; otherwise, panic is to be expected. */ #define MBUF_CLEARFLAGS (M_COPYFLAGS & ~(M_PKTHDR | M_EOR | M_PRIO)) #define MAX_LOOPS 8 /* 0 means no loop allowed */ #define MTAG_WGLOOP 0x77676c70 /* wglp; cookie for loop check */ #define MAX_STAGED_PKT 128 #define MAX_QUEUED_PKT 1024 #define MAX_QUEUED_PKT_MASK (MAX_QUEUED_PKT - 1) #define MAX_QUEUED_HANDSHAKES 4096 #define REKEY_TIMEOUT_JITTER (karc4random() % 334) /* msec */ #define MAX_TIMER_HANDSHAKES (90 / REKEY_TIMEOUT) #define NEW_HANDSHAKE_TIMEOUT (REKEY_TIMEOUT + KEEPALIVE_TIMEOUT) #define UNDERLOAD_TIMEOUT 1 /* First byte indicating packet type on the wire */ #define WG_PKT_INITIATION htole32(1) #define WG_PKT_RESPONSE htole32(2) #define WG_PKT_COOKIE htole32(3) #define WG_PKT_DATA htole32(4) #define WG_PKT_PADDING 16 #define WG_PKT_WITH_PADDING(n) \ (((n) + (WG_PKT_PADDING - 1)) & ~(WG_PKT_PADDING - 1)) #define WG_PKT_ENCRYPTED_LEN(n) \ (offsetof(struct wg_pkt_data, buf[(n)]) + NOISE_AUTHTAG_LEN) #define WG_PKT_IS_INITIATION(m) \ (*mtod((m), uint32_t *) == WG_PKT_INITIATION && \ (size_t)(m)->m_pkthdr.len == sizeof(struct wg_pkt_initiation)) #define WG_PKT_IS_RESPONSE(m) \ (*mtod((m), uint32_t *) == WG_PKT_RESPONSE && \ (size_t)(m)->m_pkthdr.len == sizeof(struct wg_pkt_response)) #define WG_PKT_IS_COOKIE(m) \ (*mtod((m), uint32_t *) == WG_PKT_COOKIE && \ (size_t)(m)->m_pkthdr.len == sizeof(struct wg_pkt_cookie)) #define WG_PKT_IS_DATA(m) \ (*mtod((m), uint32_t *) == WG_PKT_DATA && \ (size_t)(m)->m_pkthdr.len >= WG_PKT_ENCRYPTED_LEN(0)) #define DPRINTF(sc, fmt, ...) \ if (sc->sc_ifp->if_flags & IFF_DEBUG) \ if_printf(sc->sc_ifp, fmt, ##__VA_ARGS__) struct wg_pkt_initiation { uint32_t t; uint32_t s_idx; uint8_t ue[NOISE_PUBLIC_KEY_LEN]; uint8_t es[NOISE_PUBLIC_KEY_LEN + NOISE_AUTHTAG_LEN]; uint8_t ets[NOISE_TIMESTAMP_LEN + NOISE_AUTHTAG_LEN]; struct cookie_macs m; }; struct wg_pkt_response { uint32_t t; uint32_t s_idx; uint32_t r_idx; uint8_t ue[NOISE_PUBLIC_KEY_LEN]; uint8_t en[0 + NOISE_AUTHTAG_LEN]; struct cookie_macs m; }; struct wg_pkt_cookie { uint32_t t; uint32_t r_idx; uint8_t nonce[COOKIE_NONCE_SIZE]; uint8_t ec[COOKIE_ENCRYPTED_SIZE]; }; struct wg_pkt_data { uint32_t t; uint32_t r_idx; uint64_t counter; uint8_t buf[]; }; struct wg_endpoint { union { struct sockaddr r_sa; struct sockaddr_in r_sin; #ifdef INET6 struct sockaddr_in6 r_sin6; #endif } e_remote; /* * NOTE: No 'e_local' on DragonFly, because the socket upcall * and so_pru_soreceive() cannot provide the local * (i.e., destination) address of a received packet. */ }; struct aip_addr { uint8_t length; /* required by the radix code */ union { uint8_t bytes[16]; uint32_t ip; uint32_t ip6[4]; struct in_addr in; struct in6_addr in6; }; }; struct wg_aip { struct radix_node a_nodes[2]; /* make the first for casting */ LIST_ENTRY(wg_aip) a_entry; struct aip_addr a_addr; struct aip_addr a_mask; struct wg_peer *a_peer; sa_family_t a_af; }; enum wg_packet_state { WG_PACKET_DEAD, /* to be dropped */ WG_PACKET_UNCRYPTED, /* before encryption/decryption */ WG_PACKET_CRYPTED, /* after encryption/decryption */ }; struct wg_packet { STAILQ_ENTRY(wg_packet) p_serial; STAILQ_ENTRY(wg_packet) p_parallel; struct wg_endpoint p_endpoint; struct noise_keypair *p_keypair; uint64_t p_counter; struct mbuf *p_mbuf; int p_mtu; sa_family_t p_af; unsigned int p_state; /* atomic */ }; STAILQ_HEAD(wg_packet_list, wg_packet); struct wg_queue { struct lock q_mtx; struct wg_packet_list q_queue; size_t q_len; }; struct wg_peer { TAILQ_ENTRY(wg_peer) p_entry; unsigned long p_id; struct wg_softc *p_sc; char p_description[WG_PEER_DESCR_SIZE]; struct noise_remote *p_remote; struct cookie_maker *p_cookie; struct lock p_endpoint_lock; struct wg_endpoint p_endpoint; struct wg_queue p_stage_queue; struct wg_queue p_encrypt_serial; struct wg_queue p_decrypt_serial; bool p_enabled; bool p_need_another_keepalive; uint16_t p_persistent_keepalive_interval; struct callout p_new_handshake; struct callout p_send_keepalive; struct callout p_retry_handshake; struct callout p_zero_key_material; struct callout p_persistent_keepalive; struct lock p_handshake_mtx; struct timespec p_handshake_complete; /* nanotime */ int p_handshake_retries; struct task p_send_task; struct task p_recv_task; struct taskqueue *p_send_taskqueue; struct taskqueue *p_recv_taskqueue; uint64_t *p_tx_bytes; uint64_t *p_rx_bytes; LIST_HEAD(, wg_aip) p_aips; size_t p_aips_num; }; struct wg_socket { struct lock so_lock; struct socket *so_so4; struct socket *so_so6; uint32_t so_user_cookie; in_port_t so_port; }; struct wg_softc { LIST_ENTRY(wg_softc) sc_entry; struct ifnet *sc_ifp; int sc_flags; struct wg_socket sc_socket; TAILQ_HEAD(, wg_peer) sc_peers; size_t sc_peers_num; struct noise_local *sc_local; struct cookie_checker *sc_cookie; struct lock sc_aip_lock; struct radix_node_head *sc_aip4; struct radix_node_head *sc_aip6; struct taskqueue *sc_handshake_taskqueue; struct task sc_handshake_task; struct wg_queue sc_handshake_queue; struct task *sc_encrypt_tasks; /* one per CPU */ struct task *sc_decrypt_tasks; /* one per CPU */ struct wg_queue sc_encrypt_parallel; struct wg_queue sc_decrypt_parallel; int sc_encrypt_last_cpu; int sc_decrypt_last_cpu; struct lock sc_lock; }; static MALLOC_DEFINE(M_WG, "WG", "wireguard"); static MALLOC_DEFINE(M_WG_PACKET, "WG packet", "wireguard packet"); static const char wgname[] = "wg"; static struct objcache *wg_packet_zone; static struct lock wg_mtx; static struct taskqueue **wg_taskqueues; /* one taskqueue per CPU */ static struct radix_node_head *wg_maskhead; /* shared by all interfaces */ static LIST_HEAD(, wg_softc) wg_list = LIST_HEAD_INITIALIZER(wg_list); /* Timers */ static void wg_timers_enable(struct wg_peer *); static void wg_timers_disable(struct wg_peer *); /* Allowed IP */ static int wg_aip_add(struct wg_softc *, struct wg_peer *, sa_family_t, const void *, uint8_t); static struct wg_peer * wg_aip_lookup(struct wg_softc *, sa_family_t, const void *); static void wg_aip_remove_all(struct wg_softc *, struct wg_peer *); /* Handshake */ static void wg_send_initiation(struct wg_peer *); static void wg_send_response(struct wg_peer *); static void wg_send_cookie(struct wg_softc *, struct cookie_macs *, uint32_t, struct wg_endpoint *); static void wg_send_keepalive(struct wg_peer *); /* Transport Packet Functions */ static void wg_peer_send_staged(struct wg_peer *); static void wg_deliver_out(void *, int); static void wg_deliver_in(void *, int); static void wg_upcall(struct socket *, void *, int); /*----------------------------------------------------------------------------*/ /* Packet */ static struct wg_packet * wg_packet_alloc(struct mbuf *m) { struct wg_packet *pkt; if ((pkt = objcache_get(wg_packet_zone, M_NOWAIT)) == NULL) return (NULL); bzero(pkt, sizeof(*pkt)); /* objcache_get() doesn't ensure M_ZERO. */ pkt->p_mbuf = m; return (pkt); } static void wg_packet_free(struct wg_packet *pkt) { if (pkt->p_keypair != NULL) noise_keypair_put(pkt->p_keypair); if (pkt->p_mbuf != NULL) m_freem(pkt->p_mbuf); objcache_put(wg_packet_zone, pkt); } /*----------------------------------------------------------------------------*/ /* * Packet Queue Functions * * WireGuard uses the following queues: * - per-interface handshake queue: track incoming handshake packets * - per-peer staged queue: track the outgoing packets sent by that peer * - per-interface parallel encrypt and decrypt queues * - per-peer serial encrypt and decrypt queues * * For one interface, the handshake packets are only tracked in the handshake * queue and are processed in serial. However, all data packets are tracked * in two queues: a serial queue and a parallel queue. Specifically, the * outgoing packets (from the staged queue) will be queued in both the * parallel encrypt and the serial encrypt queues; the incoming packets will * be queued in both the parallel decrypt and the serial decrypt queues. * * - The parallel queues are used to distribute the encryption/decryption work * across all CPUs. The per-CPU wg_{encrypt,decrypt}_worker() work on the * parallel queues. * - The serial queues ensure that packets are not reordered and are * delivered in sequence for each peer. The per-peer wg_deliver_{in,out}() * work on the serial queues. */ static void wg_queue_purge(struct wg_queue *); static void wg_queue_init(struct wg_queue *queue, const char *name) { lockinit(&queue->q_mtx, name, 0, 0); STAILQ_INIT(&queue->q_queue); queue->q_len = 0; } static void wg_queue_deinit(struct wg_queue *queue) { wg_queue_purge(queue); lockuninit(&queue->q_mtx); } static size_t wg_queue_len(const struct wg_queue *queue) { return (queue->q_len); } static bool wg_queue_enqueue_handshake(struct wg_queue *hs, struct wg_packet *pkt) { bool ok = false; lockmgr(&hs->q_mtx, LK_EXCLUSIVE); if (hs->q_len < MAX_QUEUED_HANDSHAKES) { STAILQ_INSERT_TAIL(&hs->q_queue, pkt, p_parallel); hs->q_len++; ok = true; } lockmgr(&hs->q_mtx, LK_RELEASE); if (!ok) wg_packet_free(pkt); return (ok); } static struct wg_packet * wg_queue_dequeue_handshake(struct wg_queue *hs) { struct wg_packet *pkt; lockmgr(&hs->q_mtx, LK_EXCLUSIVE); pkt = STAILQ_FIRST(&hs->q_queue); if (pkt != NULL) { STAILQ_REMOVE_HEAD(&hs->q_queue, p_parallel); hs->q_len--; } lockmgr(&hs->q_mtx, LK_RELEASE); return (pkt); } static void wg_queue_push_staged(struct wg_queue *staged, struct wg_packet *pkt) { struct wg_packet *old = NULL; lockmgr(&staged->q_mtx, LK_EXCLUSIVE); if (staged->q_len >= MAX_STAGED_PKT) { old = STAILQ_FIRST(&staged->q_queue); STAILQ_REMOVE_HEAD(&staged->q_queue, p_parallel); staged->q_len--; } STAILQ_INSERT_TAIL(&staged->q_queue, pkt, p_parallel); staged->q_len++; lockmgr(&staged->q_mtx, LK_RELEASE); if (old != NULL) wg_packet_free(old); } static void wg_queue_enlist_staged(struct wg_queue *staged, struct wg_packet_list *list) { struct wg_packet *pkt, *tpkt; STAILQ_FOREACH_MUTABLE(pkt, list, p_parallel, tpkt) wg_queue_push_staged(staged, pkt); STAILQ_INIT(list); } static void wg_queue_delist_staged(struct wg_queue *staged, struct wg_packet_list *list) { STAILQ_INIT(list); lockmgr(&staged->q_mtx, LK_EXCLUSIVE); STAILQ_CONCAT(list, &staged->q_queue); staged->q_len = 0; lockmgr(&staged->q_mtx, LK_RELEASE); } static void wg_queue_purge(struct wg_queue *staged) { struct wg_packet_list list; struct wg_packet *pkt, *tpkt; wg_queue_delist_staged(staged, &list); STAILQ_FOREACH_MUTABLE(pkt, &list, p_parallel, tpkt) wg_packet_free(pkt); } static bool wg_queue_both(struct wg_queue *parallel, struct wg_queue *serial, struct wg_packet *pkt) { pkt->p_state = WG_PACKET_UNCRYPTED; lockmgr(&serial->q_mtx, LK_EXCLUSIVE); if (serial->q_len < MAX_QUEUED_PKT) { serial->q_len++; STAILQ_INSERT_TAIL(&serial->q_queue, pkt, p_serial); } else { lockmgr(&serial->q_mtx, LK_RELEASE); wg_packet_free(pkt); return (false); } lockmgr(&serial->q_mtx, LK_RELEASE); lockmgr(¶llel->q_mtx, LK_EXCLUSIVE); if (parallel->q_len < MAX_QUEUED_PKT) { parallel->q_len++; STAILQ_INSERT_TAIL(¶llel->q_queue, pkt, p_parallel); } else { lockmgr(¶llel->q_mtx, LK_RELEASE); /* * Cannot just free the packet because it's already queued * in the serial queue. Instead, set its state to DEAD and * let the serial worker to free it. */ pkt->p_state = WG_PACKET_DEAD; return (false); } lockmgr(¶llel->q_mtx, LK_RELEASE); return (true); } static struct wg_packet * wg_queue_dequeue_serial(struct wg_queue *serial) { struct wg_packet *pkt = NULL; lockmgr(&serial->q_mtx, LK_EXCLUSIVE); if (serial->q_len > 0 && STAILQ_FIRST(&serial->q_queue)->p_state != WG_PACKET_UNCRYPTED) { /* * Dequeue both CRYPTED packets (to be delivered) and * DEAD packets (to be freed). */ serial->q_len--; pkt = STAILQ_FIRST(&serial->q_queue); STAILQ_REMOVE_HEAD(&serial->q_queue, p_serial); } lockmgr(&serial->q_mtx, LK_RELEASE); return (pkt); } static struct wg_packet * wg_queue_dequeue_parallel(struct wg_queue *parallel) { struct wg_packet *pkt = NULL; lockmgr(¶llel->q_mtx, LK_EXCLUSIVE); if (parallel->q_len > 0) { parallel->q_len--; pkt = STAILQ_FIRST(¶llel->q_queue); STAILQ_REMOVE_HEAD(¶llel->q_queue, p_parallel); } lockmgr(¶llel->q_mtx, LK_RELEASE); return (pkt); } /*----------------------------------------------------------------------------*/ /* Peer */ static struct wg_peer * wg_peer_create(struct wg_softc *sc, const uint8_t pub_key[WG_KEY_SIZE]) { static unsigned long peer_counter = 0; struct wg_peer *peer; KKASSERT(lockstatus(&sc->sc_lock, curthread) == LK_EXCLUSIVE); peer = kmalloc(sizeof(*peer), M_WG, M_WAITOK | M_ZERO); peer->p_remote = noise_remote_alloc(sc->sc_local, pub_key, peer); if (noise_remote_enable(peer->p_remote) != 0) { kfree(peer, M_WG); return (NULL); } peer->p_cookie = cookie_maker_alloc(pub_key); peer->p_id = ++peer_counter; peer->p_sc = sc; peer->p_tx_bytes = kmalloc(sizeof(*peer->p_tx_bytes) * ncpus, M_WG, M_WAITOK | M_ZERO); peer->p_rx_bytes = kmalloc(sizeof(*peer->p_rx_bytes) * ncpus, M_WG, M_WAITOK | M_ZERO); lockinit(&peer->p_endpoint_lock, "wg_peer_endpoint", 0, 0); lockinit(&peer->p_handshake_mtx, "wg_peer_handshake", 0, 0); wg_queue_init(&peer->p_stage_queue, "stageq"); wg_queue_init(&peer->p_encrypt_serial, "txq"); wg_queue_init(&peer->p_decrypt_serial, "rxq"); callout_init_mp(&peer->p_new_handshake); callout_init_mp(&peer->p_send_keepalive); callout_init_mp(&peer->p_retry_handshake); callout_init_mp(&peer->p_persistent_keepalive); callout_init_mp(&peer->p_zero_key_material); TASK_INIT(&peer->p_send_task, 0, wg_deliver_out, peer); TASK_INIT(&peer->p_recv_task, 0, wg_deliver_in, peer); /* Randomly choose the taskqueues to distribute the load. */ peer->p_send_taskqueue = wg_taskqueues[karc4random() % ncpus]; peer->p_recv_taskqueue = wg_taskqueues[karc4random() % ncpus]; LIST_INIT(&peer->p_aips); TAILQ_INSERT_TAIL(&sc->sc_peers, peer, p_entry); sc->sc_peers_num++; if (sc->sc_ifp->if_link_state == LINK_STATE_UP) wg_timers_enable(peer); DPRINTF(sc, "Peer %ld created\n", peer->p_id); return (peer); } static void wg_peer_destroy(struct wg_peer *peer) { struct wg_softc *sc = peer->p_sc; KKASSERT(lockstatus(&sc->sc_lock, curthread) == LK_EXCLUSIVE); /* * Disable remote and timers. This will prevent any new handshakes * from occuring. */ noise_remote_disable(peer->p_remote); wg_timers_disable(peer); /* * Remove all allowed IPs, so no more packets will be routed to * this peer. */ wg_aip_remove_all(sc, peer); /* Remove peer from the interface, then free. */ sc->sc_peers_num--; TAILQ_REMOVE(&sc->sc_peers, peer, p_entry); /* * While there are no references remaining, we may still have * p_{send,recv}_task executing (think empty queue, but * wg_deliver_{in,out} needs to check the queue). We should wait * for them and then free. */ taskqueue_drain(peer->p_recv_taskqueue, &peer->p_recv_task); taskqueue_drain(peer->p_send_taskqueue, &peer->p_send_task); wg_queue_deinit(&peer->p_decrypt_serial); wg_queue_deinit(&peer->p_encrypt_serial); wg_queue_deinit(&peer->p_stage_queue); kfree(peer->p_tx_bytes, M_WG); kfree(peer->p_rx_bytes, M_WG); lockuninit(&peer->p_endpoint_lock); lockuninit(&peer->p_handshake_mtx); noise_remote_free(peer->p_remote); cookie_maker_free(peer->p_cookie); DPRINTF(sc, "Peer %ld destroyed\n", peer->p_id); kfree(peer, M_WG); } static void wg_peer_destroy_all(struct wg_softc *sc) { struct wg_peer *peer, *tpeer; TAILQ_FOREACH_MUTABLE(peer, &sc->sc_peers, p_entry, tpeer) wg_peer_destroy(peer); } static int wg_peer_set_sockaddr(struct wg_peer *peer, const struct sockaddr *remote) { int ret = 0; lockmgr(&peer->p_endpoint_lock, LK_EXCLUSIVE); memcpy(&peer->p_endpoint.e_remote, remote, sizeof(peer->p_endpoint.e_remote)); if (remote->sa_family == AF_INET) memcpy(&peer->p_endpoint.e_remote.r_sin, remote, sizeof(peer->p_endpoint.e_remote.r_sin)); #ifdef INET6 else if (remote->sa_family == AF_INET6) memcpy(&peer->p_endpoint.e_remote.r_sin6, remote, sizeof(peer->p_endpoint.e_remote.r_sin6)); #endif else ret = EAFNOSUPPORT; /* No 'e_local' to clear on DragonFly. */ lockmgr(&peer->p_endpoint_lock, LK_RELEASE); return (ret); } static int wg_peer_get_sockaddr(struct wg_peer *peer, struct sockaddr *remote) { int ret = ENOENT; lockmgr(&peer->p_endpoint_lock, LK_SHARED); if (peer->p_endpoint.e_remote.r_sa.sa_family != AF_UNSPEC) { memcpy(remote, &peer->p_endpoint.e_remote, sizeof(peer->p_endpoint.e_remote)); ret = 0; } lockmgr(&peer->p_endpoint_lock, LK_RELEASE); return (ret); } static void wg_peer_set_endpoint(struct wg_peer *peer, const struct wg_endpoint *e) { KKASSERT(e->e_remote.r_sa.sa_family != AF_UNSPEC); if (__predict_true(memcmp(e, &peer->p_endpoint, sizeof(*e)) == 0)) return; lockmgr(&peer->p_endpoint_lock, LK_EXCLUSIVE); peer->p_endpoint = *e; lockmgr(&peer->p_endpoint_lock, LK_RELEASE); } static void wg_peer_get_endpoint(struct wg_peer *peer, struct wg_endpoint *e) { if (__predict_true(memcmp(e, &peer->p_endpoint, sizeof(*e)) == 0)) return; lockmgr(&peer->p_endpoint_lock, LK_SHARED); *e = peer->p_endpoint; lockmgr(&peer->p_endpoint_lock, LK_RELEASE); } /*----------------------------------------------------------------------------*/ /* Allowed IP */ static int wg_aip_add(struct wg_softc *sc, struct wg_peer *peer, sa_family_t af, const void *addr, uint8_t cidr) { struct radix_node_head *head; struct radix_node *node; struct wg_aip *aip; int ret = 0; aip = kmalloc(sizeof(*aip), M_WG, M_WAITOK | M_ZERO); aip->a_peer = peer; aip->a_af = af; switch (af) { case AF_INET: if (cidr > 32) cidr = 32; head = sc->sc_aip4; aip->a_addr.in = *(const struct in_addr *)addr; aip->a_mask.ip = htonl(~((1LL << (32 - cidr)) - 1) & 0xffffffff); aip->a_addr.ip &= aip->a_mask.ip; aip->a_addr.length = aip->a_mask.length = offsetof(struct aip_addr, in) + sizeof(struct in_addr); break; #ifdef INET6 case AF_INET6: { int i; if (cidr > 128) cidr = 128; head = sc->sc_aip6; aip->a_addr.in6 = *(const struct in6_addr *)addr; in6_prefixlen2mask(&aip->a_mask.in6, cidr); for (i = 0; i < 4; i++) aip->a_addr.ip6[i] &= aip->a_mask.ip6[i]; aip->a_addr.length = aip->a_mask.length = offsetof(struct aip_addr, in6) + sizeof(struct in6_addr); break; } #endif default: kfree(aip, M_WG); return (EAFNOSUPPORT); } lockmgr(&sc->sc_aip_lock, LK_EXCLUSIVE); node = head->rnh_addaddr(&aip->a_addr, &aip->a_mask, head, aip->a_nodes); if (node != NULL) { KKASSERT(node == aip->a_nodes); LIST_INSERT_HEAD(&peer->p_aips, aip, a_entry); peer->p_aips_num++; } else { /* * Two possibilities: * - out of memory failure * - entry already exists */ node = head->rnh_lookup(&aip->a_addr, &aip->a_mask, head); if (node == NULL) { kfree(aip, M_WG); ret = ENOMEM; } else { KKASSERT(node != aip->a_nodes); kfree(aip, M_WG); aip = (struct wg_aip *)node; if (aip->a_peer != peer) { /* Replace the peer. */ LIST_REMOVE(aip, a_entry); aip->a_peer->p_aips_num--; aip->a_peer = peer; LIST_INSERT_HEAD(&peer->p_aips, aip, a_entry); aip->a_peer->p_aips_num++; } } } lockmgr(&sc->sc_aip_lock, LK_RELEASE); return (ret); } static struct wg_peer * wg_aip_lookup(struct wg_softc *sc, sa_family_t af, const void *a) { struct radix_node_head *head; struct radix_node *node; struct wg_peer *peer; struct aip_addr addr; switch (af) { case AF_INET: head = sc->sc_aip4; memcpy(&addr.in, a, sizeof(addr.in)); addr.length = offsetof(struct aip_addr, in) + sizeof(addr.in); break; case AF_INET6: head = sc->sc_aip6; memcpy(&addr.in6, a, sizeof(addr.in6)); addr.length = offsetof(struct aip_addr, in6) + sizeof(addr.in6); break; default: return (NULL); } lockmgr(&sc->sc_aip_lock, LK_SHARED); node = head->rnh_matchaddr(&addr, head); if (node != NULL) { peer = ((struct wg_aip *)node)->a_peer; noise_remote_ref(peer->p_remote); } else { peer = NULL; } lockmgr(&sc->sc_aip_lock, LK_RELEASE); return (peer); } static void wg_aip_remove_all(struct wg_softc *sc, struct wg_peer *peer) { struct radix_node_head *head; struct radix_node *node; struct wg_aip *aip, *taip; lockmgr(&sc->sc_aip_lock, LK_EXCLUSIVE); LIST_FOREACH_MUTABLE(aip, &peer->p_aips, a_entry, taip) { switch (aip->a_af) { case AF_INET: head = sc->sc_aip4; break; case AF_INET6: head = sc->sc_aip6; break; default: panic("%s: impossible aip %p", __func__, aip); } node = head->rnh_deladdr(&aip->a_addr, &aip->a_mask, head); if (node == NULL) panic("%s: failed to delete aip %p", __func__, aip); LIST_REMOVE(aip, a_entry); peer->p_aips_num--; kfree(aip, M_WG); } if (!LIST_EMPTY(&peer->p_aips) || peer->p_aips_num != 0) panic("%s: could not delete all aips for peer %ld", __func__, peer->p_id); lockmgr(&sc->sc_aip_lock, LK_RELEASE); } /*----------------------------------------------------------------------------*/ /* Socket */ static int wg_socket_open(struct socket **, sa_family_t, in_port_t *, void *); static int wg_socket_set_sockopt(struct socket *, struct socket *, int, void *, size_t); static int wg_socket_init(struct wg_softc *sc, in_port_t port) { struct wg_socket *so = &sc->sc_socket; struct socket *so4 = NULL, *so6 = NULL; in_port_t bound_port = port; uint32_t cookie; int ret; /* * When a host or a jail doesn't support the AF, sobind() would * return EADDRNOTAVAIL. Handle this case in order to support such * IPv4-only or IPv6-only environments. * * However, in a dual-stack environment, both IPv4 and IPv6 sockets * must bind the same port. */ ret = wg_socket_open(&so4, AF_INET, &bound_port, sc); if (ret != 0 && ret != EADDRNOTAVAIL) goto error; #ifdef INET6 ret = wg_socket_open(&so6, AF_INET6, &bound_port, sc); if (ret != 0 && ret != EADDRNOTAVAIL) goto error; #endif if (so4 == NULL && so6 == NULL) { ret = EAFNOSUPPORT; goto error; } cookie = so->so_user_cookie; if (cookie != 0) { ret = wg_socket_set_sockopt(so4, so6, SO_USER_COOKIE, &cookie, sizeof(cookie)); if (ret != 0) goto error; } KKASSERT(lockstatus(&sc->sc_lock, curthread) == LK_EXCLUSIVE); lockinit(&so->so_lock, "wg socket lock", 0, 0); if (so->so_so4 != NULL) soclose(so->so_so4, 0); if (so->so_so6 != NULL) soclose(so->so_so6, 0); so->so_so4 = so4; so->so_so6 = so6; so->so_port = bound_port; return (0); error: if (so4 != NULL) soclose(so4, 0); if (so6 != NULL) soclose(so6, 0); return (ret); } static int wg_socket_open(struct socket **so, sa_family_t af, in_port_t *port, void *upcall_arg) { struct sockaddr_in sin; #ifdef INET6 struct sockaddr_in6 sin6; #endif struct sockaddr *sa, *bound_sa; int ret; if (af == AF_INET) { bzero(&sin, sizeof(sin)); sin.sin_len = sizeof(struct sockaddr_in); sin.sin_family = AF_INET; sin.sin_port = htons(*port); sa = sintosa(&sin); #ifdef INET6 } else if (af == AF_INET6) { bzero(&sin6, sizeof(sin6)); sin6.sin6_len = sizeof(struct sockaddr_in6); sin6.sin6_family = AF_INET6; sin6.sin6_port = htons(*port); sa = sintosa(&sin6); #endif } else { return (EAFNOSUPPORT); } ret = socreate(af, so, SOCK_DGRAM, IPPROTO_UDP, curthread); if (ret != 0) return (ret); (*so)->so_upcall = wg_upcall; (*so)->so_upcallarg = upcall_arg; atomic_set_int(&(*so)->so_rcv.ssb_flags, SSB_UPCALL); ret = sobind(*so, sa, curthread); if (ret != 0) goto error; if (*port == 0) { ret = so_pru_sockaddr(*so, &bound_sa); if (ret != 0) goto error; if (bound_sa->sa_family == AF_INET) *port = ntohs(satosin(bound_sa)->sin_port); else *port = ntohs(satosin6(bound_sa)->sin6_port); kfree(bound_sa, M_SONAME); } return (0); error: if (*so != NULL) { soclose(*so, 0); *so = NULL; } return (ret); } static void wg_socket_uninit(struct wg_softc *sc) { struct wg_socket *so = &sc->sc_socket; KKASSERT(lockstatus(&sc->sc_lock, curthread) == LK_EXCLUSIVE); lockmgr(&so->so_lock, LK_EXCLUSIVE); if (so->so_so4 != NULL) { soclose(so->so_so4, 0); so->so_so4 = NULL; } if (so->so_so6 != NULL) { soclose(so->so_so6, 0); so->so_so6 = NULL; } lockmgr(&so->so_lock, LK_RELEASE); lockuninit(&so->so_lock); } static int wg_socket_set_sockopt(struct socket *so4, struct socket *so6, int name, void *val, size_t len) { struct sockopt sopt = { .sopt_dir = SOPT_SET, .sopt_level = SOL_SOCKET, .sopt_name = name, .sopt_val = val, .sopt_valsize = len, }; int ret; if (so4 != NULL) { ret = sosetopt(so4, &sopt); if (ret != 0) return (ret); } if (so6 != NULL) { ret = sosetopt(so6, &sopt); if (ret != 0) return (ret); } return (0); } static int wg_socket_set_cookie(struct wg_softc *sc, uint32_t user_cookie) { struct wg_socket *so; int ret; KKASSERT(lockstatus(&sc->sc_lock, curthread) == LK_EXCLUSIVE); so = &sc->sc_socket; lockmgr(&so->so_lock, LK_EXCLUSIVE); ret = wg_socket_set_sockopt(so->so_so4, so->so_so6, SO_USER_COOKIE, &user_cookie, sizeof(user_cookie)); if (ret == 0) so->so_user_cookie = user_cookie; lockmgr(&so->so_lock, LK_RELEASE); return (ret); } static int wg_send(struct wg_softc *sc, struct wg_endpoint *e, struct mbuf *m) { struct wg_socket *so; struct sockaddr *sa; int len, ret; so = &sc->sc_socket; sa = &e->e_remote.r_sa; len = m->m_pkthdr.len; ret = 0; /* * NOTE: DragonFly by default sends UDP packets asynchronously, * unless the 'net.inet.udp.sosend_async' sysctl MIB is set * to 0 or the 'MSG_SYNC' flag is set for so_pru_sosend(). * And in the async mode, an error code cannot really be * replied to the caller. So so_pru_sosend() may return 0 * even if the packet fails to send. */ lockmgr(&so->so_lock, LK_SHARED); if (sa->sa_family == AF_INET && so->so_so4 != NULL) { ret = so_pru_sosend(so->so_so4, sa, NULL /* uio */, m, NULL /* control */, 0 /* flags */, curthread); #ifdef INET6 } else if (sa->sa_family == AF_INET6 && so->so_so6 != NULL) { ret = so_pru_sosend(so->so_so6, sa, NULL /* uio */, m, NULL /* control */, 0 /* flags */, curthread); #endif } else { ret = ENOTCONN; m_freem(m); } lockmgr(&so->so_lock, LK_RELEASE); if (ret == 0) { IFNET_STAT_INC(sc->sc_ifp, opackets, 1); IFNET_STAT_INC(sc->sc_ifp, obytes, len); } else { IFNET_STAT_INC(sc->sc_ifp, oerrors, 1); } return (ret); } static void wg_send_buf(struct wg_softc *sc, struct wg_endpoint *e, const void *buf, size_t len) { struct mbuf *m; int ret; /* * This function only sends handshake packets of known lengths that * are <= MHLEN, so it's safe to just use m_gethdr() and memcpy(). */ KKASSERT(len <= MHLEN); m = m_gethdr(M_NOWAIT, MT_DATA); if (m == NULL) { DPRINTF(sc, "Unable to allocate mbuf\n"); return; } /* Just plain copy as it's a single mbuf. */ memcpy(mtod(m, void *), buf, len); m->m_pkthdr.len = m->m_len = len; /* Give high priority to the handshake packets. */ m->m_flags |= M_PRIO; ret = wg_send(sc, e, m); if (ret != 0) DPRINTF(sc, "Unable to send packet: %d\n", ret); } /*----------------------------------------------------------------------------*/ /* * Timers * * These functions handle the timeout callbacks for a WireGuard session, and * provide an "event-based" model for controlling WireGuard session timers. */ static void wg_timers_run_send_initiation(struct wg_peer *, bool); static void wg_timers_run_retry_handshake(void *); static void wg_timers_run_send_keepalive(void *); static void wg_timers_run_new_handshake(void *); static void wg_timers_run_zero_key_material(void *); static void wg_timers_run_persistent_keepalive(void *); static void wg_timers_enable(struct wg_peer *peer) { atomic_store_bool(&peer->p_enabled, true); wg_timers_run_persistent_keepalive(peer); } static void wg_timers_disable(struct wg_peer *peer) { atomic_store_bool(&peer->p_enabled, false); atomic_store_bool(&peer->p_need_another_keepalive, false); /* Cancel the callouts and wait for them to complete. */ callout_drain(&peer->p_new_handshake); callout_drain(&peer->p_send_keepalive); callout_drain(&peer->p_retry_handshake); callout_drain(&peer->p_persistent_keepalive); callout_drain(&peer->p_zero_key_material); } static void wg_timers_set_persistent_keepalive(struct wg_peer *peer, uint16_t interval) { atomic_store_16(&peer->p_persistent_keepalive_interval, interval); if (atomic_load_bool(&peer->p_enabled)) wg_timers_run_persistent_keepalive(peer); } static bool wg_timers_get_persistent_keepalive(struct wg_peer *peer, uint16_t *interval) { *interval = atomic_load_16(&peer->p_persistent_keepalive_interval); return (*interval > 0); } static void wg_timers_get_last_handshake(struct wg_peer *peer, struct timespec *time) { lockmgr(&peer->p_handshake_mtx, LK_EXCLUSIVE); *time = peer->p_handshake_complete; lockmgr(&peer->p_handshake_mtx, LK_RELEASE); } /* * Should be called after an authenticated data packet is sent. */ static void wg_timers_event_data_sent(struct wg_peer *peer) { int ticks; if (atomic_load_bool(&peer->p_enabled) && !callout_pending(&peer->p_new_handshake)) { ticks = NEW_HANDSHAKE_TIMEOUT * hz + REKEY_TIMEOUT_JITTER * hz / 1000; callout_reset(&peer->p_new_handshake, ticks, wg_timers_run_new_handshake, peer); } } /* * Should be called after an authenticated data packet is received. */ static void wg_timers_event_data_received(struct wg_peer *peer) { if (atomic_load_bool(&peer->p_enabled)) { if (!callout_pending(&peer->p_send_keepalive)) { callout_reset(&peer->p_send_keepalive, KEEPALIVE_TIMEOUT * hz, wg_timers_run_send_keepalive, peer); } else { atomic_store_bool(&peer->p_need_another_keepalive, true); } } } /* * Should be called before any type of authenticated packet is to be sent, * whether keepalive, data, or handshake. */ static void wg_timers_event_any_authenticated_packet_sent(struct wg_peer *peer) { callout_stop(&peer->p_send_keepalive); } /* * Should be called after any type of authenticated packet is received, * whether keepalive, data, or handshake. */ static void wg_timers_event_any_authenticated_packet_received(struct wg_peer *peer) { callout_stop(&peer->p_new_handshake); } /* * Should be called before a packet with authentication (whether keepalive, * data, or handshakem) is sent, or after one is received. */ static void wg_timers_event_any_authenticated_packet_traversal(struct wg_peer *peer) { uint16_t interval; interval = atomic_load_16(&peer->p_persistent_keepalive_interval); if (atomic_load_bool(&peer->p_enabled) && interval > 0) { callout_reset(&peer->p_persistent_keepalive, interval * hz, wg_timers_run_persistent_keepalive, peer); } } /* * Should be called after a handshake initiation message is sent. */ static void wg_timers_event_handshake_initiated(struct wg_peer *peer) { int ticks; if (atomic_load_bool(&peer->p_enabled)) { ticks = REKEY_TIMEOUT * hz + REKEY_TIMEOUT_JITTER * hz / 1000; callout_reset(&peer->p_retry_handshake, ticks, wg_timers_run_retry_handshake, peer); } } /* * Should be called after a handshake response message is received and * processed, or when getting key confirmation via the first data message. */ static void wg_timers_event_handshake_complete(struct wg_peer *peer) { if (atomic_load_bool(&peer->p_enabled)) { lockmgr(&peer->p_handshake_mtx, LK_EXCLUSIVE); callout_stop(&peer->p_retry_handshake); peer->p_handshake_retries = 0; getnanotime(&peer->p_handshake_complete); lockmgr(&peer->p_handshake_mtx, LK_RELEASE); wg_timers_run_send_keepalive(peer); } } /* * Should be called after an ephemeral key is created, which is before sending * a handshake response or after receiving a handshake response. */ static void wg_timers_event_session_derived(struct wg_peer *peer) { if (atomic_load_bool(&peer->p_enabled)) { callout_reset(&peer->p_zero_key_material, REJECT_AFTER_TIME * 3 * hz, wg_timers_run_zero_key_material, peer); } } /* * Should be called after data packet sending failure, or after the old * keypairs expiring (or near expiring). */ static void wg_timers_event_want_initiation(struct wg_peer *peer) { if (atomic_load_bool(&peer->p_enabled)) wg_timers_run_send_initiation(peer, false); } static void wg_timers_run_send_initiation(struct wg_peer *peer, bool is_retry) { if (!is_retry) peer->p_handshake_retries = 0; if (noise_remote_initiation_expired(peer->p_remote)) wg_send_initiation(peer); } static void wg_timers_run_retry_handshake(void *_peer) { struct wg_peer *peer = _peer; lockmgr(&peer->p_handshake_mtx, LK_EXCLUSIVE); if (peer->p_handshake_retries <= MAX_TIMER_HANDSHAKES) { peer->p_handshake_retries++; lockmgr(&peer->p_handshake_mtx, LK_RELEASE); DPRINTF(peer->p_sc, "Handshake for peer %ld did not complete " "after %d seconds, retrying (try %d)\n", peer->p_id, REKEY_TIMEOUT, peer->p_handshake_retries + 1); wg_timers_run_send_initiation(peer, true); } else { lockmgr(&peer->p_handshake_mtx, LK_RELEASE); DPRINTF(peer->p_sc, "Handshake for peer %ld did not complete " "after %d retries, giving up\n", peer->p_id, MAX_TIMER_HANDSHAKES + 2); callout_stop(&peer->p_send_keepalive); wg_queue_purge(&peer->p_stage_queue); if (atomic_load_bool(&peer->p_enabled) && !callout_pending(&peer->p_zero_key_material)) { callout_reset(&peer->p_zero_key_material, REJECT_AFTER_TIME * 3 * hz, wg_timers_run_zero_key_material, peer); } } } static void wg_timers_run_send_keepalive(void *_peer) { struct wg_peer *peer = _peer; wg_send_keepalive(peer); if (atomic_load_bool(&peer->p_enabled) && atomic_load_bool(&peer->p_need_another_keepalive)) { atomic_store_bool(&peer->p_need_another_keepalive, false); callout_reset(&peer->p_send_keepalive, KEEPALIVE_TIMEOUT * hz, wg_timers_run_send_keepalive, peer); } } static void wg_timers_run_persistent_keepalive(void *_peer) { struct wg_peer *peer = _peer; if (atomic_load_16(&peer->p_persistent_keepalive_interval) > 0) wg_send_keepalive(peer); } static void wg_timers_run_new_handshake(void *_peer) { struct wg_peer *peer = _peer; DPRINTF(peer->p_sc, "Retrying handshake with peer %ld, " "because we stopped hearing back after %d seconds\n", peer->p_id, NEW_HANDSHAKE_TIMEOUT); wg_timers_run_send_initiation(peer, false); } static void wg_timers_run_zero_key_material(void *_peer) { struct wg_peer *peer = _peer; DPRINTF(peer->p_sc, "Zeroing out keys for peer %ld, " "since we haven't received a new one in %d seconds\n", peer->p_id, REJECT_AFTER_TIME * 3); noise_remote_keypairs_clear(peer->p_remote); } /*----------------------------------------------------------------------------*/ /* Handshake */ static void wg_peer_send_buf(struct wg_peer *peer, const void *buf, size_t len) { struct wg_endpoint endpoint; peer->p_tx_bytes[mycpuid] += len; wg_timers_event_any_authenticated_packet_traversal(peer); wg_timers_event_any_authenticated_packet_sent(peer); wg_peer_get_endpoint(peer, &endpoint); wg_send_buf(peer->p_sc, &endpoint, buf, len); } static void wg_send_initiation(struct wg_peer *peer) { struct wg_pkt_initiation pkt; if (!noise_create_initiation(peer->p_remote, &pkt.s_idx, pkt.ue, pkt.es, pkt.ets)) return; DPRINTF(peer->p_sc, "Sending handshake initiation to peer %ld\n", peer->p_id); pkt.t = WG_PKT_INITIATION; cookie_maker_mac(peer->p_cookie, &pkt.m, &pkt, sizeof(pkt) - sizeof(pkt.m)); wg_peer_send_buf(peer, &pkt, sizeof(pkt)); wg_timers_event_handshake_initiated(peer); } static void wg_send_response(struct wg_peer *peer) { struct wg_pkt_response pkt; if (!noise_create_response(peer->p_remote, &pkt.s_idx, &pkt.r_idx, pkt.ue, pkt.en)) return; DPRINTF(peer->p_sc, "Sending handshake response to peer %ld\n", peer->p_id); wg_timers_event_session_derived(peer); pkt.t = WG_PKT_RESPONSE; cookie_maker_mac(peer->p_cookie, &pkt.m, &pkt, sizeof(pkt) - sizeof(pkt.m)); wg_peer_send_buf(peer, &pkt, sizeof(pkt)); } static void wg_send_cookie(struct wg_softc *sc, struct cookie_macs *cm, uint32_t idx, struct wg_endpoint *e) { struct wg_pkt_cookie pkt; DPRINTF(sc, "Sending cookie response for denied handshake message\n"); pkt.t = WG_PKT_COOKIE; pkt.r_idx = idx; cookie_checker_create_payload(sc->sc_cookie, cm, pkt.nonce, pkt.ec, &e->e_remote.r_sa); wg_send_buf(sc, e, &pkt, sizeof(pkt)); } static void wg_send_keepalive(struct wg_peer *peer) { struct wg_packet *pkt; struct mbuf *m; if (wg_queue_len(&peer->p_stage_queue) > 0) goto send; if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) return; if ((pkt = wg_packet_alloc(m)) == NULL) { m_freem(m); return; } wg_queue_push_staged(&peer->p_stage_queue, pkt); DPRINTF(peer->p_sc, "Sending keepalive packet to peer %ld\n", peer->p_id); send: wg_peer_send_staged(peer); } static bool wg_is_underload(struct wg_softc *sc) { /* * This is global, so that the load calculation applies to the * whole system. Don't care about races with it at all. */ static struct timespec last_underload; /* nanouptime */ struct timespec now; bool underload; underload = (wg_queue_len(&sc->sc_handshake_queue) >= MAX_QUEUED_HANDSHAKES / 8); if (underload) { getnanouptime(&last_underload); } else if (timespecisset(&last_underload)) { getnanouptime(&now); now.tv_sec -= UNDERLOAD_TIMEOUT; underload = timespeccmp(&last_underload, &now, >); if (!underload) timespecclear(&last_underload); } return (underload); } static void wg_handshake(struct wg_softc *sc, struct wg_packet *pkt) { struct wg_pkt_initiation *init; struct wg_pkt_response *resp; struct wg_pkt_cookie *cook; struct wg_endpoint *e; struct wg_peer *peer; struct mbuf *m; struct noise_remote *remote = NULL; bool underload; int ret; underload = wg_is_underload(sc); m = pkt->p_mbuf; e = &pkt->p_endpoint; /* m_pullup() may change 'm', so must update 'pkt->p_mbuf'. */ if ((pkt->p_mbuf = m = m_pullup(m, m->m_pkthdr.len)) == NULL) goto error; switch (*mtod(m, uint32_t *)) { case WG_PKT_INITIATION: init = mtod(m, struct wg_pkt_initiation *); ret = cookie_checker_validate_macs(sc->sc_cookie, &init->m, init, sizeof(*init) - sizeof(init->m), underload, &e->e_remote.r_sa); if (ret != 0) { switch (ret) { case EINVAL: DPRINTF(sc, "Invalid initiation MAC\n"); break; case ECONNREFUSED: DPRINTF(sc, "Handshake ratelimited\n"); break; case EAGAIN: wg_send_cookie(sc, &init->m, init->s_idx, e); break; default: /* * cookie_checker_validate_macs() seems could * return EAFNOSUPPORT, but that is actually * impossible, because packets of unsupported * AF have been already dropped. */ panic("%s: unexpected return: %d", __func__, ret); } goto error; } remote = noise_consume_initiation(sc->sc_local, init->s_idx, init->ue, init->es, init->ets); if (remote == NULL) { DPRINTF(sc, "Invalid handshake initiation\n"); goto error; } peer = noise_remote_arg(remote); DPRINTF(sc, "Receiving handshake initiation from peer %ld\n", peer->p_id); wg_peer_set_endpoint(peer, e); wg_send_response(peer); break; case WG_PKT_RESPONSE: resp = mtod(m, struct wg_pkt_response *); ret = cookie_checker_validate_macs(sc->sc_cookie, &resp->m, resp, sizeof(*resp) - sizeof(resp->m), underload, &e->e_remote.r_sa); if (ret != 0) { switch (ret) { case EINVAL: DPRINTF(sc, "Invalid response MAC\n"); break; case ECONNREFUSED: DPRINTF(sc, "Handshake ratelimited\n"); break; case EAGAIN: wg_send_cookie(sc, &resp->m, resp->s_idx, e); break; default: /* See also the comment above. */ panic("%s: unexpected return: %d", __func__, ret); } goto error; } remote = noise_consume_response(sc->sc_local, resp->s_idx, resp->r_idx, resp->ue, resp->en); if (remote == NULL) { DPRINTF(sc, "Invalid handshake response\n"); goto error; } peer = noise_remote_arg(remote); DPRINTF(sc, "Receiving handshake response from peer %ld\n", peer->p_id); wg_peer_set_endpoint(peer, e); wg_timers_event_session_derived(peer); wg_timers_event_handshake_complete(peer); break; case WG_PKT_COOKIE: cook = mtod(m, struct wg_pkt_cookie *); remote = noise_remote_index(sc->sc_local, cook->r_idx); if (remote == NULL) { DPRINTF(sc, "Unknown cookie index\n"); goto error; } peer = noise_remote_arg(remote); if (cookie_maker_consume_payload(peer->p_cookie, cook->nonce, cook->ec) == 0) { DPRINTF(sc, "Receiving cookie response\n"); } else { DPRINTF(sc, "Could not decrypt cookie response\n"); goto error; } goto not_authenticated; default: panic("%s: invalid packet in handshake queue", __func__); } wg_timers_event_any_authenticated_packet_received(peer); wg_timers_event_any_authenticated_packet_traversal(peer); not_authenticated: IFNET_STAT_INC(sc->sc_ifp, ipackets, 1); IFNET_STAT_INC(sc->sc_ifp, ibytes, m->m_pkthdr.len); peer->p_rx_bytes[mycpuid] += m->m_pkthdr.len; noise_remote_put(remote); wg_packet_free(pkt); return; error: IFNET_STAT_INC(sc->sc_ifp, ierrors, 1); if (remote != NULL) noise_remote_put(remote); wg_packet_free(pkt); } static void wg_handshake_worker(void *arg, int pending __unused) { struct wg_softc *sc = arg; struct wg_queue *queue = &sc->sc_handshake_queue; struct wg_packet *pkt; while ((pkt = wg_queue_dequeue_handshake(queue)) != NULL) wg_handshake(sc, pkt); } /*----------------------------------------------------------------------------*/ /* Transport Packet Functions */ static inline void wg_bpf_ptap(struct ifnet *ifp, struct mbuf *m, sa_family_t af) { uint32_t bpf_af; if (ifp->if_bpf == NULL) return; bpf_gettoken(); /* Double check after obtaining the token. */ if (ifp->if_bpf != NULL) { /* Prepend the AF as a 4-byte field for DLT_NULL. */ bpf_af = (uint32_t)af; bpf_ptap(ifp->if_bpf, m, &bpf_af, sizeof(bpf_af)); } bpf_reltoken(); } static inline unsigned int calculate_padding(struct wg_packet *pkt) { unsigned int padded_size, last_unit; last_unit = pkt->p_mbuf->m_pkthdr.len; /* Keepalive packets don't set p_mtu, but also have a length of zero. */ if (__predict_false(pkt->p_mtu == 0)) return WG_PKT_WITH_PADDING(last_unit) - last_unit; /* * Just in case the packet is bigger than the MTU and would cause * the final subtraction to overflow. */ if (__predict_false(last_unit > pkt->p_mtu)) last_unit %= pkt->p_mtu; padded_size = MIN(pkt->p_mtu, WG_PKT_WITH_PADDING(last_unit)); return (padded_size - last_unit); } static inline int determine_af_and_pullup(struct mbuf **m, sa_family_t *af) { const struct ip *ip; const struct ip6_hdr *ip6; int len; ip = mtod(*m, const struct ip *); ip6 = mtod(*m, const struct ip6_hdr *); len = (*m)->m_pkthdr.len; if (len >= sizeof(*ip) && ip->ip_v == IPVERSION) *af = AF_INET; #ifdef INET6 else if (len >= sizeof(*ip6) && (ip6->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION) *af = AF_INET6; #endif else return (EAFNOSUPPORT); *m = m_pullup(*m, (*af == AF_INET ? sizeof(*ip) : sizeof(*ip6))); if (*m == NULL) return (ENOBUFS); return (0); } static void wg_encrypt(struct wg_softc *sc, struct wg_packet *pkt) { static const uint8_t padding[WG_PKT_PADDING] = { 0 }; struct wg_pkt_data *data; struct wg_peer *peer; struct noise_remote *remote; struct mbuf *m; unsigned int padlen, state = WG_PACKET_DEAD; uint32_t idx; remote = noise_keypair_remote(pkt->p_keypair); peer = noise_remote_arg(remote); m = pkt->p_mbuf; padlen = calculate_padding(pkt); if (padlen != 0 && !m_append(m, padlen, padding)) goto out; if (noise_keypair_encrypt(pkt->p_keypair, &idx, pkt->p_counter, m) != 0) goto out; M_PREPEND(m, sizeof(struct wg_pkt_data), M_NOWAIT); if (m == NULL) goto out; data = mtod(m, struct wg_pkt_data *); data->t = WG_PKT_DATA; data->r_idx = idx; data->counter = htole64(pkt->p_counter); state = WG_PACKET_CRYPTED; out: pkt->p_mbuf = m; atomic_store_rel_int(&pkt->p_state, state); taskqueue_enqueue(peer->p_send_taskqueue, &peer->p_send_task); noise_remote_put(remote); } static void wg_decrypt(struct wg_softc *sc, struct wg_packet *pkt) { struct wg_peer *peer, *allowed_peer; struct noise_remote *remote; struct mbuf *m; unsigned int state = WG_PACKET_DEAD; int len; remote = noise_keypair_remote(pkt->p_keypair); peer = noise_remote_arg(remote); m = pkt->p_mbuf; pkt->p_counter = le64toh(mtod(m, struct wg_pkt_data *)->counter); m_adj(m, sizeof(struct wg_pkt_data)); if (noise_keypair_decrypt(pkt->p_keypair, pkt->p_counter, m) != 0) goto out; /* A packet with a length of zero is a keepalive packet. */ if (__predict_false(m->m_pkthdr.len == 0)) { DPRINTF(sc, "Receiving keepalive packet from peer %ld\n", peer->p_id); state = WG_PACKET_CRYPTED; goto out; } /* * Extract the source address for wg_aip_lookup(), and trim the * packet if it was padded before encryption. */ if (determine_af_and_pullup(&m, &pkt->p_af) != 0) goto out; if (pkt->p_af == AF_INET) { const struct ip *ip = mtod(m, const struct ip *); allowed_peer = wg_aip_lookup(sc, AF_INET, &ip->ip_src); len = ntohs(ip->ip_len); if (len >= sizeof(struct ip) && len < m->m_pkthdr.len) m_adj(m, len - m->m_pkthdr.len); } else { const struct ip6_hdr *ip6 = mtod(m, const struct ip6_hdr *); allowed_peer = wg_aip_lookup(sc, AF_INET6, &ip6->ip6_src); len = ntohs(ip6->ip6_plen) + sizeof(struct ip6_hdr); if (len < m->m_pkthdr.len) m_adj(m, len - m->m_pkthdr.len); } /* Drop the reference, since no need to dereference it. */ if (allowed_peer != NULL) noise_remote_put(allowed_peer->p_remote); if (__predict_false(peer != allowed_peer)) { DPRINTF(sc, "Packet has disallowed src IP from peer %ld\n", peer->p_id); goto out; } state = WG_PACKET_CRYPTED; out: pkt->p_mbuf = m; atomic_store_rel_int(&pkt->p_state, state); taskqueue_enqueue(peer->p_recv_taskqueue, &peer->p_recv_task); noise_remote_put(remote); } static void wg_encrypt_worker(void *arg, int pending __unused) { struct wg_softc *sc = arg; struct wg_queue *queue = &sc->sc_encrypt_parallel; struct wg_packet *pkt; while ((pkt = wg_queue_dequeue_parallel(queue)) != NULL) wg_encrypt(sc, pkt); } static void wg_decrypt_worker(void *arg, int pending __unused) { struct wg_softc *sc = arg; struct wg_queue *queue = &sc->sc_decrypt_parallel; struct wg_packet *pkt; while ((pkt = wg_queue_dequeue_parallel(queue)) != NULL) wg_decrypt(sc, pkt); } static void wg_encrypt_dispatch(struct wg_softc *sc) { int cpu; /* * The update to encrypt_last_cpu is racy such that we may * reschedule the task for the same CPU multiple times, but * the race doesn't really matter. */ cpu = (sc->sc_encrypt_last_cpu + 1) % ncpus; sc->sc_encrypt_last_cpu = cpu; taskqueue_enqueue(wg_taskqueues[cpu], &sc->sc_encrypt_tasks[cpu]); } static void wg_decrypt_dispatch(struct wg_softc *sc) { int cpu; cpu = (sc->sc_decrypt_last_cpu + 1) % ncpus; sc->sc_decrypt_last_cpu = cpu; taskqueue_enqueue(wg_taskqueues[cpu], &sc->sc_decrypt_tasks[cpu]); } static void wg_deliver_out(void *arg, int pending __unused) { struct wg_peer *peer = arg; struct wg_softc *sc = peer->p_sc; struct wg_queue *queue = &peer->p_encrypt_serial; struct wg_endpoint endpoint; struct wg_packet *pkt; struct mbuf *m; int len, cpu; cpu = mycpuid; while ((pkt = wg_queue_dequeue_serial(queue)) != NULL) { if (atomic_load_acq_int(&pkt->p_state) != WG_PACKET_CRYPTED) { IFNET_STAT_INC(sc->sc_ifp, oerrors, 1); wg_packet_free(pkt); continue; } m = pkt->p_mbuf; m->m_flags &= ~MBUF_CLEARFLAGS; len = m->m_pkthdr.len; pkt->p_mbuf = NULL; wg_packet_free(pkt); /* * The keepalive timers -- both persistent and mandatory -- * are part of the internal state machine, which needs to be * cranked whether or not the packet was actually sent. */ wg_timers_event_any_authenticated_packet_traversal(peer); wg_timers_event_any_authenticated_packet_sent(peer); wg_peer_get_endpoint(peer, &endpoint); if (wg_send(sc, &endpoint, m) == 0) { peer->p_tx_bytes[cpu] += len; if (len > WG_PKT_ENCRYPTED_LEN(0)) wg_timers_event_data_sent(peer); if (noise_keypair_should_refresh(peer->p_remote, true)) wg_timers_event_want_initiation(peer); } } } static void wg_deliver_in(void *arg, int pending __unused) { struct wg_peer *peer = arg; struct wg_softc *sc = peer->p_sc; struct wg_queue *queue = &peer->p_decrypt_serial; struct wg_packet *pkt; struct ifnet *ifp; struct mbuf *m; size_t rx_bytes; int cpu; cpu = mycpuid; ifp = sc->sc_ifp; while ((pkt = wg_queue_dequeue_serial(queue)) != NULL) { if (atomic_load_acq_int(&pkt->p_state) != WG_PACKET_CRYPTED || noise_keypair_counter_check(pkt->p_keypair, pkt->p_counter) != 0) { IFNET_STAT_INC(ifp, ierrors, 1); wg_packet_free(pkt); continue; } if (noise_keypair_received_with(pkt->p_keypair)) wg_timers_event_handshake_complete(peer); wg_timers_event_any_authenticated_packet_received(peer); wg_timers_event_any_authenticated_packet_traversal(peer); wg_peer_set_endpoint(peer, &pkt->p_endpoint); m = pkt->p_mbuf; rx_bytes = WG_PKT_ENCRYPTED_LEN(m->m_pkthdr.len); peer->p_rx_bytes[cpu] += rx_bytes; IFNET_STAT_INC(ifp, ipackets, 1); IFNET_STAT_INC(ifp, ibytes, rx_bytes); if (m->m_pkthdr.len > 0) { if (ifp->if_capenable & IFCAP_RXCSUM) { /* * The packet is authentic as ensured by the * AEAD tag, so we can tell the networking * stack that this packet has valid checksums * and thus is unnecessary to check again. */ if (m->m_pkthdr.csum_flags & CSUM_IP) m->m_pkthdr.csum_flags |= (CSUM_IP_CHECKED | CSUM_IP_VALID); if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA) { m->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); m->m_pkthdr.csum_data = 0xffff; } } m->m_flags &= ~MBUF_CLEARFLAGS; m->m_pkthdr.rcvif = ifp; wg_bpf_ptap(ifp, m, pkt->p_af); netisr_queue((pkt->p_af == AF_INET ? NETISR_IP : NETISR_IPV6), m); pkt->p_mbuf = NULL; wg_timers_event_data_received(peer); } wg_packet_free(pkt); if (noise_keypair_should_refresh(peer->p_remote, false)) wg_timers_event_want_initiation(peer); } } static void wg_input(struct wg_softc *sc, struct mbuf *m, const struct sockaddr *sa) { struct noise_remote *remote; struct wg_pkt_data *data; struct wg_packet *pkt; struct wg_peer *peer; struct mbuf *defragged; /* * Defragment mbufs early on in order to: * - make the crypto a lot faster; * - make the subsequent m_pullup()'s no-ops. */ defragged = m_defrag(m, M_NOWAIT); if (defragged != NULL) m = defragged; /* The original mbuf chain is freed. */ /* Ensure the packet is not shared before modifying it. */ m = m_unshare(m, M_NOWAIT); if (m == NULL) { IFNET_STAT_INC(sc->sc_ifp, iqdrops, 1); return; } /* Pullup enough to read packet type */ if ((m = m_pullup(m, sizeof(uint32_t))) == NULL) { IFNET_STAT_INC(sc->sc_ifp, iqdrops, 1); return; } if ((pkt = wg_packet_alloc(m)) == NULL) { IFNET_STAT_INC(sc->sc_ifp, iqdrops, 1); m_freem(m); return; } /* Save the remote address and port for later use. */ switch (sa->sa_family) { case AF_INET: pkt->p_endpoint.e_remote.r_sin = *(const struct sockaddr_in *)sa; break; #ifdef INET6 case AF_INET6: pkt->p_endpoint.e_remote.r_sin6 = *(const struct sockaddr_in6 *)sa; break; #endif default: DPRINTF(sc, "Unsupported packet address family\n"); goto error; } if (WG_PKT_IS_INITIATION(m) || WG_PKT_IS_RESPONSE(m) || WG_PKT_IS_COOKIE(m)) { if (!wg_queue_enqueue_handshake(&sc->sc_handshake_queue, pkt)) { IFNET_STAT_INC(sc->sc_ifp, iqdrops, 1); DPRINTF(sc, "Dropping handshake packet\n"); } taskqueue_enqueue(sc->sc_handshake_taskqueue, &sc->sc_handshake_task); return; } if (WG_PKT_IS_DATA(m)) { /* Pullup the whole header to read r_idx below. */ pkt->p_mbuf = m_pullup(m, sizeof(struct wg_pkt_data)); if (pkt->p_mbuf == NULL) goto error; data = mtod(pkt->p_mbuf, struct wg_pkt_data *); pkt->p_keypair = noise_keypair_lookup(sc->sc_local, data->r_idx); if (pkt->p_keypair == NULL) goto error; remote = noise_keypair_remote(pkt->p_keypair); peer = noise_remote_arg(remote); if (!wg_queue_both(&sc->sc_decrypt_parallel, &peer->p_decrypt_serial, pkt)) IFNET_STAT_INC(sc->sc_ifp, iqdrops, 1); wg_decrypt_dispatch(sc); noise_remote_put(remote); return; } error: IFNET_STAT_INC(sc->sc_ifp, ierrors, 1); wg_packet_free(pkt); } static void wg_upcall(struct socket *so, void *arg, int waitflag __unused) { struct wg_softc *sc = arg; struct sockaddr *from; struct sockbuf sio; int ret, flags; /* * For UDP, soreceive typically pulls just one packet, * so loop to get the whole batch. */ do { sbinit(&sio, 1000000000); /* really large to receive all */ flags = MSG_DONTWAIT; ret = so_pru_soreceive(so, &from, NULL, &sio, NULL, &flags); if (ret != 0 || sio.sb_mb == NULL) { if (from != NULL) kfree(from, M_SONAME); break; } wg_input(sc, sio.sb_mb, from); kfree(from, M_SONAME); } while (sio.sb_mb != NULL); } static void wg_peer_send_staged(struct wg_peer *peer) { struct wg_softc *sc = peer->p_sc; struct wg_packet *pkt, *tpkt; struct wg_packet_list list; struct noise_keypair *keypair = NULL; wg_queue_delist_staged(&peer->p_stage_queue, &list); if (STAILQ_EMPTY(&list)) return; if ((keypair = noise_keypair_current(peer->p_remote)) == NULL) goto error; /* * We now try to assign counters to all of the packets in the queue. * If we can't assign counters for all of them, we just consider it * a failure and wait for the next handshake. */ STAILQ_FOREACH(pkt, &list, p_parallel) { if (!noise_keypair_counter_next(keypair, &pkt->p_counter)) goto error; } STAILQ_FOREACH_MUTABLE(pkt, &list, p_parallel, tpkt) { pkt->p_keypair = noise_keypair_ref(keypair); if (!wg_queue_both(&sc->sc_encrypt_parallel, &peer->p_encrypt_serial, pkt)) IFNET_STAT_INC(sc->sc_ifp, oqdrops, 1); } wg_encrypt_dispatch(sc); noise_keypair_put(keypair); return; error: if (keypair != NULL) noise_keypair_put(keypair); wg_queue_enlist_staged(&peer->p_stage_queue, &list); wg_timers_event_want_initiation(peer); } static int wg_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, struct rtentry *rt) { struct wg_softc *sc = ifp->if_softc; struct wg_packet *pkt = NULL; struct wg_peer *peer = NULL; struct mbuf *defragged; sa_family_t af = AF_UNSPEC; int ret; if (dst->sa_family == AF_UNSPEC) { /* * Specially handle packets written/injected by BPF. * The packets have the same DLT_NULL link-layer type * (i.e., 4-byte link-layer header in host byte order). */ dst->sa_family = *(mtod(m, uint32_t *)); m_adj(m, sizeof(uint32_t)); } if (dst->sa_family == AF_UNSPEC) { ret = EAFNOSUPPORT; goto error; } wg_bpf_ptap(ifp, m, dst->sa_family); if (__predict_false(if_tunnel_check_nesting(ifp, m, MTAG_WGLOOP, MAX_LOOPS) != 0)) { DPRINTF(sc, "Packet looped\n"); ret = ELOOP; goto error; } defragged = m_defrag(m, M_NOWAIT); if (defragged != NULL) m = defragged; m = m_unshare(m, M_NOWAIT); if (m == NULL) { ret = ENOBUFS; goto error; } if ((ret = determine_af_and_pullup(&m, &af)) != 0) goto error; if (af != dst->sa_family) { ret = EAFNOSUPPORT; goto error; } if ((pkt = wg_packet_alloc(m)) == NULL) { ret = ENOBUFS; goto error; } pkt->p_af = af; pkt->p_mtu = ifp->if_mtu; if (rt != NULL && rt->rt_rmx.rmx_mtu > 0 && rt->rt_rmx.rmx_mtu < pkt->p_mtu) pkt->p_mtu = rt->rt_rmx.rmx_mtu; if (af == AF_INET) { peer = wg_aip_lookup(sc, AF_INET, &mtod(m, struct ip *)->ip_dst); } else { peer = wg_aip_lookup(sc, AF_INET6, &mtod(m, struct ip6_hdr *)->ip6_dst); } if (__predict_false(peer == NULL)) { ret = ENOKEY; goto error; } if (__predict_false(peer->p_endpoint.e_remote.r_sa.sa_family == AF_UNSPEC)) { DPRINTF(sc, "No valid endpoint has been configured or " "discovered for peer %ld\n", peer->p_id); ret = EHOSTUNREACH; goto error; } wg_queue_push_staged(&peer->p_stage_queue, pkt); wg_peer_send_staged(peer); noise_remote_put(peer->p_remote); return (0); error: IFNET_STAT_INC(ifp, oerrors, 1); if (ret == ELOOP) { /* Skip ICMP error for ELOOP to avoid infinite loop. */ m_freem(m); /* m cannot be NULL */ m = NULL; } if (m != NULL) { if (af == AF_INET) icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_HOST, 0, 0); #ifdef INET6 else if (af == AF_INET6) icmp6_error(m, ICMP6_DST_UNREACH, 0, 0); #endif else m_freem(m); } if (pkt != NULL) { pkt->p_mbuf = NULL; /* m already freed above */ wg_packet_free(pkt); } if (peer != NULL) noise_remote_put(peer->p_remote); return (ret); } /*----------------------------------------------------------------------------*/ /* Interface Functions */ static int wg_up(struct wg_softc *); static void wg_down(struct wg_softc *); static int wg_ioctl_get(struct wg_softc *sc, struct wg_data_io *data, bool privileged) { struct wg_interface_io *iface_p, iface_o; struct wg_peer_io *peer_p, peer_o; struct wg_aip_io *aip_p, aip_o; struct wg_peer *peer; struct wg_aip *aip; size_t size, peer_count, aip_count; int cpu, ret = 0; lockmgr(&sc->sc_lock, LK_SHARED); /* Determine the required data size. */ size = sizeof(struct wg_interface_io); size += sizeof(struct wg_peer_io) * sc->sc_peers_num; TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) size += sizeof(struct wg_aip_io) * peer->p_aips_num; /* Return the required size for userland allocation. */ if (data->wgd_size < size) { data->wgd_size = size; lockmgr(&sc->sc_lock, LK_RELEASE); return (0); } iface_p = data->wgd_interface; bzero(&iface_o, sizeof(iface_o)); /* * No need to acquire the 'sc_socket.so_lock', because 'sc_lock' * is acquired and that's enough to prevent modifications to * 'sc_socket' members. */ if (sc->sc_socket.so_port != 0) { iface_o.i_port = sc->sc_socket.so_port; iface_o.i_flags |= WG_INTERFACE_HAS_PORT; } if (sc->sc_socket.so_user_cookie != 0) { iface_o.i_cookie = sc->sc_socket.so_user_cookie; iface_o.i_flags |= WG_INTERFACE_HAS_COOKIE; } if (noise_local_keys(sc->sc_local, iface_o.i_public, iface_o.i_private)) { iface_o.i_flags |= WG_INTERFACE_HAS_PUBLIC; if (privileged) iface_o.i_flags |= WG_INTERFACE_HAS_PRIVATE; else bzero(iface_o.i_private, sizeof(iface_o.i_private)); } peer_count = 0; peer_p = &iface_p->i_peers[0]; TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { bzero(&peer_o, sizeof(peer_o)); peer_o.p_flags |= WG_PEER_HAS_PUBLIC; if (noise_remote_keys(peer->p_remote, peer_o.p_public, peer_o.p_psk)) { if (privileged) peer_o.p_flags |= WG_PEER_HAS_PSK; else bzero(peer_o.p_psk, sizeof(peer_o.p_psk)); } if (wg_timers_get_persistent_keepalive(peer, &peer_o.p_pka)) peer_o.p_flags |= WG_PEER_HAS_PKA; if (wg_peer_get_sockaddr(peer, &peer_o.p_sa) == 0) peer_o.p_flags |= WG_PEER_HAS_ENDPOINT; for (cpu = 0; cpu < ncpus; cpu++) { peer_o.p_rxbytes += peer->p_rx_bytes[cpu]; peer_o.p_txbytes += peer->p_tx_bytes[cpu]; } wg_timers_get_last_handshake(peer, &peer_o.p_last_handshake); peer_o.p_id = (uint64_t)peer->p_id; strlcpy(peer_o.p_description, peer->p_description, sizeof(peer_o.p_description)); aip_count = 0; aip_p = &peer_p->p_aips[0]; LIST_FOREACH(aip, &peer->p_aips, a_entry) { bzero(&aip_o, sizeof(aip_o)); aip_o.a_af = aip->a_af; if (aip->a_af == AF_INET) { aip_o.a_cidr = bitcount32(aip->a_mask.ip); memcpy(&aip_o.a_ipv4, &aip->a_addr.in, sizeof(aip->a_addr.in)); } else if (aip->a_af == AF_INET6) { aip_o.a_cidr = in6_mask2len(&aip->a_mask.in6, NULL); memcpy(&aip_o.a_ipv6, &aip->a_addr.in6, sizeof(aip->a_addr.in6)); } ret = copyout(&aip_o, aip_p, sizeof(aip_o)); if (ret != 0) goto out; aip_p++; aip_count++; } KKASSERT(aip_count == peer->p_aips_num); peer_o.p_aips_count = aip_count; ret = copyout(&peer_o, peer_p, sizeof(peer_o)); if (ret != 0) goto out; peer_p = (struct wg_peer_io *)aip_p; peer_count++; } KKASSERT(peer_count == sc->sc_peers_num); iface_o.i_peers_count = peer_count; ret = copyout(&iface_o, iface_p, sizeof(iface_o)); out: lockmgr(&sc->sc_lock, LK_RELEASE); explicit_bzero(&iface_o, sizeof(iface_o)); explicit_bzero(&peer_o, sizeof(peer_o)); return (ret); } static int wg_ioctl_set(struct wg_softc *sc, struct wg_data_io *data) { struct wg_interface_io *iface_p, iface_o; struct wg_peer_io *peer_p, peer_o; struct wg_aip_io *aip_p, aip_o; struct wg_peer *peer; struct noise_remote *remote; uint8_t public[WG_KEY_SIZE], private[WG_KEY_SIZE]; size_t i, j; int ret; remote = NULL; lockmgr(&sc->sc_lock, LK_EXCLUSIVE); iface_p = data->wgd_interface; if ((ret = copyin(iface_p, &iface_o, sizeof(iface_o))) != 0) goto error; if (iface_o.i_flags & WG_INTERFACE_REPLACE_PEERS) wg_peer_destroy_all(sc); if ((iface_o.i_flags & WG_INTERFACE_HAS_PRIVATE) && (!noise_local_keys(sc->sc_local, NULL, private) || timingsafe_bcmp(private, iface_o.i_private, WG_KEY_SIZE) != 0)) { if (curve25519_generate_public(public, iface_o.i_private)) { remote = noise_remote_lookup(sc->sc_local, public); if (remote != NULL) { /* Remove the conflicting peer. */ peer = noise_remote_arg(remote); wg_peer_destroy(peer); noise_remote_put(remote); } } /* * Set the private key. * * Note: we might be removing the private key. */ if (noise_local_set_private(sc->sc_local, iface_o.i_private)) cookie_checker_update(sc->sc_cookie, public); else cookie_checker_update(sc->sc_cookie, NULL); } if ((iface_o.i_flags & WG_INTERFACE_HAS_PORT) && iface_o.i_port != sc->sc_socket.so_port) { if (sc->sc_ifp->if_flags & IFF_RUNNING) { ret = wg_socket_init(sc, iface_o.i_port); if (ret != 0) goto error; } else { sc->sc_socket.so_port = iface_o.i_port; } } if (iface_o.i_flags & WG_INTERFACE_HAS_COOKIE) { ret = wg_socket_set_cookie(sc, iface_o.i_cookie); if (ret != 0) goto error; } peer_p = &iface_p->i_peers[0]; for (i = 0; i < iface_o.i_peers_count; i++) { if ((ret = copyin(peer_p, &peer_o, sizeof(peer_o))) != 0) goto error; /* Peer must have public key. */ if ((peer_o.p_flags & WG_PEER_HAS_PUBLIC) == 0) goto next_peer; /* Ignore peer that has the same public key. */ if (noise_local_keys(sc->sc_local, public, NULL) && memcmp(public, peer_o.p_public, WG_KEY_SIZE) == 0) goto next_peer; /* Lookup peer, or create if it doesn't exist. */ remote = noise_remote_lookup(sc->sc_local, peer_o.p_public); if (remote != NULL) { peer = noise_remote_arg(remote); } else { if (peer_o.p_flags & (WG_PEER_REMOVE | WG_PEER_UPDATE)) goto next_peer; peer = wg_peer_create(sc, peer_o.p_public); if (peer == NULL) { ret = ENOMEM; goto error; } /* No allowed IPs to remove for a new peer. */ peer_o.p_flags &= ~WG_PEER_REPLACE_AIPS; } if (peer_o.p_flags & WG_PEER_REMOVE) { wg_peer_destroy(peer); goto next_peer; } if (peer_o.p_flags & WG_PEER_HAS_ENDPOINT) { ret = wg_peer_set_sockaddr(peer, &peer_o.p_sa); if (ret != 0) goto error; } if (peer_o.p_flags & WG_PEER_HAS_PSK) noise_remote_set_psk(peer->p_remote, peer_o.p_psk); if (peer_o.p_flags & WG_PEER_HAS_PKA) wg_timers_set_persistent_keepalive(peer, peer_o.p_pka); if (peer_o.p_flags & WG_PEER_SET_DESCRIPTION) strlcpy(peer->p_description, peer_o.p_description, sizeof(peer->p_description)); if (peer_o.p_flags & WG_PEER_REPLACE_AIPS) wg_aip_remove_all(sc, peer); for (j = 0; j < peer_o.p_aips_count; j++) { aip_p = &peer_p->p_aips[j]; if ((ret = copyin(aip_p, &aip_o, sizeof(aip_o))) != 0) goto error; ret = wg_aip_add(sc, peer, aip_o.a_af, &aip_o.a_addr, aip_o.a_cidr); if (ret != 0) goto error; } if (sc->sc_ifp->if_link_state == LINK_STATE_UP) wg_peer_send_staged(peer); next_peer: if (remote != NULL) { noise_remote_put(remote); remote = NULL; } aip_p = &peer_p->p_aips[peer_o.p_aips_count]; peer_p = (struct wg_peer_io *)aip_p; } error: if (remote != NULL) noise_remote_put(remote); lockmgr(&sc->sc_lock, LK_RELEASE); explicit_bzero(&iface_o, sizeof(iface_o)); explicit_bzero(&peer_o, sizeof(peer_o)); explicit_bzero(&aip_o, sizeof(aip_o)); explicit_bzero(public, sizeof(public)); explicit_bzero(private, sizeof(private)); return (ret); } static int wg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cred) { struct wg_data_io *wgd; struct wg_softc *sc; struct ifreq *ifr; bool privileged; int ret, mask; sc = ifp->if_softc; ifr = (struct ifreq *)data; ret = 0; switch (cmd) { case SIOCSWG: ret = caps_priv_check(cred, SYSCAP_RESTRICTEDROOT); if (ret == 0) { wgd = (struct wg_data_io *)data; ret = wg_ioctl_set(sc, wgd); } break; case SIOCGWG: privileged = (caps_priv_check(cred, SYSCAP_RESTRICTEDROOT) == 0); wgd = (struct wg_data_io *)data; ret = wg_ioctl_get(sc, wgd, privileged); break; /* Interface IOCTLs */ case SIOCSIFADDR: /* * This differs from *BSD norms, but is more uniform with how * WireGuard behaves elsewhere. */ break; case SIOCSIFFLAGS: if (ifp->if_flags & IFF_UP) ret = wg_up(sc); else wg_down(sc); break; case SIOCSIFMTU: if (ifr->ifr_mtu <= 0 || ifr->ifr_mtu > MAX_MTU) ret = EINVAL; else ifp->if_mtu = ifr->ifr_mtu; break; case SIOCSIFCAP: mask = ifp->if_capenable ^ ifr->ifr_reqcap; if (mask & IFCAP_RXCSUM) ifp->if_capenable ^= IFCAP_RXCSUM; break; case SIOCADDMULTI: case SIOCDELMULTI: break; default: ret = ENOTTY; } return (ret); } static int wg_up(struct wg_softc *sc) { struct ifnet *ifp = sc->sc_ifp; struct wg_peer *peer; int ret = 0; lockmgr(&sc->sc_lock, LK_EXCLUSIVE); /* Silent success if we're already running. */ if (ifp->if_flags & IFF_RUNNING) goto out; ifp->if_flags |= IFF_RUNNING; ret = wg_socket_init(sc, sc->sc_socket.so_port); if (ret == 0) { TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) wg_timers_enable(peer); ifp->if_link_state = LINK_STATE_UP; if_link_state_change(ifp); } else { ifp->if_flags &= ~IFF_RUNNING; DPRINTF(sc, "Unable to initialize sockets: %d\n", ret); } out: lockmgr(&sc->sc_lock, LK_RELEASE); return (ret); } static void wg_down(struct wg_softc *sc) { struct ifnet *ifp = sc->sc_ifp; struct wg_peer *peer; int i; lockmgr(&sc->sc_lock, LK_EXCLUSIVE); if ((ifp->if_flags & IFF_RUNNING) == 0) { lockmgr(&sc->sc_lock, LK_RELEASE); return; } ifp->if_flags &= ~IFF_RUNNING; /* Cancel all tasks. */ while (taskqueue_cancel(sc->sc_handshake_taskqueue, &sc->sc_handshake_task, NULL) != 0) { taskqueue_drain(sc->sc_handshake_taskqueue, &sc->sc_handshake_task); } for (i = 0; i < ncpus; i++) { while (taskqueue_cancel(wg_taskqueues[i], &sc->sc_encrypt_tasks[i], NULL) != 0) { taskqueue_drain(wg_taskqueues[i], &sc->sc_encrypt_tasks[i]); } while (taskqueue_cancel(wg_taskqueues[i], &sc->sc_decrypt_tasks[i], NULL) != 0) { taskqueue_drain(wg_taskqueues[i], &sc->sc_decrypt_tasks[i]); } } TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { wg_queue_purge(&peer->p_stage_queue); wg_timers_disable(peer); } wg_queue_purge(&sc->sc_handshake_queue); TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { noise_remote_handshake_clear(peer->p_remote); noise_remote_keypairs_clear(peer->p_remote); } ifp->if_link_state = LINK_STATE_DOWN; if_link_state_change(ifp); wg_socket_uninit(sc); lockmgr(&sc->sc_lock, LK_RELEASE); } static int wg_clone_create(struct if_clone *ifc __unused, int unit, caddr_t params __unused, caddr_t data __unused) { struct wg_softc *sc; struct ifnet *ifp; int i; sc = kmalloc(sizeof(*sc), M_WG, M_WAITOK | M_ZERO); if (!rn_inithead(&sc->sc_aip4, wg_maskhead, offsetof(struct aip_addr, in)) || !rn_inithead(&sc->sc_aip6, wg_maskhead, offsetof(struct aip_addr, in6))) { if (sc->sc_aip4 != NULL) rn_freehead(sc->sc_aip4); if (sc->sc_aip6 != NULL) rn_freehead(sc->sc_aip6); kfree(sc, M_WG); return (ENOMEM); } lockinit(&sc->sc_lock, "wg softc lock", 0, 0); lockinit(&sc->sc_aip_lock, "wg aip lock", 0, 0); sc->sc_local = noise_local_alloc(); sc->sc_cookie = cookie_checker_alloc(); TAILQ_INIT(&sc->sc_peers); sc->sc_handshake_taskqueue = wg_taskqueues[karc4random() % ncpus]; TASK_INIT(&sc->sc_handshake_task, 0, wg_handshake_worker, sc); wg_queue_init(&sc->sc_handshake_queue, "hsq"); sc->sc_encrypt_tasks = kmalloc(sizeof(*sc->sc_encrypt_tasks) * ncpus, M_WG, M_WAITOK | M_ZERO); sc->sc_decrypt_tasks = kmalloc(sizeof(*sc->sc_decrypt_tasks) * ncpus, M_WG, M_WAITOK | M_ZERO); for (i = 0; i < ncpus; i++) { TASK_INIT(&sc->sc_encrypt_tasks[i], 0, wg_encrypt_worker, sc); TASK_INIT(&sc->sc_decrypt_tasks[i], 0, wg_decrypt_worker, sc); } wg_queue_init(&sc->sc_encrypt_parallel, "encp"); wg_queue_init(&sc->sc_decrypt_parallel, "decp"); ifp = sc->sc_ifp = if_alloc(IFT_WIREGUARD); if_initname(ifp, wgname, unit); ifp->if_softc = sc; ifp->if_mtu = DEFAULT_MTU; ifp->if_flags = IFF_NOARP | IFF_MULTICAST; ifp->if_capabilities = ifp->if_capenable = IFCAP_RXCSUM; ifp->if_output = wg_output; ifp->if_ioctl = wg_ioctl; ifq_set_maxlen(&ifp->if_snd, ifqmaxlen); ifq_set_ready(&ifp->if_snd); if_attach(ifp, NULL); /* DLT_NULL link-layer header: a 4-byte field in host byte order */ bpfattach(ifp, DLT_NULL, sizeof(uint32_t)); lockmgr(&wg_mtx, LK_EXCLUSIVE); LIST_INSERT_HEAD(&wg_list, sc, sc_entry); lockmgr(&wg_mtx, LK_RELEASE); return (0); } static int wg_clone_destroy(struct ifnet *ifp) { struct wg_softc *sc = ifp->if_softc; wg_down(sc); lockmgr(&sc->sc_lock, LK_EXCLUSIVE); kfree(sc->sc_encrypt_tasks, M_WG); kfree(sc->sc_decrypt_tasks, M_WG); wg_queue_deinit(&sc->sc_handshake_queue); wg_queue_deinit(&sc->sc_encrypt_parallel); wg_queue_deinit(&sc->sc_decrypt_parallel); wg_peer_destroy_all(sc); /* * Detach and free the interface before the sc_aip4 and sc_aip6 radix * trees, because the purge of interface's IPv6 addresses can cause * packet transmission and thus wg_aip_lookup() calls. */ bpfdetach(ifp); if_detach(ifp); if_free(ifp); /* * All peers have been removed, so the sc_aip4 and sc_aip6 radix trees * must be empty now. */ rn_freehead(sc->sc_aip4); rn_freehead(sc->sc_aip6); lockuninit(&sc->sc_aip_lock); cookie_checker_free(sc->sc_cookie); noise_local_free(sc->sc_local); lockmgr(&wg_mtx, LK_EXCLUSIVE); LIST_REMOVE(sc, sc_entry); lockmgr(&wg_mtx, LK_RELEASE); lockmgr(&sc->sc_lock, LK_RELEASE); lockuninit(&sc->sc_lock); kfree(sc, M_WG); return (0); } /*----------------------------------------------------------------------------*/ /* Module Interface */ #ifdef WG_SELFTESTS #include "selftest/allowedips.c" static bool wg_run_selftests(void) { bool ret = true; ret &= wg_allowedips_selftest(); ret &= noise_counter_selftest(); ret &= cookie_selftest(); kprintf("%s: %s\n", __func__, ret ? "pass" : "FAIL"); return (ret); } #else /* !WG_SELFTESTS */ static inline bool wg_run_selftests(void) { return (true); } #endif /* WG_SELFTESTS */ static struct if_clone wg_cloner = IF_CLONE_INITIALIZER( wgname, wg_clone_create, wg_clone_destroy, 0, IF_MAXUNIT); static int wg_module_init(void) { int i, ret; lockinit(&wg_mtx, "wg mtx lock", 0, 0); wg_packet_zone = objcache_create_simple(M_WG_PACKET, sizeof(struct wg_packet)); if (wg_packet_zone == NULL) return (ENOMEM); wg_taskqueues = kmalloc(sizeof(*wg_taskqueues) * ncpus, M_WG, M_WAITOK | M_ZERO); for (i = 0; i < ncpus; i++) { wg_taskqueues[i] = taskqueue_create("wg_taskq", M_WAITOK, taskqueue_thread_enqueue, &wg_taskqueues[i]); taskqueue_start_threads(&wg_taskqueues[i], 1, TDPRI_KERN_DAEMON, i, "wg_taskq_cpu_%d", i); } if (!rn_inithead(&wg_maskhead, NULL, 0)) return (ENOMEM); ret = cookie_init(); if (ret != 0) return (ret); ret = noise_init(); if (ret != 0) return (ret); ret = if_clone_attach(&wg_cloner); if (ret != 0) return (ret); if (!wg_run_selftests()) return (ENOTRECOVERABLE); return (0); } static int wg_module_deinit(void) { int i; lockmgr(&wg_mtx, LK_EXCLUSIVE); if (!LIST_EMPTY(&wg_list)) { lockmgr(&wg_mtx, LK_RELEASE); return (EBUSY); } if_clone_detach(&wg_cloner); noise_deinit(); cookie_deinit(); for (i = 0; i < ncpus; i++) taskqueue_free(wg_taskqueues[i]); kfree(wg_taskqueues, M_WG); rn_flush(wg_maskhead, rn_freemask); rn_freehead(wg_maskhead); if (wg_packet_zone != NULL) objcache_destroy(wg_packet_zone); lockmgr(&wg_mtx, LK_RELEASE); lockuninit(&wg_mtx); return (0); } static int wg_module_event_handler(module_t mod __unused, int what, void *arg __unused) { switch (what) { case MOD_LOAD: return wg_module_init(); case MOD_UNLOAD: return wg_module_deinit(); default: return (EOPNOTSUPP); } } static moduledata_t wg_moduledata = { "if_wg", wg_module_event_handler, NULL }; DECLARE_MODULE(if_wg, wg_moduledata, SI_SUB_PSEUDO, SI_ORDER_ANY); MODULE_VERSION(if_wg, 1); /* WireGuard version */ MODULE_DEPEND(if_wg, crypto, 1, 1, 1);