1 /*- 2 * SPDX-License-Identifier: ISC 3 * 4 * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 5 * Copyright (C) 2019-2021 Matt Dunwoodie <ncon@noconroy.net> 6 * Copyright (c) 2019-2020 Rubicon Communications, LLC (Netgate) 7 * Copyright (c) 2021 Kyle Evans <kevans@FreeBSD.org> 8 * Copyright (c) 2022 The FreeBSD Foundation 9 * Copyright (c) 2023-2024 Aaron LI <aly@aaronly.me> 10 * 11 * Permission to use, copy, modify, and distribute this software for any 12 * purpose with or without fee is hereby granted, provided that the above 13 * copyright notice and this permission notice appear in all copies. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 16 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 17 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 18 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 19 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 20 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 21 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 22 */ 23 24 #include "opt_inet6.h" 25 26 #include <sys/param.h> 27 #include <sys/systm.h> 28 #include <sys/callout.h> 29 #include <sys/caps.h> 30 #include <sys/endian.h> 31 #include <sys/kernel.h> 32 #include <sys/lock.h> 33 #include <sys/malloc.h> 34 #include <sys/mbuf.h> 35 #include <sys/module.h> 36 #include <sys/objcache.h> 37 #include <sys/queue.h> 38 #include <sys/socket.h> 39 #include <sys/socketops.h> /* so_pru_*() functions */ 40 #include <sys/socketvar.h> 41 #include <sys/sockio.h> /* SIOC* ioctl commands */ 42 #include <sys/taskqueue.h> 43 #include <sys/time.h> 44 45 #include <machine/atomic.h> 46 47 #include <net/bpf.h> 48 #include <net/ethernet.h> /* ETHERMTU */ 49 #include <net/if.h> 50 #include <net/if_clone.h> 51 #include <net/if_types.h> /* IFT_WIREGUARD */ 52 #include <net/if_var.h> 53 #include <net/ifq_var.h> 54 #include <net/netisr.h> 55 #include <net/radix.h> 56 #include <net/route.h> /* struct rtentry */ 57 58 #include <netinet/in.h> 59 #include <netinet/ip.h> 60 #include <netinet/ip_icmp.h> 61 #include <netinet/ip6.h> 62 #include <netinet/icmp6.h> 63 #include <netinet6/in6_var.h> /* in6_mask2len() */ 64 65 #include "wg_cookie.h" 66 #include "wg_noise.h" 67 #include "if_wg.h" 68 69 CTASSERT(WG_KEY_SIZE >= NOISE_PUBLIC_KEY_LEN); 70 CTASSERT(WG_KEY_SIZE >= NOISE_SYMMETRIC_KEY_LEN); 71 72 #define DEFAULT_MTU (ETHERMTU - 80) 73 #define MAX_MTU (IF_MAXMTU - 80) 74 75 #ifndef ENOKEY 76 #define ENOKEY ENOENT 77 #endif 78 79 /* 80 * mbuf flags to clear after in-place encryption/decryption, so that the 81 * mbuf can be reused for re-entering the network stack or delivering to 82 * the remote peer. 83 * 84 * For example, the M_HASH and M_LENCHECKED flags must be cleared for an 85 * inbound packet; otherwise, panic is to be expected. 86 */ 87 #define MBUF_CLEARFLAGS (M_COPYFLAGS & ~(M_PKTHDR | M_EOR | M_PRIO)) 88 89 #define MAX_LOOPS 8 /* 0 means no loop allowed */ 90 #define MTAG_WGLOOP 0x77676c70 /* wglp; cookie for loop check */ 91 92 #define MAX_STAGED_PKT 128 93 #define MAX_QUEUED_PKT 1024 94 #define MAX_QUEUED_PKT_MASK (MAX_QUEUED_PKT - 1) 95 #define MAX_QUEUED_HANDSHAKES 4096 96 97 #define REKEY_TIMEOUT_JITTER (karc4random() % 334) /* msec */ 98 #define MAX_TIMER_HANDSHAKES (90 / REKEY_TIMEOUT) 99 #define NEW_HANDSHAKE_TIMEOUT (REKEY_TIMEOUT + KEEPALIVE_TIMEOUT) 100 #define UNDERLOAD_TIMEOUT 1 101 102 /* First byte indicating packet type on the wire */ 103 #define WG_PKT_INITIATION htole32(1) 104 #define WG_PKT_RESPONSE htole32(2) 105 #define WG_PKT_COOKIE htole32(3) 106 #define WG_PKT_DATA htole32(4) 107 108 #define WG_PKT_PADDING 16 109 #define WG_PKT_WITH_PADDING(n) \ 110 (((n) + (WG_PKT_PADDING - 1)) & ~(WG_PKT_PADDING - 1)) 111 #define WG_PKT_ENCRYPTED_LEN(n) \ 112 (offsetof(struct wg_pkt_data, buf[(n)]) + NOISE_AUTHTAG_LEN) 113 #define WG_PKT_IS_INITIATION(m) \ 114 (*mtod((m), uint32_t *) == WG_PKT_INITIATION && \ 115 (size_t)(m)->m_pkthdr.len == sizeof(struct wg_pkt_initiation)) 116 #define WG_PKT_IS_RESPONSE(m) \ 117 (*mtod((m), uint32_t *) == WG_PKT_RESPONSE && \ 118 (size_t)(m)->m_pkthdr.len == sizeof(struct wg_pkt_response)) 119 #define WG_PKT_IS_COOKIE(m) \ 120 (*mtod((m), uint32_t *) == WG_PKT_COOKIE && \ 121 (size_t)(m)->m_pkthdr.len == sizeof(struct wg_pkt_cookie)) 122 #define WG_PKT_IS_DATA(m) \ 123 (*mtod((m), uint32_t *) == WG_PKT_DATA && \ 124 (size_t)(m)->m_pkthdr.len >= WG_PKT_ENCRYPTED_LEN(0)) 125 126 127 #define DPRINTF(sc, fmt, ...) \ 128 if (sc->sc_ifp->if_flags & IFF_DEBUG) \ 129 if_printf(sc->sc_ifp, fmt, ##__VA_ARGS__) 130 131 132 struct wg_pkt_initiation { 133 uint32_t t; 134 uint32_t s_idx; 135 uint8_t ue[NOISE_PUBLIC_KEY_LEN]; 136 uint8_t es[NOISE_PUBLIC_KEY_LEN + NOISE_AUTHTAG_LEN]; 137 uint8_t ets[NOISE_TIMESTAMP_LEN + NOISE_AUTHTAG_LEN]; 138 struct cookie_macs m; 139 }; 140 141 struct wg_pkt_response { 142 uint32_t t; 143 uint32_t s_idx; 144 uint32_t r_idx; 145 uint8_t ue[NOISE_PUBLIC_KEY_LEN]; 146 uint8_t en[0 + NOISE_AUTHTAG_LEN]; 147 struct cookie_macs m; 148 }; 149 150 struct wg_pkt_cookie { 151 uint32_t t; 152 uint32_t r_idx; 153 uint8_t nonce[COOKIE_NONCE_SIZE]; 154 uint8_t ec[COOKIE_ENCRYPTED_SIZE]; 155 }; 156 157 struct wg_pkt_data { 158 uint32_t t; 159 uint32_t r_idx; 160 uint64_t counter; 161 uint8_t buf[]; 162 }; 163 164 struct wg_endpoint { 165 union { 166 struct sockaddr r_sa; 167 struct sockaddr_in r_sin; 168 #ifdef INET6 169 struct sockaddr_in6 r_sin6; 170 #endif 171 } e_remote; 172 /* 173 * NOTE: No 'e_local' on DragonFly, because the socket upcall 174 * and so_pru_soreceive() cannot provide the local 175 * (i.e., destination) address of a received packet. 176 */ 177 }; 178 179 struct aip_addr { 180 uint8_t length; /* required by the radix code */ 181 union { 182 uint8_t bytes[16]; 183 uint32_t ip; 184 uint32_t ip6[4]; 185 struct in_addr in; 186 struct in6_addr in6; 187 }; 188 }; 189 190 struct wg_aip { 191 struct radix_node a_nodes[2]; /* make the first for casting */ 192 LIST_ENTRY(wg_aip) a_entry; 193 struct aip_addr a_addr; 194 struct aip_addr a_mask; 195 struct wg_peer *a_peer; 196 sa_family_t a_af; 197 }; 198 199 enum wg_packet_state { 200 WG_PACKET_DEAD, /* to be dropped */ 201 WG_PACKET_UNCRYPTED, /* before encryption/decryption */ 202 WG_PACKET_CRYPTED, /* after encryption/decryption */ 203 }; 204 205 struct wg_packet { 206 STAILQ_ENTRY(wg_packet) p_serial; 207 STAILQ_ENTRY(wg_packet) p_parallel; 208 struct wg_endpoint p_endpoint; 209 struct noise_keypair *p_keypair; 210 uint64_t p_counter; 211 struct mbuf *p_mbuf; 212 int p_mtu; 213 sa_family_t p_af; 214 unsigned int p_state; /* atomic */ 215 }; 216 217 STAILQ_HEAD(wg_packet_list, wg_packet); 218 219 struct wg_queue { 220 struct lock q_mtx; 221 struct wg_packet_list q_queue; 222 size_t q_len; 223 }; 224 225 struct wg_peer { 226 TAILQ_ENTRY(wg_peer) p_entry; 227 unsigned long p_id; 228 struct wg_softc *p_sc; 229 230 char p_description[WG_PEER_DESCR_SIZE]; 231 232 struct noise_remote *p_remote; 233 struct cookie_maker *p_cookie; 234 235 struct lock p_endpoint_lock; 236 struct wg_endpoint p_endpoint; 237 238 struct wg_queue p_stage_queue; 239 struct wg_queue p_encrypt_serial; 240 struct wg_queue p_decrypt_serial; 241 242 bool p_enabled; 243 bool p_need_another_keepalive; 244 uint16_t p_persistent_keepalive_interval; 245 struct callout p_new_handshake; 246 struct callout p_send_keepalive; 247 struct callout p_retry_handshake; 248 struct callout p_zero_key_material; 249 struct callout p_persistent_keepalive; 250 251 struct lock p_handshake_mtx; 252 struct timespec p_handshake_complete; /* nanotime */ 253 int p_handshake_retries; 254 255 struct task p_send_task; 256 struct task p_recv_task; 257 struct taskqueue *p_send_taskqueue; 258 struct taskqueue *p_recv_taskqueue; 259 260 uint64_t *p_tx_bytes; 261 uint64_t *p_rx_bytes; 262 263 LIST_HEAD(, wg_aip) p_aips; 264 size_t p_aips_num; 265 }; 266 267 struct wg_socket { 268 struct lock so_lock; 269 struct socket *so_so4; 270 struct socket *so_so6; 271 uint32_t so_user_cookie; 272 in_port_t so_port; 273 }; 274 275 struct wg_softc { 276 LIST_ENTRY(wg_softc) sc_entry; 277 struct ifnet *sc_ifp; 278 int sc_flags; 279 280 struct wg_socket sc_socket; 281 282 TAILQ_HEAD(, wg_peer) sc_peers; 283 size_t sc_peers_num; 284 285 struct noise_local *sc_local; 286 struct cookie_checker *sc_cookie; 287 288 struct lock sc_aip_lock; 289 struct radix_node_head *sc_aip4; 290 struct radix_node_head *sc_aip6; 291 292 struct taskqueue *sc_handshake_taskqueue; 293 struct task sc_handshake_task; 294 struct wg_queue sc_handshake_queue; 295 296 struct task *sc_encrypt_tasks; /* one per CPU */ 297 struct task *sc_decrypt_tasks; /* one per CPU */ 298 struct wg_queue sc_encrypt_parallel; 299 struct wg_queue sc_decrypt_parallel; 300 int sc_encrypt_last_cpu; 301 int sc_decrypt_last_cpu; 302 303 struct lock sc_lock; 304 }; 305 306 307 static MALLOC_DEFINE(M_WG, "WG", "wireguard"); 308 static MALLOC_DEFINE(M_WG_PACKET, "WG packet", "wireguard packet"); 309 310 static const char wgname[] = "wg"; 311 312 static struct objcache *wg_packet_zone; 313 static struct lock wg_mtx; 314 static struct taskqueue **wg_taskqueues; /* one taskqueue per CPU */ 315 static struct radix_node_head *wg_maskhead; /* shared by all interfaces */ 316 static LIST_HEAD(, wg_softc) wg_list = LIST_HEAD_INITIALIZER(wg_list); 317 318 319 /* Timers */ 320 static void wg_timers_enable(struct wg_peer *); 321 static void wg_timers_disable(struct wg_peer *); 322 323 /* Allowed IP */ 324 static int wg_aip_add(struct wg_softc *, struct wg_peer *, sa_family_t, 325 const void *, uint8_t); 326 static struct wg_peer * 327 wg_aip_lookup(struct wg_softc *, sa_family_t, const void *); 328 static void wg_aip_remove_all(struct wg_softc *, struct wg_peer *); 329 330 /* Handshake */ 331 static void wg_send_initiation(struct wg_peer *); 332 static void wg_send_response(struct wg_peer *); 333 static void wg_send_cookie(struct wg_softc *, struct cookie_macs *, 334 uint32_t, struct wg_endpoint *); 335 static void wg_send_keepalive(struct wg_peer *); 336 337 /* Transport Packet Functions */ 338 static void wg_peer_send_staged(struct wg_peer *); 339 static void wg_deliver_out(void *, int); 340 static void wg_deliver_in(void *, int); 341 static void wg_upcall(struct socket *, void *, int); 342 343 /*----------------------------------------------------------------------------*/ 344 /* Packet */ 345 346 static struct wg_packet * 347 wg_packet_alloc(struct mbuf *m) 348 { 349 struct wg_packet *pkt; 350 351 if ((pkt = objcache_get(wg_packet_zone, M_NOWAIT)) == NULL) 352 return (NULL); 353 354 bzero(pkt, sizeof(*pkt)); /* objcache_get() doesn't ensure M_ZERO. */ 355 pkt->p_mbuf = m; 356 357 return (pkt); 358 } 359 360 static void 361 wg_packet_free(struct wg_packet *pkt) 362 { 363 if (pkt->p_keypair != NULL) 364 noise_keypair_put(pkt->p_keypair); 365 if (pkt->p_mbuf != NULL) 366 m_freem(pkt->p_mbuf); 367 objcache_put(wg_packet_zone, pkt); 368 } 369 370 /*----------------------------------------------------------------------------*/ 371 /* 372 * Packet Queue Functions 373 * 374 * WireGuard uses the following queues: 375 * - per-interface handshake queue: track incoming handshake packets 376 * - per-peer staged queue: track the outgoing packets sent by that peer 377 * - per-interface parallel encrypt and decrypt queues 378 * - per-peer serial encrypt and decrypt queues 379 * 380 * For one interface, the handshake packets are only tracked in the handshake 381 * queue and are processed in serial. However, all data packets are tracked 382 * in two queues: a serial queue and a parallel queue. Specifically, the 383 * outgoing packets (from the staged queue) will be queued in both the 384 * parallel encrypt and the serial encrypt queues; the incoming packets will 385 * be queued in both the parallel decrypt and the serial decrypt queues. 386 * 387 * - The parallel queues are used to distribute the encryption/decryption work 388 * across all CPUs. The per-CPU wg_{encrypt,decrypt}_worker() work on the 389 * parallel queues. 390 * - The serial queues ensure that packets are not reordered and are 391 * delivered in sequence for each peer. The per-peer wg_deliver_{in,out}() 392 * work on the serial queues. 393 */ 394 395 static void wg_queue_purge(struct wg_queue *); 396 397 static void 398 wg_queue_init(struct wg_queue *queue, const char *name) 399 { 400 lockinit(&queue->q_mtx, name, 0, 0); 401 STAILQ_INIT(&queue->q_queue); 402 queue->q_len = 0; 403 } 404 405 static void 406 wg_queue_deinit(struct wg_queue *queue) 407 { 408 wg_queue_purge(queue); 409 lockuninit(&queue->q_mtx); 410 } 411 412 static size_t 413 wg_queue_len(const struct wg_queue *queue) 414 { 415 return (queue->q_len); 416 } 417 418 static bool 419 wg_queue_enqueue_handshake(struct wg_queue *hs, struct wg_packet *pkt) 420 { 421 bool ok = false; 422 423 lockmgr(&hs->q_mtx, LK_EXCLUSIVE); 424 if (hs->q_len < MAX_QUEUED_HANDSHAKES) { 425 STAILQ_INSERT_TAIL(&hs->q_queue, pkt, p_parallel); 426 hs->q_len++; 427 ok = true; 428 } 429 lockmgr(&hs->q_mtx, LK_RELEASE); 430 431 if (!ok) 432 wg_packet_free(pkt); 433 434 return (ok); 435 } 436 437 static struct wg_packet * 438 wg_queue_dequeue_handshake(struct wg_queue *hs) 439 { 440 struct wg_packet *pkt; 441 442 lockmgr(&hs->q_mtx, LK_EXCLUSIVE); 443 pkt = STAILQ_FIRST(&hs->q_queue); 444 if (pkt != NULL) { 445 STAILQ_REMOVE_HEAD(&hs->q_queue, p_parallel); 446 hs->q_len--; 447 } 448 lockmgr(&hs->q_mtx, LK_RELEASE); 449 450 return (pkt); 451 } 452 453 static void 454 wg_queue_push_staged(struct wg_queue *staged, struct wg_packet *pkt) 455 { 456 struct wg_packet *old = NULL; 457 458 lockmgr(&staged->q_mtx, LK_EXCLUSIVE); 459 if (staged->q_len >= MAX_STAGED_PKT) { 460 old = STAILQ_FIRST(&staged->q_queue); 461 STAILQ_REMOVE_HEAD(&staged->q_queue, p_parallel); 462 staged->q_len--; 463 } 464 STAILQ_INSERT_TAIL(&staged->q_queue, pkt, p_parallel); 465 staged->q_len++; 466 lockmgr(&staged->q_mtx, LK_RELEASE); 467 468 if (old != NULL) 469 wg_packet_free(old); 470 } 471 472 static void 473 wg_queue_enlist_staged(struct wg_queue *staged, struct wg_packet_list *list) 474 { 475 struct wg_packet *pkt, *tpkt; 476 477 STAILQ_FOREACH_MUTABLE(pkt, list, p_parallel, tpkt) 478 wg_queue_push_staged(staged, pkt); 479 STAILQ_INIT(list); 480 } 481 482 static void 483 wg_queue_delist_staged(struct wg_queue *staged, struct wg_packet_list *list) 484 { 485 STAILQ_INIT(list); 486 lockmgr(&staged->q_mtx, LK_EXCLUSIVE); 487 STAILQ_CONCAT(list, &staged->q_queue); 488 staged->q_len = 0; 489 lockmgr(&staged->q_mtx, LK_RELEASE); 490 } 491 492 static void 493 wg_queue_purge(struct wg_queue *staged) 494 { 495 struct wg_packet_list list; 496 struct wg_packet *pkt, *tpkt; 497 498 wg_queue_delist_staged(staged, &list); 499 STAILQ_FOREACH_MUTABLE(pkt, &list, p_parallel, tpkt) 500 wg_packet_free(pkt); 501 } 502 503 static bool 504 wg_queue_both(struct wg_queue *parallel, struct wg_queue *serial, 505 struct wg_packet *pkt) 506 { 507 pkt->p_state = WG_PACKET_UNCRYPTED; 508 509 lockmgr(&serial->q_mtx, LK_EXCLUSIVE); 510 if (serial->q_len < MAX_QUEUED_PKT) { 511 serial->q_len++; 512 STAILQ_INSERT_TAIL(&serial->q_queue, pkt, p_serial); 513 } else { 514 lockmgr(&serial->q_mtx, LK_RELEASE); 515 wg_packet_free(pkt); 516 return (false); 517 } 518 lockmgr(&serial->q_mtx, LK_RELEASE); 519 520 lockmgr(¶llel->q_mtx, LK_EXCLUSIVE); 521 if (parallel->q_len < MAX_QUEUED_PKT) { 522 parallel->q_len++; 523 STAILQ_INSERT_TAIL(¶llel->q_queue, pkt, p_parallel); 524 } else { 525 lockmgr(¶llel->q_mtx, LK_RELEASE); 526 /* 527 * Cannot just free the packet because it's already queued 528 * in the serial queue. Instead, set its state to DEAD and 529 * let the serial worker to free it. 530 */ 531 pkt->p_state = WG_PACKET_DEAD; 532 return (false); 533 } 534 lockmgr(¶llel->q_mtx, LK_RELEASE); 535 536 return (true); 537 } 538 539 static struct wg_packet * 540 wg_queue_dequeue_serial(struct wg_queue *serial) 541 { 542 struct wg_packet *pkt = NULL; 543 544 lockmgr(&serial->q_mtx, LK_EXCLUSIVE); 545 if (serial->q_len > 0 && 546 STAILQ_FIRST(&serial->q_queue)->p_state != WG_PACKET_UNCRYPTED) { 547 /* 548 * Dequeue both CRYPTED packets (to be delivered) and 549 * DEAD packets (to be freed). 550 */ 551 serial->q_len--; 552 pkt = STAILQ_FIRST(&serial->q_queue); 553 STAILQ_REMOVE_HEAD(&serial->q_queue, p_serial); 554 } 555 lockmgr(&serial->q_mtx, LK_RELEASE); 556 557 return (pkt); 558 } 559 560 static struct wg_packet * 561 wg_queue_dequeue_parallel(struct wg_queue *parallel) 562 { 563 struct wg_packet *pkt = NULL; 564 565 lockmgr(¶llel->q_mtx, LK_EXCLUSIVE); 566 if (parallel->q_len > 0) { 567 parallel->q_len--; 568 pkt = STAILQ_FIRST(¶llel->q_queue); 569 STAILQ_REMOVE_HEAD(¶llel->q_queue, p_parallel); 570 } 571 lockmgr(¶llel->q_mtx, LK_RELEASE); 572 573 return (pkt); 574 } 575 576 /*----------------------------------------------------------------------------*/ 577 /* Peer */ 578 579 static struct wg_peer * 580 wg_peer_create(struct wg_softc *sc, const uint8_t pub_key[WG_KEY_SIZE]) 581 { 582 static unsigned long peer_counter = 0; 583 struct wg_peer *peer; 584 585 KKASSERT(lockstatus(&sc->sc_lock, curthread) == LK_EXCLUSIVE); 586 587 peer = kmalloc(sizeof(*peer), M_WG, M_WAITOK | M_ZERO); 588 589 peer->p_remote = noise_remote_alloc(sc->sc_local, pub_key, peer); 590 if (noise_remote_enable(peer->p_remote) != 0) { 591 kfree(peer, M_WG); 592 return (NULL); 593 } 594 595 peer->p_cookie = cookie_maker_alloc(pub_key); 596 597 peer->p_id = ++peer_counter; 598 peer->p_sc = sc; 599 peer->p_tx_bytes = kmalloc(sizeof(*peer->p_tx_bytes) * ncpus, 600 M_WG, M_WAITOK | M_ZERO); 601 peer->p_rx_bytes = kmalloc(sizeof(*peer->p_rx_bytes) * ncpus, 602 M_WG, M_WAITOK | M_ZERO); 603 604 lockinit(&peer->p_endpoint_lock, "wg_peer_endpoint", 0, 0); 605 lockinit(&peer->p_handshake_mtx, "wg_peer_handshake", 0, 0); 606 607 wg_queue_init(&peer->p_stage_queue, "stageq"); 608 wg_queue_init(&peer->p_encrypt_serial, "txq"); 609 wg_queue_init(&peer->p_decrypt_serial, "rxq"); 610 611 callout_init_mp(&peer->p_new_handshake); 612 callout_init_mp(&peer->p_send_keepalive); 613 callout_init_mp(&peer->p_retry_handshake); 614 callout_init_mp(&peer->p_persistent_keepalive); 615 callout_init_mp(&peer->p_zero_key_material); 616 617 TASK_INIT(&peer->p_send_task, 0, wg_deliver_out, peer); 618 TASK_INIT(&peer->p_recv_task, 0, wg_deliver_in, peer); 619 620 /* Randomly choose the taskqueues to distribute the load. */ 621 peer->p_send_taskqueue = wg_taskqueues[karc4random() % ncpus]; 622 peer->p_recv_taskqueue = wg_taskqueues[karc4random() % ncpus]; 623 624 LIST_INIT(&peer->p_aips); 625 626 TAILQ_INSERT_TAIL(&sc->sc_peers, peer, p_entry); 627 sc->sc_peers_num++; 628 629 if (sc->sc_ifp->if_link_state == LINK_STATE_UP) 630 wg_timers_enable(peer); 631 632 DPRINTF(sc, "Peer %ld created\n", peer->p_id); 633 return (peer); 634 } 635 636 static void 637 wg_peer_destroy(struct wg_peer *peer) 638 { 639 struct wg_softc *sc = peer->p_sc; 640 641 KKASSERT(lockstatus(&sc->sc_lock, curthread) == LK_EXCLUSIVE); 642 643 /* 644 * Disable remote and timers. This will prevent any new handshakes 645 * from occuring. 646 */ 647 noise_remote_disable(peer->p_remote); 648 wg_timers_disable(peer); 649 650 /* 651 * Remove all allowed IPs, so no more packets will be routed to 652 * this peer. 653 */ 654 wg_aip_remove_all(sc, peer); 655 656 /* Remove peer from the interface, then free. */ 657 sc->sc_peers_num--; 658 TAILQ_REMOVE(&sc->sc_peers, peer, p_entry); 659 660 /* 661 * While there are no references remaining, we may still have 662 * p_{send,recv}_task executing (think empty queue, but 663 * wg_deliver_{in,out} needs to check the queue). We should wait 664 * for them and then free. 665 */ 666 taskqueue_drain(peer->p_recv_taskqueue, &peer->p_recv_task); 667 taskqueue_drain(peer->p_send_taskqueue, &peer->p_send_task); 668 669 wg_queue_deinit(&peer->p_decrypt_serial); 670 wg_queue_deinit(&peer->p_encrypt_serial); 671 wg_queue_deinit(&peer->p_stage_queue); 672 673 kfree(peer->p_tx_bytes, M_WG); 674 kfree(peer->p_rx_bytes, M_WG); 675 676 lockuninit(&peer->p_endpoint_lock); 677 lockuninit(&peer->p_handshake_mtx); 678 679 noise_remote_free(peer->p_remote); 680 cookie_maker_free(peer->p_cookie); 681 682 DPRINTF(sc, "Peer %ld destroyed\n", peer->p_id); 683 kfree(peer, M_WG); 684 } 685 686 static void 687 wg_peer_destroy_all(struct wg_softc *sc) 688 { 689 struct wg_peer *peer, *tpeer; 690 691 TAILQ_FOREACH_MUTABLE(peer, &sc->sc_peers, p_entry, tpeer) 692 wg_peer_destroy(peer); 693 } 694 695 static int 696 wg_peer_set_sockaddr(struct wg_peer *peer, const struct sockaddr *remote) 697 { 698 int ret = 0; 699 700 lockmgr(&peer->p_endpoint_lock, LK_EXCLUSIVE); 701 702 memcpy(&peer->p_endpoint.e_remote, remote, 703 sizeof(peer->p_endpoint.e_remote)); 704 if (remote->sa_family == AF_INET) 705 memcpy(&peer->p_endpoint.e_remote.r_sin, remote, 706 sizeof(peer->p_endpoint.e_remote.r_sin)); 707 #ifdef INET6 708 else if (remote->sa_family == AF_INET6) 709 memcpy(&peer->p_endpoint.e_remote.r_sin6, remote, 710 sizeof(peer->p_endpoint.e_remote.r_sin6)); 711 #endif 712 else 713 ret = EAFNOSUPPORT; 714 715 /* No 'e_local' to clear on DragonFly. */ 716 717 lockmgr(&peer->p_endpoint_lock, LK_RELEASE); 718 return (ret); 719 } 720 721 static int 722 wg_peer_get_sockaddr(struct wg_peer *peer, struct sockaddr *remote) 723 { 724 int ret = ENOENT; 725 726 lockmgr(&peer->p_endpoint_lock, LK_SHARED); 727 if (peer->p_endpoint.e_remote.r_sa.sa_family != AF_UNSPEC) { 728 memcpy(remote, &peer->p_endpoint.e_remote, 729 sizeof(peer->p_endpoint.e_remote)); 730 ret = 0; 731 } 732 lockmgr(&peer->p_endpoint_lock, LK_RELEASE); 733 return (ret); 734 } 735 736 static void 737 wg_peer_set_endpoint(struct wg_peer *peer, const struct wg_endpoint *e) 738 { 739 KKASSERT(e->e_remote.r_sa.sa_family != AF_UNSPEC); 740 741 if (__predict_true(memcmp(e, &peer->p_endpoint, sizeof(*e)) == 0)) 742 return; 743 744 lockmgr(&peer->p_endpoint_lock, LK_EXCLUSIVE); 745 peer->p_endpoint = *e; 746 lockmgr(&peer->p_endpoint_lock, LK_RELEASE); 747 } 748 749 static void 750 wg_peer_get_endpoint(struct wg_peer *peer, struct wg_endpoint *e) 751 { 752 if (__predict_true(memcmp(e, &peer->p_endpoint, sizeof(*e)) == 0)) 753 return; 754 755 lockmgr(&peer->p_endpoint_lock, LK_SHARED); 756 *e = peer->p_endpoint; 757 lockmgr(&peer->p_endpoint_lock, LK_RELEASE); 758 } 759 760 /*----------------------------------------------------------------------------*/ 761 /* Allowed IP */ 762 763 static int 764 wg_aip_add(struct wg_softc *sc, struct wg_peer *peer, sa_family_t af, 765 const void *addr, uint8_t cidr) 766 { 767 struct radix_node_head *head; 768 struct radix_node *node; 769 struct wg_aip *aip; 770 int ret = 0; 771 772 aip = kmalloc(sizeof(*aip), M_WG, M_WAITOK | M_ZERO); 773 aip->a_peer = peer; 774 aip->a_af = af; 775 776 switch (af) { 777 case AF_INET: 778 if (cidr > 32) 779 cidr = 32; 780 head = sc->sc_aip4; 781 aip->a_addr.in = *(const struct in_addr *)addr; 782 aip->a_mask.ip = 783 htonl(~((1LL << (32 - cidr)) - 1) & 0xffffffff); 784 aip->a_addr.ip &= aip->a_mask.ip; 785 aip->a_addr.length = aip->a_mask.length = 786 offsetof(struct aip_addr, in) + sizeof(struct in_addr); 787 break; 788 #ifdef INET6 789 case AF_INET6: 790 { 791 int i; 792 793 if (cidr > 128) 794 cidr = 128; 795 head = sc->sc_aip6; 796 aip->a_addr.in6 = *(const struct in6_addr *)addr; 797 in6_prefixlen2mask(&aip->a_mask.in6, cidr); 798 for (i = 0; i < 4; i++) 799 aip->a_addr.ip6[i] &= aip->a_mask.ip6[i]; 800 aip->a_addr.length = aip->a_mask.length = 801 offsetof(struct aip_addr, in6) + sizeof(struct in6_addr); 802 break; 803 } 804 #endif 805 default: 806 kfree(aip, M_WG); 807 return (EAFNOSUPPORT); 808 } 809 810 lockmgr(&sc->sc_aip_lock, LK_EXCLUSIVE); 811 node = head->rnh_addaddr(&aip->a_addr, &aip->a_mask, head, 812 aip->a_nodes); 813 if (node != NULL) { 814 KKASSERT(node == aip->a_nodes); 815 LIST_INSERT_HEAD(&peer->p_aips, aip, a_entry); 816 peer->p_aips_num++; 817 } else { 818 /* 819 * Two possibilities: 820 * - out of memory failure 821 * - entry already exists 822 */ 823 node = head->rnh_lookup(&aip->a_addr, &aip->a_mask, head); 824 if (node == NULL) { 825 kfree(aip, M_WG); 826 ret = ENOMEM; 827 } else { 828 KKASSERT(node != aip->a_nodes); 829 kfree(aip, M_WG); 830 aip = (struct wg_aip *)node; 831 if (aip->a_peer != peer) { 832 /* Replace the peer. */ 833 LIST_REMOVE(aip, a_entry); 834 aip->a_peer->p_aips_num--; 835 aip->a_peer = peer; 836 LIST_INSERT_HEAD(&peer->p_aips, aip, a_entry); 837 aip->a_peer->p_aips_num++; 838 } 839 } 840 } 841 lockmgr(&sc->sc_aip_lock, LK_RELEASE); 842 843 return (ret); 844 } 845 846 static struct wg_peer * 847 wg_aip_lookup(struct wg_softc *sc, sa_family_t af, const void *a) 848 { 849 struct radix_node_head *head; 850 struct radix_node *node; 851 struct wg_peer *peer; 852 struct aip_addr addr; 853 854 switch (af) { 855 case AF_INET: 856 head = sc->sc_aip4; 857 memcpy(&addr.in, a, sizeof(addr.in)); 858 addr.length = offsetof(struct aip_addr, in) + sizeof(addr.in); 859 break; 860 case AF_INET6: 861 head = sc->sc_aip6; 862 memcpy(&addr.in6, a, sizeof(addr.in6)); 863 addr.length = offsetof(struct aip_addr, in6) + sizeof(addr.in6); 864 break; 865 default: 866 return (NULL); 867 } 868 869 lockmgr(&sc->sc_aip_lock, LK_SHARED); 870 node = head->rnh_matchaddr(&addr, head); 871 if (node != NULL) { 872 peer = ((struct wg_aip *)node)->a_peer; 873 noise_remote_ref(peer->p_remote); 874 } else { 875 peer = NULL; 876 } 877 lockmgr(&sc->sc_aip_lock, LK_RELEASE); 878 879 return (peer); 880 } 881 882 static void 883 wg_aip_remove_all(struct wg_softc *sc, struct wg_peer *peer) 884 { 885 struct radix_node_head *head; 886 struct radix_node *node; 887 struct wg_aip *aip, *taip; 888 889 lockmgr(&sc->sc_aip_lock, LK_EXCLUSIVE); 890 891 LIST_FOREACH_MUTABLE(aip, &peer->p_aips, a_entry, taip) { 892 switch (aip->a_af) { 893 case AF_INET: 894 head = sc->sc_aip4; 895 break; 896 case AF_INET6: 897 head = sc->sc_aip6; 898 break; 899 default: 900 panic("%s: impossible aip %p", __func__, aip); 901 } 902 node = head->rnh_deladdr(&aip->a_addr, &aip->a_mask, head); 903 if (node == NULL) 904 panic("%s: failed to delete aip %p", __func__, aip); 905 LIST_REMOVE(aip, a_entry); 906 peer->p_aips_num--; 907 kfree(aip, M_WG); 908 } 909 910 if (!LIST_EMPTY(&peer->p_aips) || peer->p_aips_num != 0) 911 panic("%s: could not delete all aips for peer %ld", 912 __func__, peer->p_id); 913 914 lockmgr(&sc->sc_aip_lock, LK_RELEASE); 915 } 916 917 /*----------------------------------------------------------------------------*/ 918 /* Socket */ 919 920 static int wg_socket_open(struct socket **, sa_family_t, in_port_t *, 921 void *); 922 static int wg_socket_set_sockopt(struct socket *, struct socket *, 923 int, void *, size_t); 924 925 static int 926 wg_socket_init(struct wg_softc *sc, in_port_t port) 927 { 928 struct wg_socket *so = &sc->sc_socket; 929 struct socket *so4 = NULL, *so6 = NULL; 930 in_port_t bound_port = port; 931 uint32_t cookie; 932 int ret; 933 934 /* 935 * When a host or a jail doesn't support the AF, sobind() would 936 * return EADDRNOTAVAIL. Handle this case in order to support such 937 * IPv4-only or IPv6-only environments. 938 * 939 * However, in a dual-stack environment, both IPv4 and IPv6 sockets 940 * must bind the same port. 941 */ 942 ret = wg_socket_open(&so4, AF_INET, &bound_port, sc); 943 if (ret != 0 && ret != EADDRNOTAVAIL) 944 goto error; 945 946 #ifdef INET6 947 ret = wg_socket_open(&so6, AF_INET6, &bound_port, sc); 948 if (ret != 0 && ret != EADDRNOTAVAIL) 949 goto error; 950 #endif 951 952 if (so4 == NULL && so6 == NULL) { 953 ret = EAFNOSUPPORT; 954 goto error; 955 } 956 957 cookie = so->so_user_cookie; 958 if (cookie != 0) { 959 ret = wg_socket_set_sockopt(so4, so6, SO_USER_COOKIE, 960 &cookie, sizeof(cookie)); 961 if (ret != 0) 962 goto error; 963 } 964 965 KKASSERT(lockstatus(&sc->sc_lock, curthread) == LK_EXCLUSIVE); 966 967 lockinit(&so->so_lock, "wg socket lock", 0, 0); 968 969 if (so->so_so4 != NULL) 970 soclose(so->so_so4, 0); 971 if (so->so_so6 != NULL) 972 soclose(so->so_so6, 0); 973 so->so_so4 = so4; 974 so->so_so6 = so6; 975 so->so_port = bound_port; 976 977 return (0); 978 979 error: 980 if (so4 != NULL) 981 soclose(so4, 0); 982 if (so6 != NULL) 983 soclose(so6, 0); 984 return (ret); 985 } 986 987 static int 988 wg_socket_open(struct socket **so, sa_family_t af, in_port_t *port, 989 void *upcall_arg) 990 { 991 struct sockaddr_in sin; 992 #ifdef INET6 993 struct sockaddr_in6 sin6; 994 #endif 995 struct sockaddr *sa, *bound_sa; 996 int ret; 997 998 if (af == AF_INET) { 999 bzero(&sin, sizeof(sin)); 1000 sin.sin_len = sizeof(struct sockaddr_in); 1001 sin.sin_family = AF_INET; 1002 sin.sin_port = htons(*port); 1003 sa = sintosa(&sin); 1004 #ifdef INET6 1005 } else if (af == AF_INET6) { 1006 bzero(&sin6, sizeof(sin6)); 1007 sin6.sin6_len = sizeof(struct sockaddr_in6); 1008 sin6.sin6_family = AF_INET6; 1009 sin6.sin6_port = htons(*port); 1010 sa = sintosa(&sin6); 1011 #endif 1012 } else { 1013 return (EAFNOSUPPORT); 1014 } 1015 1016 ret = socreate(af, so, SOCK_DGRAM, IPPROTO_UDP, curthread); 1017 if (ret != 0) 1018 return (ret); 1019 1020 (*so)->so_upcall = wg_upcall; 1021 (*so)->so_upcallarg = upcall_arg; 1022 atomic_set_int(&(*so)->so_rcv.ssb_flags, SSB_UPCALL); 1023 1024 ret = sobind(*so, sa, curthread); 1025 if (ret != 0) 1026 goto error; 1027 1028 if (*port == 0) { 1029 ret = so_pru_sockaddr(*so, &bound_sa); 1030 if (ret != 0) 1031 goto error; 1032 if (bound_sa->sa_family == AF_INET) 1033 *port = ntohs(satosin(bound_sa)->sin_port); 1034 else 1035 *port = ntohs(satosin6(bound_sa)->sin6_port); 1036 kfree(bound_sa, M_SONAME); 1037 } 1038 1039 return (0); 1040 1041 error: 1042 if (*so != NULL) { 1043 soclose(*so, 0); 1044 *so = NULL; 1045 } 1046 return (ret); 1047 } 1048 1049 static void 1050 wg_socket_uninit(struct wg_softc *sc) 1051 { 1052 struct wg_socket *so = &sc->sc_socket; 1053 1054 KKASSERT(lockstatus(&sc->sc_lock, curthread) == LK_EXCLUSIVE); 1055 1056 lockmgr(&so->so_lock, LK_EXCLUSIVE); 1057 1058 if (so->so_so4 != NULL) { 1059 soclose(so->so_so4, 0); 1060 so->so_so4 = NULL; 1061 } 1062 if (so->so_so6 != NULL) { 1063 soclose(so->so_so6, 0); 1064 so->so_so6 = NULL; 1065 } 1066 1067 lockmgr(&so->so_lock, LK_RELEASE); 1068 lockuninit(&so->so_lock); 1069 } 1070 1071 static int 1072 wg_socket_set_sockopt(struct socket *so4, struct socket *so6, 1073 int name, void *val, size_t len) 1074 { 1075 struct sockopt sopt = { 1076 .sopt_dir = SOPT_SET, 1077 .sopt_level = SOL_SOCKET, 1078 .sopt_name = name, 1079 .sopt_val = val, 1080 .sopt_valsize = len, 1081 }; 1082 int ret; 1083 1084 if (so4 != NULL) { 1085 ret = sosetopt(so4, &sopt); 1086 if (ret != 0) 1087 return (ret); 1088 } 1089 if (so6 != NULL) { 1090 ret = sosetopt(so6, &sopt); 1091 if (ret != 0) 1092 return (ret); 1093 } 1094 1095 return (0); 1096 } 1097 1098 static int 1099 wg_socket_set_cookie(struct wg_softc *sc, uint32_t user_cookie) 1100 { 1101 struct wg_socket *so; 1102 int ret; 1103 1104 KKASSERT(lockstatus(&sc->sc_lock, curthread) == LK_EXCLUSIVE); 1105 1106 so = &sc->sc_socket; 1107 lockmgr(&so->so_lock, LK_EXCLUSIVE); 1108 1109 ret = wg_socket_set_sockopt(so->so_so4, so->so_so6, SO_USER_COOKIE, 1110 &user_cookie, sizeof(user_cookie)); 1111 if (ret == 0) 1112 so->so_user_cookie = user_cookie; 1113 1114 lockmgr(&so->so_lock, LK_RELEASE); 1115 return (ret); 1116 } 1117 1118 static int 1119 wg_send(struct wg_softc *sc, struct wg_endpoint *e, struct mbuf *m) 1120 { 1121 struct wg_socket *so; 1122 struct sockaddr *sa; 1123 int len, ret; 1124 1125 so = &sc->sc_socket; 1126 sa = &e->e_remote.r_sa; 1127 len = m->m_pkthdr.len; 1128 ret = 0; 1129 1130 /* 1131 * NOTE: DragonFly by default sends UDP packets asynchronously, 1132 * unless the 'net.inet.udp.sosend_async' sysctl MIB is set 1133 * to 0 or the 'MSG_SYNC' flag is set for so_pru_sosend(). 1134 * And in the async mode, an error code cannot really be 1135 * replied to the caller. So so_pru_sosend() may return 0 1136 * even if the packet fails to send. 1137 */ 1138 lockmgr(&so->so_lock, LK_SHARED); 1139 if (sa->sa_family == AF_INET && so->so_so4 != NULL) { 1140 ret = so_pru_sosend(so->so_so4, sa, NULL /* uio */, 1141 m, NULL /* control */, 0 /* flags */, 1142 curthread); 1143 #ifdef INET6 1144 } else if (sa->sa_family == AF_INET6 && so->so_so6 != NULL) { 1145 ret = so_pru_sosend(so->so_so6, sa, NULL /* uio */, 1146 m, NULL /* control */, 0 /* flags */, 1147 curthread); 1148 #endif 1149 } else { 1150 ret = ENOTCONN; 1151 m_freem(m); 1152 } 1153 lockmgr(&so->so_lock, LK_RELEASE); 1154 1155 if (ret == 0) { 1156 IFNET_STAT_INC(sc->sc_ifp, opackets, 1); 1157 IFNET_STAT_INC(sc->sc_ifp, obytes, len); 1158 } else { 1159 IFNET_STAT_INC(sc->sc_ifp, oerrors, 1); 1160 } 1161 1162 return (ret); 1163 } 1164 1165 static void 1166 wg_send_buf(struct wg_softc *sc, struct wg_endpoint *e, const void *buf, 1167 size_t len) 1168 { 1169 struct mbuf *m; 1170 int ret; 1171 1172 /* 1173 * This function only sends handshake packets of known lengths that 1174 * are <= MHLEN, so it's safe to just use m_gethdr() and memcpy(). 1175 */ 1176 KKASSERT(len <= MHLEN); 1177 1178 m = m_gethdr(M_NOWAIT, MT_DATA); 1179 if (m == NULL) { 1180 DPRINTF(sc, "Unable to allocate mbuf\n"); 1181 return; 1182 } 1183 1184 /* Just plain copy as it's a single mbuf. */ 1185 memcpy(mtod(m, void *), buf, len); 1186 m->m_pkthdr.len = m->m_len = len; 1187 1188 /* Give high priority to the handshake packets. */ 1189 m->m_flags |= M_PRIO; 1190 1191 ret = wg_send(sc, e, m); 1192 if (ret != 0) 1193 DPRINTF(sc, "Unable to send packet: %d\n", ret); 1194 } 1195 1196 /*----------------------------------------------------------------------------*/ 1197 /* 1198 * Timers 1199 * 1200 * These functions handle the timeout callbacks for a WireGuard session, and 1201 * provide an "event-based" model for controlling WireGuard session timers. 1202 */ 1203 1204 static void wg_timers_run_send_initiation(struct wg_peer *, bool); 1205 static void wg_timers_run_retry_handshake(void *); 1206 static void wg_timers_run_send_keepalive(void *); 1207 static void wg_timers_run_new_handshake(void *); 1208 static void wg_timers_run_zero_key_material(void *); 1209 static void wg_timers_run_persistent_keepalive(void *); 1210 1211 static void 1212 wg_timers_enable(struct wg_peer *peer) 1213 { 1214 atomic_store_bool(&peer->p_enabled, true); 1215 wg_timers_run_persistent_keepalive(peer); 1216 } 1217 1218 static void 1219 wg_timers_disable(struct wg_peer *peer) 1220 { 1221 atomic_store_bool(&peer->p_enabled, false); 1222 atomic_store_bool(&peer->p_need_another_keepalive, false); 1223 1224 /* Cancel the callouts and wait for them to complete. */ 1225 callout_drain(&peer->p_new_handshake); 1226 callout_drain(&peer->p_send_keepalive); 1227 callout_drain(&peer->p_retry_handshake); 1228 callout_drain(&peer->p_persistent_keepalive); 1229 callout_drain(&peer->p_zero_key_material); 1230 } 1231 1232 static void 1233 wg_timers_set_persistent_keepalive(struct wg_peer *peer, uint16_t interval) 1234 { 1235 atomic_store_16(&peer->p_persistent_keepalive_interval, interval); 1236 if (atomic_load_bool(&peer->p_enabled)) 1237 wg_timers_run_persistent_keepalive(peer); 1238 } 1239 1240 static bool 1241 wg_timers_get_persistent_keepalive(struct wg_peer *peer, uint16_t *interval) 1242 { 1243 *interval = atomic_load_16(&peer->p_persistent_keepalive_interval); 1244 return (*interval > 0); 1245 } 1246 1247 static void 1248 wg_timers_get_last_handshake(struct wg_peer *peer, struct timespec *time) 1249 { 1250 lockmgr(&peer->p_handshake_mtx, LK_EXCLUSIVE); 1251 *time = peer->p_handshake_complete; 1252 lockmgr(&peer->p_handshake_mtx, LK_RELEASE); 1253 } 1254 1255 /* 1256 * Should be called after an authenticated data packet is sent. 1257 */ 1258 static void 1259 wg_timers_event_data_sent(struct wg_peer *peer) 1260 { 1261 int ticks; 1262 1263 if (atomic_load_bool(&peer->p_enabled) && 1264 !callout_pending(&peer->p_new_handshake)) { 1265 ticks = NEW_HANDSHAKE_TIMEOUT * hz + 1266 REKEY_TIMEOUT_JITTER * hz / 1000; 1267 callout_reset(&peer->p_new_handshake, ticks, 1268 wg_timers_run_new_handshake, peer); 1269 } 1270 } 1271 1272 /* 1273 * Should be called after an authenticated data packet is received. 1274 */ 1275 static void 1276 wg_timers_event_data_received(struct wg_peer *peer) 1277 { 1278 if (atomic_load_bool(&peer->p_enabled)) { 1279 if (!callout_pending(&peer->p_send_keepalive)) { 1280 callout_reset(&peer->p_send_keepalive, 1281 KEEPALIVE_TIMEOUT * hz, 1282 wg_timers_run_send_keepalive, peer); 1283 } else { 1284 atomic_store_bool(&peer->p_need_another_keepalive, 1285 true); 1286 } 1287 } 1288 } 1289 1290 /* 1291 * Should be called before any type of authenticated packet is to be sent, 1292 * whether keepalive, data, or handshake. 1293 */ 1294 static void 1295 wg_timers_event_any_authenticated_packet_sent(struct wg_peer *peer) 1296 { 1297 callout_stop(&peer->p_send_keepalive); 1298 } 1299 1300 /* 1301 * Should be called after any type of authenticated packet is received, 1302 * whether keepalive, data, or handshake. 1303 */ 1304 static void 1305 wg_timers_event_any_authenticated_packet_received(struct wg_peer *peer) 1306 { 1307 callout_stop(&peer->p_new_handshake); 1308 } 1309 1310 /* 1311 * Should be called before a packet with authentication (whether keepalive, 1312 * data, or handshakem) is sent, or after one is received. 1313 */ 1314 static void 1315 wg_timers_event_any_authenticated_packet_traversal(struct wg_peer *peer) 1316 { 1317 uint16_t interval; 1318 1319 interval = atomic_load_16(&peer->p_persistent_keepalive_interval); 1320 if (atomic_load_bool(&peer->p_enabled) && interval > 0) { 1321 callout_reset(&peer->p_persistent_keepalive, interval * hz, 1322 wg_timers_run_persistent_keepalive, peer); 1323 } 1324 } 1325 1326 /* 1327 * Should be called after a handshake initiation message is sent. 1328 */ 1329 static void 1330 wg_timers_event_handshake_initiated(struct wg_peer *peer) 1331 { 1332 int ticks; 1333 1334 if (atomic_load_bool(&peer->p_enabled)) { 1335 ticks = REKEY_TIMEOUT * hz + REKEY_TIMEOUT_JITTER * hz / 1000; 1336 callout_reset(&peer->p_retry_handshake, ticks, 1337 wg_timers_run_retry_handshake, peer); 1338 } 1339 } 1340 1341 /* 1342 * Should be called after a handshake response message is received and 1343 * processed, or when getting key confirmation via the first data message. 1344 */ 1345 static void 1346 wg_timers_event_handshake_complete(struct wg_peer *peer) 1347 { 1348 if (atomic_load_bool(&peer->p_enabled)) { 1349 lockmgr(&peer->p_handshake_mtx, LK_EXCLUSIVE); 1350 callout_stop(&peer->p_retry_handshake); 1351 peer->p_handshake_retries = 0; 1352 getnanotime(&peer->p_handshake_complete); 1353 lockmgr(&peer->p_handshake_mtx, LK_RELEASE); 1354 1355 wg_timers_run_send_keepalive(peer); 1356 } 1357 } 1358 1359 /* 1360 * Should be called after an ephemeral key is created, which is before sending 1361 * a handshake response or after receiving a handshake response. 1362 */ 1363 static void 1364 wg_timers_event_session_derived(struct wg_peer *peer) 1365 { 1366 if (atomic_load_bool(&peer->p_enabled)) { 1367 callout_reset(&peer->p_zero_key_material, 1368 REJECT_AFTER_TIME * 3 * hz, 1369 wg_timers_run_zero_key_material, peer); 1370 } 1371 } 1372 1373 /* 1374 * Should be called after data packet sending failure, or after the old 1375 * keypairs expiring (or near expiring). 1376 */ 1377 static void 1378 wg_timers_event_want_initiation(struct wg_peer *peer) 1379 { 1380 if (atomic_load_bool(&peer->p_enabled)) 1381 wg_timers_run_send_initiation(peer, false); 1382 } 1383 1384 static void 1385 wg_timers_run_send_initiation(struct wg_peer *peer, bool is_retry) 1386 { 1387 if (!is_retry) 1388 peer->p_handshake_retries = 0; 1389 if (noise_remote_initiation_expired(peer->p_remote)) 1390 wg_send_initiation(peer); 1391 } 1392 1393 static void 1394 wg_timers_run_retry_handshake(void *_peer) 1395 { 1396 struct wg_peer *peer = _peer; 1397 1398 lockmgr(&peer->p_handshake_mtx, LK_EXCLUSIVE); 1399 if (peer->p_handshake_retries <= MAX_TIMER_HANDSHAKES) { 1400 peer->p_handshake_retries++; 1401 lockmgr(&peer->p_handshake_mtx, LK_RELEASE); 1402 1403 DPRINTF(peer->p_sc, "Handshake for peer %ld did not complete " 1404 "after %d seconds, retrying (try %d)\n", peer->p_id, 1405 REKEY_TIMEOUT, peer->p_handshake_retries + 1); 1406 wg_timers_run_send_initiation(peer, true); 1407 } else { 1408 lockmgr(&peer->p_handshake_mtx, LK_RELEASE); 1409 1410 DPRINTF(peer->p_sc, "Handshake for peer %ld did not complete " 1411 "after %d retries, giving up\n", peer->p_id, 1412 MAX_TIMER_HANDSHAKES + 2); 1413 callout_stop(&peer->p_send_keepalive); 1414 wg_queue_purge(&peer->p_stage_queue); 1415 if (atomic_load_bool(&peer->p_enabled) && 1416 !callout_pending(&peer->p_zero_key_material)) { 1417 callout_reset(&peer->p_zero_key_material, 1418 REJECT_AFTER_TIME * 3 * hz, 1419 wg_timers_run_zero_key_material, peer); 1420 } 1421 } 1422 } 1423 1424 static void 1425 wg_timers_run_send_keepalive(void *_peer) 1426 { 1427 struct wg_peer *peer = _peer; 1428 1429 wg_send_keepalive(peer); 1430 1431 if (atomic_load_bool(&peer->p_enabled) && 1432 atomic_load_bool(&peer->p_need_another_keepalive)) { 1433 atomic_store_bool(&peer->p_need_another_keepalive, false); 1434 callout_reset(&peer->p_send_keepalive, KEEPALIVE_TIMEOUT * hz, 1435 wg_timers_run_send_keepalive, peer); 1436 } 1437 } 1438 1439 static void 1440 wg_timers_run_persistent_keepalive(void *_peer) 1441 { 1442 struct wg_peer *peer = _peer; 1443 1444 if (atomic_load_16(&peer->p_persistent_keepalive_interval) > 0) 1445 wg_send_keepalive(peer); 1446 } 1447 1448 static void 1449 wg_timers_run_new_handshake(void *_peer) 1450 { 1451 struct wg_peer *peer = _peer; 1452 1453 DPRINTF(peer->p_sc, "Retrying handshake with peer %ld, " 1454 "because we stopped hearing back after %d seconds\n", 1455 peer->p_id, NEW_HANDSHAKE_TIMEOUT); 1456 wg_timers_run_send_initiation(peer, false); 1457 } 1458 1459 static void 1460 wg_timers_run_zero_key_material(void *_peer) 1461 { 1462 struct wg_peer *peer = _peer; 1463 1464 DPRINTF(peer->p_sc, "Zeroing out keys for peer %ld, " 1465 "since we haven't received a new one in %d seconds\n", 1466 peer->p_id, REJECT_AFTER_TIME * 3); 1467 noise_remote_keypairs_clear(peer->p_remote); 1468 } 1469 1470 /*----------------------------------------------------------------------------*/ 1471 /* Handshake */ 1472 1473 static void 1474 wg_peer_send_buf(struct wg_peer *peer, const void *buf, size_t len) 1475 { 1476 struct wg_endpoint endpoint; 1477 1478 peer->p_tx_bytes[mycpuid] += len; 1479 1480 wg_timers_event_any_authenticated_packet_traversal(peer); 1481 wg_timers_event_any_authenticated_packet_sent(peer); 1482 1483 wg_peer_get_endpoint(peer, &endpoint); 1484 wg_send_buf(peer->p_sc, &endpoint, buf, len); 1485 } 1486 1487 static void 1488 wg_send_initiation(struct wg_peer *peer) 1489 { 1490 struct wg_pkt_initiation pkt; 1491 1492 if (!noise_create_initiation(peer->p_remote, &pkt.s_idx, pkt.ue, 1493 pkt.es, pkt.ets)) 1494 return; 1495 1496 DPRINTF(peer->p_sc, "Sending handshake initiation to peer %ld\n", 1497 peer->p_id); 1498 1499 pkt.t = WG_PKT_INITIATION; 1500 cookie_maker_mac(peer->p_cookie, &pkt.m, &pkt, 1501 sizeof(pkt) - sizeof(pkt.m)); 1502 wg_peer_send_buf(peer, &pkt, sizeof(pkt)); 1503 wg_timers_event_handshake_initiated(peer); 1504 } 1505 1506 static void 1507 wg_send_response(struct wg_peer *peer) 1508 { 1509 struct wg_pkt_response pkt; 1510 1511 if (!noise_create_response(peer->p_remote, &pkt.s_idx, &pkt.r_idx, 1512 pkt.ue, pkt.en)) 1513 return; 1514 1515 DPRINTF(peer->p_sc, "Sending handshake response to peer %ld\n", 1516 peer->p_id); 1517 1518 wg_timers_event_session_derived(peer); 1519 pkt.t = WG_PKT_RESPONSE; 1520 cookie_maker_mac(peer->p_cookie, &pkt.m, &pkt, 1521 sizeof(pkt) - sizeof(pkt.m)); 1522 wg_peer_send_buf(peer, &pkt, sizeof(pkt)); 1523 } 1524 1525 static void 1526 wg_send_cookie(struct wg_softc *sc, struct cookie_macs *cm, uint32_t idx, 1527 struct wg_endpoint *e) 1528 { 1529 struct wg_pkt_cookie pkt; 1530 1531 DPRINTF(sc, "Sending cookie response for denied handshake message\n"); 1532 1533 pkt.t = WG_PKT_COOKIE; 1534 pkt.r_idx = idx; 1535 1536 cookie_checker_create_payload(sc->sc_cookie, cm, pkt.nonce, 1537 pkt.ec, &e->e_remote.r_sa); 1538 wg_send_buf(sc, e, &pkt, sizeof(pkt)); 1539 } 1540 1541 static void 1542 wg_send_keepalive(struct wg_peer *peer) 1543 { 1544 struct wg_packet *pkt; 1545 struct mbuf *m; 1546 1547 if (wg_queue_len(&peer->p_stage_queue) > 0) 1548 goto send; 1549 if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) 1550 return; 1551 if ((pkt = wg_packet_alloc(m)) == NULL) { 1552 m_freem(m); 1553 return; 1554 } 1555 1556 wg_queue_push_staged(&peer->p_stage_queue, pkt); 1557 DPRINTF(peer->p_sc, "Sending keepalive packet to peer %ld\n", 1558 peer->p_id); 1559 send: 1560 wg_peer_send_staged(peer); 1561 } 1562 1563 static bool 1564 wg_is_underload(struct wg_softc *sc) 1565 { 1566 /* 1567 * This is global, so that the load calculation applies to the 1568 * whole system. Don't care about races with it at all. 1569 */ 1570 static struct timespec last_underload; /* nanouptime */ 1571 struct timespec now; 1572 bool underload; 1573 1574 underload = (wg_queue_len(&sc->sc_handshake_queue) >= 1575 MAX_QUEUED_HANDSHAKES / 8); 1576 if (underload) { 1577 getnanouptime(&last_underload); 1578 } else if (timespecisset(&last_underload)) { 1579 getnanouptime(&now); 1580 now.tv_sec -= UNDERLOAD_TIMEOUT; 1581 underload = timespeccmp(&last_underload, &now, >); 1582 if (!underload) 1583 timespecclear(&last_underload); 1584 } 1585 1586 return (underload); 1587 } 1588 1589 static void 1590 wg_handshake(struct wg_softc *sc, struct wg_packet *pkt) 1591 { 1592 struct wg_pkt_initiation *init; 1593 struct wg_pkt_response *resp; 1594 struct wg_pkt_cookie *cook; 1595 struct wg_endpoint *e; 1596 struct wg_peer *peer; 1597 struct mbuf *m; 1598 struct noise_remote *remote = NULL; 1599 bool underload; 1600 int ret; 1601 1602 underload = wg_is_underload(sc); 1603 m = pkt->p_mbuf; 1604 e = &pkt->p_endpoint; 1605 1606 /* m_pullup() may change 'm', so must update 'pkt->p_mbuf'. */ 1607 if ((pkt->p_mbuf = m = m_pullup(m, m->m_pkthdr.len)) == NULL) 1608 goto error; 1609 1610 switch (*mtod(m, uint32_t *)) { 1611 case WG_PKT_INITIATION: 1612 init = mtod(m, struct wg_pkt_initiation *); 1613 1614 ret = cookie_checker_validate_macs(sc->sc_cookie, &init->m, 1615 init, sizeof(*init) - sizeof(init->m), underload, 1616 &e->e_remote.r_sa); 1617 if (ret != 0) { 1618 switch (ret) { 1619 case EINVAL: 1620 DPRINTF(sc, "Invalid initiation MAC\n"); 1621 break; 1622 case ECONNREFUSED: 1623 DPRINTF(sc, "Handshake ratelimited\n"); 1624 break; 1625 case EAGAIN: 1626 wg_send_cookie(sc, &init->m, init->s_idx, e); 1627 break; 1628 default: 1629 /* 1630 * cookie_checker_validate_macs() seems could 1631 * return EAFNOSUPPORT, but that is actually 1632 * impossible, because packets of unsupported 1633 * AF have been already dropped. 1634 */ 1635 panic("%s: unexpected return: %d", 1636 __func__, ret); 1637 } 1638 goto error; 1639 } 1640 1641 remote = noise_consume_initiation(sc->sc_local, init->s_idx, 1642 init->ue, init->es, 1643 init->ets); 1644 if (remote == NULL) { 1645 DPRINTF(sc, "Invalid handshake initiation\n"); 1646 goto error; 1647 } 1648 1649 peer = noise_remote_arg(remote); 1650 DPRINTF(sc, "Receiving handshake initiation from peer %ld\n", 1651 peer->p_id); 1652 1653 wg_peer_set_endpoint(peer, e); 1654 wg_send_response(peer); 1655 break; 1656 1657 case WG_PKT_RESPONSE: 1658 resp = mtod(m, struct wg_pkt_response *); 1659 1660 ret = cookie_checker_validate_macs(sc->sc_cookie, &resp->m, 1661 resp, sizeof(*resp) - sizeof(resp->m), underload, 1662 &e->e_remote.r_sa); 1663 if (ret != 0) { 1664 switch (ret) { 1665 case EINVAL: 1666 DPRINTF(sc, "Invalid response MAC\n"); 1667 break; 1668 case ECONNREFUSED: 1669 DPRINTF(sc, "Handshake ratelimited\n"); 1670 break; 1671 case EAGAIN: 1672 wg_send_cookie(sc, &resp->m, resp->s_idx, e); 1673 break; 1674 default: 1675 /* See also the comment above. */ 1676 panic("%s: unexpected return: %d", 1677 __func__, ret); 1678 } 1679 goto error; 1680 } 1681 1682 remote = noise_consume_response(sc->sc_local, resp->s_idx, 1683 resp->r_idx, resp->ue, 1684 resp->en); 1685 if (remote == NULL) { 1686 DPRINTF(sc, "Invalid handshake response\n"); 1687 goto error; 1688 } 1689 1690 peer = noise_remote_arg(remote); 1691 DPRINTF(sc, "Receiving handshake response from peer %ld\n", 1692 peer->p_id); 1693 1694 wg_peer_set_endpoint(peer, e); 1695 wg_timers_event_session_derived(peer); 1696 wg_timers_event_handshake_complete(peer); 1697 break; 1698 1699 case WG_PKT_COOKIE: 1700 cook = mtod(m, struct wg_pkt_cookie *); 1701 1702 remote = noise_remote_index(sc->sc_local, cook->r_idx); 1703 if (remote == NULL) { 1704 DPRINTF(sc, "Unknown cookie index\n"); 1705 goto error; 1706 } 1707 1708 peer = noise_remote_arg(remote); 1709 if (cookie_maker_consume_payload(peer->p_cookie, cook->nonce, 1710 cook->ec) == 0) { 1711 DPRINTF(sc, "Receiving cookie response\n"); 1712 } else { 1713 DPRINTF(sc, "Could not decrypt cookie response\n"); 1714 goto error; 1715 } 1716 1717 goto not_authenticated; 1718 1719 default: 1720 panic("%s: invalid packet in handshake queue", __func__); 1721 } 1722 1723 wg_timers_event_any_authenticated_packet_received(peer); 1724 wg_timers_event_any_authenticated_packet_traversal(peer); 1725 1726 not_authenticated: 1727 IFNET_STAT_INC(sc->sc_ifp, ipackets, 1); 1728 IFNET_STAT_INC(sc->sc_ifp, ibytes, m->m_pkthdr.len); 1729 peer->p_rx_bytes[mycpuid] += m->m_pkthdr.len; 1730 noise_remote_put(remote); 1731 wg_packet_free(pkt); 1732 1733 return; 1734 1735 error: 1736 IFNET_STAT_INC(sc->sc_ifp, ierrors, 1); 1737 if (remote != NULL) 1738 noise_remote_put(remote); 1739 wg_packet_free(pkt); 1740 } 1741 1742 static void 1743 wg_handshake_worker(void *arg, int pending __unused) 1744 { 1745 struct wg_softc *sc = arg; 1746 struct wg_queue *queue = &sc->sc_handshake_queue; 1747 struct wg_packet *pkt; 1748 1749 while ((pkt = wg_queue_dequeue_handshake(queue)) != NULL) 1750 wg_handshake(sc, pkt); 1751 } 1752 1753 /*----------------------------------------------------------------------------*/ 1754 /* Transport Packet Functions */ 1755 1756 static inline void 1757 wg_bpf_ptap(struct ifnet *ifp, struct mbuf *m, sa_family_t af) 1758 { 1759 uint32_t bpf_af; 1760 1761 if (ifp->if_bpf == NULL) 1762 return; 1763 1764 bpf_gettoken(); 1765 /* Double check after obtaining the token. */ 1766 if (ifp->if_bpf != NULL) { 1767 /* Prepend the AF as a 4-byte field for DLT_NULL. */ 1768 bpf_af = (uint32_t)af; 1769 bpf_ptap(ifp->if_bpf, m, &bpf_af, sizeof(bpf_af)); 1770 } 1771 bpf_reltoken(); 1772 } 1773 1774 static inline unsigned int 1775 calculate_padding(struct wg_packet *pkt) 1776 { 1777 unsigned int padded_size, last_unit; 1778 1779 last_unit = pkt->p_mbuf->m_pkthdr.len; 1780 1781 /* Keepalive packets don't set p_mtu, but also have a length of zero. */ 1782 if (__predict_false(pkt->p_mtu == 0)) 1783 return WG_PKT_WITH_PADDING(last_unit) - last_unit; 1784 1785 /* 1786 * Just in case the packet is bigger than the MTU and would cause 1787 * the final subtraction to overflow. 1788 */ 1789 if (__predict_false(last_unit > pkt->p_mtu)) 1790 last_unit %= pkt->p_mtu; 1791 1792 padded_size = MIN(pkt->p_mtu, WG_PKT_WITH_PADDING(last_unit)); 1793 return (padded_size - last_unit); 1794 } 1795 1796 static inline int 1797 determine_af_and_pullup(struct mbuf **m, sa_family_t *af) 1798 { 1799 const struct ip *ip; 1800 const struct ip6_hdr *ip6; 1801 int len; 1802 1803 ip = mtod(*m, const struct ip *); 1804 ip6 = mtod(*m, const struct ip6_hdr *); 1805 len = (*m)->m_pkthdr.len; 1806 1807 if (len >= sizeof(*ip) && ip->ip_v == IPVERSION) 1808 *af = AF_INET; 1809 #ifdef INET6 1810 else if (len >= sizeof(*ip6) && 1811 (ip6->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION) 1812 *af = AF_INET6; 1813 #endif 1814 else 1815 return (EAFNOSUPPORT); 1816 1817 *m = m_pullup(*m, (*af == AF_INET ? sizeof(*ip) : sizeof(*ip6))); 1818 if (*m == NULL) 1819 return (ENOBUFS); 1820 1821 return (0); 1822 } 1823 1824 static void 1825 wg_encrypt(struct wg_softc *sc, struct wg_packet *pkt) 1826 { 1827 static const uint8_t padding[WG_PKT_PADDING] = { 0 }; 1828 struct wg_pkt_data *data; 1829 struct wg_peer *peer; 1830 struct noise_remote *remote; 1831 struct mbuf *m; 1832 unsigned int padlen, state = WG_PACKET_DEAD; 1833 uint32_t idx; 1834 1835 remote = noise_keypair_remote(pkt->p_keypair); 1836 peer = noise_remote_arg(remote); 1837 m = pkt->p_mbuf; 1838 1839 padlen = calculate_padding(pkt); 1840 if (padlen != 0 && !m_append(m, padlen, padding)) 1841 goto out; 1842 1843 if (noise_keypair_encrypt(pkt->p_keypair, &idx, pkt->p_counter, m) != 0) 1844 goto out; 1845 1846 M_PREPEND(m, sizeof(struct wg_pkt_data), M_NOWAIT); 1847 if (m == NULL) 1848 goto out; 1849 data = mtod(m, struct wg_pkt_data *); 1850 data->t = WG_PKT_DATA; 1851 data->r_idx = idx; 1852 data->counter = htole64(pkt->p_counter); 1853 1854 state = WG_PACKET_CRYPTED; 1855 1856 out: 1857 pkt->p_mbuf = m; 1858 atomic_store_rel_int(&pkt->p_state, state); 1859 taskqueue_enqueue(peer->p_send_taskqueue, &peer->p_send_task); 1860 noise_remote_put(remote); 1861 } 1862 1863 static void 1864 wg_decrypt(struct wg_softc *sc, struct wg_packet *pkt) 1865 { 1866 struct wg_peer *peer, *allowed_peer; 1867 struct noise_remote *remote; 1868 struct mbuf *m; 1869 unsigned int state = WG_PACKET_DEAD; 1870 int len; 1871 1872 remote = noise_keypair_remote(pkt->p_keypair); 1873 peer = noise_remote_arg(remote); 1874 m = pkt->p_mbuf; 1875 1876 pkt->p_counter = le64toh(mtod(m, struct wg_pkt_data *)->counter); 1877 m_adj(m, sizeof(struct wg_pkt_data)); 1878 1879 if (noise_keypair_decrypt(pkt->p_keypair, pkt->p_counter, m) != 0) 1880 goto out; 1881 1882 /* A packet with a length of zero is a keepalive packet. */ 1883 if (__predict_false(m->m_pkthdr.len == 0)) { 1884 DPRINTF(sc, "Receiving keepalive packet from peer %ld\n", 1885 peer->p_id); 1886 state = WG_PACKET_CRYPTED; 1887 goto out; 1888 } 1889 1890 /* 1891 * Extract the source address for wg_aip_lookup(), and trim the 1892 * packet if it was padded before encryption. 1893 */ 1894 if (determine_af_and_pullup(&m, &pkt->p_af) != 0) 1895 goto out; 1896 if (pkt->p_af == AF_INET) { 1897 const struct ip *ip = mtod(m, const struct ip *); 1898 allowed_peer = wg_aip_lookup(sc, AF_INET, &ip->ip_src); 1899 len = ntohs(ip->ip_len); 1900 if (len >= sizeof(struct ip) && len < m->m_pkthdr.len) 1901 m_adj(m, len - m->m_pkthdr.len); 1902 } else { 1903 const struct ip6_hdr *ip6 = mtod(m, const struct ip6_hdr *); 1904 allowed_peer = wg_aip_lookup(sc, AF_INET6, &ip6->ip6_src); 1905 len = ntohs(ip6->ip6_plen) + sizeof(struct ip6_hdr); 1906 if (len < m->m_pkthdr.len) 1907 m_adj(m, len - m->m_pkthdr.len); 1908 } 1909 1910 /* Drop the reference, since no need to dereference it. */ 1911 if (allowed_peer != NULL) 1912 noise_remote_put(allowed_peer->p_remote); 1913 1914 if (__predict_false(peer != allowed_peer)) { 1915 DPRINTF(sc, "Packet has disallowed src IP from peer %ld\n", 1916 peer->p_id); 1917 goto out; 1918 } 1919 1920 state = WG_PACKET_CRYPTED; 1921 1922 out: 1923 pkt->p_mbuf = m; 1924 atomic_store_rel_int(&pkt->p_state, state); 1925 taskqueue_enqueue(peer->p_recv_taskqueue, &peer->p_recv_task); 1926 noise_remote_put(remote); 1927 } 1928 1929 static void 1930 wg_encrypt_worker(void *arg, int pending __unused) 1931 { 1932 struct wg_softc *sc = arg; 1933 struct wg_queue *queue = &sc->sc_encrypt_parallel; 1934 struct wg_packet *pkt; 1935 1936 while ((pkt = wg_queue_dequeue_parallel(queue)) != NULL) 1937 wg_encrypt(sc, pkt); 1938 } 1939 1940 static void 1941 wg_decrypt_worker(void *arg, int pending __unused) 1942 { 1943 struct wg_softc *sc = arg; 1944 struct wg_queue *queue = &sc->sc_decrypt_parallel; 1945 struct wg_packet *pkt; 1946 1947 while ((pkt = wg_queue_dequeue_parallel(queue)) != NULL) 1948 wg_decrypt(sc, pkt); 1949 } 1950 1951 static void 1952 wg_encrypt_dispatch(struct wg_softc *sc) 1953 { 1954 int cpu; 1955 1956 /* 1957 * The update to encrypt_last_cpu is racy such that we may 1958 * reschedule the task for the same CPU multiple times, but 1959 * the race doesn't really matter. 1960 */ 1961 cpu = (sc->sc_encrypt_last_cpu + 1) % ncpus; 1962 sc->sc_encrypt_last_cpu = cpu; 1963 taskqueue_enqueue(wg_taskqueues[cpu], &sc->sc_encrypt_tasks[cpu]); 1964 } 1965 1966 static void 1967 wg_decrypt_dispatch(struct wg_softc *sc) 1968 { 1969 int cpu; 1970 1971 cpu = (sc->sc_decrypt_last_cpu + 1) % ncpus; 1972 sc->sc_decrypt_last_cpu = cpu; 1973 taskqueue_enqueue(wg_taskqueues[cpu], &sc->sc_decrypt_tasks[cpu]); 1974 } 1975 1976 static void 1977 wg_deliver_out(void *arg, int pending __unused) 1978 { 1979 struct wg_peer *peer = arg; 1980 struct wg_softc *sc = peer->p_sc; 1981 struct wg_queue *queue = &peer->p_encrypt_serial; 1982 struct wg_endpoint endpoint; 1983 struct wg_packet *pkt; 1984 struct mbuf *m; 1985 int len, cpu; 1986 1987 cpu = mycpuid; 1988 1989 while ((pkt = wg_queue_dequeue_serial(queue)) != NULL) { 1990 if (atomic_load_acq_int(&pkt->p_state) != WG_PACKET_CRYPTED) { 1991 IFNET_STAT_INC(sc->sc_ifp, oerrors, 1); 1992 wg_packet_free(pkt); 1993 continue; 1994 } 1995 1996 m = pkt->p_mbuf; 1997 m->m_flags &= ~MBUF_CLEARFLAGS; 1998 len = m->m_pkthdr.len; 1999 2000 pkt->p_mbuf = NULL; 2001 wg_packet_free(pkt); 2002 2003 /* 2004 * The keepalive timers -- both persistent and mandatory -- 2005 * are part of the internal state machine, which needs to be 2006 * cranked whether or not the packet was actually sent. 2007 */ 2008 wg_timers_event_any_authenticated_packet_traversal(peer); 2009 wg_timers_event_any_authenticated_packet_sent(peer); 2010 2011 wg_peer_get_endpoint(peer, &endpoint); 2012 if (wg_send(sc, &endpoint, m) == 0) { 2013 peer->p_tx_bytes[cpu] += len; 2014 if (len > WG_PKT_ENCRYPTED_LEN(0)) 2015 wg_timers_event_data_sent(peer); 2016 if (noise_keypair_should_refresh(peer->p_remote, true)) 2017 wg_timers_event_want_initiation(peer); 2018 } 2019 } 2020 } 2021 2022 static void 2023 wg_deliver_in(void *arg, int pending __unused) 2024 { 2025 struct wg_peer *peer = arg; 2026 struct wg_softc *sc = peer->p_sc; 2027 struct wg_queue *queue = &peer->p_decrypt_serial; 2028 struct wg_packet *pkt; 2029 struct ifnet *ifp; 2030 struct mbuf *m; 2031 size_t rx_bytes; 2032 int cpu; 2033 2034 cpu = mycpuid; 2035 ifp = sc->sc_ifp; 2036 2037 while ((pkt = wg_queue_dequeue_serial(queue)) != NULL) { 2038 if (atomic_load_acq_int(&pkt->p_state) != WG_PACKET_CRYPTED || 2039 noise_keypair_counter_check(pkt->p_keypair, pkt->p_counter) 2040 != 0) { 2041 IFNET_STAT_INC(ifp, ierrors, 1); 2042 wg_packet_free(pkt); 2043 continue; 2044 } 2045 2046 if (noise_keypair_received_with(pkt->p_keypair)) 2047 wg_timers_event_handshake_complete(peer); 2048 2049 wg_timers_event_any_authenticated_packet_received(peer); 2050 wg_timers_event_any_authenticated_packet_traversal(peer); 2051 wg_peer_set_endpoint(peer, &pkt->p_endpoint); 2052 2053 m = pkt->p_mbuf; 2054 rx_bytes = WG_PKT_ENCRYPTED_LEN(m->m_pkthdr.len); 2055 peer->p_rx_bytes[cpu] += rx_bytes; 2056 IFNET_STAT_INC(ifp, ipackets, 1); 2057 IFNET_STAT_INC(ifp, ibytes, rx_bytes); 2058 2059 if (m->m_pkthdr.len > 0) { 2060 if (ifp->if_capenable & IFCAP_RXCSUM) { 2061 /* 2062 * The packet is authentic as ensured by the 2063 * AEAD tag, so we can tell the networking 2064 * stack that this packet has valid checksums 2065 * and thus is unnecessary to check again. 2066 */ 2067 if (m->m_pkthdr.csum_flags & CSUM_IP) 2068 m->m_pkthdr.csum_flags |= 2069 (CSUM_IP_CHECKED | CSUM_IP_VALID); 2070 if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA) { 2071 m->m_pkthdr.csum_flags |= 2072 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 2073 m->m_pkthdr.csum_data = 0xffff; 2074 } 2075 } 2076 m->m_flags &= ~MBUF_CLEARFLAGS; 2077 m->m_pkthdr.rcvif = ifp; 2078 2079 wg_bpf_ptap(ifp, m, pkt->p_af); 2080 2081 netisr_queue((pkt->p_af == AF_INET ? 2082 NETISR_IP : NETISR_IPV6), m); 2083 pkt->p_mbuf = NULL; 2084 2085 wg_timers_event_data_received(peer); 2086 } 2087 2088 wg_packet_free(pkt); 2089 2090 if (noise_keypair_should_refresh(peer->p_remote, false)) 2091 wg_timers_event_want_initiation(peer); 2092 } 2093 } 2094 2095 static void 2096 wg_input(struct wg_softc *sc, struct mbuf *m, const struct sockaddr *sa) 2097 { 2098 struct noise_remote *remote; 2099 struct wg_pkt_data *data; 2100 struct wg_packet *pkt; 2101 struct wg_peer *peer; 2102 struct mbuf *defragged; 2103 2104 /* 2105 * Defragment mbufs early on in order to: 2106 * - make the crypto a lot faster; 2107 * - make the subsequent m_pullup()'s no-ops. 2108 */ 2109 defragged = m_defrag(m, M_NOWAIT); 2110 if (defragged != NULL) 2111 m = defragged; /* The original mbuf chain is freed. */ 2112 2113 /* Ensure the packet is not shared before modifying it. */ 2114 m = m_unshare(m, M_NOWAIT); 2115 if (m == NULL) { 2116 IFNET_STAT_INC(sc->sc_ifp, iqdrops, 1); 2117 return; 2118 } 2119 2120 /* Pullup enough to read packet type */ 2121 if ((m = m_pullup(m, sizeof(uint32_t))) == NULL) { 2122 IFNET_STAT_INC(sc->sc_ifp, iqdrops, 1); 2123 return; 2124 } 2125 2126 if ((pkt = wg_packet_alloc(m)) == NULL) { 2127 IFNET_STAT_INC(sc->sc_ifp, iqdrops, 1); 2128 m_freem(m); 2129 return; 2130 } 2131 2132 /* Save the remote address and port for later use. */ 2133 switch (sa->sa_family) { 2134 case AF_INET: 2135 pkt->p_endpoint.e_remote.r_sin = 2136 *(const struct sockaddr_in *)sa; 2137 break; 2138 #ifdef INET6 2139 case AF_INET6: 2140 pkt->p_endpoint.e_remote.r_sin6 = 2141 *(const struct sockaddr_in6 *)sa; 2142 break; 2143 #endif 2144 default: 2145 DPRINTF(sc, "Unsupported packet address family\n"); 2146 goto error; 2147 } 2148 2149 if (WG_PKT_IS_INITIATION(m) || 2150 WG_PKT_IS_RESPONSE(m) || 2151 WG_PKT_IS_COOKIE(m)) { 2152 if (!wg_queue_enqueue_handshake(&sc->sc_handshake_queue, pkt)) { 2153 IFNET_STAT_INC(sc->sc_ifp, iqdrops, 1); 2154 DPRINTF(sc, "Dropping handshake packet\n"); 2155 } 2156 taskqueue_enqueue(sc->sc_handshake_taskqueue, 2157 &sc->sc_handshake_task); 2158 return; 2159 } 2160 2161 if (WG_PKT_IS_DATA(m)) { 2162 /* Pullup the whole header to read r_idx below. */ 2163 pkt->p_mbuf = m_pullup(m, sizeof(struct wg_pkt_data)); 2164 if (pkt->p_mbuf == NULL) 2165 goto error; 2166 2167 data = mtod(pkt->p_mbuf, struct wg_pkt_data *); 2168 pkt->p_keypair = noise_keypair_lookup(sc->sc_local, 2169 data->r_idx); 2170 if (pkt->p_keypair == NULL) 2171 goto error; 2172 2173 remote = noise_keypair_remote(pkt->p_keypair); 2174 peer = noise_remote_arg(remote); 2175 if (!wg_queue_both(&sc->sc_decrypt_parallel, 2176 &peer->p_decrypt_serial, pkt)) 2177 IFNET_STAT_INC(sc->sc_ifp, iqdrops, 1); 2178 2179 wg_decrypt_dispatch(sc); 2180 noise_remote_put(remote); 2181 return; 2182 } 2183 2184 error: 2185 IFNET_STAT_INC(sc->sc_ifp, ierrors, 1); 2186 wg_packet_free(pkt); 2187 } 2188 2189 static void 2190 wg_upcall(struct socket *so, void *arg, int waitflag __unused) 2191 { 2192 struct wg_softc *sc = arg; 2193 struct sockaddr *from; 2194 struct sockbuf sio; 2195 int ret, flags; 2196 2197 /* 2198 * For UDP, soreceive typically pulls just one packet, 2199 * so loop to get the whole batch. 2200 */ 2201 do { 2202 sbinit(&sio, 1000000000); /* really large to receive all */ 2203 flags = MSG_DONTWAIT; 2204 ret = so_pru_soreceive(so, &from, NULL, &sio, NULL, &flags); 2205 if (ret != 0 || sio.sb_mb == NULL) { 2206 if (from != NULL) 2207 kfree(from, M_SONAME); 2208 break; 2209 } 2210 wg_input(sc, sio.sb_mb, from); 2211 kfree(from, M_SONAME); 2212 } while (sio.sb_mb != NULL); 2213 } 2214 2215 static void 2216 wg_peer_send_staged(struct wg_peer *peer) 2217 { 2218 struct wg_softc *sc = peer->p_sc; 2219 struct wg_packet *pkt, *tpkt; 2220 struct wg_packet_list list; 2221 struct noise_keypair *keypair = NULL; 2222 2223 wg_queue_delist_staged(&peer->p_stage_queue, &list); 2224 2225 if (STAILQ_EMPTY(&list)) 2226 return; 2227 2228 if ((keypair = noise_keypair_current(peer->p_remote)) == NULL) 2229 goto error; 2230 2231 /* 2232 * We now try to assign counters to all of the packets in the queue. 2233 * If we can't assign counters for all of them, we just consider it 2234 * a failure and wait for the next handshake. 2235 */ 2236 STAILQ_FOREACH(pkt, &list, p_parallel) { 2237 if (!noise_keypair_counter_next(keypair, &pkt->p_counter)) 2238 goto error; 2239 } 2240 STAILQ_FOREACH_MUTABLE(pkt, &list, p_parallel, tpkt) { 2241 pkt->p_keypair = noise_keypair_ref(keypair); 2242 if (!wg_queue_both(&sc->sc_encrypt_parallel, 2243 &peer->p_encrypt_serial, pkt)) 2244 IFNET_STAT_INC(sc->sc_ifp, oqdrops, 1); 2245 } 2246 2247 wg_encrypt_dispatch(sc); 2248 noise_keypair_put(keypair); 2249 return; 2250 2251 error: 2252 if (keypair != NULL) 2253 noise_keypair_put(keypair); 2254 wg_queue_enlist_staged(&peer->p_stage_queue, &list); 2255 wg_timers_event_want_initiation(peer); 2256 } 2257 2258 static int 2259 wg_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, 2260 struct rtentry *rt) 2261 { 2262 struct wg_softc *sc = ifp->if_softc; 2263 struct wg_packet *pkt = NULL; 2264 struct wg_peer *peer = NULL; 2265 struct mbuf *defragged; 2266 sa_family_t af = AF_UNSPEC; 2267 int ret; 2268 2269 if (dst->sa_family == AF_UNSPEC) { 2270 /* 2271 * Specially handle packets written/injected by BPF. 2272 * The packets have the same DLT_NULL link-layer type 2273 * (i.e., 4-byte link-layer header in host byte order). 2274 */ 2275 dst->sa_family = *(mtod(m, uint32_t *)); 2276 m_adj(m, sizeof(uint32_t)); 2277 } 2278 if (dst->sa_family == AF_UNSPEC) { 2279 ret = EAFNOSUPPORT; 2280 goto error; 2281 } 2282 2283 wg_bpf_ptap(ifp, m, dst->sa_family); 2284 2285 if (__predict_false(if_tunnel_check_nesting(ifp, m, MTAG_WGLOOP, 2286 MAX_LOOPS) != 0)) { 2287 DPRINTF(sc, "Packet looped\n"); 2288 ret = ELOOP; 2289 goto error; 2290 } 2291 2292 defragged = m_defrag(m, M_NOWAIT); 2293 if (defragged != NULL) 2294 m = defragged; 2295 2296 m = m_unshare(m, M_NOWAIT); 2297 if (m == NULL) { 2298 ret = ENOBUFS; 2299 goto error; 2300 } 2301 2302 if ((ret = determine_af_and_pullup(&m, &af)) != 0) 2303 goto error; 2304 if (af != dst->sa_family) { 2305 ret = EAFNOSUPPORT; 2306 goto error; 2307 } 2308 2309 if ((pkt = wg_packet_alloc(m)) == NULL) { 2310 ret = ENOBUFS; 2311 goto error; 2312 } 2313 2314 pkt->p_af = af; 2315 pkt->p_mtu = ifp->if_mtu; 2316 if (rt != NULL && rt->rt_rmx.rmx_mtu > 0 && 2317 rt->rt_rmx.rmx_mtu < pkt->p_mtu) 2318 pkt->p_mtu = rt->rt_rmx.rmx_mtu; 2319 2320 if (af == AF_INET) { 2321 peer = wg_aip_lookup(sc, AF_INET, 2322 &mtod(m, struct ip *)->ip_dst); 2323 } else { 2324 peer = wg_aip_lookup(sc, AF_INET6, 2325 &mtod(m, struct ip6_hdr *)->ip6_dst); 2326 } 2327 if (__predict_false(peer == NULL)) { 2328 ret = ENOKEY; 2329 goto error; 2330 } 2331 if (__predict_false(peer->p_endpoint.e_remote.r_sa.sa_family 2332 == AF_UNSPEC)) { 2333 DPRINTF(sc, "No valid endpoint has been configured or " 2334 "discovered for peer %ld\n", peer->p_id); 2335 ret = EHOSTUNREACH; 2336 goto error; 2337 } 2338 2339 wg_queue_push_staged(&peer->p_stage_queue, pkt); 2340 wg_peer_send_staged(peer); 2341 noise_remote_put(peer->p_remote); 2342 2343 return (0); 2344 2345 error: 2346 IFNET_STAT_INC(ifp, oerrors, 1); 2347 if (ret == ELOOP) { 2348 /* Skip ICMP error for ELOOP to avoid infinite loop. */ 2349 m_freem(m); /* m cannot be NULL */ 2350 m = NULL; 2351 } 2352 if (m != NULL) { 2353 if (af == AF_INET) 2354 icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_HOST, 0, 0); 2355 #ifdef INET6 2356 else if (af == AF_INET6) 2357 icmp6_error(m, ICMP6_DST_UNREACH, 0, 0); 2358 #endif 2359 else 2360 m_freem(m); 2361 } 2362 if (pkt != NULL) { 2363 pkt->p_mbuf = NULL; /* m already freed above */ 2364 wg_packet_free(pkt); 2365 } 2366 if (peer != NULL) 2367 noise_remote_put(peer->p_remote); 2368 return (ret); 2369 } 2370 2371 /*----------------------------------------------------------------------------*/ 2372 /* Interface Functions */ 2373 2374 static int wg_up(struct wg_softc *); 2375 static void wg_down(struct wg_softc *); 2376 2377 static int 2378 wg_ioctl_get(struct wg_softc *sc, struct wg_data_io *data, bool privileged) 2379 { 2380 struct wg_interface_io *iface_p, iface_o; 2381 struct wg_peer_io *peer_p, peer_o; 2382 struct wg_aip_io *aip_p, aip_o; 2383 struct wg_peer *peer; 2384 struct wg_aip *aip; 2385 size_t size, peer_count, aip_count; 2386 int cpu, ret = 0; 2387 2388 lockmgr(&sc->sc_lock, LK_SHARED); 2389 2390 /* Determine the required data size. */ 2391 size = sizeof(struct wg_interface_io); 2392 size += sizeof(struct wg_peer_io) * sc->sc_peers_num; 2393 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) 2394 size += sizeof(struct wg_aip_io) * peer->p_aips_num; 2395 2396 /* Return the required size for userland allocation. */ 2397 if (data->wgd_size < size) { 2398 data->wgd_size = size; 2399 lockmgr(&sc->sc_lock, LK_RELEASE); 2400 return (0); 2401 } 2402 2403 iface_p = data->wgd_interface; 2404 bzero(&iface_o, sizeof(iface_o)); 2405 /* 2406 * No need to acquire the 'sc_socket.so_lock', because 'sc_lock' 2407 * is acquired and that's enough to prevent modifications to 2408 * 'sc_socket' members. 2409 */ 2410 if (sc->sc_socket.so_port != 0) { 2411 iface_o.i_port = sc->sc_socket.so_port; 2412 iface_o.i_flags |= WG_INTERFACE_HAS_PORT; 2413 } 2414 if (sc->sc_socket.so_user_cookie != 0) { 2415 iface_o.i_cookie = sc->sc_socket.so_user_cookie; 2416 iface_o.i_flags |= WG_INTERFACE_HAS_COOKIE; 2417 } 2418 if (noise_local_keys(sc->sc_local, iface_o.i_public, 2419 iface_o.i_private)) { 2420 iface_o.i_flags |= WG_INTERFACE_HAS_PUBLIC; 2421 if (privileged) 2422 iface_o.i_flags |= WG_INTERFACE_HAS_PRIVATE; 2423 else 2424 bzero(iface_o.i_private, sizeof(iface_o.i_private)); 2425 } 2426 2427 peer_count = 0; 2428 peer_p = &iface_p->i_peers[0]; 2429 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { 2430 bzero(&peer_o, sizeof(peer_o)); 2431 2432 peer_o.p_flags |= WG_PEER_HAS_PUBLIC; 2433 if (noise_remote_keys(peer->p_remote, peer_o.p_public, 2434 peer_o.p_psk)) { 2435 if (privileged) 2436 peer_o.p_flags |= WG_PEER_HAS_PSK; 2437 else 2438 bzero(peer_o.p_psk, sizeof(peer_o.p_psk)); 2439 } 2440 if (wg_timers_get_persistent_keepalive(peer, &peer_o.p_pka)) 2441 peer_o.p_flags |= WG_PEER_HAS_PKA; 2442 if (wg_peer_get_sockaddr(peer, &peer_o.p_sa) == 0) 2443 peer_o.p_flags |= WG_PEER_HAS_ENDPOINT; 2444 for (cpu = 0; cpu < ncpus; cpu++) { 2445 peer_o.p_rxbytes += peer->p_rx_bytes[cpu]; 2446 peer_o.p_txbytes += peer->p_tx_bytes[cpu]; 2447 } 2448 wg_timers_get_last_handshake(peer, &peer_o.p_last_handshake); 2449 peer_o.p_id = (uint64_t)peer->p_id; 2450 strlcpy(peer_o.p_description, peer->p_description, 2451 sizeof(peer_o.p_description)); 2452 2453 aip_count = 0; 2454 aip_p = &peer_p->p_aips[0]; 2455 LIST_FOREACH(aip, &peer->p_aips, a_entry) { 2456 bzero(&aip_o, sizeof(aip_o)); 2457 aip_o.a_af = aip->a_af; 2458 if (aip->a_af == AF_INET) { 2459 aip_o.a_cidr = bitcount32(aip->a_mask.ip); 2460 memcpy(&aip_o.a_ipv4, &aip->a_addr.in, 2461 sizeof(aip->a_addr.in)); 2462 } else if (aip->a_af == AF_INET6) { 2463 aip_o.a_cidr = in6_mask2len(&aip->a_mask.in6, 2464 NULL); 2465 memcpy(&aip_o.a_ipv6, &aip->a_addr.in6, 2466 sizeof(aip->a_addr.in6)); 2467 } 2468 2469 ret = copyout(&aip_o, aip_p, sizeof(aip_o)); 2470 if (ret != 0) 2471 goto out; 2472 2473 aip_p++; 2474 aip_count++; 2475 } 2476 KKASSERT(aip_count == peer->p_aips_num); 2477 peer_o.p_aips_count = aip_count; 2478 2479 ret = copyout(&peer_o, peer_p, sizeof(peer_o)); 2480 if (ret != 0) 2481 goto out; 2482 2483 peer_p = (struct wg_peer_io *)aip_p; 2484 peer_count++; 2485 } 2486 KKASSERT(peer_count == sc->sc_peers_num); 2487 iface_o.i_peers_count = peer_count; 2488 2489 ret = copyout(&iface_o, iface_p, sizeof(iface_o)); 2490 2491 out: 2492 lockmgr(&sc->sc_lock, LK_RELEASE); 2493 explicit_bzero(&iface_o, sizeof(iface_o)); 2494 explicit_bzero(&peer_o, sizeof(peer_o)); 2495 return (ret); 2496 } 2497 2498 static int 2499 wg_ioctl_set(struct wg_softc *sc, struct wg_data_io *data) 2500 { 2501 struct wg_interface_io *iface_p, iface_o; 2502 struct wg_peer_io *peer_p, peer_o; 2503 struct wg_aip_io *aip_p, aip_o; 2504 struct wg_peer *peer; 2505 struct noise_remote *remote; 2506 uint8_t public[WG_KEY_SIZE], private[WG_KEY_SIZE]; 2507 size_t i, j; 2508 int ret; 2509 2510 remote = NULL; 2511 lockmgr(&sc->sc_lock, LK_EXCLUSIVE); 2512 2513 iface_p = data->wgd_interface; 2514 if ((ret = copyin(iface_p, &iface_o, sizeof(iface_o))) != 0) 2515 goto error; 2516 2517 if (iface_o.i_flags & WG_INTERFACE_REPLACE_PEERS) 2518 wg_peer_destroy_all(sc); 2519 2520 if ((iface_o.i_flags & WG_INTERFACE_HAS_PRIVATE) && 2521 (!noise_local_keys(sc->sc_local, NULL, private) || 2522 timingsafe_bcmp(private, iface_o.i_private, WG_KEY_SIZE) != 0)) { 2523 if (curve25519_generate_public(public, iface_o.i_private)) { 2524 remote = noise_remote_lookup(sc->sc_local, public); 2525 if (remote != NULL) { 2526 /* Remove the conflicting peer. */ 2527 peer = noise_remote_arg(remote); 2528 wg_peer_destroy(peer); 2529 noise_remote_put(remote); 2530 } 2531 } 2532 2533 /* 2534 * Set the private key. 2535 * 2536 * Note: we might be removing the private key. 2537 */ 2538 if (noise_local_set_private(sc->sc_local, iface_o.i_private)) 2539 cookie_checker_update(sc->sc_cookie, public); 2540 else 2541 cookie_checker_update(sc->sc_cookie, NULL); 2542 } 2543 2544 if ((iface_o.i_flags & WG_INTERFACE_HAS_PORT) && 2545 iface_o.i_port != sc->sc_socket.so_port) { 2546 if (sc->sc_ifp->if_flags & IFF_RUNNING) { 2547 ret = wg_socket_init(sc, iface_o.i_port); 2548 if (ret != 0) 2549 goto error; 2550 } else { 2551 sc->sc_socket.so_port = iface_o.i_port; 2552 } 2553 } 2554 2555 if (iface_o.i_flags & WG_INTERFACE_HAS_COOKIE) { 2556 ret = wg_socket_set_cookie(sc, iface_o.i_cookie); 2557 if (ret != 0) 2558 goto error; 2559 } 2560 2561 peer_p = &iface_p->i_peers[0]; 2562 for (i = 0; i < iface_o.i_peers_count; i++) { 2563 if ((ret = copyin(peer_p, &peer_o, sizeof(peer_o))) != 0) 2564 goto error; 2565 2566 /* Peer must have public key. */ 2567 if ((peer_o.p_flags & WG_PEER_HAS_PUBLIC) == 0) 2568 goto next_peer; 2569 /* Ignore peer that has the same public key. */ 2570 if (noise_local_keys(sc->sc_local, public, NULL) && 2571 memcmp(public, peer_o.p_public, WG_KEY_SIZE) == 0) 2572 goto next_peer; 2573 2574 /* Lookup peer, or create if it doesn't exist. */ 2575 remote = noise_remote_lookup(sc->sc_local, peer_o.p_public); 2576 if (remote != NULL) { 2577 peer = noise_remote_arg(remote); 2578 } else { 2579 if (peer_o.p_flags & (WG_PEER_REMOVE | WG_PEER_UPDATE)) 2580 goto next_peer; 2581 2582 peer = wg_peer_create(sc, peer_o.p_public); 2583 if (peer == NULL) { 2584 ret = ENOMEM; 2585 goto error; 2586 } 2587 2588 /* No allowed IPs to remove for a new peer. */ 2589 peer_o.p_flags &= ~WG_PEER_REPLACE_AIPS; 2590 } 2591 2592 if (peer_o.p_flags & WG_PEER_REMOVE) { 2593 wg_peer_destroy(peer); 2594 goto next_peer; 2595 } 2596 2597 if (peer_o.p_flags & WG_PEER_HAS_ENDPOINT) { 2598 ret = wg_peer_set_sockaddr(peer, &peer_o.p_sa); 2599 if (ret != 0) 2600 goto error; 2601 } 2602 if (peer_o.p_flags & WG_PEER_HAS_PSK) 2603 noise_remote_set_psk(peer->p_remote, peer_o.p_psk); 2604 if (peer_o.p_flags & WG_PEER_HAS_PKA) 2605 wg_timers_set_persistent_keepalive(peer, peer_o.p_pka); 2606 if (peer_o.p_flags & WG_PEER_SET_DESCRIPTION) 2607 strlcpy(peer->p_description, peer_o.p_description, 2608 sizeof(peer->p_description)); 2609 2610 if (peer_o.p_flags & WG_PEER_REPLACE_AIPS) 2611 wg_aip_remove_all(sc, peer); 2612 2613 for (j = 0; j < peer_o.p_aips_count; j++) { 2614 aip_p = &peer_p->p_aips[j]; 2615 if ((ret = copyin(aip_p, &aip_o, sizeof(aip_o))) != 0) 2616 goto error; 2617 ret = wg_aip_add(sc, peer, aip_o.a_af, &aip_o.a_addr, 2618 aip_o.a_cidr); 2619 if (ret != 0) 2620 goto error; 2621 } 2622 2623 if (sc->sc_ifp->if_link_state == LINK_STATE_UP) 2624 wg_peer_send_staged(peer); 2625 2626 next_peer: 2627 if (remote != NULL) { 2628 noise_remote_put(remote); 2629 remote = NULL; 2630 } 2631 aip_p = &peer_p->p_aips[peer_o.p_aips_count]; 2632 peer_p = (struct wg_peer_io *)aip_p; 2633 } 2634 2635 error: 2636 if (remote != NULL) 2637 noise_remote_put(remote); 2638 lockmgr(&sc->sc_lock, LK_RELEASE); 2639 explicit_bzero(&iface_o, sizeof(iface_o)); 2640 explicit_bzero(&peer_o, sizeof(peer_o)); 2641 explicit_bzero(&aip_o, sizeof(aip_o)); 2642 explicit_bzero(public, sizeof(public)); 2643 explicit_bzero(private, sizeof(private)); 2644 return (ret); 2645 } 2646 2647 static int 2648 wg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cred) 2649 { 2650 struct wg_data_io *wgd; 2651 struct wg_softc *sc; 2652 struct ifreq *ifr; 2653 bool privileged; 2654 int ret, mask; 2655 2656 sc = ifp->if_softc; 2657 ifr = (struct ifreq *)data; 2658 ret = 0; 2659 2660 switch (cmd) { 2661 case SIOCSWG: 2662 ret = caps_priv_check(cred, SYSCAP_RESTRICTEDROOT); 2663 if (ret == 0) { 2664 wgd = (struct wg_data_io *)data; 2665 ret = wg_ioctl_set(sc, wgd); 2666 } 2667 break; 2668 case SIOCGWG: 2669 privileged = 2670 (caps_priv_check(cred, SYSCAP_RESTRICTEDROOT) == 0); 2671 wgd = (struct wg_data_io *)data; 2672 ret = wg_ioctl_get(sc, wgd, privileged); 2673 break; 2674 /* Interface IOCTLs */ 2675 case SIOCSIFADDR: 2676 /* 2677 * This differs from *BSD norms, but is more uniform with how 2678 * WireGuard behaves elsewhere. 2679 */ 2680 break; 2681 case SIOCSIFFLAGS: 2682 if (ifp->if_flags & IFF_UP) 2683 ret = wg_up(sc); 2684 else 2685 wg_down(sc); 2686 break; 2687 case SIOCSIFMTU: 2688 if (ifr->ifr_mtu <= 0 || ifr->ifr_mtu > MAX_MTU) 2689 ret = EINVAL; 2690 else 2691 ifp->if_mtu = ifr->ifr_mtu; 2692 break; 2693 case SIOCSIFCAP: 2694 mask = ifp->if_capenable ^ ifr->ifr_reqcap; 2695 if (mask & IFCAP_RXCSUM) 2696 ifp->if_capenable ^= IFCAP_RXCSUM; 2697 break; 2698 case SIOCADDMULTI: 2699 case SIOCDELMULTI: 2700 break; 2701 default: 2702 ret = ENOTTY; 2703 } 2704 2705 return (ret); 2706 } 2707 2708 static int 2709 wg_up(struct wg_softc *sc) 2710 { 2711 struct ifnet *ifp = sc->sc_ifp; 2712 struct wg_peer *peer; 2713 int ret = 0; 2714 2715 lockmgr(&sc->sc_lock, LK_EXCLUSIVE); 2716 2717 /* Silent success if we're already running. */ 2718 if (ifp->if_flags & IFF_RUNNING) 2719 goto out; 2720 ifp->if_flags |= IFF_RUNNING; 2721 2722 ret = wg_socket_init(sc, sc->sc_socket.so_port); 2723 if (ret == 0) { 2724 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) 2725 wg_timers_enable(peer); 2726 ifp->if_link_state = LINK_STATE_UP; 2727 if_link_state_change(ifp); 2728 } else { 2729 ifp->if_flags &= ~IFF_RUNNING; 2730 DPRINTF(sc, "Unable to initialize sockets: %d\n", ret); 2731 } 2732 2733 out: 2734 lockmgr(&sc->sc_lock, LK_RELEASE); 2735 return (ret); 2736 } 2737 2738 static void 2739 wg_down(struct wg_softc *sc) 2740 { 2741 struct ifnet *ifp = sc->sc_ifp; 2742 struct wg_peer *peer; 2743 int i; 2744 2745 lockmgr(&sc->sc_lock, LK_EXCLUSIVE); 2746 2747 if ((ifp->if_flags & IFF_RUNNING) == 0) { 2748 lockmgr(&sc->sc_lock, LK_RELEASE); 2749 return; 2750 } 2751 ifp->if_flags &= ~IFF_RUNNING; 2752 2753 /* Cancel all tasks. */ 2754 while (taskqueue_cancel(sc->sc_handshake_taskqueue, 2755 &sc->sc_handshake_task, NULL) != 0) { 2756 taskqueue_drain(sc->sc_handshake_taskqueue, 2757 &sc->sc_handshake_task); 2758 } 2759 for (i = 0; i < ncpus; i++) { 2760 while (taskqueue_cancel(wg_taskqueues[i], 2761 &sc->sc_encrypt_tasks[i], NULL) != 0) { 2762 taskqueue_drain(wg_taskqueues[i], 2763 &sc->sc_encrypt_tasks[i]); 2764 } 2765 while (taskqueue_cancel(wg_taskqueues[i], 2766 &sc->sc_decrypt_tasks[i], NULL) != 0) { 2767 taskqueue_drain(wg_taskqueues[i], 2768 &sc->sc_decrypt_tasks[i]); 2769 } 2770 } 2771 2772 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { 2773 wg_queue_purge(&peer->p_stage_queue); 2774 wg_timers_disable(peer); 2775 } 2776 2777 wg_queue_purge(&sc->sc_handshake_queue); 2778 2779 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { 2780 noise_remote_handshake_clear(peer->p_remote); 2781 noise_remote_keypairs_clear(peer->p_remote); 2782 } 2783 2784 ifp->if_link_state = LINK_STATE_DOWN; 2785 if_link_state_change(ifp); 2786 wg_socket_uninit(sc); 2787 2788 lockmgr(&sc->sc_lock, LK_RELEASE); 2789 } 2790 2791 static int 2792 wg_clone_create(struct if_clone *ifc __unused, int unit, 2793 caddr_t params __unused, caddr_t data __unused) 2794 { 2795 struct wg_softc *sc; 2796 struct ifnet *ifp; 2797 int i; 2798 2799 sc = kmalloc(sizeof(*sc), M_WG, M_WAITOK | M_ZERO); 2800 2801 if (!rn_inithead(&sc->sc_aip4, wg_maskhead, 2802 offsetof(struct aip_addr, in)) || 2803 !rn_inithead(&sc->sc_aip6, wg_maskhead, 2804 offsetof(struct aip_addr, in6))) { 2805 if (sc->sc_aip4 != NULL) 2806 rn_freehead(sc->sc_aip4); 2807 if (sc->sc_aip6 != NULL) 2808 rn_freehead(sc->sc_aip6); 2809 kfree(sc, M_WG); 2810 return (ENOMEM); 2811 } 2812 2813 lockinit(&sc->sc_lock, "wg softc lock", 0, 0); 2814 lockinit(&sc->sc_aip_lock, "wg aip lock", 0, 0); 2815 2816 sc->sc_local = noise_local_alloc(); 2817 sc->sc_cookie = cookie_checker_alloc(); 2818 2819 TAILQ_INIT(&sc->sc_peers); 2820 2821 sc->sc_handshake_taskqueue = wg_taskqueues[karc4random() % ncpus]; 2822 TASK_INIT(&sc->sc_handshake_task, 0, wg_handshake_worker, sc); 2823 wg_queue_init(&sc->sc_handshake_queue, "hsq"); 2824 2825 sc->sc_encrypt_tasks = kmalloc(sizeof(*sc->sc_encrypt_tasks) * ncpus, 2826 M_WG, M_WAITOK | M_ZERO); 2827 sc->sc_decrypt_tasks = kmalloc(sizeof(*sc->sc_decrypt_tasks) * ncpus, 2828 M_WG, M_WAITOK | M_ZERO); 2829 for (i = 0; i < ncpus; i++) { 2830 TASK_INIT(&sc->sc_encrypt_tasks[i], 0, wg_encrypt_worker, sc); 2831 TASK_INIT(&sc->sc_decrypt_tasks[i], 0, wg_decrypt_worker, sc); 2832 } 2833 wg_queue_init(&sc->sc_encrypt_parallel, "encp"); 2834 wg_queue_init(&sc->sc_decrypt_parallel, "decp"); 2835 2836 ifp = sc->sc_ifp = if_alloc(IFT_WIREGUARD); 2837 if_initname(ifp, wgname, unit); 2838 ifp->if_softc = sc; 2839 ifp->if_mtu = DEFAULT_MTU; 2840 ifp->if_flags = IFF_NOARP | IFF_MULTICAST; 2841 ifp->if_capabilities = ifp->if_capenable = IFCAP_RXCSUM; 2842 ifp->if_output = wg_output; 2843 ifp->if_ioctl = wg_ioctl; 2844 ifq_set_maxlen(&ifp->if_snd, ifqmaxlen); 2845 ifq_set_ready(&ifp->if_snd); 2846 2847 if_attach(ifp, NULL); 2848 2849 /* DLT_NULL link-layer header: a 4-byte field in host byte order */ 2850 bpfattach(ifp, DLT_NULL, sizeof(uint32_t)); 2851 2852 lockmgr(&wg_mtx, LK_EXCLUSIVE); 2853 LIST_INSERT_HEAD(&wg_list, sc, sc_entry); 2854 lockmgr(&wg_mtx, LK_RELEASE); 2855 2856 return (0); 2857 } 2858 2859 static int 2860 wg_clone_destroy(struct ifnet *ifp) 2861 { 2862 struct wg_softc *sc = ifp->if_softc; 2863 2864 wg_down(sc); 2865 2866 lockmgr(&sc->sc_lock, LK_EXCLUSIVE); 2867 2868 kfree(sc->sc_encrypt_tasks, M_WG); 2869 kfree(sc->sc_decrypt_tasks, M_WG); 2870 wg_queue_deinit(&sc->sc_handshake_queue); 2871 wg_queue_deinit(&sc->sc_encrypt_parallel); 2872 wg_queue_deinit(&sc->sc_decrypt_parallel); 2873 2874 wg_peer_destroy_all(sc); 2875 2876 /* 2877 * Detach and free the interface before the sc_aip4 and sc_aip6 radix 2878 * trees, because the purge of interface's IPv6 addresses can cause 2879 * packet transmission and thus wg_aip_lookup() calls. 2880 */ 2881 bpfdetach(ifp); 2882 if_detach(ifp); 2883 if_free(ifp); 2884 2885 /* 2886 * All peers have been removed, so the sc_aip4 and sc_aip6 radix trees 2887 * must be empty now. 2888 */ 2889 rn_freehead(sc->sc_aip4); 2890 rn_freehead(sc->sc_aip6); 2891 lockuninit(&sc->sc_aip_lock); 2892 2893 cookie_checker_free(sc->sc_cookie); 2894 noise_local_free(sc->sc_local); 2895 2896 lockmgr(&wg_mtx, LK_EXCLUSIVE); 2897 LIST_REMOVE(sc, sc_entry); 2898 lockmgr(&wg_mtx, LK_RELEASE); 2899 2900 lockmgr(&sc->sc_lock, LK_RELEASE); 2901 lockuninit(&sc->sc_lock); 2902 kfree(sc, M_WG); 2903 2904 return (0); 2905 } 2906 2907 /*----------------------------------------------------------------------------*/ 2908 /* Module Interface */ 2909 2910 #ifdef WG_SELFTESTS 2911 #include "selftest/allowedips.c" 2912 static bool 2913 wg_run_selftests(void) 2914 { 2915 bool ret = true; 2916 2917 ret &= wg_allowedips_selftest(); 2918 ret &= noise_counter_selftest(); 2919 ret &= cookie_selftest(); 2920 2921 kprintf("%s: %s\n", __func__, ret ? "pass" : "FAIL"); 2922 return (ret); 2923 } 2924 #else /* !WG_SELFTESTS */ 2925 static inline bool 2926 wg_run_selftests(void) 2927 { 2928 return (true); 2929 } 2930 #endif /* WG_SELFTESTS */ 2931 2932 static struct if_clone wg_cloner = IF_CLONE_INITIALIZER( 2933 wgname, wg_clone_create, wg_clone_destroy, 0, IF_MAXUNIT); 2934 2935 static int 2936 wg_module_init(void) 2937 { 2938 int i, ret; 2939 2940 lockinit(&wg_mtx, "wg mtx lock", 0, 0); 2941 2942 wg_packet_zone = objcache_create_simple(M_WG_PACKET, 2943 sizeof(struct wg_packet)); 2944 if (wg_packet_zone == NULL) 2945 return (ENOMEM); 2946 2947 wg_taskqueues = kmalloc(sizeof(*wg_taskqueues) * ncpus, M_WG, 2948 M_WAITOK | M_ZERO); 2949 for (i = 0; i < ncpus; i++) { 2950 wg_taskqueues[i] = taskqueue_create("wg_taskq", M_WAITOK, 2951 taskqueue_thread_enqueue, 2952 &wg_taskqueues[i]); 2953 taskqueue_start_threads(&wg_taskqueues[i], 1, 2954 TDPRI_KERN_DAEMON, i, 2955 "wg_taskq_cpu_%d", i); 2956 } 2957 2958 if (!rn_inithead(&wg_maskhead, NULL, 0)) 2959 return (ENOMEM); 2960 2961 ret = cookie_init(); 2962 if (ret != 0) 2963 return (ret); 2964 ret = noise_init(); 2965 if (ret != 0) 2966 return (ret); 2967 2968 ret = if_clone_attach(&wg_cloner); 2969 if (ret != 0) 2970 return (ret); 2971 2972 if (!wg_run_selftests()) 2973 return (ENOTRECOVERABLE); 2974 2975 return (0); 2976 } 2977 2978 static int 2979 wg_module_deinit(void) 2980 { 2981 int i; 2982 2983 lockmgr(&wg_mtx, LK_EXCLUSIVE); 2984 2985 if (!LIST_EMPTY(&wg_list)) { 2986 lockmgr(&wg_mtx, LK_RELEASE); 2987 return (EBUSY); 2988 } 2989 2990 if_clone_detach(&wg_cloner); 2991 2992 noise_deinit(); 2993 cookie_deinit(); 2994 2995 for (i = 0; i < ncpus; i++) 2996 taskqueue_free(wg_taskqueues[i]); 2997 kfree(wg_taskqueues, M_WG); 2998 2999 rn_flush(wg_maskhead, rn_freemask); 3000 rn_freehead(wg_maskhead); 3001 3002 if (wg_packet_zone != NULL) 3003 objcache_destroy(wg_packet_zone); 3004 3005 lockmgr(&wg_mtx, LK_RELEASE); 3006 lockuninit(&wg_mtx); 3007 3008 return (0); 3009 } 3010 3011 static int 3012 wg_module_event_handler(module_t mod __unused, int what, void *arg __unused) 3013 { 3014 switch (what) { 3015 case MOD_LOAD: 3016 return wg_module_init(); 3017 case MOD_UNLOAD: 3018 return wg_module_deinit(); 3019 default: 3020 return (EOPNOTSUPP); 3021 } 3022 } 3023 3024 static moduledata_t wg_moduledata = { 3025 "if_wg", 3026 wg_module_event_handler, 3027 NULL 3028 }; 3029 3030 DECLARE_MODULE(if_wg, wg_moduledata, SI_SUB_PSEUDO, SI_ORDER_ANY); 3031 MODULE_VERSION(if_wg, 1); /* WireGuard version */ 3032 MODULE_DEPEND(if_wg, crypto, 1, 1, 1); 3033