xref: /dragonfly/sys/netinet/ip_carp.c (revision c03f08f3)
1 /*
2  * Copyright (c) 2002 Michael Shalayeff. All rights reserved.
3  * Copyright (c) 2003 Ryan McBride. All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17  * IN NO EVENT SHALL THE AUTHOR OR HIS RELATIVES BE LIABLE FOR ANY DIRECT,
18  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
19  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
20  * SERVICES; LOSS OF MIND, USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
22  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
23  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
24  * THE POSSIBILITY OF SUCH DAMAGE.
25  */
26 /*
27  * $FreeBSD: src/sys/netinet/ip_carp.c,v 1.48 2007/02/02 09:39:09 glebius Exp $
28  * $DragonFly: src/sys/netinet/ip_carp.c,v 1.3 2007/08/27 16:15:42 hasso Exp $
29  */
30 
31 #include "opt_carp.h"
32 /*#include "opt_bpf.h"*/
33 #include "opt_inet.h"
34 #include "opt_inet6.h"
35 
36 #include <sys/types.h>
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/conf.h>
40 #include <sys/kernel.h>
41 #include <machine/limits.h>
42 #include <sys/malloc.h>
43 #include <sys/mbuf.h>
44 #include <sys/module.h>
45 #include <sys/time.h>
46 #include <sys/proc.h>
47 #include <sys/sysctl.h>
48 #include <sys/syslog.h>
49 #include <sys/signalvar.h>
50 #include <sys/filio.h>
51 #include <sys/sockio.h>
52 #include <sys/in_cksum.h>
53 #include <sys/socket.h>
54 #include <sys/vnode.h>
55 
56 #include <machine/stdarg.h>
57 
58 #include <net/bpf.h>
59 #include <net/ethernet.h>
60 #include <net/if.h>
61 #include <net/if_dl.h>
62 #include <net/if_types.h>
63 #include <net/route.h>
64 
65 #ifdef INET
66 #include <netinet/in.h>
67 #include <netinet/in_var.h>
68 #include <netinet/in_systm.h>
69 #include <netinet/ip.h>
70 #include <netinet/ip_var.h>
71 #include <netinet/if_ether.h>
72 #endif
73 
74 #ifdef INET6
75 #include <netinet/icmp6.h>
76 #include <netinet/ip6.h>
77 #include <netinet6/ip6_var.h>
78 #include <netinet6/scope6_var.h>
79 #include <netinet6/nd6.h>
80 #endif
81 
82 #include <crypto/sha1.h>
83 #include <netinet/ip_carp.h>
84 #include <sys/lock.h>
85 
86 #define	CARP_IFNAME	"carp"
87 static MALLOC_DEFINE(M_CARP, "CARP", "CARP interfaces");
88 static MALLOC_DEFINE(M_IFNET, "IFNET", "IFNET CARP?");
89 SYSCTL_DECL(_net_inet_carp);
90 
91 struct carp_softc {
92 	struct ifnet	 	*sc_ifp;	/* Interface clue */
93 	struct ifnet		*sc_carpdev;	/* Pointer to parent interface */
94 	struct in_ifaddr 	*sc_ia;		/* primary iface address */
95 	struct ip_moptions 	 sc_imo;
96 #ifdef INET6
97 	struct in6_ifaddr 	*sc_ia6;	/* primary iface address v6 */
98 	struct ip6_moptions 	 sc_im6o;
99 #endif /* INET6 */
100 	TAILQ_ENTRY(carp_softc)	 sc_list;
101 
102 	enum { INIT = 0, BACKUP, MASTER }	sc_state;
103 
104 	int			 sc_flags_backup;
105 	int			 sc_suppress;
106 
107 	int			 sc_sendad_errors;
108 #define	CARP_SENDAD_MAX_ERRORS	3
109 	int			 sc_sendad_success;
110 #define	CARP_SENDAD_MIN_SUCCESS 3
111 
112 	int			 sc_vhid;
113 	int			 sc_advskew;
114 	int			 sc_naddrs;
115 	int			 sc_naddrs6;
116 	int			 sc_advbase;	/* seconds */
117 	int			 sc_init_counter;
118 	u_int64_t		 sc_counter;
119 
120 	/* authentication */
121 #define CARP_HMAC_PAD	64
122 	unsigned char sc_key[CARP_KEY_LEN];
123 	unsigned char sc_pad[CARP_HMAC_PAD];
124 	SHA1_CTX sc_sha1;
125 
126 	struct callout		 sc_ad_tmo;	/* advertisement timeout */
127 	struct callout		 sc_md_tmo;	/* master down timeout */
128 	struct callout 		 sc_md6_tmo;	/* master down timeout */
129 
130 	LIST_ENTRY(carp_softc)	 sc_next;	/* Interface clue */
131 };
132 #define	SC2IFP(sc)	((sc)->sc_ifp)
133 
134 int carp_suppress_preempt = 0;
135 int carp_opts[CARPCTL_MAXID] = { 0, 1, 0, 1, 0, 0 };	/* XXX for now */
136 SYSCTL_INT(_net_inet_carp, CARPCTL_ALLOW, allow, CTLFLAG_RW,
137     &carp_opts[CARPCTL_ALLOW], 0, "Accept incoming CARP packets");
138 SYSCTL_INT(_net_inet_carp, CARPCTL_PREEMPT, preempt, CTLFLAG_RW,
139     &carp_opts[CARPCTL_PREEMPT], 0, "high-priority backup preemption mode");
140 SYSCTL_INT(_net_inet_carp, CARPCTL_LOG, log, CTLFLAG_RW,
141     &carp_opts[CARPCTL_LOG], 0, "log bad carp packets");
142 SYSCTL_INT(_net_inet_carp, CARPCTL_ARPBALANCE, arpbalance, CTLFLAG_RW,
143     &carp_opts[CARPCTL_ARPBALANCE], 0, "balance arp responses");
144 SYSCTL_INT(_net_inet_carp, OID_AUTO, suppress_preempt, CTLFLAG_RD,
145     &carp_suppress_preempt, 0, "Preemption is suppressed");
146 
147 struct carpstats carpstats;
148 SYSCTL_STRUCT(_net_inet_carp, CARPCTL_STATS, stats, CTLFLAG_RW,
149     &carpstats, carpstats,
150     "CARP statistics (struct carpstats, netinet/ip_carp.h)");
151 
152 struct carp_if {
153 	TAILQ_HEAD(, carp_softc) vhif_vrs;
154 	int vhif_nvrs;
155 
156 	struct ifnet 	*vhif_ifp;
157 	struct lock	vhif_lock;
158 };
159 
160 /* Get carp_if from softc. Valid after carp_set_addr{,6}. */
161 #define	SC2CIF(sc)		((struct carp_if *)(sc)->sc_carpdev->if_carp)
162 
163 #define	CARP_LOCK_INIT(cif)	lockinit(&(cif)->vhif_lock, "carp_if", 0,  LK_NOWAIT);
164 #define	CARP_LOCK_DESTROY(cif)	;
165 #define	CARP_LOCK_ASSERT(cif)	;
166 #define	CARP_LOCK(cif)		lockmgr(&(cif)->vhif_lock, LK_EXCLUSIVE);
167 #define	CARP_UNLOCK(cif)	lockmgr(&(cif)->vhif_lock, LK_RELEASE);
168 
169 #define	CARP_SCLOCK(sc)		lockmgr(&SC2CIF(sc)->vhif_lock, LK_EXCLUSIVE);
170 #define	CARP_SCUNLOCK(sc)	lockmgr(&SC2CIF(sc)->vhif_lock, LK_RELEASE);
171 #define	CARP_SCLOCK_ASSERT(sc)	;
172 
173 #define	CARP_LOG(...)	do {				\
174 	if (carp_opts[CARPCTL_LOG] > 0)			\
175 		log(LOG_INFO, __VA_ARGS__);		\
176 } while (0)
177 
178 #define	CARP_DEBUG(...)	do {				\
179 	if (carp_opts[CARPCTL_LOG] > 1)			\
180 		log(LOG_DEBUG, __VA_ARGS__);		\
181 } while (0)
182 
183 static void	carp_hmac_prepare(struct carp_softc *);
184 static void	carp_hmac_generate(struct carp_softc *, u_int32_t *,
185 		    unsigned char *);
186 static int	carp_hmac_verify(struct carp_softc *, u_int32_t *,
187 		    unsigned char *);
188 static void	carp_setroute(struct carp_softc *, int);
189 static void	carp_input_c(struct mbuf *, struct carp_header *, sa_family_t);
190 static int 	carp_clone_create(struct if_clone *, int);
191 static void 	carp_clone_destroy(struct ifnet *);
192 static void	carpdetach(struct carp_softc *, int);
193 static int	carp_prepare_ad(struct mbuf *, struct carp_softc *,
194 		    struct carp_header *);
195 static void	carp_send_ad_all(void);
196 static void	carp_send_ad(void *);
197 static void	carp_send_ad_locked(struct carp_softc *);
198 static void	carp_send_arp(struct carp_softc *);
199 static void	carp_master_down(void *);
200 static void	carp_master_down_locked(struct carp_softc *);
201 static int	carp_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
202 static int	carp_looutput(struct ifnet *, struct mbuf *, struct sockaddr *,
203 		    struct rtentry *);
204 static void	carp_start(struct ifnet *);
205 static void	carp_setrun(struct carp_softc *, sa_family_t);
206 static void	carp_set_state(struct carp_softc *, int);
207 static int	carp_addrcount(struct carp_if *, struct in_ifaddr *, int);
208 enum	{ CARP_COUNT_MASTER, CARP_COUNT_RUNNING };
209 
210 static void	carp_multicast_cleanup(struct carp_softc *);
211 static int	carp_set_addr(struct carp_softc *, struct sockaddr_in *);
212 static int	carp_del_addr(struct carp_softc *, struct sockaddr_in *);
213 static void	carp_carpdev_state_locked(struct carp_if *);
214 static void	carp_sc_state_locked(struct carp_softc *);
215 #ifdef INET6
216 static void	carp_send_na(struct carp_softc *);
217 static int	carp_set_addr6(struct carp_softc *, struct sockaddr_in6 *);
218 static int	carp_del_addr6(struct carp_softc *, struct sockaddr_in6 *);
219 static void	carp_multicast6_cleanup(struct carp_softc *);
220 #endif
221 
222 static LIST_HEAD(, carp_softc) carpif_list;
223 
224 struct if_clone carp_cloner = IF_CLONE_INITIALIZER(CARP_IFNAME, carp_clone_create, carp_clone_destroy, 0, IF_MAXUNIT);
225 
226 static eventhandler_tag if_detach_event_tag;
227 
228 static __inline u_int16_t
229 carp_cksum(struct mbuf *m, int len)
230 {
231 	return (in_cksum(m, len));
232 }
233 
234 static void
235 carp_hmac_prepare(struct carp_softc *sc)
236 {
237 	u_int8_t version = CARP_VERSION, type = CARP_ADVERTISEMENT;
238 	u_int8_t vhid = sc->sc_vhid & 0xff;
239 	struct ifaddr *ifa;
240 	int i;
241 #ifdef INET6
242 	struct in6_addr in6;
243 #endif
244 
245 	if (sc->sc_carpdev)
246 		CARP_SCLOCK(sc);
247 
248 	/* XXX: possible race here */
249 
250 	/* compute ipad from key */
251 	bzero(sc->sc_pad, sizeof(sc->sc_pad));
252 	bcopy(sc->sc_key, sc->sc_pad, sizeof(sc->sc_key));
253 	for (i = 0; i < sizeof(sc->sc_pad); i++)
254 		sc->sc_pad[i] ^= 0x36;
255 
256 	/* precompute first part of inner hash */
257 	SHA1Init(&sc->sc_sha1);
258 	SHA1Update(&sc->sc_sha1, sc->sc_pad, sizeof(sc->sc_pad));
259 	SHA1Update(&sc->sc_sha1, (void *)&version, sizeof(version));
260 	SHA1Update(&sc->sc_sha1, (void *)&type, sizeof(type));
261 	SHA1Update(&sc->sc_sha1, (void *)&vhid, sizeof(vhid));
262 #ifdef INET
263 	TAILQ_FOREACH(ifa, &SC2IFP(sc)->if_addrlist, ifa_list) {
264 		if (ifa->ifa_addr->sa_family == AF_INET)
265 			SHA1Update(&sc->sc_sha1,
266 			    (void *)&ifatoia(ifa)->ia_addr.sin_addr.s_addr,
267 			    sizeof(struct in_addr));
268 	}
269 #endif /* INET */
270 #ifdef INET6
271 	TAILQ_FOREACH(ifa, &SC2IFP(sc)->if_addrlist, ifa_list) {
272 		if (ifa->ifa_addr->sa_family == AF_INET6) {
273 			in6 = ifatoia6(ifa)->ia_addr.sin6_addr;
274 			in6_clearscope(&in6);
275 			SHA1Update(&sc->sc_sha1, (void *)&in6, sizeof(in6));
276 		}
277 	}
278 #endif /* INET6 */
279 
280 	/* convert ipad to opad */
281 	for (i = 0; i < sizeof(sc->sc_pad); i++)
282 		sc->sc_pad[i] ^= 0x36 ^ 0x5c;
283 
284 	if (sc->sc_carpdev)
285 		CARP_SCUNLOCK(sc);
286 }
287 
288 static void
289 carp_hmac_generate(struct carp_softc *sc, u_int32_t counter[2],
290     unsigned char md[20])
291 {
292 	SHA1_CTX sha1ctx;
293 
294 	/* fetch first half of inner hash */
295 	bcopy(&sc->sc_sha1, &sha1ctx, sizeof(sha1ctx));
296 
297 	SHA1Update(&sha1ctx, (void *)counter, sizeof(sc->sc_counter));
298 	SHA1Final(md, &sha1ctx);
299 
300 	/* outer hash */
301 	SHA1Init(&sha1ctx);
302 	SHA1Update(&sha1ctx, sc->sc_pad, sizeof(sc->sc_pad));
303 	SHA1Update(&sha1ctx, md, 20);
304 	SHA1Final(md, &sha1ctx);
305 }
306 
307 static int
308 carp_hmac_verify(struct carp_softc *sc, u_int32_t counter[2],
309     unsigned char md[20])
310 {
311 	unsigned char md2[20];
312 
313 	CARP_SCLOCK_ASSERT(sc);
314 
315 	carp_hmac_generate(sc, counter, md2);
316 
317 	return (bcmp(md, md2, sizeof(md2)));
318 }
319 
320 static void
321 carp_setroute(struct carp_softc *sc, int cmd)
322 {
323 	struct ifaddr *ifa;
324 
325 	if (sc->sc_carpdev)
326 		CARP_SCLOCK_ASSERT(sc);
327 
328 	crit_enter();
329 	TAILQ_FOREACH(ifa, &SC2IFP(sc)->if_addrlist, ifa_list) {
330 		if (ifa->ifa_addr->sa_family == AF_INET &&
331 		    sc->sc_carpdev != NULL) {
332 			int count = carp_addrcount(
333 			    (struct carp_if *)sc->sc_carpdev->if_carp,
334 			    ifatoia(ifa), CARP_COUNT_MASTER);
335 
336 			if ((cmd == RTM_ADD && count == 1) ||
337 			    (cmd == RTM_DELETE && count == 0))
338 				rtinit(ifa, cmd, RTF_UP | RTF_HOST);
339 		}
340 #ifdef INET6
341 		if (ifa->ifa_addr->sa_family == AF_INET6) {
342 			if (cmd == RTM_ADD)
343 				in6_ifaddloop(ifa);
344 			else
345 				in6_ifremloop(ifa);
346 		}
347 #endif /* INET6 */
348 	}
349 	crit_exit();
350 
351 }
352 
353 static int
354 carp_clone_create(struct if_clone *ifc, int unit)
355 {
356 
357 	struct carp_softc *sc;
358 	struct ifnet *ifp;
359 
360         MALLOC(sc, struct carp_softc *, sizeof(*sc), M_CARP, M_WAITOK|M_ZERO);
361 	ifp = SC2IFP(sc) = kmalloc(sizeof(struct ifnet), M_IFNET, M_WAITOK|M_ZERO);
362 
363 	if (ifp == NULL) {
364 		FREE(sc, M_CARP);
365 		return (ENOSPC);
366 	}
367 
368 	sc->sc_flags_backup = 0;
369 	sc->sc_suppress = 0;
370 	sc->sc_advbase = CARP_DFLTINTV;
371 	sc->sc_vhid = -1;	/* required setting */
372 	sc->sc_advskew = 0;
373 	sc->sc_init_counter = 1;
374 	sc->sc_naddrs = sc->sc_naddrs6 = 0; /* M_ZERO? */
375 
376 #ifdef INET6
377 	sc->sc_im6o.im6o_multicast_hlim = CARP_DFLTTL;
378 #endif
379 
380 /*	sc->sc_imo.imo_membership = kmalloc((sizeof(struct in_multi) * IP_MAX_MEMBERSHIPS), M_CARP,M_WAITOK);*/
381 /*
382 	sc->sc_imo.imo_max_memberships = IP_MAX_MEMBERSHIPS;
383 	sc->sc_imo.imo_multicast_vif = -1;
384 */
385 	callout_init(&sc->sc_ad_tmo);
386 	callout_init(&sc->sc_md_tmo);
387 	callout_init(&sc->sc_md6_tmo);
388 
389 	ifp->if_softc = sc;
390         if_initname(ifp, CARP_IFNAME, unit);
391 	ifp->if_mtu = ETHERMTU;
392 	ifp->if_flags = IFF_LOOPBACK;
393 	ifp->if_ioctl = carp_ioctl;
394 	ifp->if_output = carp_looutput;
395 	ifp->if_start = carp_start;
396 	ifp->if_type = IFT_CARP;
397 	ifp->if_snd.ifq_maxlen = ifqmaxlen;
398 	ifp->if_hdrlen = 0;
399 	if_attach(ifp, NULL);
400 	bpfattach(ifp, DLT_NULL, sizeof(u_int));
401 
402 	crit_enter();
403 	LIST_INSERT_HEAD(&carpif_list, sc, sc_next);
404 	crit_exit();
405 
406 	return (0);
407 }
408 
409 static void
410 carp_clone_destroy(struct ifnet *ifp)
411 {
412 	struct carp_softc *sc = ifp->if_softc;
413 
414 	if (sc->sc_carpdev)
415 		CARP_SCLOCK(sc);
416 	carpdetach(sc, 1);	/* Returns unlocked. */
417 
418 	crit_enter();
419 	LIST_REMOVE(sc, sc_next);
420 	crit_exit();
421 	bpfdetach(ifp);
422 	if_detach(ifp);
423 /*	if_free_type(ifp, IFT_ETHER);*/
424 /*	kfree(sc->sc_imo.imo_membership, M_CARP); */
425 	kfree(sc, M_CARP);
426 }
427 
428 /*
429  * This function can be called on CARP interface destroy path,
430  * and in case of the removal of the underlying interface as
431  * well. We differentiate these two cases. In the latter case
432  * we do not cleanup our multicast memberships, since they
433  * are already freed. Also, in the latter case we do not
434  * release the lock on return, because the function will be
435  * called once more, for another CARP instance on the same
436  * interface.
437  */
438 static void
439 carpdetach(struct carp_softc *sc, int unlock)
440 {
441 	struct carp_if *cif;
442 
443 	callout_stop(&sc->sc_ad_tmo);
444 	callout_stop(&sc->sc_md_tmo);
445 	callout_stop(&sc->sc_md6_tmo);
446 
447 	if (sc->sc_suppress)
448 		carp_suppress_preempt--;
449 	sc->sc_suppress = 0;
450 
451 	if (sc->sc_sendad_errors >= CARP_SENDAD_MAX_ERRORS)
452 		carp_suppress_preempt--;
453 	sc->sc_sendad_errors = 0;
454 
455 	carp_set_state(sc, INIT);
456 	SC2IFP(sc)->if_flags &= ~IFF_UP;
457 	carp_setrun(sc, 0);
458 	if (unlock)
459 		carp_multicast_cleanup(sc);
460 #ifdef INET6
461 	carp_multicast6_cleanup(sc);
462 #endif
463 
464 	if (sc->sc_carpdev != NULL) {
465 		cif = (struct carp_if *)sc->sc_carpdev->if_carp;
466 		CARP_LOCK_ASSERT(cif);
467 		TAILQ_REMOVE(&cif->vhif_vrs, sc, sc_list);
468 		if (!--cif->vhif_nvrs) {
469 			ifpromisc(sc->sc_carpdev, 0);
470 			sc->sc_carpdev->if_carp = NULL;
471 			CARP_LOCK_DESTROY(cif);
472 			FREE(cif, M_IFADDR);
473 		} else if (unlock)
474 			CARP_UNLOCK(cif);
475 		sc->sc_carpdev = NULL;
476 	}
477 }
478 
479 /* Detach an interface from the carp. */
480 static void
481 carp_ifdetach(void *arg __unused, struct ifnet *ifp)
482 {
483 	struct carp_if *cif = (struct carp_if *)ifp->if_carp;
484 	struct carp_softc *sc, *nextsc;
485 
486 	if (cif == NULL)
487 		return;
488 
489 	/*
490 	 * XXX: At the end of for() cycle the lock will be destroyed.
491 	 */
492 	CARP_LOCK(cif);
493 	for (sc = TAILQ_FIRST(&cif->vhif_vrs); sc; sc = nextsc) {
494 		nextsc = TAILQ_NEXT(sc, sc_list);
495 		carpdetach(sc, 0);
496 	}
497 	CARP_UNLOCK(cif);
498 }
499 
500 /*
501  * process input packet.
502  * we have rearranged checks order compared to the rfc,
503  * but it seems more efficient this way or not possible otherwise.
504  */
505 void
506 carp_input(struct mbuf *m, int hlen)
507 {
508 	struct ip *ip = mtod(m, struct ip *);
509 	struct carp_header *ch;
510 	int iplen, len;
511 
512 	carpstats.carps_ipackets++;
513 
514 	if (!carp_opts[CARPCTL_ALLOW]) {
515 		m_freem(m);
516 		return;
517 	}
518 
519 	/* check if received on a valid carp interface */
520 	if (m->m_pkthdr.rcvif->if_carp == NULL) {
521 		carpstats.carps_badif++;
522 		CARP_LOG("carp_input: packet received on non-carp "
523 		    "interface: %s\n",
524 		    m->m_pkthdr.rcvif->if_xname);
525 		m_freem(m);
526 		return;
527 	}
528 
529 	/* verify that the IP TTL is 255.  */
530 	if (ip->ip_ttl != CARP_DFLTTL) {
531 		carpstats.carps_badttl++;
532 		CARP_LOG("carp_input: received ttl %d != 255i on %s\n",
533 		    ip->ip_ttl,
534 		    m->m_pkthdr.rcvif->if_xname);
535 		m_freem(m);
536 		return;
537 	}
538 
539 	iplen = ip->ip_hl << 2;
540 
541 	if (m->m_pkthdr.len < iplen + sizeof(*ch)) {
542 		carpstats.carps_badlen++;
543 		CARP_LOG("carp_input: received len %zd < "
544 		    "sizeof(struct carp_header)\n",
545 		    m->m_len - sizeof(struct ip));
546 		m_freem(m);
547 		return;
548 	}
549 
550 	if (iplen + sizeof(*ch) < m->m_len) {
551 		if ((m = m_pullup(m, iplen + sizeof(*ch))) == NULL) {
552 			carpstats.carps_hdrops++;
553 			CARP_LOG("carp_input: pullup failed\n");
554 			return;
555 		}
556 		ip = mtod(m, struct ip *);
557 	}
558 	ch = (struct carp_header *)((char *)ip + iplen);
559 
560 	/*
561 	 * verify that the received packet length is
562 	 * equal to the CARP header
563 	 */
564 	len = iplen + sizeof(*ch);
565 	if (len > m->m_pkthdr.len) {
566 		carpstats.carps_badlen++;
567 		CARP_LOG("carp_input: packet too short %d on %s\n",
568 		    m->m_pkthdr.len,
569 		    m->m_pkthdr.rcvif->if_xname);
570 		m_freem(m);
571 		return;
572 	}
573 
574 	if ((m = m_pullup(m, len)) == NULL) {
575 		carpstats.carps_hdrops++;
576 		return;
577 	}
578 	ip = mtod(m, struct ip *);
579 	ch = (struct carp_header *)((char *)ip + iplen);
580 
581 	/* verify the CARP checksum */
582 	m->m_data += iplen;
583 	if (carp_cksum(m, len - iplen)) {
584 		carpstats.carps_badsum++;
585 		CARP_LOG("carp_input: checksum failed on %s\n",
586 		    m->m_pkthdr.rcvif->if_xname);
587 		m_freem(m);
588 		return;
589 	}
590 	m->m_data -= iplen;
591 
592 	carp_input_c(m, ch, AF_INET);
593 }
594 
595 #ifdef INET6
596 int
597 carp6_input(struct mbuf **mp, int *offp, int proto)
598 {
599 	struct mbuf *m = *mp;
600 	struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
601 	struct carp_header *ch;
602 	u_int len;
603 
604 	carpstats.carps_ipackets6++;
605 
606 	if (!carp_opts[CARPCTL_ALLOW]) {
607 		m_freem(m);
608 		return (IPPROTO_DONE);
609 	}
610 
611 	/* check if received on a valid carp interface */
612 	if (m->m_pkthdr.rcvif->if_carp == NULL) {
613 		carpstats.carps_badif++;
614 		CARP_LOG("carp6_input: packet received on non-carp "
615 		    "interface: %s\n",
616 		    m->m_pkthdr.rcvif->if_xname);
617 		m_freem(m);
618 		return (IPPROTO_DONE);
619 	}
620 
621 	/* verify that the IP TTL is 255 */
622 	if (ip6->ip6_hlim != CARP_DFLTTL) {
623 		carpstats.carps_badttl++;
624 		CARP_LOG("carp6_input: received ttl %d != 255 on %s\n",
625 		    ip6->ip6_hlim,
626 		    m->m_pkthdr.rcvif->if_xname);
627 		m_freem(m);
628 		return (IPPROTO_DONE);
629 	}
630 
631 	/* verify that we have a complete carp packet */
632 	len = m->m_len;
633 	IP6_EXTHDR_GET(ch, struct carp_header *, m, *offp, sizeof(*ch));
634 	if (ch == NULL) {
635 		carpstats.carps_badlen++;
636 		CARP_LOG("carp6_input: packet size %u too small\n", len);
637 		return (IPPROTO_DONE);
638 	}
639 
640 
641 	/* verify the CARP checksum */
642 	m->m_data += *offp;
643 	if (carp_cksum(m, sizeof(*ch))) {
644 		carpstats.carps_badsum++;
645 		CARP_LOG("carp6_input: checksum failed, on %s\n",
646 		    m->m_pkthdr.rcvif->if_xname);
647 		m_freem(m);
648 		return (IPPROTO_DONE);
649 	}
650 	m->m_data -= *offp;
651 
652 	carp_input_c(m, ch, AF_INET6);
653 	return (IPPROTO_DONE);
654 }
655 #endif /* INET6 */
656 
657 static void
658 carp_input_c(struct mbuf *m, struct carp_header *ch, sa_family_t af)
659 {
660 	struct ifnet *ifp = m->m_pkthdr.rcvif;
661 	struct carp_softc *sc;
662 	u_int64_t tmp_counter;
663 	struct timeval sc_tv, ch_tv;
664 
665 	/* verify that the VHID is valid on the receiving interface */
666 	CARP_LOCK(ifp->if_carp);
667 	TAILQ_FOREACH(sc, &((struct carp_if *)ifp->if_carp)->vhif_vrs, sc_list)
668 		if (sc->sc_vhid == ch->carp_vhid)
669 			break;
670 
671 	if (!sc || !((SC2IFP(sc)->if_flags & IFF_UP) && (SC2IFP(sc)->if_flags & IFF_RUNNING))) {
672 		carpstats.carps_badvhid++;
673 		CARP_UNLOCK(ifp->if_carp);
674 		m_freem(m);
675 		return;
676 	}
677 
678 	getmicrotime(&SC2IFP(sc)->if_lastchange);
679 	SC2IFP(sc)->if_ipackets++;
680 	SC2IFP(sc)->if_ibytes += m->m_pkthdr.len;
681 
682 	if (SC2IFP(sc)->if_bpf) {
683 		struct ip *ip = mtod(m, struct ip *);
684 
685 		/* BPF wants net byte order */
686 		ip->ip_len = htons(ip->ip_len + (ip->ip_hl << 2));
687 		ip->ip_off = htons(ip->ip_off);
688 		bpf_mtap(SC2IFP(sc)->if_bpf, m);
689 	}
690 
691 	/* verify the CARP version. */
692 	if (ch->carp_version != CARP_VERSION) {
693 		carpstats.carps_badver++;
694 		SC2IFP(sc)->if_ierrors++;
695 		CARP_UNLOCK(ifp->if_carp);
696 		CARP_LOG("%s; invalid version %d\n",
697 		    SC2IFP(sc)->if_xname,
698 		    ch->carp_version);
699 		m_freem(m);
700 		return;
701 	}
702 
703 	/* verify the hash */
704 	if (carp_hmac_verify(sc, ch->carp_counter, ch->carp_md)) {
705 		carpstats.carps_badauth++;
706 		SC2IFP(sc)->if_ierrors++;
707 		CARP_UNLOCK(ifp->if_carp);
708 		CARP_LOG("%s: incorrect hash\n", SC2IFP(sc)->if_xname);
709 		m_freem(m);
710 		return;
711 	}
712 
713 	tmp_counter = ntohl(ch->carp_counter[0]);
714 	tmp_counter = tmp_counter<<32;
715 	tmp_counter += ntohl(ch->carp_counter[1]);
716 
717 	/* XXX Replay protection goes here */
718 
719 	sc->sc_init_counter = 0;
720 	sc->sc_counter = tmp_counter;
721 
722 	sc_tv.tv_sec = sc->sc_advbase;
723 	if (carp_suppress_preempt && sc->sc_advskew <  240)
724 		sc_tv.tv_usec = 240 * 1000000 / 256;
725 	else
726 		sc_tv.tv_usec = sc->sc_advskew * 1000000 / 256;
727 	ch_tv.tv_sec = ch->carp_advbase;
728 	ch_tv.tv_usec = ch->carp_advskew * 1000000 / 256;
729 
730 	switch (sc->sc_state) {
731 	case INIT:
732 		break;
733 	case MASTER:
734 		/*
735 		 * If we receive an advertisement from a master who's going to
736 		 * be more frequent than us, go into BACKUP state.
737 		 */
738 		if (timevalcmp(&sc_tv, &ch_tv, >) ||
739 		    timevalcmp(&sc_tv, &ch_tv, ==)) {
740 			callout_stop(&sc->sc_ad_tmo);
741 			CARP_DEBUG("%s: MASTER -> BACKUP "
742 			   "(more frequent advertisement received)\n",
743 			   SC2IFP(sc)->if_xname);
744 			carp_set_state(sc, BACKUP);
745 			carp_setrun(sc, 0);
746 			carp_setroute(sc, RTM_DELETE);
747 		}
748 		break;
749 	case BACKUP:
750 		/*
751 		 * If we're pre-empting masters who advertise slower than us,
752 		 * and this one claims to be slower, treat him as down.
753 		 */
754 		if (carp_opts[CARPCTL_PREEMPT] &&
755 		    timevalcmp(&sc_tv, &ch_tv, <)) {
756 			CARP_DEBUG("%s: BACKUP -> MASTER "
757 			    "(preempting a slower master)\n",
758 			    SC2IFP(sc)->if_xname);
759 			carp_master_down_locked(sc);
760 			break;
761 		}
762 
763 		/*
764 		 *  If the master is going to advertise at such a low frequency
765 		 *  that he's guaranteed to time out, we'd might as well just
766 		 *  treat him as timed out now.
767 		 */
768 		sc_tv.tv_sec = sc->sc_advbase * 3;
769 		if (timevalcmp(&sc_tv, &ch_tv, <)) {
770 			CARP_DEBUG("%s: BACKUP -> MASTER "
771 			    "(master timed out)\n",
772 			    SC2IFP(sc)->if_xname);
773 			carp_master_down_locked(sc);
774 			break;
775 		}
776 
777 		/*
778 		 * Otherwise, we reset the counter and wait for the next
779 		 * advertisement.
780 		 */
781 		carp_setrun(sc, af);
782 		break;
783 	}
784 
785 	CARP_UNLOCK(ifp->if_carp);
786 
787 	m_freem(m);
788 	return;
789 }
790 
791 static int
792 carp_prepare_ad(struct mbuf *m, struct carp_softc *sc, struct carp_header *ch)
793 {
794 	struct m_tag *mtag;
795 	struct ifnet *ifp = SC2IFP(sc);
796 
797 	if (sc->sc_init_counter) {
798 		/* this could also be seconds since unix epoch */
799 		sc->sc_counter = karc4random();
800 		sc->sc_counter = sc->sc_counter << 32;
801 		sc->sc_counter += karc4random();
802 	} else
803 		sc->sc_counter++;
804 
805 	ch->carp_counter[0] = htonl((sc->sc_counter>>32)&0xffffffff);
806 	ch->carp_counter[1] = htonl(sc->sc_counter&0xffffffff);
807 
808 	carp_hmac_generate(sc, ch->carp_counter, ch->carp_md);
809 
810 	/* Tag packet for carp_output */
811 	mtag = m_tag_get(PACKET_TAG_CARP, sizeof(struct ifnet *), M_NOWAIT);
812 	if (mtag == NULL) {
813 		m_freem(m);
814 		SC2IFP(sc)->if_oerrors++;
815 		return (ENOMEM);
816 	}
817 	bcopy(&ifp, (caddr_t)(mtag + 1), sizeof(struct ifnet *));
818 	m_tag_prepend(m, mtag);
819 
820 	return (0);
821 }
822 
823 static void
824 carp_send_ad_all(void)
825 {
826 	struct carp_softc *sc;
827 
828 	LIST_FOREACH(sc, &carpif_list, sc_next) {
829 		if (sc->sc_carpdev == NULL)
830 			continue;
831 		CARP_SCLOCK(sc);
832 		if ((SC2IFP(sc)->if_flags & IFF_UP) && (SC2IFP(sc)->if_flags & IFF_RUNNING) &&
833 		     sc->sc_state == MASTER)
834 			carp_send_ad_locked(sc);
835 		CARP_SCUNLOCK(sc);
836 	}
837 }
838 
839 static void
840 carp_send_ad(void *v)
841 {
842 	struct carp_softc *sc = v;
843 
844 	CARP_SCLOCK(sc);
845 	carp_send_ad_locked(sc);
846 	CARP_SCUNLOCK(sc);
847 }
848 
849 static void
850 carp_send_ad_locked(struct carp_softc *sc)
851 {
852 	struct carp_header ch;
853 	struct timeval tv;
854 	struct carp_header *ch_ptr;
855 	struct mbuf *m;
856 	int len, advbase, advskew;
857 
858 
859 	/* bow out if we've lost our UPness or RUNNINGuiness */
860 	if (!((SC2IFP(sc)->if_flags & IFF_UP) && (SC2IFP(sc)->if_flags & IFF_RUNNING))) {
861 		advbase = 255;
862 		advskew = 255;
863 	} else {
864 		advbase = sc->sc_advbase;
865 		if (!carp_suppress_preempt || sc->sc_advskew > 240)
866 			advskew = sc->sc_advskew;
867 		else
868 			advskew = 240;
869 		tv.tv_sec = advbase;
870 		tv.tv_usec = advskew * 1000000 / 256;
871 	}
872 
873 	ch.carp_version = CARP_VERSION;
874 	ch.carp_type = CARP_ADVERTISEMENT;
875 	ch.carp_vhid = sc->sc_vhid;
876 	ch.carp_advbase = advbase;
877 	ch.carp_advskew = advskew;
878 	ch.carp_authlen = 7;	/* XXX DEFINE */
879 	ch.carp_pad1 = 0;	/* must be zero */
880 	ch.carp_cksum = 0;
881 
882 #ifdef INET
883 	if (sc->sc_ia) {
884 		struct ip *ip;
885 
886 		MGETHDR(m, M_NOWAIT, MT_HEADER);
887 		if (m == NULL) {
888 			SC2IFP(sc)->if_oerrors++;
889 			carpstats.carps_onomem++;
890 			/* XXX maybe less ? */
891 			if (advbase != 255 || advskew != 255)
892 				callout_reset(&sc->sc_ad_tmo, tvtohz_high(&tv),
893 				    carp_send_ad, sc);
894 			return;
895 		}
896 		len = sizeof(*ip) + sizeof(ch);
897 		m->m_pkthdr.len = len;
898 		m->m_pkthdr.rcvif = NULL;
899 		m->m_len = len;
900 		MH_ALIGN(m, m->m_len);
901 		m->m_flags |= M_MCAST;
902 		ip = mtod(m, struct ip *);
903 		ip->ip_v = IPVERSION;
904 		ip->ip_hl = sizeof(*ip) >> 2;
905 		ip->ip_tos = IPTOS_LOWDELAY;
906 		ip->ip_len = len;
907 		ip->ip_id = ip_newid();
908 		ip->ip_off = IP_DF;
909 		ip->ip_ttl = CARP_DFLTTL;
910 		ip->ip_p = IPPROTO_CARP;
911 		ip->ip_sum = 0;
912 		ip->ip_src.s_addr = sc->sc_ia->ia_addr.sin_addr.s_addr;
913 		ip->ip_dst.s_addr = htonl(INADDR_CARP_GROUP);
914 
915 		ch_ptr = (struct carp_header *)(&ip[1]);
916 		bcopy(&ch, ch_ptr, sizeof(ch));
917 		if (carp_prepare_ad(m, sc, ch_ptr))
918 			return;
919 
920 		m->m_data += sizeof(*ip);
921 		ch_ptr->carp_cksum = carp_cksum(m, len - sizeof(*ip));
922 		m->m_data -= sizeof(*ip);
923 
924 		getmicrotime(&SC2IFP(sc)->if_lastchange);
925 		SC2IFP(sc)->if_opackets++;
926 		SC2IFP(sc)->if_obytes += len;
927 		carpstats.carps_opackets++;
928 
929 		if (ip_output(m, NULL, NULL, IP_RAWOUTPUT, &sc->sc_imo, NULL)) {
930 			SC2IFP(sc)->if_oerrors++;
931 			if (sc->sc_sendad_errors < INT_MAX)
932 				sc->sc_sendad_errors++;
933 			if (sc->sc_sendad_errors == CARP_SENDAD_MAX_ERRORS) {
934 				carp_suppress_preempt++;
935 				if (carp_suppress_preempt == 1) {
936 					CARP_SCUNLOCK(sc);
937 					carp_send_ad_all();
938 					CARP_SCLOCK(sc);
939 				}
940 			}
941 			sc->sc_sendad_success = 0;
942 		} else {
943 			if (sc->sc_sendad_errors >= CARP_SENDAD_MAX_ERRORS) {
944 				if (++sc->sc_sendad_success >=
945 				    CARP_SENDAD_MIN_SUCCESS) {
946 					carp_suppress_preempt--;
947 					sc->sc_sendad_errors = 0;
948 				}
949 			} else
950 				sc->sc_sendad_errors = 0;
951 		}
952 	}
953 #endif /* INET */
954 #ifdef INET6
955 	if (sc->sc_ia6) {
956 		struct ip6_hdr *ip6;
957 
958 		MGETHDR(m, M_NOWAIT, MT_HEADER);
959 		if (m == NULL) {
960 			SC2IFP(sc)->if_oerrors++;
961 			carpstats.carps_onomem++;
962 			/* XXX maybe less ? */
963 			if (advbase != 255 || advskew != 255)
964 				callout_reset(&sc->sc_ad_tmo, tvtohz_high(&tv),
965 				    carp_send_ad, sc);
966 			return;
967 		}
968 		len = sizeof(*ip6) + sizeof(ch);
969 		m->m_pkthdr.len = len;
970 		m->m_pkthdr.rcvif = NULL;
971 		m->m_len = len;
972 		MH_ALIGN(m, m->m_len);
973 		m->m_flags |= M_MCAST;
974 		ip6 = mtod(m, struct ip6_hdr *);
975 		bzero(ip6, sizeof(*ip6));
976 		ip6->ip6_vfc |= IPV6_VERSION;
977 		ip6->ip6_hlim = CARP_DFLTTL;
978 		ip6->ip6_nxt = IPPROTO_CARP;
979 		bcopy(&sc->sc_ia6->ia_addr.sin6_addr, &ip6->ip6_src,
980 		    sizeof(struct in6_addr));
981 		/* set the multicast destination */
982 
983 		ip6->ip6_dst.s6_addr16[0] = htons(0xff02);
984 		ip6->ip6_dst.s6_addr8[15] = 0x12;
985 		if (in6_setscope(&ip6->ip6_dst, sc->sc_carpdev, NULL) != 0) {
986 			SC2IFP(sc)->if_oerrors++;
987 			m_freem(m);
988 			CARP_LOG("%s: in6_setscope failed\n", __func__);
989 			return;
990 		}
991 
992 		ch_ptr = (struct carp_header *)(&ip6[1]);
993 		bcopy(&ch, ch_ptr, sizeof(ch));
994 		if (carp_prepare_ad(m, sc, ch_ptr))
995 			return;
996 
997 		m->m_data += sizeof(*ip6);
998 		ch_ptr->carp_cksum = carp_cksum(m, len - sizeof(*ip6));
999 		m->m_data -= sizeof(*ip6);
1000 
1001 		getmicrotime(&SC2IFP(sc)->if_lastchange);
1002 		SC2IFP(sc)->if_opackets++;
1003 		SC2IFP(sc)->if_obytes += len;
1004 		carpstats.carps_opackets6++;
1005 
1006 		if (ip6_output(m, NULL, NULL, 0, &sc->sc_im6o, NULL, NULL)) {
1007 			SC2IFP(sc)->if_oerrors++;
1008 			if (sc->sc_sendad_errors < INT_MAX)
1009 				sc->sc_sendad_errors++;
1010 			if (sc->sc_sendad_errors == CARP_SENDAD_MAX_ERRORS) {
1011 				carp_suppress_preempt++;
1012 				if (carp_suppress_preempt == 1) {
1013 					CARP_SCUNLOCK(sc);
1014 					carp_send_ad_all();
1015 					CARP_SCLOCK(sc);
1016 				}
1017 			}
1018 			sc->sc_sendad_success = 0;
1019 		} else {
1020 			if (sc->sc_sendad_errors >= CARP_SENDAD_MAX_ERRORS) {
1021 				if (++sc->sc_sendad_success >=
1022 				    CARP_SENDAD_MIN_SUCCESS) {
1023 					carp_suppress_preempt--;
1024 					sc->sc_sendad_errors = 0;
1025 				}
1026 			} else
1027 				sc->sc_sendad_errors = 0;
1028 		}
1029 	}
1030 #endif /* INET6 */
1031 
1032 	if (advbase != 255 || advskew != 255)
1033 		callout_reset(&sc->sc_ad_tmo, tvtohz_high(&tv),
1034 		    carp_send_ad, sc);
1035 
1036 }
1037 
1038 /*
1039  * Broadcast a gratuitous ARP request containing
1040  * the virtual router MAC address for each IP address
1041  * associated with the virtual router.
1042  */
1043 static void
1044 carp_send_arp(struct carp_softc *sc)
1045 {
1046 	struct ifaddr *ifa;
1047 
1048 	TAILQ_FOREACH(ifa, &SC2IFP(sc)->if_addrlist, ifa_list) {
1049 
1050 		if (ifa->ifa_addr->sa_family != AF_INET)
1051 			continue;
1052 		lwkt_serialize_enter(sc->sc_carpdev->if_serializer);
1053 	        arp_ifinit2(sc->sc_carpdev, ifa, IF_LLADDR(sc->sc_ifp));
1054 		lwkt_serialize_exit(sc->sc_carpdev->if_serializer);
1055 
1056 		DELAY(1000);	/* XXX */
1057 	}
1058 }
1059 
1060 #ifdef INET6
1061 static void
1062 carp_send_na(struct carp_softc *sc)
1063 {
1064 	struct ifaddr *ifa;
1065 	struct in6_addr *in6;
1066 	static struct in6_addr mcast = IN6ADDR_LINKLOCAL_ALLNODES_INIT;
1067 
1068 	TAILQ_FOREACH(ifa, &SC2IFP(sc)->if_addrlist, ifa_list) {
1069 
1070 		if (ifa->ifa_addr->sa_family != AF_INET6)
1071 			continue;
1072 
1073 		in6 = &ifatoia6(ifa)->ia_addr.sin6_addr;
1074 		nd6_na_output(sc->sc_carpdev, &mcast, in6,
1075 		    ND_NA_FLAG_OVERRIDE, 1, NULL);
1076 		DELAY(1000);	/* XXX */
1077 	}
1078 }
1079 #endif /* INET6 */
1080 
1081 static int
1082 carp_addrcount(struct carp_if *cif, struct in_ifaddr *ia, int type)
1083 {
1084 	struct carp_softc *vh;
1085 	struct ifaddr *ifa;
1086 	int count = 0;
1087 
1088 	CARP_LOCK_ASSERT(cif);
1089 
1090 	TAILQ_FOREACH(vh, &cif->vhif_vrs, sc_list) {
1091 		if ((type == CARP_COUNT_RUNNING &&
1092 		    (SC2IFP(vh)->if_flags & IFF_UP) && (SC2IFP(vh)->if_flags & IFF_RUNNING)) ||
1093 		    (type == CARP_COUNT_MASTER && vh->sc_state == MASTER)) {
1094 			TAILQ_FOREACH(ifa, &SC2IFP(vh)->if_addrlist,
1095 			    ifa_list) {
1096 				if (ifa->ifa_addr->sa_family == AF_INET &&
1097 				    ia->ia_addr.sin_addr.s_addr ==
1098 				    ifatoia(ifa)->ia_addr.sin_addr.s_addr)
1099 					count++;
1100 			}
1101 		}
1102 	}
1103 	return (count);
1104 }
1105 
1106 int
1107 carp_iamatch(void *v, struct in_ifaddr *ia,
1108     struct in_addr *isaddr, u_int8_t **enaddr)
1109 {
1110 	struct carp_if *cif = v;
1111 	struct carp_softc *vh;
1112 	int index, count = 0;
1113 	struct ifaddr *ifa;
1114 
1115 	CARP_LOCK(cif);
1116 
1117 	if (carp_opts[CARPCTL_ARPBALANCE]) {
1118 		/*
1119 		 * XXX proof of concept implementation.
1120 		 * We use the source ip to decide which virtual host should
1121 		 * handle the request. If we're master of that virtual host,
1122 		 * then we respond, otherwise, just drop the arp packet on
1123 		 * the floor.
1124 		 */
1125 		count = carp_addrcount(cif, ia, CARP_COUNT_RUNNING);
1126 		if (count == 0) {
1127 			/* should never reach this */
1128 			CARP_UNLOCK(cif);
1129 			return (0);
1130 		}
1131 
1132 		/* this should be a hash, like pf_hash() */
1133 		index = ntohl(isaddr->s_addr) % count;
1134 		count = 0;
1135 
1136 		TAILQ_FOREACH(vh, &cif->vhif_vrs, sc_list) {
1137 			if ((SC2IFP(vh)->if_flags & IFF_UP) && (SC2IFP(vh)->if_flags & IFF_RUNNING)) {
1138 				TAILQ_FOREACH(ifa, &SC2IFP(vh)->if_addrlist,
1139 				    ifa_list) {
1140 					if (ifa->ifa_addr->sa_family ==
1141 					    AF_INET &&
1142 					    ia->ia_addr.sin_addr.s_addr ==
1143 					    ifatoia(ifa)->ia_addr.sin_addr.s_addr) {
1144 						if (count == index) {
1145 							if (vh->sc_state ==
1146 							    MASTER) {
1147 								*enaddr = IF_LLADDR(vh->sc_ifp);
1148 								CARP_UNLOCK(cif);
1149 								return (1);
1150 							} else {
1151 								CARP_UNLOCK(cif);
1152 								return (0);
1153 							}
1154 						}
1155 						count++;
1156 					}
1157 				}
1158 			}
1159 		}
1160 	} else {
1161 		TAILQ_FOREACH(vh, &cif->vhif_vrs, sc_list) {
1162 			if ((SC2IFP(vh)->if_flags & IFF_UP) && (SC2IFP(vh)->if_flags & IFF_RUNNING) &&
1163 			    vh->sc_state == MASTER) {
1164 				*enaddr = IF_LLADDR(vh->sc_ifp);
1165 				CARP_UNLOCK(cif);
1166 				return (1);
1167 			}
1168 		}
1169 	}
1170 	CARP_UNLOCK(cif);
1171 	return(0);
1172 }
1173 
1174 #ifdef INET6
1175 struct ifaddr *
1176 carp_iamatch6(void *v, struct in6_addr *taddr)
1177 {
1178 	struct carp_if *cif = v;
1179 	struct carp_softc *vh;
1180 	struct ifaddr *ifa;
1181 
1182 	CARP_LOCK(cif);
1183 	TAILQ_FOREACH(vh, &cif->vhif_vrs, sc_list) {
1184 		TAILQ_FOREACH(ifa, &SC2IFP(vh)->if_addrlist, ifa_list) {
1185 			if (IN6_ARE_ADDR_EQUAL(taddr,
1186 			    &ifatoia6(ifa)->ia_addr.sin6_addr) &&
1187  			    (SC2IFP(vh)->if_flags & IFF_UP) && (SC2IFP(vh)->if_flags & IFF_RUNNING) &&
1188 			    vh->sc_state == MASTER) {
1189 			    	CARP_UNLOCK(cif);
1190 				return (ifa);
1191 			}
1192 		}
1193 	}
1194 	CARP_UNLOCK(cif);
1195 
1196 	return (NULL);
1197 }
1198 
1199 void *
1200 carp_macmatch6(void *v, struct mbuf *m, const struct in6_addr *taddr)
1201 {
1202 	struct m_tag *mtag;
1203 	struct carp_if *cif = v;
1204 	struct carp_softc *sc;
1205 	struct ifaddr *ifa;
1206 
1207 	CARP_LOCK(cif);
1208 	TAILQ_FOREACH(sc, &cif->vhif_vrs, sc_list) {
1209 		TAILQ_FOREACH(ifa, &SC2IFP(sc)->if_addrlist, ifa_list) {
1210 			if (IN6_ARE_ADDR_EQUAL(taddr,
1211 			    &ifatoia6(ifa)->ia_addr.sin6_addr) &&
1212  			    (SC2IFP(sc)->if_flags & IFF_UP) && (SC2IFP(sc)->if_flags & IFF_RUNNING)) {
1213 				struct ifnet *ifp = SC2IFP(sc);
1214 				mtag = m_tag_get(PACKET_TAG_CARP,
1215 				    sizeof(struct ifnet *), M_NOWAIT);
1216 				if (mtag == NULL) {
1217 					/* better a bit than nothing */
1218 					CARP_UNLOCK(cif);
1219 					return (IF_LLADDR(sc->sc_ifp));
1220 				}
1221 				bcopy(&ifp, (caddr_t)(mtag + 1),
1222 				    sizeof(struct ifnet *));
1223 				m_tag_prepend(m, mtag);
1224 
1225 				CARP_UNLOCK(cif);
1226 				return (IF_LLADDR(sc->sc_ifp));
1227 			}
1228 		}
1229 	}
1230 	CARP_UNLOCK(cif);
1231 
1232 	return (NULL);
1233 }
1234 #endif
1235 
1236 struct ifnet *
1237 carp_forus(void *v, void *dhost)
1238 {
1239 	struct carp_if *cif = v;
1240 	struct carp_softc *vh;
1241 	u_int8_t *ena = dhost;
1242 
1243 	/**
1244          * XXX: See here for check on MAC adr is not for virtual use
1245          *
1246          **/
1247 
1248 	if (ena[0] || ena[1] || ena[2] != 0x5e || ena[3] || ena[4] != 1)
1249 	{
1250 		return (NULL);
1251 	}
1252 
1253 	CARP_LOCK(cif);
1254 	TAILQ_FOREACH(vh, &cif->vhif_vrs, sc_list)
1255 		if ((SC2IFP(vh)->if_flags & IFF_UP) && (SC2IFP(vh)->if_flags & IFF_RUNNING) &&
1256 		    vh->sc_state == MASTER &&
1257 		    !bcmp(dhost, IF_LLADDR(vh->sc_ifp), ETHER_ADDR_LEN)) {
1258 		    	CARP_UNLOCK(cif);
1259 			return (SC2IFP(vh));
1260 		}
1261 
1262     	CARP_UNLOCK(cif);
1263 	return (NULL);
1264 }
1265 
1266 static void
1267 carp_master_down(void *v)
1268 {
1269 	struct carp_softc *sc = v;
1270 
1271 	lwkt_serialize_enter(sc->sc_ifp->if_serializer);
1272 	carp_master_down_locked(sc);
1273 	lwkt_serialize_exit(sc->sc_ifp->if_serializer);
1274 }
1275 
1276 static void
1277 carp_master_down_locked(struct carp_softc *sc)
1278 {
1279 	if (sc->sc_carpdev)
1280 		CARP_SCLOCK_ASSERT(sc);
1281 
1282 	switch (sc->sc_state) {
1283 	case INIT:
1284 		kprintf("%s: master_down event in INIT state\n",
1285 		    SC2IFP(sc)->if_xname);
1286 		break;
1287 	case MASTER:
1288 		break;
1289 	case BACKUP:
1290 		carp_set_state(sc, MASTER);
1291 		carp_send_ad_locked(sc);
1292 		carp_send_arp(sc);
1293 #ifdef INET6
1294 		carp_send_na(sc);
1295 #endif /* INET6 */
1296 		carp_setrun(sc, 0);
1297 		carp_setroute(sc, RTM_ADD);
1298 		break;
1299 	}
1300 }
1301 
1302 /*
1303  * When in backup state, af indicates whether to reset the master down timer
1304  * for v4 or v6. If it's set to zero, reset the ones which are already pending.
1305  */
1306 static void
1307 carp_setrun(struct carp_softc *sc, sa_family_t af)
1308 {
1309 	struct timeval tv;
1310 
1311 	if (sc->sc_carpdev == NULL) {
1312 		SC2IFP(sc)->if_flags &= ~IFF_RUNNING;
1313 		carp_set_state(sc, INIT);
1314 		return;
1315 	}
1316 
1317 	if (SC2IFP(sc)->if_flags & IFF_UP &&
1318 	    sc->sc_vhid > 0 && (sc->sc_naddrs || sc->sc_naddrs6))
1319 		SC2IFP(sc)->if_flags |= IFF_RUNNING;
1320 	else {
1321 		SC2IFP(sc)->if_flags &= ~IFF_RUNNING;
1322 		carp_setroute(sc, RTM_DELETE);
1323 		return;
1324 	}
1325 
1326 	switch (sc->sc_state) {
1327 	case INIT:
1328 		if (carp_opts[CARPCTL_PREEMPT] && !carp_suppress_preempt) {
1329 			carp_send_ad_locked(sc);
1330 			carp_send_arp(sc);
1331 #ifdef INET6
1332 			carp_send_na(sc);
1333 #endif /* INET6 */
1334 			CARP_DEBUG("%s: INIT -> MASTER (preempting)\n",
1335 			    SC2IFP(sc)->if_xname);
1336 			carp_set_state(sc, MASTER);
1337 			carp_setroute(sc, RTM_ADD);
1338 		} else {
1339 			CARP_DEBUG("%s: INIT -> BACKUP\n", SC2IFP(sc)->if_xname);
1340 			carp_set_state(sc, BACKUP);
1341 			carp_setroute(sc, RTM_DELETE);
1342 			carp_setrun(sc, 0);
1343 		}
1344 		break;
1345 	case BACKUP:
1346 		callout_stop(&sc->sc_ad_tmo);
1347 		tv.tv_sec = 3 * sc->sc_advbase;
1348 		tv.tv_usec = sc->sc_advskew * 1000000 / 256;
1349 		switch (af) {
1350 #ifdef INET
1351 		case AF_INET:
1352 			callout_reset(&sc->sc_md_tmo, tvtohz_high(&tv),
1353 			    carp_master_down, sc);
1354 			break;
1355 #endif /* INET */
1356 #ifdef INET6
1357 		case AF_INET6:
1358 			callout_reset(&sc->sc_md6_tmo, tvtohz_high(&tv),
1359 			    carp_master_down, sc);
1360 			break;
1361 #endif /* INET6 */
1362 		default:
1363 			if (sc->sc_naddrs)
1364 				callout_reset(&sc->sc_md_tmo, tvtohz_high(&tv),
1365 				    carp_master_down, sc);
1366 			if (sc->sc_naddrs6)
1367 				callout_reset(&sc->sc_md6_tmo, tvtohz_high(&tv),
1368 				    carp_master_down, sc);
1369 			break;
1370 		}
1371 		break;
1372 	case MASTER:
1373 		tv.tv_sec = sc->sc_advbase;
1374 		tv.tv_usec = sc->sc_advskew * 1000000 / 256;
1375 		callout_reset(&sc->sc_ad_tmo, tvtohz_high(&tv),
1376 		    carp_send_ad, sc);
1377 		break;
1378 	}
1379 }
1380 
1381 static void
1382 carp_multicast_cleanup(struct carp_softc *sc)
1383 {
1384 	struct ip_moptions *imo = &sc->sc_imo;
1385 	u_int16_t n = imo->imo_num_memberships;
1386 
1387 	/* Clean up our own multicast memberships */
1388 	while (n-- > 0) {
1389 		if (imo->imo_membership[n] != NULL) {
1390 			in_delmulti(imo->imo_membership[n]);
1391 			imo->imo_membership[n] = NULL;
1392 		}
1393 	}
1394 	imo->imo_num_memberships = 0;
1395 	imo->imo_multicast_ifp = NULL;
1396 }
1397 
1398 #ifdef INET6
1399 static void
1400 carp_multicast6_cleanup(struct carp_softc *sc)
1401 {
1402 	struct ip6_moptions *im6o = &sc->sc_im6o;
1403 
1404 	while (!LIST_EMPTY(&im6o->im6o_memberships)) {
1405 		struct in6_multi_mship *imm =
1406 		    LIST_FIRST(&im6o->im6o_memberships);
1407 
1408 		LIST_REMOVE(imm, i6mm_chain);
1409 		in6_leavegroup(imm);
1410 	}
1411 	im6o->im6o_multicast_ifp = NULL;
1412 }
1413 #endif
1414 
1415 static int
1416 carp_set_addr(struct carp_softc *sc, struct sockaddr_in *sin)
1417 {
1418 	struct ifnet *ifp;
1419 	struct carp_if *cif;
1420 	struct in_ifaddr *ia, *ia_if;
1421 	struct ip_moptions *imo = &sc->sc_imo;
1422 	struct in_addr addr;
1423 	u_long iaddr = htonl(sin->sin_addr.s_addr);
1424 	int own, error;
1425 
1426 	if (sin->sin_addr.s_addr == 0)
1427 	{
1428 		if (!(SC2IFP(sc)->if_flags & IFF_UP))
1429 		{
1430 			carp_set_state(sc, INIT);
1431 		}
1432 		if (sc->sc_naddrs)
1433 		{
1434 			SC2IFP(sc)->if_flags |= IFF_UP;
1435 		}
1436 		carp_setrun(sc, 0);
1437 		return (0);
1438 	}
1439 	/* we have to do it by hands to check we won't match on us */
1440 	ia_if = NULL; own = 0;
1441 	TAILQ_FOREACH(ia, &in_ifaddrhead, ia_link) {
1442 		/* and, yeah, we need a multicast-capable iface too */
1443 		if (ia->ia_ifp != SC2IFP(sc) &&
1444 		    (ia->ia_ifp->if_flags & IFF_MULTICAST) &&
1445 		    (iaddr & ia->ia_subnetmask) == ia->ia_subnet) {
1446 			if (!ia_if)
1447 				ia_if = ia;
1448 			if (sin->sin_addr.s_addr ==
1449 			    ia->ia_addr.sin_addr.s_addr)
1450 				own++;
1451 		}
1452 	}
1453 
1454 
1455 	if (!ia_if)
1456 		return (EADDRNOTAVAIL);
1457 
1458 	ia = ia_if;
1459 	ifp = ia->ia_ifp;
1460 
1461 	if (ifp == NULL || (ifp->if_flags & IFF_MULTICAST) == 0 ||
1462 	    (imo->imo_multicast_ifp && imo->imo_multicast_ifp != ifp))
1463 		return (EADDRNOTAVAIL);
1464 
1465 	if (imo->imo_num_memberships == 0) {
1466 		addr.s_addr = htonl(INADDR_CARP_GROUP);
1467 		if ((imo->imo_membership[0] = in_addmulti(&addr, ifp)) == NULL)
1468 			return (ENOBUFS);
1469 		imo->imo_num_memberships++;
1470 		imo->imo_multicast_ifp = ifp;
1471 		imo->imo_multicast_ttl = CARP_DFLTTL;
1472 		imo->imo_multicast_loop = 0;
1473 	}
1474 
1475 	if (!ifp->if_carp) {
1476 
1477 		MALLOC(cif, struct carp_if *, sizeof(*cif), M_CARP,
1478 		    M_WAITOK|M_ZERO);
1479 		if (!cif) {
1480 			error = ENOBUFS;
1481 			goto cleanup;
1482 		}
1483 		if ((error = ifpromisc(ifp, 1))) {
1484 			FREE(cif, M_CARP);
1485 			goto cleanup;
1486 		}
1487 
1488 		CARP_LOCK_INIT(cif);
1489 		CARP_LOCK(cif);
1490 		cif->vhif_ifp = ifp;
1491 		TAILQ_INIT(&cif->vhif_vrs);
1492 		ifp->if_carp = cif;
1493 
1494 	} else {
1495 		struct carp_softc *vr;
1496 
1497 		cif = (struct carp_if *)ifp->if_carp;
1498 		CARP_LOCK(cif);
1499 		TAILQ_FOREACH(vr, &cif->vhif_vrs, sc_list)
1500 			if (vr != sc && vr->sc_vhid == sc->sc_vhid) {
1501 				CARP_UNLOCK(cif);
1502 				error = EINVAL;
1503 				goto cleanup;
1504 			}
1505 	}
1506 	sc->sc_ia = ia;
1507 	sc->sc_carpdev = ifp;
1508 
1509 	{ /* XXX prevent endless loop if already in queue */
1510 	struct carp_softc *vr, *after = NULL;
1511 	int myself = 0;
1512 	cif = (struct carp_if *)ifp->if_carp;
1513 
1514 	/* XXX: cif should not change, right? So we still hold the lock */
1515 	CARP_LOCK_ASSERT(cif);
1516 
1517 	TAILQ_FOREACH(vr, &cif->vhif_vrs, sc_list) {
1518 		if (vr == sc)
1519 			myself = 1;
1520 		if (vr->sc_vhid < sc->sc_vhid)
1521 			after = vr;
1522 	}
1523 
1524 	if (!myself) {
1525 		/* We're trying to keep things in order */
1526 		if (after == NULL) {
1527 			TAILQ_INSERT_TAIL(&cif->vhif_vrs, sc, sc_list);
1528 		} else {
1529 			TAILQ_INSERT_AFTER(&cif->vhif_vrs, after, sc, sc_list);
1530 		}
1531 		cif->vhif_nvrs++;
1532 	}
1533 	}
1534 
1535 	sc->sc_naddrs++;
1536 	SC2IFP(sc)->if_flags |= IFF_UP;
1537 	if (own)
1538 		sc->sc_advskew = 0;
1539 
1540 
1541 	carp_sc_state_locked(sc);
1542 	carp_setrun(sc, 0);
1543 
1544 	CARP_UNLOCK(cif);
1545 
1546 	return (0);
1547 
1548 cleanup:
1549 	in_delmulti(imo->imo_membership[--imo->imo_num_memberships]);
1550 	return (error);
1551 
1552 }
1553 
1554 static int
1555 carp_del_addr(struct carp_softc *sc, struct sockaddr_in *sin)
1556 {
1557 	int error = 0;
1558 
1559 	if (!--sc->sc_naddrs) {
1560 		struct carp_if *cif = (struct carp_if *)sc->sc_carpdev->if_carp;
1561 		struct ip_moptions *imo = &sc->sc_imo;
1562 
1563 		CARP_LOCK(cif);
1564 		callout_stop(&sc->sc_ad_tmo);
1565 		SC2IFP(sc)->if_flags &= ~IFF_UP;
1566 		SC2IFP(sc)->if_flags &= ~IFF_RUNNING;
1567 		sc->sc_vhid = -1;
1568 		in_delmulti(imo->imo_membership[--imo->imo_num_memberships]);
1569 		imo->imo_multicast_ifp = NULL;
1570 		TAILQ_REMOVE(&cif->vhif_vrs, sc, sc_list);
1571 		if (!--cif->vhif_nvrs) {
1572 			sc->sc_carpdev->if_carp = NULL;
1573 			CARP_LOCK_DESTROY(cif);
1574 			FREE(cif, M_IFADDR);
1575 		} else {
1576 			CARP_UNLOCK(cif);
1577 		}
1578 	}
1579 
1580 	return (error);
1581 }
1582 
1583 #ifdef INET6
1584 static int
1585 carp_set_addr6(struct carp_softc *sc, struct sockaddr_in6 *sin6)
1586 {
1587 	struct ifnet *ifp;
1588 	struct carp_if *cif;
1589 	struct in6_ifaddr *ia, *ia_if;
1590 	struct ip6_moptions *im6o = &sc->sc_im6o;
1591 	struct in6_multi_mship *imm;
1592 	struct in6_addr in6;
1593 	int own, error;
1594 
1595 	if (IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) {
1596 		if (!(SC2IFP(sc)->if_flags & IFF_UP))
1597 			carp_set_state(sc, INIT);
1598 		if (sc->sc_naddrs6)
1599 			SC2IFP(sc)->if_flags |= IFF_UP;
1600 		carp_setrun(sc, 0);
1601 		return (0);
1602 	}
1603 
1604 	/* we have to do it by hands to check we won't match on us */
1605 	ia_if = NULL; own = 0;
1606 	for (ia = in6_ifaddr; ia; ia = ia->ia_next) {
1607 		int i;
1608 
1609 		for (i = 0; i < 4; i++) {
1610 			if ((sin6->sin6_addr.s6_addr32[i] &
1611 			    ia->ia_prefixmask.sin6_addr.s6_addr32[i]) !=
1612 			    (ia->ia_addr.sin6_addr.s6_addr32[i] &
1613 			    ia->ia_prefixmask.sin6_addr.s6_addr32[i]))
1614 				break;
1615 		}
1616 		/* and, yeah, we need a multicast-capable iface too */
1617 		if (ia->ia_ifp != SC2IFP(sc) &&
1618 		    (ia->ia_ifp->if_flags & IFF_MULTICAST) &&
1619 		    (i == 4)) {
1620 			if (!ia_if)
1621 				ia_if = ia;
1622 			if (IN6_ARE_ADDR_EQUAL(&sin6->sin6_addr,
1623 			    &ia->ia_addr.sin6_addr))
1624 				own++;
1625 		}
1626 	}
1627 
1628 	if (!ia_if)
1629 		return (EADDRNOTAVAIL);
1630 	ia = ia_if;
1631 	ifp = ia->ia_ifp;
1632 
1633 	if (ifp == NULL || (ifp->if_flags & IFF_MULTICAST) == 0 ||
1634 	    (im6o->im6o_multicast_ifp && im6o->im6o_multicast_ifp != ifp))
1635 		return (EADDRNOTAVAIL);
1636 
1637 	if (!sc->sc_naddrs6) {
1638 		im6o->im6o_multicast_ifp = ifp;
1639 
1640 		/* join CARP multicast address */
1641 		bzero(&in6, sizeof(in6));
1642 		in6.s6_addr16[0] = htons(0xff02);
1643 		in6.s6_addr8[15] = 0x12;
1644 		if (in6_setscope(&in6, ifp, NULL) != 0)
1645 			goto cleanup;
1646 		if ((imm = in6_joingroup(ifp, &in6, &error)) == NULL)
1647 			goto cleanup;
1648 		LIST_INSERT_HEAD(&im6o->im6o_memberships, imm, i6mm_chain);
1649 
1650 		/* join solicited multicast address */
1651 		bzero(&in6, sizeof(in6));
1652 		in6.s6_addr16[0] = htons(0xff02);
1653 		in6.s6_addr32[1] = 0;
1654 		in6.s6_addr32[2] = htonl(1);
1655 		in6.s6_addr32[3] = sin6->sin6_addr.s6_addr32[3];
1656 		in6.s6_addr8[12] = 0xff;
1657 		if (in6_setscope(&in6, ifp, NULL) != 0)
1658 			goto cleanup;
1659 		if ((imm = in6_joingroup(ifp, &in6, &error)) == NULL)
1660 			goto cleanup;
1661 		LIST_INSERT_HEAD(&im6o->im6o_memberships, imm, i6mm_chain);
1662 	}
1663 
1664 	if (!ifp->if_carp) {
1665 		MALLOC(cif, struct carp_if *, sizeof(*cif), M_CARP,
1666 		    M_WAITOK|M_ZERO);
1667 		if (!cif) {
1668 			error = ENOBUFS;
1669 			goto cleanup;
1670 		}
1671 		if ((error = ifpromisc(ifp, 1))) {
1672 			FREE(cif, M_CARP);
1673 			goto cleanup;
1674 		}
1675 
1676 		CARP_LOCK_INIT(cif);
1677 		CARP_LOCK(cif);
1678 		cif->vhif_ifp = ifp;
1679 		TAILQ_INIT(&cif->vhif_vrs);
1680 		ifp->if_carp = cif;
1681 
1682 	} else {
1683 		struct carp_softc *vr;
1684 
1685 		cif = (struct carp_if *)ifp->if_carp;
1686 		CARP_LOCK(cif);
1687 		TAILQ_FOREACH(vr, &cif->vhif_vrs, sc_list)
1688 			if (vr != sc && vr->sc_vhid == sc->sc_vhid) {
1689 				CARP_UNLOCK(cif);
1690 				error = EINVAL;
1691 				goto cleanup;
1692 			}
1693 	}
1694 	sc->sc_ia6 = ia;
1695 	sc->sc_carpdev = ifp;
1696 
1697 	{ /* XXX prevent endless loop if already in queue */
1698 	struct carp_softc *vr, *after = NULL;
1699 	int myself = 0;
1700 	cif = (struct carp_if *)ifp->if_carp;
1701 	CARP_LOCK_ASSERT(cif);
1702 
1703 	TAILQ_FOREACH(vr, &cif->vhif_vrs, sc_list) {
1704 		if (vr == sc)
1705 			myself = 1;
1706 		if (vr->sc_vhid < sc->sc_vhid)
1707 			after = vr;
1708 	}
1709 
1710 	if (!myself) {
1711 		/* We're trying to keep things in order */
1712 		if (after == NULL) {
1713 			TAILQ_INSERT_TAIL(&cif->vhif_vrs, sc, sc_list);
1714 		} else {
1715 			TAILQ_INSERT_AFTER(&cif->vhif_vrs, after, sc, sc_list);
1716 		}
1717 		cif->vhif_nvrs++;
1718 	}
1719 	}
1720 
1721 	sc->sc_naddrs6++;
1722 	SC2IFP(sc)->if_flags |= IFF_UP;
1723 	if (own)
1724 		sc->sc_advskew = 0;
1725 	carp_sc_state_locked(sc);
1726 	carp_setrun(sc, 0);
1727 
1728 	CARP_UNLOCK(cif);
1729 
1730 	return (0);
1731 
1732 cleanup:
1733 	/* clean up multicast memberships */
1734 	if (!sc->sc_naddrs6) {
1735 		while (!LIST_EMPTY(&im6o->im6o_memberships)) {
1736 			imm = LIST_FIRST(&im6o->im6o_memberships);
1737 			LIST_REMOVE(imm, i6mm_chain);
1738 			in6_leavegroup(imm);
1739 		}
1740 	}
1741 	return (error);
1742 }
1743 
1744 static int
1745 carp_del_addr6(struct carp_softc *sc, struct sockaddr_in6 *sin6)
1746 {
1747 	int error = 0;
1748 
1749 	if (!--sc->sc_naddrs6) {
1750 		struct carp_if *cif = (struct carp_if *)sc->sc_carpdev->if_carp;
1751 		struct ip6_moptions *im6o = &sc->sc_im6o;
1752 
1753 		CARP_LOCK(cif);
1754 		callout_stop(&sc->sc_ad_tmo);
1755 		SC2IFP(sc)->if_flags &= ~IFF_UP;
1756 		SC2IFP(sc)->if_flags &= ~IFF_RUNNING;
1757 		sc->sc_vhid = -1;
1758 		while (!LIST_EMPTY(&im6o->im6o_memberships)) {
1759 			struct in6_multi_mship *imm =
1760 			    LIST_FIRST(&im6o->im6o_memberships);
1761 
1762 			LIST_REMOVE(imm, i6mm_chain);
1763 			in6_leavegroup(imm);
1764 		}
1765 		im6o->im6o_multicast_ifp = NULL;
1766 		TAILQ_REMOVE(&cif->vhif_vrs, sc, sc_list);
1767 		if (!--cif->vhif_nvrs) {
1768 			CARP_LOCK_DESTROY(cif);
1769 			sc->sc_carpdev->if_carp = NULL;
1770 			FREE(cif, M_IFADDR);
1771 		} else
1772 			CARP_UNLOCK(cif);
1773 	}
1774 
1775 	return (error);
1776 }
1777 #endif /* INET6 */
1778 
1779 static int
1780 carp_ioctl(struct ifnet *ifp, u_long cmd, caddr_t addr, struct ucred *creds)
1781 {
1782 	struct carp_softc *sc = ifp->if_softc, *vr;
1783 	struct carpreq carpr;
1784 	struct ifaddr *ifa;
1785 	struct ifreq *ifr;
1786 	struct ifaliasreq *ifra;
1787 	int locked = 0, error = 0;
1788 
1789 	ifa = (struct ifaddr *)addr;
1790 	ifra = (struct ifaliasreq *)addr;
1791 	ifr = (struct ifreq *)addr;
1792 
1793 
1794 	switch (cmd) {
1795 	case SIOCSIFADDR:
1796 		switch (ifa->ifa_addr->sa_family) {
1797 #ifdef INET
1798 		case AF_INET:
1799 			SC2IFP(sc)->if_flags |= IFF_UP;
1800 			bcopy(ifa->ifa_addr, ifa->ifa_dstaddr,
1801 			    sizeof(struct sockaddr));
1802 			error = carp_set_addr(sc, satosin(ifa->ifa_addr));
1803 			break;
1804 #endif /* INET */
1805 #ifdef INET6
1806 		case AF_INET6:
1807 			SC2IFP(sc)->if_flags |= IFF_UP;
1808 			error = carp_set_addr6(sc, satosin6(ifa->ifa_addr));
1809 			break;
1810 #endif /* INET6 */
1811 		default:
1812 			error = EAFNOSUPPORT;
1813 			break;
1814 		}
1815 		break;
1816 
1817 	case SIOCAIFADDR:
1818 		switch (ifa->ifa_addr->sa_family) {
1819 #ifdef INET
1820 		case AF_INET:
1821 			SC2IFP(sc)->if_flags |= IFF_UP;
1822 			bcopy(ifa->ifa_addr, ifa->ifa_dstaddr,
1823 			    sizeof(struct sockaddr));
1824 			error = carp_set_addr(sc, satosin(&ifra->ifra_addr));
1825 			break;
1826 #endif /* INET */
1827 #ifdef INET6
1828 		case AF_INET6:
1829 			SC2IFP(sc)->if_flags |= IFF_UP;
1830 			error = carp_set_addr6(sc, satosin6(&ifra->ifra_addr));
1831 			break;
1832 #endif /* INET6 */
1833 		default:
1834 			error = EAFNOSUPPORT;
1835 			break;
1836 		}
1837 		break;
1838 
1839 	case SIOCDIFADDR:
1840 		switch (ifa->ifa_addr->sa_family) {
1841 #ifdef INET
1842 		case AF_INET:
1843 			error = carp_del_addr(sc, satosin(&ifra->ifra_addr));
1844 			break;
1845 #endif /* INET */
1846 #ifdef INET6
1847 		case AF_INET6:
1848 			error = carp_del_addr6(sc, satosin6(&ifra->ifra_addr));
1849 			break;
1850 #endif /* INET6 */
1851 		default:
1852 			error = EAFNOSUPPORT;
1853 			break;
1854 		}
1855 		break;
1856 
1857 	case SIOCSIFFLAGS:
1858 		if (sc->sc_carpdev) {
1859 			locked = 1;
1860 			CARP_SCLOCK(sc);
1861 		}
1862 		if (sc->sc_state != INIT && !(ifr->ifr_flags & IFF_UP)) {
1863  			callout_stop(&sc->sc_ad_tmo);
1864  			callout_stop(&sc->sc_md_tmo);
1865  			callout_stop(&sc->sc_md6_tmo);
1866 			if (sc->sc_state == MASTER)
1867 				carp_send_ad_locked(sc);
1868 			carp_set_state(sc, INIT);
1869 			carp_setrun(sc, 0);
1870 		} else if (sc->sc_state == INIT && (ifr->ifr_flags & IFF_UP)) {
1871 			SC2IFP(sc)->if_flags |= IFF_UP;
1872 			carp_setrun(sc, 0);
1873 		}
1874 		break;
1875 
1876 	case SIOCSVH:
1877 		error = suser(curthread);
1878 		if (error)
1879 			break;
1880 		if ((error = copyin(ifr->ifr_data, &carpr, sizeof carpr)))
1881 			break;
1882 		error = 1;
1883 		if (sc->sc_carpdev) {
1884 			locked = 1;
1885 			CARP_SCLOCK(sc);
1886 		}
1887 		if (sc->sc_state != INIT && carpr.carpr_state != sc->sc_state) {
1888 			switch (carpr.carpr_state) {
1889 			case BACKUP:
1890 				callout_stop(&sc->sc_ad_tmo);
1891 				carp_set_state(sc, BACKUP);
1892 				carp_setrun(sc, 0);
1893 				carp_setroute(sc, RTM_DELETE);
1894 				break;
1895 			case MASTER:
1896 				carp_master_down_locked(sc);
1897 				break;
1898 			default:
1899 				break;
1900 			}
1901 		}
1902 		if (carpr.carpr_vhid > 0) {
1903 			if (carpr.carpr_vhid > 255) {
1904 				error = EINVAL;
1905 				break;
1906 			}
1907 			if (sc->sc_carpdev) {
1908 				struct carp_if *cif;
1909 				cif = (struct carp_if *)sc->sc_carpdev->if_carp;
1910 				TAILQ_FOREACH(vr, &cif->vhif_vrs, sc_list)
1911 					if (vr != sc &&
1912 					    vr->sc_vhid == carpr.carpr_vhid)
1913 						return EEXIST;
1914 			}
1915 			sc->sc_vhid = carpr.carpr_vhid;
1916 			IF_LLADDR(sc->sc_ifp)[0] = 0;
1917 			IF_LLADDR(sc->sc_ifp)[1] = 0;
1918 			IF_LLADDR(sc->sc_ifp)[2] = 0x5e;
1919 			IF_LLADDR(sc->sc_ifp)[3] = 0;
1920 			IF_LLADDR(sc->sc_ifp)[4] = 1;
1921 			IF_LLADDR(sc->sc_ifp)[5] = sc->sc_vhid;
1922 			error--;
1923 		}
1924 		if (carpr.carpr_advbase > 0 || carpr.carpr_advskew > 0) {
1925 			if (carpr.carpr_advskew >= 255) {
1926 				error = EINVAL;
1927 				break;
1928 			}
1929 			if (carpr.carpr_advbase > 255) {
1930 				error = EINVAL;
1931 				break;
1932 			}
1933 			sc->sc_advbase = carpr.carpr_advbase;
1934 			sc->sc_advskew = carpr.carpr_advskew;
1935 			error--;
1936 		}
1937 		bcopy(carpr.carpr_key, sc->sc_key, sizeof(sc->sc_key));
1938 		if (error > 0)
1939 			error = EINVAL;
1940 		else {
1941 			error = 0;
1942 			carp_setrun(sc, 0);
1943 		}
1944 		break;
1945 
1946 	case SIOCGVH:
1947 		/* XXX: lockless read */
1948 		bzero(&carpr, sizeof(carpr));
1949 		carpr.carpr_state = sc->sc_state;
1950 		carpr.carpr_vhid = sc->sc_vhid;
1951 		carpr.carpr_advbase = sc->sc_advbase;
1952 		carpr.carpr_advskew = sc->sc_advskew;
1953 		error = suser(curthread);
1954 		if (error == 0)
1955 			bcopy(sc->sc_key, carpr.carpr_key,
1956 			    sizeof(carpr.carpr_key));
1957 		error = copyout(&carpr, ifr->ifr_data, sizeof(carpr));
1958 		break;
1959 
1960 	default:
1961 		error = EINVAL;
1962 	}
1963 
1964 	if (locked)
1965 		CARP_SCUNLOCK(sc);
1966 
1967 	carp_hmac_prepare(sc);
1968 
1969 	return (error);
1970 }
1971 
1972 /*
1973  * XXX: this is looutput. We should eventually use it from there.
1974  */
1975 static int
1976 carp_looutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
1977     struct rtentry *rt)
1978 {
1979 	u_int32_t af;
1980 
1981 	M_ASSERTPKTHDR(m); /* check if we have the packet header */
1982 
1983 	if (rt && rt->rt_flags & (RTF_REJECT|RTF_BLACKHOLE)) {
1984 		m_freem(m);
1985 		return (rt->rt_flags & RTF_BLACKHOLE ? 0 :
1986 			rt->rt_flags & RTF_HOST ? EHOSTUNREACH : ENETUNREACH);
1987 	}
1988 
1989 	ifp->if_opackets++;
1990 	ifp->if_obytes += m->m_pkthdr.len;
1991 
1992 	/* BPF writes need to be handled specially. */
1993 	if (dst->sa_family == AF_UNSPEC) {
1994 		bcopy(dst->sa_data, &af, sizeof(af));
1995 		dst->sa_family = af;
1996 	}
1997 
1998 #if 1	/* XXX */
1999 	switch (dst->sa_family) {
2000 	case AF_INET:
2001 	case AF_INET6:
2002 	case AF_IPX:
2003 	case AF_APPLETALK:
2004 		break;
2005 	default:
2006 		m_freem(m);
2007 		return (EAFNOSUPPORT);
2008 	}
2009 #endif
2010 	return(if_simloop(ifp, m, dst->sa_family, 0));
2011 }
2012 
2013 /*
2014  * Start output on carp interface. This function should never be called.
2015  */
2016 static void
2017 carp_start(struct ifnet *ifp)
2018 {
2019 #ifdef DEBUG
2020 	kprintf("%s: start called\n", ifp->if_xname);
2021 #endif
2022 }
2023 
2024 int
2025 carp_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
2026     struct rtentry *rt)
2027 {
2028 	struct m_tag *mtag;
2029 	struct carp_softc *sc;
2030 	struct ifnet *carp_ifp;
2031 
2032 	if (!sa)
2033 		return (0);
2034 
2035 	switch (sa->sa_family) {
2036 #ifdef INET
2037 	case AF_INET:
2038 		break;
2039 #endif /* INET */
2040 #ifdef INET6
2041 	case AF_INET6:
2042 		break;
2043 #endif /* INET6 */
2044 	default:
2045 		return (0);
2046 	}
2047 
2048 	mtag = m_tag_find(m, PACKET_TAG_CARP, NULL);
2049 	if (mtag == NULL)
2050 		return (0);
2051 
2052 	bcopy(mtag + 1, &carp_ifp, sizeof(struct ifnet *));
2053 	sc = carp_ifp->if_softc;
2054 
2055 	/* Set the source MAC address to Virtual Router MAC Address */
2056 	switch (ifp->if_type) {
2057 	case IFT_ETHER:
2058 	case IFT_L2VLAN: {
2059 			struct ether_header *eh;
2060 
2061 			eh = mtod(m, struct ether_header *);
2062 			eh->ether_shost[0] = 0;
2063 			eh->ether_shost[1] = 0;
2064 			eh->ether_shost[2] = 0x5e;
2065 			eh->ether_shost[3] = 0;
2066 			eh->ether_shost[4] = 1;
2067 			eh->ether_shost[5] = sc->sc_vhid;
2068 		}
2069 		break;
2070 	default:
2071 		kprintf("%s: carp is not supported for this interface type\n",
2072 		    ifp->if_xname);
2073 		return (EOPNOTSUPP);
2074 	}
2075 
2076 	return (0);
2077 
2078 }
2079 
2080 static void
2081 carp_set_state(struct carp_softc *sc, int state)
2082 {
2083 
2084 	if (sc->sc_carpdev)
2085 		CARP_SCLOCK_ASSERT(sc);
2086 
2087 	if (sc->sc_state == state)
2088 		return;
2089 
2090 	sc->sc_state = state;
2091 	switch (state) {
2092 	case BACKUP:
2093 		SC2IFP(sc)->if_link_state = LINK_STATE_DOWN;
2094 		break;
2095 	case MASTER:
2096 		SC2IFP(sc)->if_link_state = LINK_STATE_UP;
2097 		break;
2098 	default:
2099 		SC2IFP(sc)->if_link_state = LINK_STATE_UNKNOWN;
2100 		break;
2101 	}
2102 	rt_ifmsg(SC2IFP(sc));
2103 }
2104 
2105 void
2106 carp_carpdev_state(void *v)
2107 {
2108 	struct carp_if *cif = v;
2109 
2110 	CARP_LOCK(cif);
2111 	carp_carpdev_state_locked(cif);
2112 	CARP_UNLOCK(cif);
2113 }
2114 
2115 static void
2116 carp_carpdev_state_locked(struct carp_if *cif)
2117 {
2118 	struct carp_softc *sc;
2119 
2120 	TAILQ_FOREACH(sc, &cif->vhif_vrs, sc_list)
2121 		carp_sc_state_locked(sc);
2122 }
2123 
2124 static void
2125 carp_sc_state_locked(struct carp_softc *sc)
2126 {
2127 	CARP_SCLOCK_ASSERT(sc);
2128 
2129 	if ( !(sc->sc_carpdev->if_flags & IFF_UP)) {
2130 		sc->sc_flags_backup = SC2IFP(sc)->if_flags;
2131 		SC2IFP(sc)->if_flags &= ~IFF_UP;
2132 		SC2IFP(sc)->if_flags &= ~IFF_RUNNING;
2133 		callout_stop(&sc->sc_ad_tmo);
2134 		callout_stop(&sc->sc_md_tmo);
2135 		callout_stop(&sc->sc_md6_tmo);
2136 		carp_set_state(sc, INIT);
2137 		carp_setrun(sc, 0);
2138 		if (!sc->sc_suppress) {
2139 			carp_suppress_preempt++;
2140 			if (carp_suppress_preempt == 1) {
2141 				CARP_SCUNLOCK(sc);
2142 				carp_send_ad_all();
2143 				CARP_SCLOCK(sc);
2144 			}
2145 		}
2146 		sc->sc_suppress = 1;
2147 	} else {
2148 		SC2IFP(sc)->if_flags |= sc->sc_flags_backup;
2149 		carp_set_state(sc, INIT);
2150 		carp_setrun(sc, 0);
2151 		if (sc->sc_suppress)
2152 			carp_suppress_preempt--;
2153 		sc->sc_suppress = 0;
2154 	}
2155 
2156 	return;
2157 }
2158 
2159 static int
2160 carp_modevent(module_t mod, int type, void *data)
2161 {
2162 	switch (type) {
2163 	case MOD_LOAD:
2164 		if_detach_event_tag = EVENTHANDLER_REGISTER(ifnet_departure_event,
2165 		    carp_ifdetach, NULL, EVENTHANDLER_PRI_ANY);
2166 		if (if_detach_event_tag == NULL)
2167 			return (ENOMEM);
2168 
2169 		LIST_INIT(&carpif_list);
2170 		if_clone_attach(&carp_cloner);
2171 		break;
2172 
2173 	case MOD_UNLOAD:
2174 		EVENTHANDLER_DEREGISTER(ifnet_departure_event, if_detach_event_tag);
2175 		if_clone_detach(&carp_cloner);
2176 		break;
2177 
2178 	default:
2179 		return (EINVAL);
2180 	}
2181 
2182 	return (0);
2183 }
2184 
2185 static moduledata_t carp_mod = {
2186 	"carp",
2187 	carp_modevent,
2188 	0
2189 };
2190 
2191 DECLARE_MODULE(carp, carp_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
2192