xref: /netbsd/sys/net/if_mpls.c (revision d10e2ccb)
1 /*	$NetBSD: if_mpls.c,v 1.40 2022/09/03 02:47:59 thorpej Exp $ */
2 
3 /*
4  * Copyright (c) 2010 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Mihai Chelaru <kefren@NetBSD.org>
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 #include <sys/cdefs.h>
33 __KERNEL_RCSID(0, "$NetBSD: if_mpls.c,v 1.40 2022/09/03 02:47:59 thorpej Exp $");
34 
35 #ifdef _KERNEL_OPT
36 #include "opt_inet.h"
37 #include "opt_mpls.h"
38 #endif
39 
40 #include <sys/param.h>
41 
42 #include <sys/errno.h>
43 #include <sys/malloc.h>
44 #include <sys/mbuf.h>
45 #include <sys/sysctl.h>
46 
47 #include <net/bpf.h>
48 #include <net/if.h>
49 #include <net/if_types.h>
50 #include <net/route.h>
51 #include <sys/device.h>
52 #include <sys/module.h>
53 #include <sys/atomic.h>
54 
55 #ifdef INET
56 #include <netinet/in.h>
57 #include <netinet/in_systm.h>
58 #include <netinet/in_var.h>
59 #include <netinet/ip.h>
60 #include <netinet/ip_var.h>
61 #endif
62 
63 #ifdef INET6
64 #include <netinet/ip6.h>
65 #include <netinet6/in6_var.h>
66 #include <netinet6/ip6_var.h>
67 #endif
68 
69 #include <netmpls/mpls.h>
70 #include <netmpls/mpls_var.h>
71 
72 #include "if_mpls.h"
73 
74 #include "ioconf.h"
75 
76 static int mpls_clone_create(struct if_clone *, int);
77 static int mpls_clone_destroy(struct ifnet *);
78 
79 static struct if_clone mpls_if_cloner =
80 	IF_CLONE_INITIALIZER("mpls", mpls_clone_create, mpls_clone_destroy);
81 
82 static void mpls_input(struct ifnet *, struct mbuf *);
83 static int mpls_output(struct ifnet *, struct mbuf *, const struct sockaddr *,
84     const struct rtentry *);
85 static int mpls_ioctl(struct ifnet *, u_long, void *);
86 static int mpls_send_frame(struct mbuf *, struct ifnet *,
87     const struct rtentry *);
88 static int mpls_lse(struct mbuf *);
89 
90 #ifdef INET
91 static struct mbuf *mpls_unlabel_inet(struct mbuf *, int *error);
92 static struct mbuf *mpls_label_inet(struct mbuf *, union mpls_shim *, uint);
93 #endif
94 
95 #ifdef INET6
96 static struct mbuf *mpls_unlabel_inet6(struct mbuf *, int *error);
97 static struct mbuf *mpls_label_inet6(struct mbuf *, union mpls_shim *, uint);
98 #endif
99 
100 static struct mbuf *mpls_prepend_shim(struct mbuf *, union mpls_shim *);
101 
102 extern int mpls_defttl, mpls_mapttl_inet, mpls_mapttl_inet6, mpls_icmp_respond,
103     mpls_forwarding, mpls_frame_accept, mpls_mapprec_inet, mpls_mapclass_inet6,
104     mpls_rfc4182;
105 
106 static u_int mpls_count;
107 /* ARGSUSED */
108 void
109 mplsattach(int count)
110 {
111 	/*
112 	 * Nothing to do here, initialization is handled by the
113 	 * module initialization code in mplsinit() below).
114 	 */
115 }
116 
117 static void
118 mplsinit(void)
119 {
120 	if_clone_attach(&mpls_if_cloner);
121 }
122 
123 static int
124 mplsdetach(void)
125 {
126 	int error = 0;
127 
128 	if (mpls_count != 0)
129 		error = EBUSY;
130 
131 	if (error == 0)
132 		if_clone_detach(&mpls_if_cloner);
133 
134 	return error;
135 }
136 
137 static int
138 mpls_clone_create(struct if_clone *ifc, int unit)
139 {
140 	struct mpls_softc *sc;
141 
142 	atomic_inc_uint(&mpls_count);
143 	sc = malloc(sizeof(*sc), M_DEVBUF, M_WAITOK | M_ZERO);
144 
145 	if_initname(&sc->sc_if, ifc->ifc_name, unit);
146 	sc->sc_if.if_softc = sc;
147 	sc->sc_if.if_type = IFT_MPLS;
148 	sc->sc_if.if_addrlen = 0;
149 	sc->sc_if.if_hdrlen = sizeof(union mpls_shim);
150 	sc->sc_if.if_dlt = DLT_NULL;
151 	sc->sc_if.if_mtu = 1500;
152 	sc->sc_if.if_flags = 0;
153 	sc->sc_if._if_input = mpls_input;
154 	sc->sc_if.if_output = mpls_output;
155 	sc->sc_if.if_ioctl = mpls_ioctl;
156 
157 	if_attach(&sc->sc_if);
158 	if_alloc_sadl(&sc->sc_if);
159 	bpf_attach(&sc->sc_if, DLT_NULL, sizeof(uint32_t));
160 	return 0;
161 }
162 
163 static int
164 mpls_clone_destroy(struct ifnet *ifp)
165 {
166 
167 	bpf_detach(ifp);
168 	if_detach(ifp);
169 
170 	free(ifp->if_softc, M_DEVBUF);
171 	atomic_dec_uint(&mpls_count);
172 	return 0;
173 }
174 
175 static void
176 mpls_input(struct ifnet *ifp, struct mbuf *m)
177 {
178 #if 0
179 	/*
180 	 * TODO - kefren
181 	 * I'd love to unshim the packet, guess family
182 	 * and pass it to bpf
183 	 */
184 	bpf_mtap_af(ifp, AF_MPLS, m, BPF_D_IN);
185 #endif
186 
187 	mpls_lse(m);
188 }
189 
190 void
191 mplsintr(void *arg __unused)
192 {
193 	struct mbuf *m;
194 
195 	while ((m = pktq_dequeue(mpls_pktq)) != NULL) {
196 		if (((m->m_flags & M_PKTHDR) == 0) ||
197 		    (m->m_pkthdr.rcvif_index == 0))
198 			panic("mplsintr(): no pkthdr or rcvif");
199 
200 #ifdef MBUFTRACE
201 		m_claimm(m, &mpls_owner);
202 #endif
203 		mpls_input(m_get_rcvif_NOMPSAFE(m), m);
204 	}
205 }
206 
207 /*
208  * prepend shim and deliver
209  */
210 static int
211 mpls_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
212     const struct rtentry *rt)
213 {
214 	union mpls_shim mh, *pms;
215 	struct rtentry *rt1;
216 	int err;
217 	uint psize = sizeof(struct sockaddr_mpls);
218 
219 	KASSERT(KERNEL_LOCKED_P());
220 
221 	if ((ifp->if_flags & (IFF_UP|IFF_RUNNING)) != (IFF_UP|IFF_RUNNING)) {
222 		m_freem(m);
223 		return ENETDOWN;
224 	}
225 
226 	if (rt_gettag(rt) == NULL || rt_gettag(rt)->sa_family != AF_MPLS) {
227 		m_freem(m);
228 		return EINVAL;
229 	}
230 
231 	bpf_mtap_af(ifp, dst->sa_family, m, BPF_D_OUT);
232 
233 	memset(&mh, 0, sizeof(mh));
234 	mh.s_addr = MPLS_GETSADDR(rt);
235 	mh.shim.bos = 1;
236 	mh.shim.exp = 0;
237 	mh.shim.ttl = mpls_defttl;
238 
239 	pms = &((struct sockaddr_mpls*)rt_gettag(rt))->smpls_addr;
240 
241 	while (psize <= rt_gettag(rt)->sa_len - sizeof(mh)) {
242 		pms++;
243 		if (mh.shim.label != MPLS_LABEL_IMPLNULL &&
244 		    ((m = mpls_prepend_shim(m, &mh)) == NULL))
245 			return ENOBUFS;
246 		memset(&mh, 0, sizeof(mh));
247 		mh.s_addr = ntohl(pms->s_addr);
248 		mh.shim.bos = mh.shim.exp = 0;
249 		mh.shim.ttl = mpls_defttl;
250 		psize += sizeof(mh);
251 	}
252 
253 	switch (dst->sa_family) {
254 #ifdef INET
255 	case AF_INET:
256 		m = mpls_label_inet(m, &mh, psize - sizeof(struct sockaddr_mpls));
257 		break;
258 #endif
259 #ifdef INET6
260 	case AF_INET6:
261 		m = mpls_label_inet6(m, &mh, psize - sizeof(struct sockaddr_mpls));
262 		break;
263 #endif
264 	default:
265 		m = mpls_prepend_shim(m, &mh);
266 		break;
267 	}
268 
269 	if (m == NULL) {
270 		IF_DROP(&ifp->if_snd);
271 		if_statinc(ifp, if_oerrors);
272 		return ENOBUFS;
273 	}
274 
275 	if_statadd2(ifp, if_opackets, 1, if_obytes, m->m_pkthdr.len);
276 
277 	if ((rt1 = rtalloc1(rt->rt_gateway, 1)) == NULL) {
278 		m_freem(m);
279 		return EHOSTUNREACH;
280 	}
281 
282 	err = mpls_send_frame(m, rt1->rt_ifp, rt);
283 	rt_unref(rt1);
284 	return err;
285 }
286 
287 static int
288 mpls_ioctl(struct ifnet *ifp, u_long cmd, void *data)
289 {
290 	int error = 0, s = splnet();
291 	struct ifreq *ifr = data;
292 
293 	switch(cmd) {
294 	case SIOCINITIFADDR:
295 		ifp->if_flags |= IFF_UP | IFF_RUNNING;
296 		break;
297 	case SIOCSIFMTU:
298 		if (ifr != NULL && ifr->ifr_mtu < 576) {
299 			error = EINVAL;
300 			break;
301 		}
302 		/* FALLTHROUGH */
303 	case SIOCGIFMTU:
304 		if ((error = ifioctl_common(ifp, cmd, data)) == ENETRESET)
305 			error = 0;
306 		break;
307 	case SIOCSIFFLAGS:
308 		if ((error = ifioctl_common(ifp, cmd, data)) != 0)
309 			break;
310 		if (ifp->if_flags & IFF_UP)
311 			ifp->if_flags |= IFF_RUNNING;
312 		break;
313 	default:
314 		error = ifioctl_common(ifp, cmd, data);
315 		break;
316 	}
317 	splx(s);
318 	return error;
319 }
320 
321 static inline struct mbuf *
322 mpls_trim_label(struct mbuf *m, union mpls_shim *sh)
323 {
324 	m_adj(m, sizeof(union mpls_shim));
325 
326 	if (m->m_len < sizeof(union mpls_shim) &&
327 	    (m = m_pullup(m, sizeof(union mpls_shim))) == NULL)
328 		return NULL;
329 
330 	sh->s_addr = ntohl(mtod(m, union mpls_shim *)->s_addr);
331 
332 	return m;
333 }
334 
335 /*
336  * MPLS Label Switch Engine
337  */
338 static int
339 mpls_lse(struct mbuf *m)
340 {
341 	struct sockaddr_mpls dst;
342 	union mpls_shim tshim, *htag;
343 	struct rtentry *rt = NULL;
344 	int error = ENOBUFS;
345 	uint psize = sizeof(struct sockaddr_mpls);
346 	bool push_back_alert = false;
347 
348 	/* If we're not accepting MPLS frames, leave now. */
349 	if (!mpls_frame_accept) {
350 		error = EINVAL;
351 		goto done;
352 	}
353 
354 	if (m->m_len < sizeof(union mpls_shim) &&
355 	    (m = m_pullup(m, sizeof(union mpls_shim))) == NULL)
356 		goto done;
357 
358 	dst.smpls_len = sizeof(struct sockaddr_mpls);
359 	dst.smpls_family = AF_MPLS;
360 	dst.smpls_addr.s_addr = ntohl(mtod(m, union mpls_shim *)->s_addr);
361 
362 	error = EINVAL;
363 
364 	/* TTL decrement */
365 	if ((m = mpls_ttl_dec(m)) == NULL)
366 		goto done;
367 
368 	/* RFC 4182 */
369 	if (mpls_rfc4182 != 0) {
370 		while ((dst.smpls_addr.shim.label == MPLS_LABEL_IPV4NULL ||
371 		    dst.smpls_addr.shim.label == MPLS_LABEL_IPV6NULL) &&
372 		    __predict_false(dst.smpls_addr.shim.bos == 0)) {
373 			m = mpls_trim_label(m, &dst.smpls_addr);
374 			if (m == NULL) {
375 				goto done;
376 			}
377 		}
378 	}
379 
380 	/* RFC 3032 Section 2.1 Page 4 */
381 	if (__predict_false(dst.smpls_addr.shim.label == MPLS_LABEL_RTALERT) &&
382 	    dst.smpls_addr.shim.bos == 0) {
383 		m = mpls_trim_label(m, &dst.smpls_addr);
384 		if (m == NULL) {
385 			goto done;
386 		}
387 		push_back_alert = true;
388 	}
389 
390 	if (dst.smpls_addr.shim.label <= MPLS_LABEL_RESMAX) {
391 		/* Don't swap reserved labels */
392 		switch (dst.smpls_addr.shim.label) {
393 #ifdef INET
394 		case MPLS_LABEL_IPV4NULL:
395 			/* Pop shim and push mbuf to IP stack */
396 			if (dst.smpls_addr.shim.bos) {
397 				m = mpls_unlabel_inet(m, &error);
398 			}
399 			break;
400 #endif
401 #ifdef INET6
402 		case MPLS_LABEL_IPV6NULL:
403 			/* Pop shim and push mbuf to IPv6 stack */
404 			if (dst.smpls_addr.shim.bos) {
405 				m = mpls_unlabel_inet6(m, &error);
406 			}
407 			break;
408 #endif
409 		case MPLS_LABEL_RTALERT:	/* Yeah, I'm all alerted */
410 		case MPLS_LABEL_IMPLNULL:	/* This is logical only */
411 		default:			/* Rest are not allowed */
412 			break;
413 		}
414 		goto done;
415 	}
416 
417 	/* Check if we should do MPLS forwarding */
418 	error = EHOSTUNREACH;
419 	if (!mpls_forwarding)
420 		goto done;
421 
422 	/* Get a route to dst */
423 	dst.smpls_addr.shim.ttl = 0;
424 	dst.smpls_addr.shim.bos = 0;
425 	dst.smpls_addr.shim.exp = 0;
426 	dst.smpls_addr.s_addr = htonl(dst.smpls_addr.s_addr);
427 	if ((rt = rtalloc1((const struct sockaddr*)&dst, 1)) == NULL)
428 		goto done;
429 
430 	/* MPLS packet with no MPLS tagged route ? */
431 	if ((rt->rt_flags & RTF_GATEWAY) == 0 ||
432 	     rt_gettag(rt) == NULL ||
433 	     rt_gettag(rt)->sa_family != AF_MPLS)
434 		goto done;
435 
436 	tshim.s_addr = MPLS_GETSADDR(rt);
437 
438 	/* Swap labels */
439 	if ((m->m_len < sizeof(union mpls_shim)) &&
440 	    (m = m_pullup(m, sizeof(union mpls_shim))) == 0) {
441 		error = ENOBUFS;
442 		goto done;
443 	}
444 
445 	/* Replace only the label */
446 	htag = mtod(m, union mpls_shim *);
447 	htag->s_addr = ntohl(htag->s_addr);
448 	htag->shim.label = tshim.shim.label;
449 	htag->s_addr = htonl(htag->s_addr);
450 
451 	/* check if there is anything more to prepend */
452 	htag = &((struct sockaddr_mpls*)rt_gettag(rt))->smpls_addr;
453 	while (psize <= rt_gettag(rt)->sa_len - sizeof(tshim)) {
454 		htag++;
455 		memset(&tshim, 0, sizeof(tshim));
456 		tshim.s_addr = ntohl(htag->s_addr);
457 		tshim.shim.bos = tshim.shim.exp = 0;
458 		tshim.shim.ttl = mpls_defttl;
459 		if (tshim.shim.label != MPLS_LABEL_IMPLNULL &&
460 		    ((m = mpls_prepend_shim(m, &tshim)) == NULL)) {
461 			error = ENOBUFS;
462 			goto done;
463 		}
464 		psize += sizeof(tshim);
465 	}
466 
467 	if (__predict_false(push_back_alert == true)) {
468 		/* re-add the router alert label */
469 		memset(&tshim, 0, sizeof(tshim));
470 		tshim.s_addr = MPLS_LABEL_RTALERT;
471 		tshim.shim.bos = tshim.shim.exp = 0;
472 		tshim.shim.ttl = mpls_defttl;
473 		if ((m = mpls_prepend_shim(m, &tshim)) == NULL) {
474 			error = ENOBUFS;
475 			goto done;
476 		}
477 	}
478 
479 	if ((rt->rt_flags & RTF_GATEWAY) == 0) {
480 		error = EHOSTUNREACH;
481 		goto done;
482 	}
483 
484 	rt->rt_use++;
485 	error = mpls_send_frame(m, rt->rt_ifp, rt);
486 
487 done:
488 	if (error != 0 && m != NULL)
489 		m_freem(m);
490 	if (rt != NULL)
491 		rt_unref(rt);
492 
493 	return error;
494 }
495 
496 static int
497 mpls_send_frame(struct mbuf *m, struct ifnet *ifp, const struct rtentry *rt)
498 {
499 	union mpls_shim msh;
500 	int ret;
501 
502 	msh.s_addr = MPLS_GETSADDR(rt);
503 	if (msh.shim.label == MPLS_LABEL_IMPLNULL ||
504 	    (m->m_flags & (M_MCAST | M_BCAST))) {
505 		m_adj(m, sizeof(union mpls_shim));
506 		m->m_pkthdr.csum_flags = 0;
507 	}
508 
509 	switch(ifp->if_type) {
510 	/* only these are supported for now */
511 	case IFT_ETHER:
512 	case IFT_TUNNEL:
513 	case IFT_LOOP:
514 #ifdef INET
515 		ret = ip_if_output(ifp, m, rt->rt_gateway, rt);
516 #else
517 		ret = if_output_lock(ifp, ifp, m, rt->rt_gateway, rt);
518 #endif
519 		return ret;
520 		break;
521 	default:
522 		return ENETUNREACH;
523 	}
524 	return 0;
525 }
526 
527 #ifdef INET
528 static struct mbuf *
529 mpls_unlabel_inet(struct mbuf *m, int *error)
530 {
531 	struct ip *iph;
532 	union mpls_shim ms;
533 	int iphlen;
534 
535 	if (mpls_mapttl_inet || mpls_mapprec_inet) {
536 		/* get shim info */
537 		ms.s_addr = ntohl(mtod(m, union mpls_shim *)->s_addr);
538 
539 		/* and get rid of it */
540 		m_adj(m, sizeof(union mpls_shim));
541 
542 		/* get ip header */
543 		if (m->m_len < sizeof(struct ip) &&
544 		    (m = m_pullup(m, sizeof(struct ip))) == NULL) {
545 			*error = ENOBUFS;
546 			return NULL;
547 		}
548 
549 		iph = mtod(m, struct ip *);
550 		iphlen = iph->ip_hl << 2;
551 
552 		/* get it all */
553 		if (m->m_len < iphlen) {
554 			if ((m = m_pullup(m, iphlen)) == NULL) {
555 				*error = ENOBUFS;
556 				return NULL;
557 			}
558 			iph = mtod(m, struct ip *);
559 		}
560 
561 		/* check ipsum */
562 		if (in_cksum(m, iphlen) != 0) {
563 			m_freem(m);
564 			*error = EINVAL;
565 			return NULL;
566 		}
567 
568 		/* set IP ttl from MPLS ttl */
569 		if (mpls_mapttl_inet)
570 			iph->ip_ttl = ms.shim.ttl;
571 
572 		/* set IP Precedence from MPLS Exp */
573 		if (mpls_mapprec_inet) {
574 			iph->ip_tos = (iph->ip_tos << 3) >> 3;
575 			iph->ip_tos |= ms.shim.exp << 5;
576 		}
577 
578 		/* reset ipsum because we modified TTL and TOS */
579 		iph->ip_sum = 0;
580 		iph->ip_sum = in_cksum(m, iphlen);
581 	} else {
582 		m_adj(m, sizeof(union mpls_shim));
583 	}
584 
585 	/* Put it on IP queue */
586 	if (__predict_false(!pktq_enqueue(ip_pktq, m, 0))) {
587 		m_freem(m);
588 		*error = ENOBUFS;
589 		return NULL;
590 	}
591 
592 	*error = 0;
593 	return m;
594 }
595 
596 /*
597  * Prepend MPLS label
598  */
599 static struct mbuf *
600 mpls_label_inet(struct mbuf *m, union mpls_shim *ms, uint offset)
601 {
602 	struct ip iphdr;
603 
604 	if (mpls_mapttl_inet || mpls_mapprec_inet) {
605 		/* XXX Maybe just check m->m_pkthdr.len instead? */
606 		if ((m->m_len < offset + sizeof(struct ip)) &&
607 		    (m = m_pullup(m, offset + sizeof(struct ip))) == 0)
608 			return NULL;
609 
610 		m_copydata(m, offset, sizeof(struct ip), &iphdr);
611 
612 		/* Map TTL */
613 		if (mpls_mapttl_inet)
614 			ms->shim.ttl = iphdr.ip_ttl;
615 
616 		/* Copy IP precedence to EXP */
617 		if (mpls_mapprec_inet)
618 			ms->shim.exp = ((u_int8_t)iphdr.ip_tos) >> 5;
619 	}
620 
621 	if ((m = mpls_prepend_shim(m, ms)) == NULL)
622 		return NULL;
623 
624 	return m;
625 }
626 #endif	/* INET */
627 
628 #ifdef INET6
629 static struct mbuf *
630 mpls_unlabel_inet6(struct mbuf *m, int *error)
631 {
632 	struct ip6_hdr *ip6hdr;
633 	union mpls_shim ms;
634 
635 	/* TODO: mapclass */
636 	if (mpls_mapttl_inet6) {
637 		ms.s_addr = ntohl(mtod(m, union mpls_shim *)->s_addr);
638 		m_adj(m, sizeof(union mpls_shim));
639 
640 		if (m->m_len < sizeof(struct ip6_hdr) &&
641 		    (m = m_pullup(m, sizeof(struct ip6_hdr))) == 0) {
642 			*error = ENOBUFS;
643 			return NULL;
644 		}
645 		ip6hdr = mtod(m, struct ip6_hdr *);
646 
647 		/* Because we just decremented this in mpls_lse */
648 		ip6hdr->ip6_hlim = ms.shim.ttl + 1;
649 	} else {
650 		m_adj(m, sizeof(union mpls_shim));
651 	}
652 
653 	/* Put it back on IPv6 queue. */
654 	if (__predict_false(!pktq_enqueue(ip6_pktq, m, 0))) {
655 		m_freem(m);
656 		*error = ENOBUFS;
657 		return NULL;
658 	}
659 
660 	*error = 0;
661 	return m;
662 }
663 
664 static struct mbuf *
665 mpls_label_inet6(struct mbuf *m, union mpls_shim *ms, uint offset)
666 {
667 	struct ip6_hdr ip6h;
668 
669 	if (mpls_mapttl_inet6 || mpls_mapclass_inet6) {
670 		/* XXX Maybe just check m->m_pkthdr.len instead? */
671 		if ((m->m_len < offset + sizeof(struct ip6_hdr)) &&
672 		    (m = m_pullup(m, offset + sizeof(struct ip6_hdr))) == 0)
673 			return NULL;
674 
675 		m_copydata(m, offset, sizeof(struct ip6_hdr), &ip6h);
676 
677 		if (mpls_mapttl_inet6)
678 			ms->shim.ttl = ip6h.ip6_hlim;
679 
680 		if (mpls_mapclass_inet6)
681 			ms->shim.exp = ip6h.ip6_vfc << 1 >> 5;
682 	}
683 
684 	if ((m = mpls_prepend_shim(m, ms)) == NULL)
685 		return NULL;
686 
687 	return m;
688 }
689 #endif	/* INET6 */
690 
691 static struct mbuf *
692 mpls_prepend_shim(struct mbuf *m, union mpls_shim *ms)
693 {
694 	union mpls_shim *shim;
695 
696 	M_PREPEND(m, sizeof(*ms), M_DONTWAIT);
697 	if (m == NULL)
698 		return NULL;
699 
700 	if (m->m_len < sizeof(union mpls_shim) &&
701 	    (m = m_pullup(m, sizeof(union mpls_shim))) == 0)
702 		return NULL;
703 
704 	shim = mtod(m, union mpls_shim *);
705 
706 	memcpy(shim, ms, sizeof(*shim));
707 	shim->s_addr = htonl(shim->s_addr);
708 
709 	return m;
710 }
711 
712 /*
713  * Module infrastructure
714  */
715 #include "if_module.h"
716 
717 IF_MODULE(MODULE_CLASS_DRIVER, mpls, NULL)
718