xref: /dragonfly/tools/tools/netrate/pktgen/pktgen.c (revision 5f39c7e7) 
1 /*
2  * Copyright (c) 2012 The DragonFly Project.  All rights reserved.
3  *
4  * This code is derived from software contributed to The DragonFly Project
5  * by Sepherosa Ziehau <sepherosa@gmail.com>
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in
15  *    the documentation and/or other materials provided with the
16  *    distribution.
17  * 3. Neither the name of The DragonFly Project nor the names of its
18  *    contributors may be used to endorse or promote products derived
19  *    from this software without specific, prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  * $DragonFly: src/tools/tools/netrate/pktgen/pktgen.c,v 1.4 2008/04/02 14:18:55 sephe Exp $
35  */
36 
37 #define _IP_VHL
38 
39 #include <sys/param.h>
40 #include <sys/conf.h>
41 #include <sys/device.h>
42 #include <sys/in_cksum.h>
43 #include <sys/kernel.h>
44 #include <sys/malloc.h>
45 #include <sys/mbuf.h>
46 #include <sys/proc.h>
47 #include <sys/priv.h>
48 #include <sys/queue.h>
49 #include <sys/socket.h>
50 #include <sys/systm.h>
51 #include <sys/serialize.h>
52 
53 #include <net/if.h>
54 #include <net/if_dl.h>
55 #include <net/if_var.h>
56 #include <net/ifq_var.h>
57 #include <net/ethernet.h>
58 #include <net/netmsg2.h>
59 #include <net/netisr2.h>
60 
61 #include <netinet/in.h>
62 #include <netinet/ip.h>
63 #include <netinet/udp_var.h>
64 
65 #include "pktgen.h"
66 
67 #define CDEV_NAME	"pktg"
68 
69 #define PKTGEN_BUFSZ	2048
70 
71 #ifndef PKTGEN_DEVCNT
72 #define PKTGEN_DEVCNT	4
73 #endif
74 
75 struct pktgen;
76 
77 struct netmsg_pktgen {
78 	struct netmsg_base	np_base;
79 	struct pktgen		*np_pktg;
80 	struct ifaltq_subque	*np_ifsq;
81 };
82 
83 struct pktgen_buf {
84 	struct netmsg_base	pb_nmsg;	/* MUST BE THE FIRST */
85 	void			*pb_buf;
86 	volatile int		pb_done;
87 	int			pb_inuse;
88 	struct ifnet		*pb_ifp;
89 	struct ifaltq_subque	*pb_ifsq;
90 	int			pb_len;
91 	int			pb_cpuid;
92 	struct pktgen		*pb_pktg;
93 	LIST_ENTRY(pktgen_buf)	pb_link;
94 };
95 
96 struct pktgen_pcpu {
97 	struct callout		pktg_stop;
98 	LIST_HEAD(, pktgen_buf)	pktg_buflist;
99 };
100 
101 struct pktgen {
102 	uint32_t		pktg_flags;	/* PKTG_F_ */
103 	int			pktg_refcnt;
104 
105 	int			pktg_duration;
106 
107 	int			pktg_datalen;
108 	struct ifnet		*pktg_ifp;
109 
110 	int			pktg_pktenq;
111 
112 	struct sockaddr_in	pktg_src;
113 	int			pktg_ndst;
114 	struct sockaddr_in	*pktg_dst;
115 	uint8_t			pktg_dst_lladdr[ETHER_ADDR_LEN];
116 
117 	struct pktgen_pcpu	pktg_pcpu[MAXCPU];
118 };
119 
120 #define PKTG_F_CONFIG		0x1
121 #define PKTG_F_RUNNING		0x4
122 #define PKTG_F_SWITCH_SRCDST	0x8
123 
124 static int 		pktgen_modevent(module_t, int, void *);
125 
126 static void		pktgen_buf_free(void *);
127 static void		pktgen_buf_ref(void *);
128 static void		pktgen_buf_send(netmsg_t);
129 
130 static int		pktgen_config(struct pktgen *,
131 			    const struct pktgen_conf *);
132 static int		pktgen_start(struct pktgen *, int);
133 static void		pktgen_free(struct pktgen *);
134 static void		pktgen_ref(struct pktgen *);
135 static void		pktgen_pcpu_stop_cb(void *);
136 static void		pktgen_mbuf(struct pktgen_buf *, struct mbuf *);
137 static void		pktgen_start_ifsq(struct pktgen *,
138 			    struct ifaltq_subque *);
139 static void		pktgen_start_ifsq_handler(netmsg_t);
140 
141 static d_open_t		pktgen_open;
142 static d_close_t	pktgen_close;
143 static d_ioctl_t	pktgen_ioctl;
144 
145 static struct dev_ops	pktgen_ops = {
146 	{ CDEV_NAME, 0, D_MPSAFE },
147 	.d_open =	pktgen_open,
148 	.d_close =	pktgen_close,
149 	.d_ioctl =	pktgen_ioctl,
150 };
151 
152 static volatile int		pktgen_refcnt;
153 static struct lwkt_token pktgen_tok = LWKT_TOKEN_INITIALIZER(pktgen_token);
154 
155 MALLOC_DECLARE(M_PKTGEN);
156 MALLOC_DEFINE(M_PKTGEN, CDEV_NAME, "Packet generator");
157 
158 DEV_MODULE(pktgen, pktgen_modevent, NULL);
159 
160 static int
161 pktgen_modevent(module_t mod, int type, void *data)
162 {
163 	int error = 0, i;
164 
165 	switch (type) {
166 	case MOD_LOAD:
167 		for (i = 0; i < PKTGEN_DEVCNT; ++i) {
168 			make_dev(&pktgen_ops, 0, UID_ROOT, GID_WHEEL, 0600,
169 			    CDEV_NAME"%d", i);
170 		}
171 		break;
172 
173 	case MOD_UNLOAD:
174 		if (pktgen_refcnt > 0)
175 			return EBUSY;
176 		dev_ops_remove_all(&pktgen_ops);
177 		break;
178 
179 	default:
180 		error = EOPNOTSUPP;
181 		break;
182 	}
183 	return error;
184 }
185 
186 static int
187 pktgen_open(struct dev_open_args *ap)
188 {
189 	cdev_t dev = ap->a_head.a_dev;
190 	struct pktgen *pktg;
191 	int error, i;
192 
193 	error = priv_check_cred(ap->a_cred, PRIV_ROOT, 0);
194 	if (error)
195 		return error;
196 
197 	lwkt_gettoken(&pktgen_tok);
198 
199 	if (dev->si_drv1 != NULL) {
200 		lwkt_reltoken(&pktgen_tok);
201 		return EBUSY;
202 	}
203 
204 	pktg = kmalloc(sizeof(*pktg), M_PKTGEN, M_ZERO | M_WAITOK);
205 	for (i = 0; i < ncpus; ++i) {
206 		struct pktgen_pcpu *p = &pktg->pktg_pcpu[i];
207 
208 		callout_init_mp(&p->pktg_stop);
209 		LIST_INIT(&p->pktg_buflist);
210 	}
211 
212 	dev->si_drv1 = pktg;
213 	pktg->pktg_refcnt = 1;
214 
215 	atomic_add_int(&pktgen_refcnt, 1);
216 
217 	lwkt_reltoken(&pktgen_tok);
218 	return 0;
219 }
220 
221 static int
222 pktgen_close(struct dev_close_args *ap)
223 {
224 	cdev_t dev = ap->a_head.a_dev;
225 	struct pktgen *pktg = dev->si_drv1;
226 
227 	lwkt_gettoken(&pktgen_tok);
228 	dev->si_drv1 = NULL;
229 	lwkt_reltoken(&pktgen_tok);
230 
231 	pktgen_free(pktg);
232 
233 	return 0;
234 }
235 
236 static int
237 pktgen_ioctl(struct dev_ioctl_args *ap __unused)
238 {
239 	cdev_t dev = ap->a_head.a_dev;
240 	caddr_t data = ap->a_data;
241 	struct pktgen *pktg = dev->si_drv1;
242 	int error;
243 
244 	lwkt_gettoken(&pktgen_tok);
245 
246 	switch (ap->a_cmd) {
247 	case PKTGENSTART:
248 		error = pktgen_start(pktg, 0);
249 		break;
250 
251 	case PKTGENMQSTART:
252 		error = pktgen_start(pktg, 1);
253 		break;
254 
255 	case PKTGENSCONF:
256 		error = pktgen_config(pktg, (const struct pktgen_conf *)data);
257 		break;
258 
259 	default:
260 		error = EOPNOTSUPP;
261 		break;
262 	}
263 
264 	lwkt_reltoken(&pktgen_tok);
265 	return error;
266 }
267 
268 static int
269 pktgen_config(struct pktgen *pktg, const struct pktgen_conf *conf)
270 {
271 	const struct sockaddr_in *sin;
272 	struct sockaddr_in *dst = NULL;
273 	const struct sockaddr *sa;
274 	struct ifnet *ifp;
275 	size_t dst_size;
276 	int i, error, pktenq;
277 
278 	if (pktg->pktg_flags & (PKTG_F_RUNNING | PKTG_F_CONFIG))
279 		return EBUSY;
280 
281 	if (conf->pc_datalen <= 0 ||
282 	    conf->pc_datalen > ETHERMTU - sizeof(struct udpiphdr))
283 		return EINVAL;
284 	if (conf->pc_duration <= 0)
285 		return EINVAL;
286 
287 	sin = &conf->pc_src;
288 	if (sin->sin_family != AF_INET)
289 		return EPROTONOSUPPORT;
290 	if (sin->sin_port == 0)
291 		return EINVAL;
292 
293 	if (conf->pc_ndst <= 0)
294 		return EINVAL;
295 	dst_size = conf->pc_ndst * sizeof(struct sockaddr_in);
296 
297 	dst = kmalloc(dst_size, M_PKTGEN, M_WAITOK | M_NULLOK);
298 	if (dst == NULL)
299 		return ENOMEM;
300 
301 	error = copyin(conf->pc_dst, dst, dst_size);
302 	if (error)
303 		goto failed;
304 
305 	for (i = 0; i < conf->pc_ndst; ++i) {
306 		sin = &dst[i];
307 		if (sin->sin_family != AF_INET) {
308 			error = EPROTONOSUPPORT;
309 			goto failed;
310 		}
311 		if (sin->sin_port == 0) {
312 			error = EINVAL;
313 			goto failed;
314 		}
315 	}
316 
317 	ifp = ifunit(conf->pc_ifname);
318 	if (ifp == NULL) {
319 		error = ENXIO;
320 		goto failed;
321 	}
322 
323 	pktenq = conf->pc_pktenq;
324 	if (pktenq < 0 || pktenq > ifp->if_snd.altq_maxlen) {
325 		error = ENOBUFS;
326 		goto failed;
327 	} else if (pktenq == 0) {
328 		pktenq = (ifp->if_snd.altq_maxlen * 3) / 4;
329 	}
330 
331 	sa = &conf->pc_dst_lladdr;
332 	if (sa->sa_family != AF_LINK) {
333 		error = EPROTONOSUPPORT;
334 		goto failed;
335 	}
336 	if (sa->sa_len != ETHER_ADDR_LEN) {
337 		error = EPROTONOSUPPORT;
338 		goto failed;
339 	}
340 	if (ETHER_IS_MULTICAST(sa->sa_data) ||
341 	    bcmp(sa->sa_data, ifp->if_broadcastaddr, ifp->if_addrlen) == 0) {
342 		error = EADDRNOTAVAIL;
343 		goto failed;
344 	}
345 
346 	/*
347 	 * Accept the config
348 	 */
349 	pktg->pktg_flags |= PKTG_F_CONFIG;
350 
351 	if (conf->pc_flags & PKTGEN_FLAG_SWITCH_SRCDST)
352 		pktg->pktg_flags |= PKTG_F_SWITCH_SRCDST;
353 	pktg->pktg_duration = conf->pc_duration;
354 	pktg->pktg_datalen = conf->pc_datalen;
355 	pktg->pktg_pktenq = pktenq;
356 	pktg->pktg_ifp = ifp;
357 	pktg->pktg_src = conf->pc_src;
358 	pktg->pktg_ndst = conf->pc_ndst;
359 	KKASSERT(pktg->pktg_dst == NULL);
360 	pktg->pktg_dst = dst;
361 	bcopy(sa->sa_data, pktg->pktg_dst_lladdr, ETHER_ADDR_LEN);
362 
363 	return 0;
364 
365 failed:
366 	if (dst != NULL)
367 		kfree(dst, M_PKTGEN);
368 	return error;
369 }
370 
371 static void
372 pktgen_start_ifsq(struct pktgen *pktg, struct ifaltq_subque *ifsq)
373 {
374 	struct netmsg_pktgen *np;
375 
376 	np = kmalloc(sizeof(*np), M_LWKTMSG, M_WAITOK);
377 	netmsg_init(&np->np_base, NULL, &netisr_afree_rport, 0,
378 	    pktgen_start_ifsq_handler);
379 	np->np_pktg = pktg;
380 	np->np_ifsq = ifsq;
381 
382 	lwkt_sendmsg(netisr_cpuport(ifsq_get_cpuid(ifsq)), &np->np_base.lmsg);
383 }
384 
385 static int
386 pktgen_start(struct pktgen *pktg, int mq)
387 {
388 	struct ifaltq *ifq;
389 
390 	if ((pktg->pktg_flags & PKTG_F_CONFIG) == 0)
391 		return EINVAL;
392 	if (pktg->pktg_flags & PKTG_F_RUNNING)
393 		return EBUSY;
394 	pktg->pktg_flags |= PKTG_F_RUNNING;
395 
396 	ifq = &pktg->pktg_ifp->if_snd;
397 	if (!mq) {
398 		pktgen_ref(pktg);
399 		pktgen_start_ifsq(pktg, ifq_get_subq_default(ifq));
400 	} else {
401 		int i;
402 
403 		for (i = 0; i < ifq->altq_subq_cnt; ++i)
404 			pktgen_ref(pktg);
405 		for (i = 0; i < ifq->altq_subq_cnt; ++i)
406 			pktgen_start_ifsq(pktg, ifq_get_subq(ifq, i));
407 	}
408 	return 0;
409 }
410 
411 static void
412 pktgen_start_ifsq_handler(netmsg_t nmsg)
413 {
414 	struct netmsg_pktgen *np = (struct netmsg_pktgen *)nmsg;
415 	struct pktgen *pktg = np->np_pktg;
416 	struct ifaltq_subque *ifsq = np->np_ifsq;
417 
418 	struct mbuf *m, *head = NULL, **next;
419 	struct ifnet *ifp;
420 	struct pktgen_pcpu *p;
421 	int cpuid, i, alloc_cnt, keep_cnt;
422 
423 	u_short ulen, psum;
424 	int len, ip_len;
425 
426 	/* Reply ASAP */
427 	lwkt_replymsg(&np->np_base.lmsg, 0);
428 
429 	ifp = pktg->pktg_ifp;
430 
431 	cpuid = ifsq_get_cpuid(ifsq);
432 	KKASSERT(cpuid == mycpuid);
433 
434 	p = &pktg->pktg_pcpu[cpuid];
435 
436 	keep_cnt = pktg->pktg_pktenq;
437 	alloc_cnt = keep_cnt * 2;
438 
439 	/*
440 	 * Prefault enough mbuf into mbuf objcache
441 	 */
442 	next = &head;
443 	for (i = 0; i < alloc_cnt; ++i) {
444 		MGETHDR(m, MB_WAIT, MT_DATA);
445 		*next = m;
446 		next = &m->m_nextpkt;
447 	}
448 
449 	for (i = 0; i < alloc_cnt - keep_cnt; ++i) {
450 		m = head;
451 		head = m->m_nextpkt;
452 		m->m_nextpkt = NULL;
453 		m_freem(m);
454 	}
455 	KKASSERT(head != NULL);
456 
457 	/*
458 	 * Setup the packets' data
459 	 */
460 	ip_len = pktg->pktg_datalen + sizeof(struct udpiphdr);
461 	len = ip_len + ETHER_HDR_LEN;
462 
463 	psum = htons((u_short)pktg->pktg_datalen + sizeof(struct udphdr) +
464 	    IPPROTO_UDP);
465 	ulen = htons(pktg->pktg_datalen + sizeof(struct udphdr));
466 
467 	m = head;
468 	i = 0;
469 	while (m != NULL) {
470 		struct mbuf *nextm;
471 		const struct sockaddr_in *dst;
472 		struct pktgen_buf *pb;
473 		struct ip *ip;
474 		struct udpiphdr *ui;
475 		struct ether_header *eh;
476 
477 		pktgen_ref(pktg);
478 
479 		pb = kmalloc(sizeof(*pb), M_PKTGEN, M_WAITOK | M_ZERO);
480 		pb->pb_ifp = ifp;
481 		pb->pb_ifsq = ifsq;
482 		pb->pb_inuse = 1;
483 		pb->pb_buf = kmalloc(PKTGEN_BUFSZ, M_PKTGEN, M_WAITOK);
484 		pb->pb_len = len;
485 		pb->pb_cpuid = cpuid;
486 		pb->pb_pktg = pktg;
487 		netmsg_init(&pb->pb_nmsg, NULL, &netisr_adone_rport, 0,
488 		    pktgen_buf_send);
489 		LIST_INSERT_HEAD(&p->pktg_buflist, pb, pb_link);
490 
491 		dst = &pktg->pktg_dst[i % pktg->pktg_ndst];
492 		++i;
493 
494 		m->m_ext.ext_arg = pb;
495 		m->m_ext.ext_buf = pb->pb_buf;
496 		m->m_ext.ext_free = pktgen_buf_free;
497 		m->m_ext.ext_ref = pktgen_buf_ref;
498 		m->m_ext.ext_size = PKTGEN_BUFSZ;
499 
500 		m->m_data = m->m_ext.ext_buf;
501 		m->m_flags |= M_EXT;
502 		m->m_len = m->m_pkthdr.len = len;
503 
504 		m->m_data += ETHER_HDR_LEN;
505 		m->m_len -= ETHER_HDR_LEN;
506 		m->m_pkthdr.len -= ETHER_HDR_LEN;
507 
508 		ui = mtod(m, struct udpiphdr *);
509 		ui->ui_pr = IPPROTO_UDP;
510 		if (pktg->pktg_flags & PKTG_F_SWITCH_SRCDST) {
511 			ui->ui_src.s_addr = dst->sin_addr.s_addr;
512 			ui->ui_dst.s_addr = pktg->pktg_src.sin_addr.s_addr;
513 			ui->ui_sport = dst->sin_port;
514 			ui->ui_dport = pktg->pktg_src.sin_port;
515 		} else {
516 			ui->ui_src.s_addr = pktg->pktg_src.sin_addr.s_addr;
517 			ui->ui_dst.s_addr = dst->sin_addr.s_addr;
518 			ui->ui_sport = pktg->pktg_src.sin_port;
519 			ui->ui_dport = dst->sin_port;
520 		}
521 		ui->ui_ulen = ulen;
522 		ui->ui_sum = in_pseudo(ui->ui_src.s_addr, ui->ui_dst.s_addr,
523 		    psum);
524 		m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
525 
526 		ip = (struct ip *)ui;
527 		ip->ip_len = ip_len;
528 		ip->ip_ttl = 64;	/* XXX */
529 		ip->ip_tos = 0;		/* XXX */
530 		ip->ip_vhl = IP_VHL_BORING;
531 		ip->ip_off = 0;
532 		ip->ip_sum = 0;
533 		ip->ip_id = ip_newid();
534 
535 		in_delayed_cksum(m);
536 
537 		ip->ip_len = htons(ip->ip_len);
538 		ip->ip_sum = in_cksum_hdr(ip);
539 
540 		m->m_data -= ETHER_HDR_LEN;
541 		m->m_len += ETHER_HDR_LEN;
542 		m->m_pkthdr.len += ETHER_HDR_LEN;
543 
544 		eh = mtod(m, struct ether_header *);
545 		bcopy(pktg->pktg_dst_lladdr, eh->ether_dhost, ETHER_ADDR_LEN);
546 		bcopy(IF_LLADDR(ifp), eh->ether_shost, ETHER_ADDR_LEN);
547 		eh->ether_type = htons(ETHERTYPE_IP);
548 
549 		nextm = m->m_nextpkt;
550 		m->m_nextpkt = NULL;
551 
552 		ifq_dispatch(ifp, m, NULL);
553 
554 		m = nextm;
555 	}
556 
557 	callout_reset(&p->pktg_stop, pktg->pktg_duration * hz,
558 	    pktgen_pcpu_stop_cb, p);
559 
560 	pktgen_free(pktg);
561 }
562 
563 static void
564 pktgen_mbuf(struct pktgen_buf *pb, struct mbuf *m)
565 {
566 	m->m_ext.ext_arg = pb;
567 	m->m_ext.ext_buf = pb->pb_buf;
568 	m->m_ext.ext_free = pktgen_buf_free;
569 	m->m_ext.ext_ref = pktgen_buf_ref;
570 	m->m_ext.ext_size = PKTGEN_BUFSZ;
571 
572 	m->m_data = m->m_ext.ext_buf;
573 	m->m_flags |= M_EXT;
574 	m->m_len = m->m_pkthdr.len = pb->pb_len;
575 }
576 
577 static void
578 pktgen_buf_send(netmsg_t msg)
579 {
580 	struct pktgen_buf *pb = (struct pktgen_buf *)msg;
581 	struct mbuf *m;
582 
583 	KKASSERT(&curthread->td_msgport == netisr_cpuport(pb->pb_cpuid));
584 
585 	crit_enter();
586 	lwkt_replymsg(&pb->pb_nmsg.lmsg, 0);
587 	crit_exit();
588 
589 	MGETHDR(m, MB_WAIT, MT_DATA);
590 	pktgen_mbuf(pb, m);
591 	ifq_dispatch(pb->pb_ifp, m, NULL);
592 }
593 
594 static void
595 pktgen_buf_free(void *arg)
596 {
597 	struct pktgen_buf *pb = arg;
598 	struct mbuf *m;
599 
600 	KKASSERT(pb->pb_inuse > 0);
601 	if (pb->pb_done) {
602 		if (atomic_fetchadd_int(&pb->pb_inuse, -1) == 1) {
603 			struct pktgen *pktg;
604 
605 			pktg = pb->pb_pktg;
606 			crit_enter();
607 			LIST_REMOVE(pb, pb_link);
608 			crit_exit();
609 			kfree(pb->pb_buf, M_PKTGEN);
610 			kfree(pb, M_PKTGEN);
611 
612 			pktgen_free(pktg);
613 		}
614 		return;
615 	}
616 
617 	if (&curthread->td_msgport != netisr_cpuport(pb->pb_cpuid)) {
618 		KKASSERT(pb->pb_cpuid == mycpuid);
619 		crit_enter();
620 		KKASSERT(pb->pb_nmsg.lmsg.ms_flags & MSGF_DONE);
621 		lwkt_sendmsg(netisr_cpuport(pb->pb_cpuid), &pb->pb_nmsg.lmsg);
622 		crit_exit();
623 		return;
624 	}
625 
626 	MGETHDR(m, MB_WAIT, MT_DATA);
627 	pktgen_mbuf(pb, m);
628 	ifsq_enqueue(pb->pb_ifsq, m, NULL);
629 }
630 
631 static void
632 pktgen_buf_ref(void *arg)
633 {
634 	struct pktgen_buf *pb = arg;
635 
636 	panic("%s should never be called\n", __func__);
637 
638 	KKASSERT(pb->pb_inuse > 0);
639 	atomic_add_int(&pb->pb_inuse, 1);
640 }
641 
642 static void
643 pktgen_free(struct pktgen *pktg)
644 {
645 	KKASSERT(pktg->pktg_refcnt > 0);
646 	if (atomic_fetchadd_int(&pktg->pktg_refcnt, -1) == 1) {
647 		int i;
648 
649 		if (pktg->pktg_dst != NULL)
650 			kfree(pktg->pktg_dst, M_PKTGEN);
651 
652 		for (i = 0; i < ncpus; ++i)
653 			KKASSERT(LIST_EMPTY(&pktg->pktg_pcpu[i].pktg_buflist));
654 		kfree(pktg, M_PKTGEN);
655 	}
656 
657 	KKASSERT(pktgen_refcnt > 0);
658 	atomic_subtract_int(&pktgen_refcnt, 1);
659 }
660 
661 static void
662 pktgen_pcpu_stop_cb(void *arg)
663 {
664 	struct pktgen_pcpu *p = arg;
665 	struct pktgen_buf *pb;
666 
667 	crit_enter();
668 	LIST_FOREACH(pb, &p->pktg_buflist, pb_link)
669 		pb->pb_done = 1;
670 	crit_exit();
671 }
672 
673 static void
674 pktgen_ref(struct pktgen *pktg)
675 {
676 	atomic_add_int(&pktg->pktg_refcnt, 1);
677 	atomic_add_int(&pktgen_refcnt, 1);
678 }
679