xref: /dragonfly/tools/tools/netrate/pktgen/pktgen.c (revision e7d467f4) 
1 /*
2  * Copyright (c) 2012 The DragonFly Project.  All rights reserved.
3  *
4  * This code is derived from software contributed to The DragonFly Project
5  * by Sepherosa Ziehau <sepherosa@gmail.com>
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in
15  *    the documentation and/or other materials provided with the
16  *    distribution.
17  * 3. Neither the name of The DragonFly Project nor the names of its
18  *    contributors may be used to endorse or promote products derived
19  *    from this software without specific, prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  * $DragonFly: src/tools/tools/netrate/pktgen/pktgen.c,v 1.4 2008/04/02 14:18:55 sephe Exp $
35  */
36 
37 #define _IP_VHL
38 
39 #include <sys/param.h>
40 #include <sys/conf.h>
41 #include <sys/device.h>
42 #include <sys/in_cksum.h>
43 #include <sys/kernel.h>
44 #include <sys/malloc.h>
45 #include <sys/mbuf.h>
46 #include <sys/proc.h>
47 #include <sys/priv.h>
48 #include <sys/queue.h>
49 #include <sys/socket.h>
50 #include <sys/systm.h>
51 #include <sys/serialize.h>
52 
53 #include <net/if.h>
54 #include <net/if_dl.h>
55 #include <net/if_var.h>
56 #include <net/ifq_var.h>
57 #include <net/ethernet.h>
58 #include <net/netmsg2.h>
59 
60 #include <netinet/in.h>
61 #include <netinet/ip.h>
62 #include <netinet/udp_var.h>
63 
64 #include "pktgen.h"
65 
66 #define CDEV_NAME	"pktg"
67 
68 #define PKTGEN_BUFSZ	2048
69 
70 #ifndef PKTGEN_DEVCNT
71 #define PKTGEN_DEVCNT	4
72 #endif
73 
74 struct pktgen;
75 
76 struct netmsg_pktgen {
77 	struct netmsg_base	np_base;
78 	struct pktgen		*np_pktg;
79 	struct ifaltq_subque	*np_ifsq;
80 };
81 
82 struct pktgen_buf {
83 	struct netmsg_base	pb_nmsg;	/* MUST BE THE FIRST */
84 	void			*pb_buf;
85 	volatile int		pb_done;
86 	int			pb_inuse;
87 	struct ifnet		*pb_ifp;
88 	struct ifaltq_subque	*pb_ifsq;
89 	int			pb_len;
90 	int			pb_cpuid;
91 	struct pktgen		*pb_pktg;
92 	LIST_ENTRY(pktgen_buf)	pb_link;
93 };
94 
95 struct pktgen_pcpu {
96 	struct callout		pktg_stop;
97 	LIST_HEAD(, pktgen_buf)	pktg_buflist;
98 };
99 
100 struct pktgen {
101 	uint32_t		pktg_flags;	/* PKTG_F_ */
102 	int			pktg_refcnt;
103 
104 	int			pktg_duration;
105 
106 	int			pktg_datalen;
107 	struct ifnet		*pktg_ifp;
108 
109 	int			pktg_pktenq;
110 
111 	struct sockaddr_in	pktg_src;
112 	int			pktg_ndst;
113 	struct sockaddr_in	*pktg_dst;
114 	uint8_t			pktg_dst_lladdr[ETHER_ADDR_LEN];
115 
116 	struct pktgen_pcpu	pktg_pcpu[MAXCPU];
117 };
118 
119 #define PKTG_F_CONFIG	0x1
120 #define PKTG_F_RUNNING	0x4
121 
122 static int 		pktgen_modevent(module_t, int, void *);
123 
124 static void		pktgen_buf_free(void *);
125 static void		pktgen_buf_ref(void *);
126 static void		pktgen_buf_send(netmsg_t);
127 
128 static int		pktgen_config(struct pktgen *,
129 			    const struct pktgen_conf *);
130 static int		pktgen_start(struct pktgen *, int);
131 static void		pktgen_free(struct pktgen *);
132 static void		pktgen_ref(struct pktgen *);
133 static void		pktgen_pcpu_stop_cb(void *);
134 static void		pktgen_mbuf(struct pktgen_buf *, struct mbuf *);
135 static void		pktgen_start_ifsq(struct pktgen *,
136 			    struct ifaltq_subque *);
137 static void		pktgen_start_ifsq_handler(netmsg_t);
138 
139 static d_open_t		pktgen_open;
140 static d_close_t	pktgen_close;
141 static d_ioctl_t	pktgen_ioctl;
142 
143 static struct dev_ops	pktgen_ops = {
144 	{ CDEV_NAME, 0, D_MPSAFE },
145 	.d_open =	pktgen_open,
146 	.d_close =	pktgen_close,
147 	.d_ioctl =	pktgen_ioctl,
148 };
149 
150 static volatile int		pktgen_refcnt;
151 static struct lwkt_token pktgen_tok = LWKT_TOKEN_INITIALIZER(pktgen_token);
152 
153 MALLOC_DECLARE(M_PKTGEN);
154 MALLOC_DEFINE(M_PKTGEN, CDEV_NAME, "Packet generator");
155 
156 DEV_MODULE(pktgen, pktgen_modevent, NULL);
157 
158 static int
159 pktgen_modevent(module_t mod, int type, void *data)
160 {
161 	int error = 0, i;
162 
163 	switch (type) {
164 	case MOD_LOAD:
165 		for (i = 0; i < PKTGEN_DEVCNT; ++i) {
166 			make_dev(&pktgen_ops, 0, UID_ROOT, GID_WHEEL, 0600,
167 			    CDEV_NAME"%d", i);
168 		}
169 		break;
170 
171 	case MOD_UNLOAD:
172 		if (pktgen_refcnt > 0)
173 			return EBUSY;
174 		dev_ops_remove_all(&pktgen_ops);
175 		break;
176 
177 	default:
178 		error = EOPNOTSUPP;
179 		break;
180 	}
181 	return error;
182 }
183 
184 static int
185 pktgen_open(struct dev_open_args *ap)
186 {
187 	cdev_t dev = ap->a_head.a_dev;
188 	struct pktgen *pktg;
189 	int error, i;
190 
191 	error = priv_check_cred(ap->a_cred, PRIV_ROOT, 0);
192 	if (error)
193 		return error;
194 
195 	lwkt_gettoken(&pktgen_tok);
196 
197 	if (dev->si_drv1 != NULL) {
198 		lwkt_reltoken(&pktgen_tok);
199 		return EBUSY;
200 	}
201 
202 	pktg = kmalloc(sizeof(*pktg), M_PKTGEN, M_ZERO | M_WAITOK);
203 	for (i = 0; i < ncpus; ++i) {
204 		struct pktgen_pcpu *p = &pktg->pktg_pcpu[i];
205 
206 		callout_init_mp(&p->pktg_stop);
207 		LIST_INIT(&p->pktg_buflist);
208 	}
209 
210 	dev->si_drv1 = pktg;
211 	pktg->pktg_refcnt = 1;
212 
213 	atomic_add_int(&pktgen_refcnt, 1);
214 
215 	lwkt_reltoken(&pktgen_tok);
216 	return 0;
217 }
218 
219 static int
220 pktgen_close(struct dev_close_args *ap)
221 {
222 	cdev_t dev = ap->a_head.a_dev;
223 	struct pktgen *pktg = dev->si_drv1;
224 
225 	lwkt_gettoken(&pktgen_tok);
226 	dev->si_drv1 = NULL;
227 	lwkt_reltoken(&pktgen_tok);
228 
229 	pktgen_free(pktg);
230 
231 	return 0;
232 }
233 
234 static int
235 pktgen_ioctl(struct dev_ioctl_args *ap __unused)
236 {
237 	cdev_t dev = ap->a_head.a_dev;
238 	caddr_t data = ap->a_data;
239 	struct pktgen *pktg = dev->si_drv1;
240 	int error;
241 
242 	lwkt_gettoken(&pktgen_tok);
243 
244 	switch (ap->a_cmd) {
245 	case PKTGENSTART:
246 		error = pktgen_start(pktg, 0);
247 		break;
248 
249 	case PKTGENMQSTART:
250 		error = pktgen_start(pktg, 1);
251 		break;
252 
253 	case PKTGENSCONF:
254 		error = pktgen_config(pktg, (const struct pktgen_conf *)data);
255 		break;
256 
257 	default:
258 		error = EOPNOTSUPP;
259 		break;
260 	}
261 
262 	lwkt_reltoken(&pktgen_tok);
263 	return error;
264 }
265 
266 static int
267 pktgen_config(struct pktgen *pktg, const struct pktgen_conf *conf)
268 {
269 	const struct sockaddr_in *sin;
270 	struct sockaddr_in *dst = NULL;
271 	const struct sockaddr *sa;
272 	struct ifnet *ifp;
273 	size_t dst_size;
274 	int i, error, pktenq;
275 
276 	if (pktg->pktg_flags & (PKTG_F_RUNNING | PKTG_F_CONFIG))
277 		return EBUSY;
278 
279 	if (conf->pc_datalen <= 0 ||
280 	    conf->pc_datalen > ETHERMTU - sizeof(struct udpiphdr))
281 		return EINVAL;
282 	if (conf->pc_duration <= 0)
283 		return EINVAL;
284 
285 	sin = &conf->pc_src;
286 	if (sin->sin_family != AF_INET)
287 		return EPROTONOSUPPORT;
288 	if (sin->sin_port == 0)
289 		return EINVAL;
290 
291 	if (conf->pc_ndst <= 0)
292 		return EINVAL;
293 	dst_size = conf->pc_ndst * sizeof(struct sockaddr_in);
294 
295 	dst = kmalloc(dst_size, M_PKTGEN, M_WAITOK | M_NULLOK);
296 	if (dst == NULL)
297 		return ENOMEM;
298 
299 	error = copyin(conf->pc_dst, dst, dst_size);
300 	if (error)
301 		goto failed;
302 
303 	for (i = 0; i < conf->pc_ndst; ++i) {
304 		sin = &dst[i];
305 		if (sin->sin_family != AF_INET) {
306 			error = EPROTONOSUPPORT;
307 			goto failed;
308 		}
309 		if (sin->sin_port == 0) {
310 			error = EINVAL;
311 			goto failed;
312 		}
313 	}
314 
315 	ifp = ifunit(conf->pc_ifname);
316 	if (ifp == NULL) {
317 		error = ENXIO;
318 		goto failed;
319 	}
320 
321 	pktenq = conf->pc_pktenq;
322 	if (pktenq < 0 || pktenq > ifp->if_snd.altq_maxlen) {
323 		error = ENOBUFS;
324 		goto failed;
325 	} else if (pktenq == 0) {
326 		pktenq = (ifp->if_snd.altq_maxlen * 3) / 4;
327 	}
328 
329 	sa = &conf->pc_dst_lladdr;
330 	if (sa->sa_family != AF_LINK) {
331 		error = EPROTONOSUPPORT;
332 		goto failed;
333 	}
334 	if (sa->sa_len != ETHER_ADDR_LEN) {
335 		error = EPROTONOSUPPORT;
336 		goto failed;
337 	}
338 	if (ETHER_IS_MULTICAST(sa->sa_data) ||
339 	    bcmp(sa->sa_data, ifp->if_broadcastaddr, ifp->if_addrlen) == 0) {
340 		error = EADDRNOTAVAIL;
341 		goto failed;
342 	}
343 
344 	/*
345 	 * Accept the config
346 	 */
347 	pktg->pktg_flags |= PKTG_F_CONFIG;
348 
349 	pktg->pktg_duration = conf->pc_duration;
350 	pktg->pktg_datalen = conf->pc_datalen;
351 	pktg->pktg_pktenq = pktenq;
352 	pktg->pktg_ifp = ifp;
353 	pktg->pktg_src = conf->pc_src;
354 	pktg->pktg_ndst = conf->pc_ndst;
355 	KKASSERT(pktg->pktg_dst == NULL);
356 	pktg->pktg_dst = dst;
357 	bcopy(sa->sa_data, pktg->pktg_dst_lladdr, ETHER_ADDR_LEN);
358 
359 	return 0;
360 
361 failed:
362 	if (dst != NULL)
363 		kfree(dst, M_PKTGEN);
364 	return error;
365 }
366 
367 static void
368 pktgen_start_ifsq(struct pktgen *pktg, struct ifaltq_subque *ifsq)
369 {
370 	struct netmsg_pktgen *np;
371 
372 	np = kmalloc(sizeof(*np), M_LWKTMSG, M_WAITOK);
373 	netmsg_init(&np->np_base, NULL, &netisr_afree_rport, 0,
374 	    pktgen_start_ifsq_handler);
375 	np->np_pktg = pktg;
376 	np->np_ifsq = ifsq;
377 
378 	lwkt_sendmsg(netisr_portfn(ifsq_get_cpuid(ifsq)), &np->np_base.lmsg);
379 }
380 
381 static int
382 pktgen_start(struct pktgen *pktg, int mq)
383 {
384 	struct ifaltq *ifq;
385 
386 	if ((pktg->pktg_flags & PKTG_F_CONFIG) == 0)
387 		return EINVAL;
388 	if (pktg->pktg_flags & PKTG_F_RUNNING)
389 		return EBUSY;
390 	pktg->pktg_flags |= PKTG_F_RUNNING;
391 
392 	ifq = &pktg->pktg_ifp->if_snd;
393 	if (!mq) {
394 		pktgen_ref(pktg);
395 		pktgen_start_ifsq(pktg, ifq_get_subq_default(ifq));
396 	} else {
397 		int i;
398 
399 		for (i = 0; i < ifq->altq_subq_cnt; ++i)
400 			pktgen_ref(pktg);
401 		for (i = 0; i < ifq->altq_subq_cnt; ++i)
402 			pktgen_start_ifsq(pktg, ifq_get_subq(ifq, i));
403 	}
404 	return 0;
405 }
406 
407 static void
408 pktgen_start_ifsq_handler(netmsg_t nmsg)
409 {
410 	struct netmsg_pktgen *np = (struct netmsg_pktgen *)nmsg;
411 	struct pktgen *pktg = np->np_pktg;
412 	struct ifaltq_subque *ifsq = np->np_ifsq;
413 
414 	struct mbuf *m, *head = NULL, **next;
415 	struct ifnet *ifp;
416 	struct pktgen_pcpu *p;
417 	int cpuid, i, alloc_cnt, keep_cnt;
418 
419 	u_short ulen, psum;
420 	int len, ip_len;
421 
422 	/* Reply ASAP */
423 	lwkt_replymsg(&np->np_base.lmsg, 0);
424 
425 	ifp = pktg->pktg_ifp;
426 
427 	cpuid = ifsq_get_cpuid(ifsq);
428 	KKASSERT(cpuid == mycpuid);
429 
430 	p = &pktg->pktg_pcpu[cpuid];
431 
432 	keep_cnt = pktg->pktg_pktenq;
433 	alloc_cnt = keep_cnt * 2;
434 
435 	/*
436 	 * Prefault enough mbuf into mbuf objcache
437 	 */
438 	next = &head;
439 	for (i = 0; i < alloc_cnt; ++i) {
440 		MGETHDR(m, MB_WAIT, MT_DATA);
441 		*next = m;
442 		next = &m->m_nextpkt;
443 	}
444 
445 	for (i = 0; i < alloc_cnt - keep_cnt; ++i) {
446 		m = head;
447 		head = m->m_nextpkt;
448 		m->m_nextpkt = NULL;
449 		m_freem(m);
450 	}
451 	KKASSERT(head != NULL);
452 
453 	/*
454 	 * Setup the packets' data
455 	 */
456 	ip_len = pktg->pktg_datalen + sizeof(struct udpiphdr);
457 	len = ip_len + ETHER_HDR_LEN;
458 
459 	psum = htons((u_short)pktg->pktg_datalen + sizeof(struct udphdr) +
460 	    IPPROTO_UDP);
461 	ulen = htons(pktg->pktg_datalen + sizeof(struct udphdr));
462 
463 	m = head;
464 	i = 0;
465 	while (m != NULL) {
466 		struct mbuf *nextm;
467 		const struct sockaddr_in *dst;
468 		struct pktgen_buf *pb;
469 		struct ip *ip;
470 		struct udpiphdr *ui;
471 		struct ether_header *eh;
472 
473 		pktgen_ref(pktg);
474 
475 		pb = kmalloc(sizeof(*pb), M_PKTGEN, M_WAITOK | M_ZERO);
476 		pb->pb_ifp = ifp;
477 		pb->pb_ifsq = ifsq;
478 		pb->pb_inuse = 1;
479 		pb->pb_buf = kmalloc(PKTGEN_BUFSZ, M_PKTGEN, M_WAITOK);
480 		pb->pb_len = len;
481 		pb->pb_cpuid = cpuid;
482 		pb->pb_pktg = pktg;
483 		netmsg_init(&pb->pb_nmsg, NULL, &netisr_adone_rport, 0,
484 		    pktgen_buf_send);
485 		LIST_INSERT_HEAD(&p->pktg_buflist, pb, pb_link);
486 
487 		dst = &pktg->pktg_dst[i % pktg->pktg_ndst];
488 		++i;
489 
490 		m->m_ext.ext_arg = pb;
491 		m->m_ext.ext_buf = pb->pb_buf;
492 		m->m_ext.ext_free = pktgen_buf_free;
493 		m->m_ext.ext_ref = pktgen_buf_ref;
494 		m->m_ext.ext_size = PKTGEN_BUFSZ;
495 
496 		m->m_data = m->m_ext.ext_buf;
497 		m->m_flags |= M_EXT;
498 		m->m_len = m->m_pkthdr.len = len;
499 
500 		m->m_data += ETHER_HDR_LEN;
501 		m->m_len -= ETHER_HDR_LEN;
502 		m->m_pkthdr.len -= ETHER_HDR_LEN;
503 
504 		ui = mtod(m, struct udpiphdr *);
505 		ui->ui_pr = IPPROTO_UDP;
506 		ui->ui_src.s_addr = pktg->pktg_src.sin_addr.s_addr;
507 		ui->ui_dst.s_addr = dst->sin_addr.s_addr;
508 		ui->ui_sport = pktg->pktg_src.sin_port;
509 		ui->ui_dport = dst->sin_port;
510 		ui->ui_ulen = ulen;
511 		ui->ui_sum = in_pseudo(ui->ui_src.s_addr, ui->ui_dst.s_addr,
512 		    psum);
513 		m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
514 
515 		ip = (struct ip *)ui;
516 		ip->ip_len = ip_len;
517 		ip->ip_ttl = 64;	/* XXX */
518 		ip->ip_tos = 0;		/* XXX */
519 		ip->ip_vhl = IP_VHL_BORING;
520 		ip->ip_off = 0;
521 		ip->ip_sum = 0;
522 		ip->ip_id = ip_newid();
523 
524 		in_delayed_cksum(m);
525 
526 		ip->ip_len = htons(ip->ip_len);
527 		ip->ip_sum = in_cksum_hdr(ip);
528 
529 		m->m_data -= ETHER_HDR_LEN;
530 		m->m_len += ETHER_HDR_LEN;
531 		m->m_pkthdr.len += ETHER_HDR_LEN;
532 
533 		eh = mtod(m, struct ether_header *);
534 		bcopy(pktg->pktg_dst_lladdr, eh->ether_dhost, ETHER_ADDR_LEN);
535 		bcopy(IF_LLADDR(ifp), eh->ether_shost, ETHER_ADDR_LEN);
536 		eh->ether_type = htons(ETHERTYPE_IP);
537 
538 		nextm = m->m_nextpkt;
539 		m->m_nextpkt = NULL;
540 
541 		ifq_dispatch(ifp, m, NULL);
542 
543 		m = nextm;
544 	}
545 
546 	callout_reset(&p->pktg_stop, pktg->pktg_duration * hz,
547 	    pktgen_pcpu_stop_cb, p);
548 
549 	pktgen_free(pktg);
550 }
551 
552 static void
553 pktgen_mbuf(struct pktgen_buf *pb, struct mbuf *m)
554 {
555 	m->m_ext.ext_arg = pb;
556 	m->m_ext.ext_buf = pb->pb_buf;
557 	m->m_ext.ext_free = pktgen_buf_free;
558 	m->m_ext.ext_ref = pktgen_buf_ref;
559 	m->m_ext.ext_size = PKTGEN_BUFSZ;
560 
561 	m->m_data = m->m_ext.ext_buf;
562 	m->m_flags |= M_EXT;
563 	m->m_len = m->m_pkthdr.len = pb->pb_len;
564 }
565 
566 static void
567 pktgen_buf_send(netmsg_t msg)
568 {
569 	struct pktgen_buf *pb = (struct pktgen_buf *)msg;
570 	struct mbuf *m;
571 
572 	KKASSERT(&curthread->td_msgport == netisr_portfn(pb->pb_cpuid));
573 
574 	crit_enter();
575 	lwkt_replymsg(&pb->pb_nmsg.lmsg, 0);
576 	crit_exit();
577 
578 	MGETHDR(m, MB_WAIT, MT_DATA);
579 	pktgen_mbuf(pb, m);
580 	ifq_dispatch(pb->pb_ifp, m, NULL);
581 }
582 
583 static void
584 pktgen_buf_free(void *arg)
585 {
586 	struct pktgen_buf *pb = arg;
587 	struct mbuf *m;
588 
589 	KKASSERT(pb->pb_inuse > 0);
590 	if (pb->pb_done) {
591 		if (atomic_fetchadd_int(&pb->pb_inuse, -1) == 1) {
592 			struct pktgen *pktg;
593 
594 			pktg = pb->pb_pktg;
595 			crit_enter();
596 			LIST_REMOVE(pb, pb_link);
597 			crit_exit();
598 			kfree(pb->pb_buf, M_PKTGEN);
599 			kfree(pb, M_PKTGEN);
600 
601 			pktgen_free(pktg);
602 		}
603 		return;
604 	}
605 
606 	if (&curthread->td_msgport != netisr_portfn(pb->pb_cpuid)) {
607 		KKASSERT(pb->pb_cpuid == mycpuid);
608 		crit_enter();
609 		KKASSERT(pb->pb_nmsg.lmsg.ms_flags & MSGF_DONE);
610 		lwkt_sendmsg(netisr_portfn(pb->pb_cpuid), &pb->pb_nmsg.lmsg);
611 		crit_exit();
612 		return;
613 	}
614 
615 	MGETHDR(m, MB_WAIT, MT_DATA);
616 	pktgen_mbuf(pb, m);
617 	ifsq_enqueue(pb->pb_ifsq, m, NULL);
618 }
619 
620 static void
621 pktgen_buf_ref(void *arg)
622 {
623 	struct pktgen_buf *pb = arg;
624 
625 	panic("%s should never be called\n", __func__);
626 
627 	KKASSERT(pb->pb_inuse > 0);
628 	atomic_add_int(&pb->pb_inuse, 1);
629 }
630 
631 static void
632 pktgen_free(struct pktgen *pktg)
633 {
634 	KKASSERT(pktg->pktg_refcnt > 0);
635 	if (atomic_fetchadd_int(&pktg->pktg_refcnt, -1) == 1) {
636 		int i;
637 
638 		if (pktg->pktg_dst != NULL)
639 			kfree(pktg->pktg_dst, M_PKTGEN);
640 
641 		for (i = 0; i < ncpus; ++i)
642 			KKASSERT(LIST_EMPTY(&pktg->pktg_pcpu[i].pktg_buflist));
643 		kfree(pktg, M_PKTGEN);
644 	}
645 
646 	KKASSERT(pktgen_refcnt > 0);
647 	atomic_subtract_int(&pktgen_refcnt, 1);
648 }
649 
650 static void
651 pktgen_pcpu_stop_cb(void *arg)
652 {
653 	struct pktgen_pcpu *p = arg;
654 	struct pktgen_buf *pb;
655 
656 	crit_enter();
657 	LIST_FOREACH(pb, &p->pktg_buflist, pb_link)
658 		pb->pb_done = 1;
659 	crit_exit();
660 }
661 
662 static void
663 pktgen_ref(struct pktgen *pktg)
664 {
665 	atomic_add_int(&pktg->pktg_refcnt, 1);
666 	atomic_add_int(&pktgen_refcnt, 1);
667 }
668