xref: /dragonfly/tools/tools/netrate/pktgen/pktgen.c (revision 279dd846) 
1 /*
2  * Copyright (c) 2012 The DragonFly Project.  All rights reserved.
3  *
4  * This code is derived from software contributed to The DragonFly Project
5  * by Sepherosa Ziehau <sepherosa@gmail.com>
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in
15  *    the documentation and/or other materials provided with the
16  *    distribution.
17  * 3. Neither the name of The DragonFly Project nor the names of its
18  *    contributors may be used to endorse or promote products derived
19  *    from this software without specific, prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  */
34 
35 #define _IP_VHL
36 
37 #include <sys/param.h>
38 #include <sys/conf.h>
39 #include <sys/device.h>
40 #include <sys/in_cksum.h>
41 #include <sys/kernel.h>
42 #include <sys/malloc.h>
43 #include <sys/mbuf.h>
44 #include <sys/proc.h>
45 #include <sys/priv.h>
46 #include <sys/queue.h>
47 #include <sys/socket.h>
48 #include <sys/systm.h>
49 #include <sys/serialize.h>
50 
51 #include <net/if.h>
52 #include <net/if_dl.h>
53 #include <net/if_var.h>
54 #include <net/ifq_var.h>
55 #include <net/ethernet.h>
56 #include <net/netmsg2.h>
57 #include <net/netisr2.h>
58 
59 #include <netinet/in.h>
60 #include <netinet/in_pcb.h>
61 #include <netinet/ip.h>
62 #include <netinet/udp_var.h>
63 
64 #include "pktgen.h"
65 
66 #define CDEV_NAME	"pktg"
67 
68 #define PKTGEN_BUFSZ	2048
69 
70 #ifndef PKTGEN_DEVCNT
71 #define PKTGEN_DEVCNT	4
72 #endif
73 
74 struct pktgen;
75 
76 struct netmsg_pktgen {
77 	struct netmsg_base	np_base;
78 	struct pktgen		*np_pktg;
79 	struct ifaltq_subque	*np_ifsq;
80 };
81 
82 struct pktgen_buf {
83 	struct netmsg_base	pb_nmsg;	/* MUST BE THE FIRST */
84 	void			*pb_buf;
85 	volatile int		pb_done;
86 	int			pb_inuse;
87 	struct ifnet		*pb_ifp;
88 	struct ifaltq_subque	*pb_ifsq;
89 	int			pb_len;
90 	int			pb_cpuid;
91 	struct pktgen		*pb_pktg;
92 	LIST_ENTRY(pktgen_buf)	pb_link;
93 };
94 
95 struct pktgen_pcpu {
96 	struct callout		pktg_stop;
97 	LIST_HEAD(, pktgen_buf)	pktg_buflist;
98 };
99 
100 struct pktgen {
101 	uint32_t		pktg_flags;	/* PKTG_F_ */
102 	int			pktg_refcnt;
103 
104 	int			pktg_duration;
105 
106 	int			pktg_datalen;
107 	struct ifnet		*pktg_ifp;
108 
109 	int			pktg_pktenq;
110 
111 	struct sockaddr_in	pktg_src;
112 	int			pktg_ndst;
113 	struct sockaddr_in	*pktg_dst;
114 	uint8_t			pktg_dst_lladdr[ETHER_ADDR_LEN];
115 
116 	struct pktgen_pcpu	pktg_pcpu[MAXCPU];
117 };
118 
119 #define PKTG_F_CONFIG		0x1
120 #define PKTG_F_RUNNING		0x4
121 #define PKTG_F_SWITCH_SRCDST	0x8
122 
123 static int 		pktgen_modevent(module_t, int, void *);
124 
125 static void		pktgen_buf_free(void *);
126 static void		pktgen_buf_ref(void *);
127 static void		pktgen_buf_send(netmsg_t);
128 
129 static int		pktgen_config(struct pktgen *,
130 			    const struct pktgen_conf *);
131 static int		pktgen_start(struct pktgen *, int);
132 static void		pktgen_free(struct pktgen *);
133 static void		pktgen_ref(struct pktgen *);
134 static void		pktgen_pcpu_stop_cb(void *);
135 static void		pktgen_mbuf(struct pktgen_buf *, struct mbuf *);
136 static void		pktgen_start_ifsq(struct pktgen *,
137 			    struct ifaltq_subque *);
138 static void		pktgen_start_ifsq_handler(netmsg_t);
139 
140 static d_open_t		pktgen_open;
141 static d_close_t	pktgen_close;
142 static d_ioctl_t	pktgen_ioctl;
143 
144 static struct dev_ops	pktgen_ops = {
145 	{ CDEV_NAME, 0, D_MPSAFE },
146 	.d_open =	pktgen_open,
147 	.d_close =	pktgen_close,
148 	.d_ioctl =	pktgen_ioctl,
149 };
150 
151 static volatile int		pktgen_refcnt;
152 static struct lwkt_token pktgen_tok = LWKT_TOKEN_INITIALIZER(pktgen_token);
153 
154 MALLOC_DECLARE(M_PKTGEN);
155 MALLOC_DEFINE(M_PKTGEN, CDEV_NAME, "Packet generator");
156 
157 DEV_MODULE(pktgen, pktgen_modevent, NULL);
158 
159 static int
160 pktgen_modevent(module_t mod, int type, void *data)
161 {
162 	int error = 0, i;
163 
164 	switch (type) {
165 	case MOD_LOAD:
166 		for (i = 0; i < PKTGEN_DEVCNT; ++i) {
167 			make_dev(&pktgen_ops, 0, UID_ROOT, GID_WHEEL, 0600,
168 			    CDEV_NAME"%d", i);
169 		}
170 		break;
171 
172 	case MOD_UNLOAD:
173 		if (pktgen_refcnt > 0)
174 			return EBUSY;
175 		dev_ops_remove_all(&pktgen_ops);
176 		break;
177 
178 	default:
179 		error = EOPNOTSUPP;
180 		break;
181 	}
182 	return error;
183 }
184 
185 static int
186 pktgen_open(struct dev_open_args *ap)
187 {
188 	cdev_t dev = ap->a_head.a_dev;
189 	struct pktgen *pktg;
190 	int error, i;
191 
192 	error = priv_check_cred(ap->a_cred, PRIV_ROOT, 0);
193 	if (error)
194 		return error;
195 
196 	lwkt_gettoken(&pktgen_tok);
197 
198 	if (dev->si_drv1 != NULL) {
199 		lwkt_reltoken(&pktgen_tok);
200 		return EBUSY;
201 	}
202 
203 	pktg = kmalloc(sizeof(*pktg), M_PKTGEN, M_ZERO | M_WAITOK);
204 	for (i = 0; i < ncpus; ++i) {
205 		struct pktgen_pcpu *p = &pktg->pktg_pcpu[i];
206 
207 		callout_init_mp(&p->pktg_stop);
208 		LIST_INIT(&p->pktg_buflist);
209 	}
210 
211 	dev->si_drv1 = pktg;
212 	pktg->pktg_refcnt = 1;
213 
214 	atomic_add_int(&pktgen_refcnt, 1);
215 
216 	lwkt_reltoken(&pktgen_tok);
217 	return 0;
218 }
219 
220 static int
221 pktgen_close(struct dev_close_args *ap)
222 {
223 	cdev_t dev = ap->a_head.a_dev;
224 	struct pktgen *pktg = dev->si_drv1;
225 
226 	lwkt_gettoken(&pktgen_tok);
227 	dev->si_drv1 = NULL;
228 	lwkt_reltoken(&pktgen_tok);
229 
230 	pktgen_free(pktg);
231 
232 	return 0;
233 }
234 
235 static int
236 pktgen_ioctl(struct dev_ioctl_args *ap __unused)
237 {
238 	cdev_t dev = ap->a_head.a_dev;
239 	caddr_t data = ap->a_data;
240 	struct pktgen *pktg = dev->si_drv1;
241 	int error;
242 
243 	lwkt_gettoken(&pktgen_tok);
244 
245 	switch (ap->a_cmd) {
246 	case PKTGENSTART:
247 		error = pktgen_start(pktg, 0);
248 		break;
249 
250 	case PKTGENMQSTART:
251 		error = pktgen_start(pktg, 1);
252 		break;
253 
254 	case PKTGENSCONF:
255 		error = pktgen_config(pktg, (const struct pktgen_conf *)data);
256 		break;
257 
258 	default:
259 		error = EOPNOTSUPP;
260 		break;
261 	}
262 
263 	lwkt_reltoken(&pktgen_tok);
264 	return error;
265 }
266 
267 static int
268 pktgen_config(struct pktgen *pktg, const struct pktgen_conf *conf)
269 {
270 	const struct sockaddr_in *sin;
271 	struct sockaddr_in *dst = NULL;
272 	const struct sockaddr *sa;
273 	struct ifnet *ifp;
274 	size_t dst_size;
275 	int i, error, pktenq;
276 
277 	if (pktg->pktg_flags & (PKTG_F_RUNNING | PKTG_F_CONFIG))
278 		return EBUSY;
279 
280 	if (conf->pc_datalen <= 0 ||
281 	    conf->pc_datalen > ETHERMTU - sizeof(struct udpiphdr))
282 		return EINVAL;
283 	if (conf->pc_duration <= 0)
284 		return EINVAL;
285 
286 	sin = &conf->pc_src;
287 	if (sin->sin_family != AF_INET)
288 		return EPROTONOSUPPORT;
289 	if (sin->sin_port == 0)
290 		return EINVAL;
291 
292 	if (conf->pc_ndst <= 0)
293 		return EINVAL;
294 	dst_size = conf->pc_ndst * sizeof(struct sockaddr_in);
295 
296 	dst = kmalloc(dst_size, M_PKTGEN, M_WAITOK | M_NULLOK);
297 	if (dst == NULL)
298 		return ENOMEM;
299 
300 	error = copyin(conf->pc_dst, dst, dst_size);
301 	if (error)
302 		goto failed;
303 
304 	for (i = 0; i < conf->pc_ndst; ++i) {
305 		sin = &dst[i];
306 		if (sin->sin_family != AF_INET) {
307 			error = EPROTONOSUPPORT;
308 			goto failed;
309 		}
310 		if (sin->sin_port == 0) {
311 			error = EINVAL;
312 			goto failed;
313 		}
314 	}
315 
316 	ifp = ifunit(conf->pc_ifname);
317 	if (ifp == NULL) {
318 		error = ENXIO;
319 		goto failed;
320 	}
321 
322 	pktenq = conf->pc_pktenq;
323 	if (pktenq < 0 || pktenq > ifp->if_snd.altq_maxlen) {
324 		error = ENOBUFS;
325 		goto failed;
326 	} else if (pktenq == 0) {
327 		pktenq = (ifp->if_snd.altq_maxlen * 3) / 4;
328 	}
329 
330 	sa = &conf->pc_dst_lladdr;
331 	if (sa->sa_family != AF_LINK) {
332 		error = EPROTONOSUPPORT;
333 		goto failed;
334 	}
335 	if (sa->sa_len != ETHER_ADDR_LEN) {
336 		error = EPROTONOSUPPORT;
337 		goto failed;
338 	}
339 	if (ETHER_IS_MULTICAST(sa->sa_data) ||
340 	    bcmp(sa->sa_data, ifp->if_broadcastaddr, ifp->if_addrlen) == 0) {
341 		error = EADDRNOTAVAIL;
342 		goto failed;
343 	}
344 
345 	/*
346 	 * Accept the config
347 	 */
348 	pktg->pktg_flags |= PKTG_F_CONFIG;
349 
350 	if (conf->pc_flags & PKTGEN_FLAG_SWITCH_SRCDST)
351 		pktg->pktg_flags |= PKTG_F_SWITCH_SRCDST;
352 	pktg->pktg_duration = conf->pc_duration;
353 	pktg->pktg_datalen = conf->pc_datalen;
354 	pktg->pktg_pktenq = pktenq;
355 	pktg->pktg_ifp = ifp;
356 	pktg->pktg_src = conf->pc_src;
357 	pktg->pktg_ndst = conf->pc_ndst;
358 	KKASSERT(pktg->pktg_dst == NULL);
359 	pktg->pktg_dst = dst;
360 	bcopy(sa->sa_data, pktg->pktg_dst_lladdr, ETHER_ADDR_LEN);
361 
362 	return 0;
363 
364 failed:
365 	if (dst != NULL)
366 		kfree(dst, M_PKTGEN);
367 	return error;
368 }
369 
370 static void
371 pktgen_start_ifsq(struct pktgen *pktg, struct ifaltq_subque *ifsq)
372 {
373 	struct netmsg_pktgen *np;
374 
375 	np = kmalloc(sizeof(*np), M_LWKTMSG, M_WAITOK);
376 	netmsg_init(&np->np_base, NULL, &netisr_afree_rport, 0,
377 	    pktgen_start_ifsq_handler);
378 	np->np_pktg = pktg;
379 	np->np_ifsq = ifsq;
380 
381 	lwkt_sendmsg(netisr_cpuport(ifsq_get_cpuid(ifsq)), &np->np_base.lmsg);
382 }
383 
384 static int
385 pktgen_start(struct pktgen *pktg, int mq)
386 {
387 	struct ifaltq *ifq;
388 
389 	if ((pktg->pktg_flags & PKTG_F_CONFIG) == 0)
390 		return EINVAL;
391 	if (pktg->pktg_flags & PKTG_F_RUNNING)
392 		return EBUSY;
393 	pktg->pktg_flags |= PKTG_F_RUNNING;
394 
395 	ifq = &pktg->pktg_ifp->if_snd;
396 	if (!mq) {
397 		pktgen_ref(pktg);
398 		pktgen_start_ifsq(pktg, ifq_get_subq_default(ifq));
399 	} else {
400 		int i;
401 
402 		for (i = 0; i < ifq->altq_subq_cnt; ++i)
403 			pktgen_ref(pktg);
404 		for (i = 0; i < ifq->altq_subq_cnt; ++i)
405 			pktgen_start_ifsq(pktg, ifq_get_subq(ifq, i));
406 	}
407 	return 0;
408 }
409 
410 static void
411 pktgen_start_ifsq_handler(netmsg_t nmsg)
412 {
413 	struct netmsg_pktgen *np = (struct netmsg_pktgen *)nmsg;
414 	struct pktgen *pktg = np->np_pktg;
415 	struct ifaltq_subque *ifsq = np->np_ifsq;
416 
417 	struct mbuf *m, *head = NULL, **next;
418 	struct ifnet *ifp;
419 	struct pktgen_pcpu *p;
420 	int cpuid, i, alloc_cnt, keep_cnt;
421 
422 	u_short ulen, psum;
423 	int len, ip_len;
424 
425 	/* Reply ASAP */
426 	lwkt_replymsg(&np->np_base.lmsg, 0);
427 
428 	ifp = pktg->pktg_ifp;
429 
430 	cpuid = ifsq_get_cpuid(ifsq);
431 	KKASSERT(cpuid == mycpuid);
432 
433 	p = &pktg->pktg_pcpu[cpuid];
434 
435 	keep_cnt = pktg->pktg_pktenq;
436 	alloc_cnt = keep_cnt * 2;
437 
438 	/*
439 	 * Prefault enough mbuf into mbuf objcache
440 	 */
441 	next = &head;
442 	for (i = 0; i < alloc_cnt; ++i) {
443 		MGETHDR(m, M_WAITOK, MT_DATA);
444 		*next = m;
445 		next = &m->m_nextpkt;
446 	}
447 
448 	for (i = 0; i < alloc_cnt - keep_cnt; ++i) {
449 		m = head;
450 		head = m->m_nextpkt;
451 		m->m_nextpkt = NULL;
452 		m_freem(m);
453 	}
454 	KKASSERT(head != NULL);
455 
456 	/*
457 	 * Setup the packets' data
458 	 */
459 	ip_len = pktg->pktg_datalen + sizeof(struct udpiphdr);
460 	len = ip_len + ETHER_HDR_LEN;
461 
462 	psum = htons((u_short)pktg->pktg_datalen + sizeof(struct udphdr) +
463 	    IPPROTO_UDP);
464 	ulen = htons(pktg->pktg_datalen + sizeof(struct udphdr));
465 
466 	m = head;
467 	i = 0;
468 	while (m != NULL) {
469 		struct mbuf *nextm;
470 		const struct sockaddr_in *dst;
471 		struct pktgen_buf *pb;
472 		struct ip *ip;
473 		struct udpiphdr *ui;
474 		struct ether_header *eh;
475 
476 		pktgen_ref(pktg);
477 
478 		pb = kmalloc(sizeof(*pb), M_PKTGEN, M_WAITOK | M_ZERO);
479 		pb->pb_ifp = ifp;
480 		pb->pb_ifsq = ifsq;
481 		pb->pb_inuse = 1;
482 		pb->pb_buf = kmalloc(PKTGEN_BUFSZ, M_PKTGEN, M_WAITOK);
483 		pb->pb_len = len;
484 		pb->pb_cpuid = cpuid;
485 		pb->pb_pktg = pktg;
486 		netmsg_init(&pb->pb_nmsg, NULL, &netisr_adone_rport, 0,
487 		    pktgen_buf_send);
488 		LIST_INSERT_HEAD(&p->pktg_buflist, pb, pb_link);
489 
490 		dst = &pktg->pktg_dst[i % pktg->pktg_ndst];
491 		++i;
492 
493 		m->m_ext.ext_arg = pb;
494 		m->m_ext.ext_buf = pb->pb_buf;
495 		m->m_ext.ext_free = pktgen_buf_free;
496 		m->m_ext.ext_ref = pktgen_buf_ref;
497 		m->m_ext.ext_size = PKTGEN_BUFSZ;
498 
499 		m->m_data = m->m_ext.ext_buf;
500 		m->m_flags |= M_EXT;
501 		m->m_len = m->m_pkthdr.len = len;
502 
503 		m->m_data += ETHER_HDR_LEN;
504 		m->m_len -= ETHER_HDR_LEN;
505 		m->m_pkthdr.len -= ETHER_HDR_LEN;
506 
507 		ui = mtod(m, struct udpiphdr *);
508 		ui->ui_pr = IPPROTO_UDP;
509 		if (pktg->pktg_flags & PKTG_F_SWITCH_SRCDST) {
510 			ui->ui_src.s_addr = dst->sin_addr.s_addr;
511 			ui->ui_dst.s_addr = pktg->pktg_src.sin_addr.s_addr;
512 			ui->ui_sport = dst->sin_port;
513 			ui->ui_dport = pktg->pktg_src.sin_port;
514 		} else {
515 			ui->ui_src.s_addr = pktg->pktg_src.sin_addr.s_addr;
516 			ui->ui_dst.s_addr = dst->sin_addr.s_addr;
517 			ui->ui_sport = pktg->pktg_src.sin_port;
518 			ui->ui_dport = dst->sin_port;
519 		}
520 		ui->ui_ulen = ulen;
521 		ui->ui_sum = in_pseudo(ui->ui_src.s_addr, ui->ui_dst.s_addr,
522 		    psum);
523 		m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
524 
525 		ip = (struct ip *)ui;
526 		ip->ip_len = ip_len;
527 		ip->ip_ttl = 64;	/* XXX */
528 		ip->ip_tos = 0;		/* XXX */
529 		ip->ip_vhl = IP_VHL_BORING;
530 		ip->ip_off = 0;
531 		ip->ip_sum = 0;
532 		ip->ip_id = ip_newid();
533 
534 		in_delayed_cksum(m);
535 
536 		ip->ip_len = htons(ip->ip_len);
537 		ip->ip_sum = in_cksum_hdr(ip);
538 
539 		m->m_data -= ETHER_HDR_LEN;
540 		m->m_len += ETHER_HDR_LEN;
541 		m->m_pkthdr.len += ETHER_HDR_LEN;
542 
543 		eh = mtod(m, struct ether_header *);
544 		bcopy(pktg->pktg_dst_lladdr, eh->ether_dhost, ETHER_ADDR_LEN);
545 		bcopy(IF_LLADDR(ifp), eh->ether_shost, ETHER_ADDR_LEN);
546 		eh->ether_type = htons(ETHERTYPE_IP);
547 
548 		nextm = m->m_nextpkt;
549 		m->m_nextpkt = NULL;
550 
551 		ifq_dispatch(ifp, m, NULL);
552 
553 		m = nextm;
554 	}
555 
556 	callout_reset(&p->pktg_stop, pktg->pktg_duration * hz,
557 	    pktgen_pcpu_stop_cb, p);
558 
559 	pktgen_free(pktg);
560 }
561 
562 static void
563 pktgen_mbuf(struct pktgen_buf *pb, struct mbuf *m)
564 {
565 	m->m_ext.ext_arg = pb;
566 	m->m_ext.ext_buf = pb->pb_buf;
567 	m->m_ext.ext_free = pktgen_buf_free;
568 	m->m_ext.ext_ref = pktgen_buf_ref;
569 	m->m_ext.ext_size = PKTGEN_BUFSZ;
570 
571 	m->m_data = m->m_ext.ext_buf;
572 	m->m_flags |= M_EXT;
573 	m->m_len = m->m_pkthdr.len = pb->pb_len;
574 }
575 
576 static void
577 pktgen_buf_send(netmsg_t msg)
578 {
579 	struct pktgen_buf *pb = (struct pktgen_buf *)msg;
580 	struct mbuf *m;
581 
582 	KKASSERT(&curthread->td_msgport == netisr_cpuport(pb->pb_cpuid));
583 
584 	crit_enter();
585 	lwkt_replymsg(&pb->pb_nmsg.lmsg, 0);
586 	crit_exit();
587 
588 	MGETHDR(m, M_WAITOK, MT_DATA);
589 	pktgen_mbuf(pb, m);
590 	ifq_dispatch(pb->pb_ifp, m, NULL);
591 }
592 
593 static void
594 pktgen_buf_free(void *arg)
595 {
596 	struct pktgen_buf *pb = arg;
597 	struct mbuf *m;
598 
599 	KKASSERT(pb->pb_inuse > 0);
600 	if (pb->pb_done) {
601 		if (atomic_fetchadd_int(&pb->pb_inuse, -1) == 1) {
602 			struct pktgen *pktg;
603 
604 			pktg = pb->pb_pktg;
605 			crit_enter();
606 			LIST_REMOVE(pb, pb_link);
607 			crit_exit();
608 			kfree(pb->pb_buf, M_PKTGEN);
609 			kfree(pb, M_PKTGEN);
610 
611 			pktgen_free(pktg);
612 		}
613 		return;
614 	}
615 
616 	if (&curthread->td_msgport != netisr_cpuport(pb->pb_cpuid)) {
617 		KKASSERT(pb->pb_cpuid == mycpuid);
618 		crit_enter();
619 		KKASSERT(pb->pb_nmsg.lmsg.ms_flags & MSGF_DONE);
620 		lwkt_sendmsg(netisr_cpuport(pb->pb_cpuid), &pb->pb_nmsg.lmsg);
621 		crit_exit();
622 		return;
623 	}
624 
625 	MGETHDR(m, M_WAITOK, MT_DATA);
626 	pktgen_mbuf(pb, m);
627 	ifsq_enqueue(pb->pb_ifsq, m, NULL);
628 }
629 
630 static void
631 pktgen_buf_ref(void *arg)
632 {
633 	struct pktgen_buf *pb = arg;
634 
635 	panic("%s should never be called\n", __func__);
636 
637 	KKASSERT(pb->pb_inuse > 0);
638 	atomic_add_int(&pb->pb_inuse, 1);
639 }
640 
641 static void
642 pktgen_free(struct pktgen *pktg)
643 {
644 	KKASSERT(pktg->pktg_refcnt > 0);
645 	if (atomic_fetchadd_int(&pktg->pktg_refcnt, -1) == 1) {
646 		int i;
647 
648 		if (pktg->pktg_dst != NULL)
649 			kfree(pktg->pktg_dst, M_PKTGEN);
650 
651 		for (i = 0; i < ncpus; ++i)
652 			KKASSERT(LIST_EMPTY(&pktg->pktg_pcpu[i].pktg_buflist));
653 		kfree(pktg, M_PKTGEN);
654 	}
655 
656 	KKASSERT(pktgen_refcnt > 0);
657 	atomic_subtract_int(&pktgen_refcnt, 1);
658 }
659 
660 static void
661 pktgen_pcpu_stop_cb(void *arg)
662 {
663 	struct pktgen_pcpu *p = arg;
664 	struct pktgen_buf *pb;
665 
666 	crit_enter();
667 	LIST_FOREACH(pb, &p->pktg_buflist, pb_link)
668 		pb->pb_done = 1;
669 	crit_exit();
670 }
671 
672 static void
673 pktgen_ref(struct pktgen *pktg)
674 {
675 	atomic_add_int(&pktg->pktg_refcnt, 1);
676 	atomic_add_int(&pktgen_refcnt, 1);
677 }
678