xref: /dragonfly/tools/tools/netrate/pktgen/pktgen.c (revision d4ef6694) 
1 /*
2  * Copyright (c) 2012 The DragonFly Project.  All rights reserved.
3  *
4  * This code is derived from software contributed to The DragonFly Project
5  * by Sepherosa Ziehau <sepherosa@gmail.com>
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in
15  *    the documentation and/or other materials provided with the
16  *    distribution.
17  * 3. Neither the name of The DragonFly Project nor the names of its
18  *    contributors may be used to endorse or promote products derived
19  *    from this software without specific, prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  * $DragonFly: src/tools/tools/netrate/pktgen/pktgen.c,v 1.4 2008/04/02 14:18:55 sephe Exp $
35  */
36 
37 #define _IP_VHL
38 
39 #include <sys/param.h>
40 #include <sys/conf.h>
41 #include <sys/device.h>
42 #include <sys/in_cksum.h>
43 #include <sys/kernel.h>
44 #include <sys/malloc.h>
45 #include <sys/mbuf.h>
46 #include <sys/proc.h>
47 #include <sys/priv.h>
48 #include <sys/queue.h>
49 #include <sys/socket.h>
50 #include <sys/systm.h>
51 #include <sys/serialize.h>
52 
53 #include <net/if.h>
54 #include <net/if_dl.h>
55 #include <net/if_var.h>
56 #include <net/ifq_var.h>
57 #include <net/ethernet.h>
58 #include <net/netmsg2.h>
59 #include <net/netisr2.h>
60 
61 #include <netinet/in.h>
62 #include <netinet/in_pcb.h>
63 #include <netinet/ip.h>
64 #include <netinet/udp_var.h>
65 
66 #include "pktgen.h"
67 
68 #define CDEV_NAME	"pktg"
69 
70 #define PKTGEN_BUFSZ	2048
71 
72 #ifndef PKTGEN_DEVCNT
73 #define PKTGEN_DEVCNT	4
74 #endif
75 
76 struct pktgen;
77 
78 struct netmsg_pktgen {
79 	struct netmsg_base	np_base;
80 	struct pktgen		*np_pktg;
81 	struct ifaltq_subque	*np_ifsq;
82 };
83 
84 struct pktgen_buf {
85 	struct netmsg_base	pb_nmsg;	/* MUST BE THE FIRST */
86 	void			*pb_buf;
87 	volatile int		pb_done;
88 	int			pb_inuse;
89 	struct ifnet		*pb_ifp;
90 	struct ifaltq_subque	*pb_ifsq;
91 	int			pb_len;
92 	int			pb_cpuid;
93 	struct pktgen		*pb_pktg;
94 	LIST_ENTRY(pktgen_buf)	pb_link;
95 };
96 
97 struct pktgen_pcpu {
98 	struct callout		pktg_stop;
99 	LIST_HEAD(, pktgen_buf)	pktg_buflist;
100 };
101 
102 struct pktgen {
103 	uint32_t		pktg_flags;	/* PKTG_F_ */
104 	int			pktg_refcnt;
105 
106 	int			pktg_duration;
107 
108 	int			pktg_datalen;
109 	struct ifnet		*pktg_ifp;
110 
111 	int			pktg_pktenq;
112 
113 	struct sockaddr_in	pktg_src;
114 	int			pktg_ndst;
115 	struct sockaddr_in	*pktg_dst;
116 	uint8_t			pktg_dst_lladdr[ETHER_ADDR_LEN];
117 
118 	struct pktgen_pcpu	pktg_pcpu[MAXCPU];
119 };
120 
121 #define PKTG_F_CONFIG		0x1
122 #define PKTG_F_RUNNING		0x4
123 #define PKTG_F_SWITCH_SRCDST	0x8
124 
125 static int 		pktgen_modevent(module_t, int, void *);
126 
127 static void		pktgen_buf_free(void *);
128 static void		pktgen_buf_ref(void *);
129 static void		pktgen_buf_send(netmsg_t);
130 
131 static int		pktgen_config(struct pktgen *,
132 			    const struct pktgen_conf *);
133 static int		pktgen_start(struct pktgen *, int);
134 static void		pktgen_free(struct pktgen *);
135 static void		pktgen_ref(struct pktgen *);
136 static void		pktgen_pcpu_stop_cb(void *);
137 static void		pktgen_mbuf(struct pktgen_buf *, struct mbuf *);
138 static void		pktgen_start_ifsq(struct pktgen *,
139 			    struct ifaltq_subque *);
140 static void		pktgen_start_ifsq_handler(netmsg_t);
141 
142 static d_open_t		pktgen_open;
143 static d_close_t	pktgen_close;
144 static d_ioctl_t	pktgen_ioctl;
145 
146 static struct dev_ops	pktgen_ops = {
147 	{ CDEV_NAME, 0, D_MPSAFE },
148 	.d_open =	pktgen_open,
149 	.d_close =	pktgen_close,
150 	.d_ioctl =	pktgen_ioctl,
151 };
152 
153 static volatile int		pktgen_refcnt;
154 static struct lwkt_token pktgen_tok = LWKT_TOKEN_INITIALIZER(pktgen_token);
155 
156 MALLOC_DECLARE(M_PKTGEN);
157 MALLOC_DEFINE(M_PKTGEN, CDEV_NAME, "Packet generator");
158 
159 DEV_MODULE(pktgen, pktgen_modevent, NULL);
160 
161 static int
162 pktgen_modevent(module_t mod, int type, void *data)
163 {
164 	int error = 0, i;
165 
166 	switch (type) {
167 	case MOD_LOAD:
168 		for (i = 0; i < PKTGEN_DEVCNT; ++i) {
169 			make_dev(&pktgen_ops, 0, UID_ROOT, GID_WHEEL, 0600,
170 			    CDEV_NAME"%d", i);
171 		}
172 		break;
173 
174 	case MOD_UNLOAD:
175 		if (pktgen_refcnt > 0)
176 			return EBUSY;
177 		dev_ops_remove_all(&pktgen_ops);
178 		break;
179 
180 	default:
181 		error = EOPNOTSUPP;
182 		break;
183 	}
184 	return error;
185 }
186 
187 static int
188 pktgen_open(struct dev_open_args *ap)
189 {
190 	cdev_t dev = ap->a_head.a_dev;
191 	struct pktgen *pktg;
192 	int error, i;
193 
194 	error = priv_check_cred(ap->a_cred, PRIV_ROOT, 0);
195 	if (error)
196 		return error;
197 
198 	lwkt_gettoken(&pktgen_tok);
199 
200 	if (dev->si_drv1 != NULL) {
201 		lwkt_reltoken(&pktgen_tok);
202 		return EBUSY;
203 	}
204 
205 	pktg = kmalloc(sizeof(*pktg), M_PKTGEN, M_ZERO | M_WAITOK);
206 	for (i = 0; i < ncpus; ++i) {
207 		struct pktgen_pcpu *p = &pktg->pktg_pcpu[i];
208 
209 		callout_init_mp(&p->pktg_stop);
210 		LIST_INIT(&p->pktg_buflist);
211 	}
212 
213 	dev->si_drv1 = pktg;
214 	pktg->pktg_refcnt = 1;
215 
216 	atomic_add_int(&pktgen_refcnt, 1);
217 
218 	lwkt_reltoken(&pktgen_tok);
219 	return 0;
220 }
221 
222 static int
223 pktgen_close(struct dev_close_args *ap)
224 {
225 	cdev_t dev = ap->a_head.a_dev;
226 	struct pktgen *pktg = dev->si_drv1;
227 
228 	lwkt_gettoken(&pktgen_tok);
229 	dev->si_drv1 = NULL;
230 	lwkt_reltoken(&pktgen_tok);
231 
232 	pktgen_free(pktg);
233 
234 	return 0;
235 }
236 
237 static int
238 pktgen_ioctl(struct dev_ioctl_args *ap __unused)
239 {
240 	cdev_t dev = ap->a_head.a_dev;
241 	caddr_t data = ap->a_data;
242 	struct pktgen *pktg = dev->si_drv1;
243 	int error;
244 
245 	lwkt_gettoken(&pktgen_tok);
246 
247 	switch (ap->a_cmd) {
248 	case PKTGENSTART:
249 		error = pktgen_start(pktg, 0);
250 		break;
251 
252 	case PKTGENMQSTART:
253 		error = pktgen_start(pktg, 1);
254 		break;
255 
256 	case PKTGENSCONF:
257 		error = pktgen_config(pktg, (const struct pktgen_conf *)data);
258 		break;
259 
260 	default:
261 		error = EOPNOTSUPP;
262 		break;
263 	}
264 
265 	lwkt_reltoken(&pktgen_tok);
266 	return error;
267 }
268 
269 static int
270 pktgen_config(struct pktgen *pktg, const struct pktgen_conf *conf)
271 {
272 	const struct sockaddr_in *sin;
273 	struct sockaddr_in *dst = NULL;
274 	const struct sockaddr *sa;
275 	struct ifnet *ifp;
276 	size_t dst_size;
277 	int i, error, pktenq;
278 
279 	if (pktg->pktg_flags & (PKTG_F_RUNNING | PKTG_F_CONFIG))
280 		return EBUSY;
281 
282 	if (conf->pc_datalen <= 0 ||
283 	    conf->pc_datalen > ETHERMTU - sizeof(struct udpiphdr))
284 		return EINVAL;
285 	if (conf->pc_duration <= 0)
286 		return EINVAL;
287 
288 	sin = &conf->pc_src;
289 	if (sin->sin_family != AF_INET)
290 		return EPROTONOSUPPORT;
291 	if (sin->sin_port == 0)
292 		return EINVAL;
293 
294 	if (conf->pc_ndst <= 0)
295 		return EINVAL;
296 	dst_size = conf->pc_ndst * sizeof(struct sockaddr_in);
297 
298 	dst = kmalloc(dst_size, M_PKTGEN, M_WAITOK | M_NULLOK);
299 	if (dst == NULL)
300 		return ENOMEM;
301 
302 	error = copyin(conf->pc_dst, dst, dst_size);
303 	if (error)
304 		goto failed;
305 
306 	for (i = 0; i < conf->pc_ndst; ++i) {
307 		sin = &dst[i];
308 		if (sin->sin_family != AF_INET) {
309 			error = EPROTONOSUPPORT;
310 			goto failed;
311 		}
312 		if (sin->sin_port == 0) {
313 			error = EINVAL;
314 			goto failed;
315 		}
316 	}
317 
318 	ifp = ifunit(conf->pc_ifname);
319 	if (ifp == NULL) {
320 		error = ENXIO;
321 		goto failed;
322 	}
323 
324 	pktenq = conf->pc_pktenq;
325 	if (pktenq < 0 || pktenq > ifp->if_snd.altq_maxlen) {
326 		error = ENOBUFS;
327 		goto failed;
328 	} else if (pktenq == 0) {
329 		pktenq = (ifp->if_snd.altq_maxlen * 3) / 4;
330 	}
331 
332 	sa = &conf->pc_dst_lladdr;
333 	if (sa->sa_family != AF_LINK) {
334 		error = EPROTONOSUPPORT;
335 		goto failed;
336 	}
337 	if (sa->sa_len != ETHER_ADDR_LEN) {
338 		error = EPROTONOSUPPORT;
339 		goto failed;
340 	}
341 	if (ETHER_IS_MULTICAST(sa->sa_data) ||
342 	    bcmp(sa->sa_data, ifp->if_broadcastaddr, ifp->if_addrlen) == 0) {
343 		error = EADDRNOTAVAIL;
344 		goto failed;
345 	}
346 
347 	/*
348 	 * Accept the config
349 	 */
350 	pktg->pktg_flags |= PKTG_F_CONFIG;
351 
352 	if (conf->pc_flags & PKTGEN_FLAG_SWITCH_SRCDST)
353 		pktg->pktg_flags |= PKTG_F_SWITCH_SRCDST;
354 	pktg->pktg_duration = conf->pc_duration;
355 	pktg->pktg_datalen = conf->pc_datalen;
356 	pktg->pktg_pktenq = pktenq;
357 	pktg->pktg_ifp = ifp;
358 	pktg->pktg_src = conf->pc_src;
359 	pktg->pktg_ndst = conf->pc_ndst;
360 	KKASSERT(pktg->pktg_dst == NULL);
361 	pktg->pktg_dst = dst;
362 	bcopy(sa->sa_data, pktg->pktg_dst_lladdr, ETHER_ADDR_LEN);
363 
364 	return 0;
365 
366 failed:
367 	if (dst != NULL)
368 		kfree(dst, M_PKTGEN);
369 	return error;
370 }
371 
372 static void
373 pktgen_start_ifsq(struct pktgen *pktg, struct ifaltq_subque *ifsq)
374 {
375 	struct netmsg_pktgen *np;
376 
377 	np = kmalloc(sizeof(*np), M_LWKTMSG, M_WAITOK);
378 	netmsg_init(&np->np_base, NULL, &netisr_afree_rport, 0,
379 	    pktgen_start_ifsq_handler);
380 	np->np_pktg = pktg;
381 	np->np_ifsq = ifsq;
382 
383 	lwkt_sendmsg(netisr_cpuport(ifsq_get_cpuid(ifsq)), &np->np_base.lmsg);
384 }
385 
386 static int
387 pktgen_start(struct pktgen *pktg, int mq)
388 {
389 	struct ifaltq *ifq;
390 
391 	if ((pktg->pktg_flags & PKTG_F_CONFIG) == 0)
392 		return EINVAL;
393 	if (pktg->pktg_flags & PKTG_F_RUNNING)
394 		return EBUSY;
395 	pktg->pktg_flags |= PKTG_F_RUNNING;
396 
397 	ifq = &pktg->pktg_ifp->if_snd;
398 	if (!mq) {
399 		pktgen_ref(pktg);
400 		pktgen_start_ifsq(pktg, ifq_get_subq_default(ifq));
401 	} else {
402 		int i;
403 
404 		for (i = 0; i < ifq->altq_subq_cnt; ++i)
405 			pktgen_ref(pktg);
406 		for (i = 0; i < ifq->altq_subq_cnt; ++i)
407 			pktgen_start_ifsq(pktg, ifq_get_subq(ifq, i));
408 	}
409 	return 0;
410 }
411 
412 static void
413 pktgen_start_ifsq_handler(netmsg_t nmsg)
414 {
415 	struct netmsg_pktgen *np = (struct netmsg_pktgen *)nmsg;
416 	struct pktgen *pktg = np->np_pktg;
417 	struct ifaltq_subque *ifsq = np->np_ifsq;
418 
419 	struct mbuf *m, *head = NULL, **next;
420 	struct ifnet *ifp;
421 	struct pktgen_pcpu *p;
422 	int cpuid, i, alloc_cnt, keep_cnt;
423 
424 	u_short ulen, psum;
425 	int len, ip_len;
426 
427 	/* Reply ASAP */
428 	lwkt_replymsg(&np->np_base.lmsg, 0);
429 
430 	ifp = pktg->pktg_ifp;
431 
432 	cpuid = ifsq_get_cpuid(ifsq);
433 	KKASSERT(cpuid == mycpuid);
434 
435 	p = &pktg->pktg_pcpu[cpuid];
436 
437 	keep_cnt = pktg->pktg_pktenq;
438 	alloc_cnt = keep_cnt * 2;
439 
440 	/*
441 	 * Prefault enough mbuf into mbuf objcache
442 	 */
443 	next = &head;
444 	for (i = 0; i < alloc_cnt; ++i) {
445 		MGETHDR(m, MB_WAIT, MT_DATA);
446 		*next = m;
447 		next = &m->m_nextpkt;
448 	}
449 
450 	for (i = 0; i < alloc_cnt - keep_cnt; ++i) {
451 		m = head;
452 		head = m->m_nextpkt;
453 		m->m_nextpkt = NULL;
454 		m_freem(m);
455 	}
456 	KKASSERT(head != NULL);
457 
458 	/*
459 	 * Setup the packets' data
460 	 */
461 	ip_len = pktg->pktg_datalen + sizeof(struct udpiphdr);
462 	len = ip_len + ETHER_HDR_LEN;
463 
464 	psum = htons((u_short)pktg->pktg_datalen + sizeof(struct udphdr) +
465 	    IPPROTO_UDP);
466 	ulen = htons(pktg->pktg_datalen + sizeof(struct udphdr));
467 
468 	m = head;
469 	i = 0;
470 	while (m != NULL) {
471 		struct mbuf *nextm;
472 		const struct sockaddr_in *dst;
473 		struct pktgen_buf *pb;
474 		struct ip *ip;
475 		struct udpiphdr *ui;
476 		struct ether_header *eh;
477 
478 		pktgen_ref(pktg);
479 
480 		pb = kmalloc(sizeof(*pb), M_PKTGEN, M_WAITOK | M_ZERO);
481 		pb->pb_ifp = ifp;
482 		pb->pb_ifsq = ifsq;
483 		pb->pb_inuse = 1;
484 		pb->pb_buf = kmalloc(PKTGEN_BUFSZ, M_PKTGEN, M_WAITOK);
485 		pb->pb_len = len;
486 		pb->pb_cpuid = cpuid;
487 		pb->pb_pktg = pktg;
488 		netmsg_init(&pb->pb_nmsg, NULL, &netisr_adone_rport, 0,
489 		    pktgen_buf_send);
490 		LIST_INSERT_HEAD(&p->pktg_buflist, pb, pb_link);
491 
492 		dst = &pktg->pktg_dst[i % pktg->pktg_ndst];
493 		++i;
494 
495 		m->m_ext.ext_arg = pb;
496 		m->m_ext.ext_buf = pb->pb_buf;
497 		m->m_ext.ext_free = pktgen_buf_free;
498 		m->m_ext.ext_ref = pktgen_buf_ref;
499 		m->m_ext.ext_size = PKTGEN_BUFSZ;
500 
501 		m->m_data = m->m_ext.ext_buf;
502 		m->m_flags |= M_EXT;
503 		m->m_len = m->m_pkthdr.len = len;
504 
505 		m->m_data += ETHER_HDR_LEN;
506 		m->m_len -= ETHER_HDR_LEN;
507 		m->m_pkthdr.len -= ETHER_HDR_LEN;
508 
509 		ui = mtod(m, struct udpiphdr *);
510 		ui->ui_pr = IPPROTO_UDP;
511 		if (pktg->pktg_flags & PKTG_F_SWITCH_SRCDST) {
512 			ui->ui_src.s_addr = dst->sin_addr.s_addr;
513 			ui->ui_dst.s_addr = pktg->pktg_src.sin_addr.s_addr;
514 			ui->ui_sport = dst->sin_port;
515 			ui->ui_dport = pktg->pktg_src.sin_port;
516 		} else {
517 			ui->ui_src.s_addr = pktg->pktg_src.sin_addr.s_addr;
518 			ui->ui_dst.s_addr = dst->sin_addr.s_addr;
519 			ui->ui_sport = pktg->pktg_src.sin_port;
520 			ui->ui_dport = dst->sin_port;
521 		}
522 		ui->ui_ulen = ulen;
523 		ui->ui_sum = in_pseudo(ui->ui_src.s_addr, ui->ui_dst.s_addr,
524 		    psum);
525 		m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
526 
527 		ip = (struct ip *)ui;
528 		ip->ip_len = ip_len;
529 		ip->ip_ttl = 64;	/* XXX */
530 		ip->ip_tos = 0;		/* XXX */
531 		ip->ip_vhl = IP_VHL_BORING;
532 		ip->ip_off = 0;
533 		ip->ip_sum = 0;
534 		ip->ip_id = ip_newid();
535 
536 		in_delayed_cksum(m);
537 
538 		ip->ip_len = htons(ip->ip_len);
539 		ip->ip_sum = in_cksum_hdr(ip);
540 
541 		m->m_data -= ETHER_HDR_LEN;
542 		m->m_len += ETHER_HDR_LEN;
543 		m->m_pkthdr.len += ETHER_HDR_LEN;
544 
545 		eh = mtod(m, struct ether_header *);
546 		bcopy(pktg->pktg_dst_lladdr, eh->ether_dhost, ETHER_ADDR_LEN);
547 		bcopy(IF_LLADDR(ifp), eh->ether_shost, ETHER_ADDR_LEN);
548 		eh->ether_type = htons(ETHERTYPE_IP);
549 
550 		nextm = m->m_nextpkt;
551 		m->m_nextpkt = NULL;
552 
553 		ifq_dispatch(ifp, m, NULL);
554 
555 		m = nextm;
556 	}
557 
558 	callout_reset(&p->pktg_stop, pktg->pktg_duration * hz,
559 	    pktgen_pcpu_stop_cb, p);
560 
561 	pktgen_free(pktg);
562 }
563 
564 static void
565 pktgen_mbuf(struct pktgen_buf *pb, struct mbuf *m)
566 {
567 	m->m_ext.ext_arg = pb;
568 	m->m_ext.ext_buf = pb->pb_buf;
569 	m->m_ext.ext_free = pktgen_buf_free;
570 	m->m_ext.ext_ref = pktgen_buf_ref;
571 	m->m_ext.ext_size = PKTGEN_BUFSZ;
572 
573 	m->m_data = m->m_ext.ext_buf;
574 	m->m_flags |= M_EXT;
575 	m->m_len = m->m_pkthdr.len = pb->pb_len;
576 }
577 
578 static void
579 pktgen_buf_send(netmsg_t msg)
580 {
581 	struct pktgen_buf *pb = (struct pktgen_buf *)msg;
582 	struct mbuf *m;
583 
584 	KKASSERT(&curthread->td_msgport == netisr_cpuport(pb->pb_cpuid));
585 
586 	crit_enter();
587 	lwkt_replymsg(&pb->pb_nmsg.lmsg, 0);
588 	crit_exit();
589 
590 	MGETHDR(m, MB_WAIT, MT_DATA);
591 	pktgen_mbuf(pb, m);
592 	ifq_dispatch(pb->pb_ifp, m, NULL);
593 }
594 
595 static void
596 pktgen_buf_free(void *arg)
597 {
598 	struct pktgen_buf *pb = arg;
599 	struct mbuf *m;
600 
601 	KKASSERT(pb->pb_inuse > 0);
602 	if (pb->pb_done) {
603 		if (atomic_fetchadd_int(&pb->pb_inuse, -1) == 1) {
604 			struct pktgen *pktg;
605 
606 			pktg = pb->pb_pktg;
607 			crit_enter();
608 			LIST_REMOVE(pb, pb_link);
609 			crit_exit();
610 			kfree(pb->pb_buf, M_PKTGEN);
611 			kfree(pb, M_PKTGEN);
612 
613 			pktgen_free(pktg);
614 		}
615 		return;
616 	}
617 
618 	if (&curthread->td_msgport != netisr_cpuport(pb->pb_cpuid)) {
619 		KKASSERT(pb->pb_cpuid == mycpuid);
620 		crit_enter();
621 		KKASSERT(pb->pb_nmsg.lmsg.ms_flags & MSGF_DONE);
622 		lwkt_sendmsg(netisr_cpuport(pb->pb_cpuid), &pb->pb_nmsg.lmsg);
623 		crit_exit();
624 		return;
625 	}
626 
627 	MGETHDR(m, MB_WAIT, MT_DATA);
628 	pktgen_mbuf(pb, m);
629 	ifsq_enqueue(pb->pb_ifsq, m, NULL);
630 }
631 
632 static void
633 pktgen_buf_ref(void *arg)
634 {
635 	struct pktgen_buf *pb = arg;
636 
637 	panic("%s should never be called\n", __func__);
638 
639 	KKASSERT(pb->pb_inuse > 0);
640 	atomic_add_int(&pb->pb_inuse, 1);
641 }
642 
643 static void
644 pktgen_free(struct pktgen *pktg)
645 {
646 	KKASSERT(pktg->pktg_refcnt > 0);
647 	if (atomic_fetchadd_int(&pktg->pktg_refcnt, -1) == 1) {
648 		int i;
649 
650 		if (pktg->pktg_dst != NULL)
651 			kfree(pktg->pktg_dst, M_PKTGEN);
652 
653 		for (i = 0; i < ncpus; ++i)
654 			KKASSERT(LIST_EMPTY(&pktg->pktg_pcpu[i].pktg_buflist));
655 		kfree(pktg, M_PKTGEN);
656 	}
657 
658 	KKASSERT(pktgen_refcnt > 0);
659 	atomic_subtract_int(&pktgen_refcnt, 1);
660 }
661 
662 static void
663 pktgen_pcpu_stop_cb(void *arg)
664 {
665 	struct pktgen_pcpu *p = arg;
666 	struct pktgen_buf *pb;
667 
668 	crit_enter();
669 	LIST_FOREACH(pb, &p->pktg_buflist, pb_link)
670 		pb->pb_done = 1;
671 	crit_exit();
672 }
673 
674 static void
675 pktgen_ref(struct pktgen *pktg)
676 {
677 	atomic_add_int(&pktg->pktg_refcnt, 1);
678 	atomic_add_int(&pktgen_refcnt, 1);
679 }
680