1 /*
2 * Copyright (c) 2012 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Sepherosa Ziehau <sepherosa@gmail.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35 #define _IP_VHL
36
37 #include <sys/param.h>
38 #include <sys/conf.h>
39 #include <sys/device.h>
40 #include <sys/in_cksum.h>
41 #include <sys/kernel.h>
42 #include <sys/malloc.h>
43 #include <sys/mbuf.h>
44 #include <sys/proc.h>
45 #include <sys/caps.h>
46 #include <sys/queue.h>
47 #include <sys/socket.h>
48 #include <sys/systm.h>
49 #include <sys/serialize.h>
50
51 #include <net/if.h>
52 #include <net/if_dl.h>
53 #include <net/if_var.h>
54 #include <net/ifq_var.h>
55 #include <net/ethernet.h>
56 #include <net/netmsg2.h>
57 #include <net/netisr2.h>
58
59 #include <netinet/in.h>
60 #include <netinet/in_pcb.h>
61 #include <netinet/ip.h>
62 #include <netinet/udp_var.h>
63
64 #include "pktgen.h"
65
66 #define CDEV_NAME "pktg"
67
68 #define PKTGEN_BUFSZ 2048
69
70 #ifndef PKTGEN_DEVCNT
71 #define PKTGEN_DEVCNT 4
72 #endif
73
74 struct pktgen;
75
76 struct netmsg_pktgen {
77 struct netmsg_base np_base;
78 struct pktgen *np_pktg;
79 struct ifaltq_subque *np_ifsq;
80 };
81
82 struct pktgen_buf {
83 struct netmsg_base pb_nmsg; /* MUST BE THE FIRST */
84 void *pb_buf;
85 volatile int pb_done;
86 int pb_inuse;
87 struct ifnet *pb_ifp;
88 struct ifaltq_subque *pb_ifsq;
89 int pb_len;
90 int pb_cpuid;
91 struct pktgen *pb_pktg;
92 LIST_ENTRY(pktgen_buf) pb_link;
93 };
94
95 struct pktgen_pcpu {
96 struct callout pktg_stop;
97 LIST_HEAD(, pktgen_buf) pktg_buflist;
98 };
99
100 struct pktgen {
101 uint32_t pktg_flags; /* PKTG_F_ */
102 int pktg_refcnt;
103
104 int pktg_duration;
105
106 int pktg_datalen;
107 struct ifnet *pktg_ifp;
108
109 int pktg_pktenq;
110
111 struct sockaddr_in pktg_src;
112 int pktg_ndst;
113 struct sockaddr_in *pktg_dst;
114 uint8_t pktg_dst_lladdr[ETHER_ADDR_LEN];
115
116 struct pktgen_pcpu pktg_pcpu[MAXCPU];
117 };
118
119 #define PKTG_F_CONFIG 0x1
120 #define PKTG_F_RUNNING 0x4
121 #define PKTG_F_SWITCH_SRCDST 0x8
122
123 static int pktgen_modevent(module_t, int, void *);
124
125 static void pktgen_buf_free(void *);
126 static void pktgen_buf_ref(void *);
127 static void pktgen_buf_send(netmsg_t);
128
129 static int pktgen_config(struct pktgen *,
130 const struct pktgen_conf *);
131 static int pktgen_start(struct pktgen *, int);
132 static void pktgen_free(struct pktgen *);
133 static void pktgen_ref(struct pktgen *);
134 static void pktgen_pcpu_stop_cb(void *);
135 static void pktgen_mbuf(struct pktgen_buf *, struct mbuf *);
136 static void pktgen_start_ifsq(struct pktgen *,
137 struct ifaltq_subque *);
138 static void pktgen_start_ifsq_handler(netmsg_t);
139
140 static d_open_t pktgen_open;
141 static d_close_t pktgen_close;
142 static d_ioctl_t pktgen_ioctl;
143
144 static struct dev_ops pktgen_ops = {
145 { CDEV_NAME, 0, D_MPSAFE },
146 .d_open = pktgen_open,
147 .d_close = pktgen_close,
148 .d_ioctl = pktgen_ioctl,
149 };
150
151 static volatile int pktgen_refcnt;
152 static struct lwkt_token pktgen_tok = LWKT_TOKEN_INITIALIZER(pktgen_token);
153
154 MALLOC_DECLARE(M_PKTGEN);
155 MALLOC_DEFINE(M_PKTGEN, CDEV_NAME, "Packet generator");
156
157 DEV_MODULE(pktgen, pktgen_modevent, NULL);
158
159 static int
pktgen_modevent(module_t mod,int type,void * data)160 pktgen_modevent(module_t mod, int type, void *data)
161 {
162 int error = 0, i;
163
164 switch (type) {
165 case MOD_LOAD:
166 for (i = 0; i < PKTGEN_DEVCNT; ++i) {
167 make_dev(&pktgen_ops, 0, UID_ROOT, GID_WHEEL, 0600,
168 CDEV_NAME"%d", i);
169 }
170 break;
171
172 case MOD_UNLOAD:
173 if (pktgen_refcnt > 0)
174 return EBUSY;
175 dev_ops_remove_all(&pktgen_ops);
176 break;
177
178 default:
179 error = EOPNOTSUPP;
180 break;
181 }
182 return error;
183 }
184
185 static int
pktgen_open(struct dev_open_args * ap)186 pktgen_open(struct dev_open_args *ap)
187 {
188 cdev_t dev = ap->a_head.a_dev;
189 struct pktgen *pktg;
190 int error, i;
191
192 error = priv_check_cred(ap->a_cred, PRIV_ROOT, 0);
193 if (error)
194 return error;
195
196 lwkt_gettoken(&pktgen_tok);
197
198 if (dev->si_drv1 != NULL) {
199 lwkt_reltoken(&pktgen_tok);
200 return EBUSY;
201 }
202
203 pktg = kmalloc(sizeof(*pktg), M_PKTGEN, M_ZERO | M_WAITOK);
204 for (i = 0; i < ncpus; ++i) {
205 struct pktgen_pcpu *p = &pktg->pktg_pcpu[i];
206
207 callout_init_mp(&p->pktg_stop);
208 LIST_INIT(&p->pktg_buflist);
209 }
210
211 dev->si_drv1 = pktg;
212 pktg->pktg_refcnt = 1;
213
214 atomic_add_int(&pktgen_refcnt, 1);
215
216 lwkt_reltoken(&pktgen_tok);
217 return 0;
218 }
219
220 static int
pktgen_close(struct dev_close_args * ap)221 pktgen_close(struct dev_close_args *ap)
222 {
223 cdev_t dev = ap->a_head.a_dev;
224 struct pktgen *pktg = dev->si_drv1;
225
226 lwkt_gettoken(&pktgen_tok);
227 dev->si_drv1 = NULL;
228 lwkt_reltoken(&pktgen_tok);
229
230 pktgen_free(pktg);
231
232 return 0;
233 }
234
235 static int
pktgen_ioctl(struct dev_ioctl_args * ap __unused)236 pktgen_ioctl(struct dev_ioctl_args *ap __unused)
237 {
238 cdev_t dev = ap->a_head.a_dev;
239 caddr_t data = ap->a_data;
240 struct pktgen *pktg = dev->si_drv1;
241 int error;
242
243 lwkt_gettoken(&pktgen_tok);
244
245 switch (ap->a_cmd) {
246 case PKTGENSTART:
247 error = pktgen_start(pktg, 0);
248 break;
249
250 case PKTGENMQSTART:
251 error = pktgen_start(pktg, 1);
252 break;
253
254 case PKTGENSCONF:
255 error = pktgen_config(pktg, (const struct pktgen_conf *)data);
256 break;
257
258 default:
259 error = EOPNOTSUPP;
260 break;
261 }
262
263 lwkt_reltoken(&pktgen_tok);
264 return error;
265 }
266
267 static int
pktgen_config(struct pktgen * pktg,const struct pktgen_conf * conf)268 pktgen_config(struct pktgen *pktg, const struct pktgen_conf *conf)
269 {
270 const struct sockaddr_in *sin;
271 struct sockaddr_in *dst = NULL;
272 const struct sockaddr *sa;
273 struct ifnet *ifp;
274 size_t dst_size;
275 int i, error, pktenq;
276
277 if (pktg->pktg_flags & (PKTG_F_RUNNING | PKTG_F_CONFIG))
278 return EBUSY;
279
280 if (conf->pc_datalen <= 0 ||
281 conf->pc_datalen > ETHERMTU - sizeof(struct udpiphdr))
282 return EINVAL;
283 if (conf->pc_duration <= 0)
284 return EINVAL;
285
286 sin = &conf->pc_src;
287 if (sin->sin_family != AF_INET)
288 return EPROTONOSUPPORT;
289 if (sin->sin_port == 0)
290 return EINVAL;
291
292 if (conf->pc_ndst <= 0)
293 return EINVAL;
294 dst_size = conf->pc_ndst * sizeof(struct sockaddr_in);
295
296 dst = kmalloc(dst_size, M_PKTGEN, M_WAITOK | M_NULLOK);
297 if (dst == NULL)
298 return ENOMEM;
299
300 error = copyin(conf->pc_dst, dst, dst_size);
301 if (error)
302 goto failed;
303
304 for (i = 0; i < conf->pc_ndst; ++i) {
305 sin = &dst[i];
306 if (sin->sin_family != AF_INET) {
307 error = EPROTONOSUPPORT;
308 goto failed;
309 }
310 if (sin->sin_port == 0) {
311 error = EINVAL;
312 goto failed;
313 }
314 }
315
316 /*
317 * XXX NOT MPSAFE.
318 * Make sure that pc_ifname is not gone.
319 */
320 ifnet_lock();
321 ifp = ifunit(conf->pc_ifname);
322 ifnet_unlock();
323 if (ifp == NULL) {
324 error = ENXIO;
325 goto failed;
326 }
327
328 pktenq = conf->pc_pktenq;
329 if (pktenq < 0 || pktenq > ifp->if_snd.altq_maxlen) {
330 error = ENOBUFS;
331 goto failed;
332 } else if (pktenq == 0) {
333 pktenq = (ifp->if_snd.altq_maxlen * 3) / 4;
334 }
335
336 sa = &conf->pc_dst_lladdr;
337 if (sa->sa_family != AF_LINK) {
338 error = EPROTONOSUPPORT;
339 goto failed;
340 }
341 if (sa->sa_len != ETHER_ADDR_LEN) {
342 error = EPROTONOSUPPORT;
343 goto failed;
344 }
345 if (ETHER_IS_MULTICAST(sa->sa_data) ||
346 bcmp(sa->sa_data, ifp->if_broadcastaddr, ifp->if_addrlen) == 0) {
347 error = EADDRNOTAVAIL;
348 goto failed;
349 }
350
351 /*
352 * Accept the config
353 */
354 pktg->pktg_flags |= PKTG_F_CONFIG;
355
356 if (conf->pc_flags & PKTGEN_FLAG_SWITCH_SRCDST)
357 pktg->pktg_flags |= PKTG_F_SWITCH_SRCDST;
358 pktg->pktg_duration = conf->pc_duration;
359 pktg->pktg_datalen = conf->pc_datalen;
360 pktg->pktg_pktenq = pktenq;
361 pktg->pktg_ifp = ifp;
362 pktg->pktg_src = conf->pc_src;
363 pktg->pktg_ndst = conf->pc_ndst;
364 KKASSERT(pktg->pktg_dst == NULL);
365 pktg->pktg_dst = dst;
366 bcopy(sa->sa_data, pktg->pktg_dst_lladdr, ETHER_ADDR_LEN);
367
368 return 0;
369
370 failed:
371 if (dst != NULL)
372 kfree(dst, M_PKTGEN);
373 return error;
374 }
375
376 static void
pktgen_start_ifsq(struct pktgen * pktg,struct ifaltq_subque * ifsq)377 pktgen_start_ifsq(struct pktgen *pktg, struct ifaltq_subque *ifsq)
378 {
379 struct netmsg_pktgen *np;
380
381 np = kmalloc(sizeof(*np), M_LWKTMSG, M_WAITOK);
382 netmsg_init(&np->np_base, NULL, &netisr_afree_rport, 0,
383 pktgen_start_ifsq_handler);
384 np->np_pktg = pktg;
385 np->np_ifsq = ifsq;
386
387 lwkt_sendmsg(netisr_cpuport(ifsq_get_cpuid(ifsq)), &np->np_base.lmsg);
388 }
389
390 static int
pktgen_start(struct pktgen * pktg,int mq)391 pktgen_start(struct pktgen *pktg, int mq)
392 {
393 struct ifaltq *ifq;
394
395 if ((pktg->pktg_flags & PKTG_F_CONFIG) == 0)
396 return EINVAL;
397 if (pktg->pktg_flags & PKTG_F_RUNNING)
398 return EBUSY;
399 pktg->pktg_flags |= PKTG_F_RUNNING;
400
401 ifq = &pktg->pktg_ifp->if_snd;
402 if (!mq) {
403 pktgen_ref(pktg);
404 pktgen_start_ifsq(pktg, ifq_get_subq_default(ifq));
405 } else {
406 int i;
407
408 for (i = 0; i < ifq->altq_subq_cnt; ++i)
409 pktgen_ref(pktg);
410 for (i = 0; i < ifq->altq_subq_cnt; ++i)
411 pktgen_start_ifsq(pktg, ifq_get_subq(ifq, i));
412 }
413 return 0;
414 }
415
416 static void
pktgen_start_ifsq_handler(netmsg_t nmsg)417 pktgen_start_ifsq_handler(netmsg_t nmsg)
418 {
419 struct netmsg_pktgen *np = (struct netmsg_pktgen *)nmsg;
420 struct pktgen *pktg = np->np_pktg;
421 struct ifaltq_subque *ifsq = np->np_ifsq;
422
423 struct mbuf *m, *head = NULL, **next;
424 struct ifnet *ifp;
425 struct pktgen_pcpu *p;
426 int cpuid, i, alloc_cnt, keep_cnt;
427
428 u_short ulen, psum;
429 int len, ip_len;
430
431 /* Reply ASAP */
432 lwkt_replymsg(&np->np_base.lmsg, 0);
433
434 ifp = pktg->pktg_ifp;
435
436 cpuid = ifsq_get_cpuid(ifsq);
437 KKASSERT(cpuid == mycpuid);
438
439 p = &pktg->pktg_pcpu[cpuid];
440
441 keep_cnt = pktg->pktg_pktenq;
442 alloc_cnt = keep_cnt * 2;
443
444 /*
445 * Prefault enough mbuf into mbuf objcache
446 */
447 next = &head;
448 for (i = 0; i < alloc_cnt; ++i) {
449 MGETHDR(m, M_WAITOK, MT_DATA);
450 *next = m;
451 next = &m->m_nextpkt;
452 }
453
454 for (i = 0; i < alloc_cnt - keep_cnt; ++i) {
455 m = head;
456 head = m->m_nextpkt;
457 m->m_nextpkt = NULL;
458 m_freem(m);
459 }
460 KKASSERT(head != NULL);
461
462 /*
463 * Setup the packets' data
464 */
465 ip_len = pktg->pktg_datalen + sizeof(struct udpiphdr);
466 len = ip_len + ETHER_HDR_LEN;
467
468 psum = htons((u_short)pktg->pktg_datalen + sizeof(struct udphdr) +
469 IPPROTO_UDP);
470 ulen = htons(pktg->pktg_datalen + sizeof(struct udphdr));
471
472 m = head;
473 i = 0;
474 while (m != NULL) {
475 struct mbuf *nextm;
476 const struct sockaddr_in *dst;
477 struct pktgen_buf *pb;
478 struct ip *ip;
479 struct udpiphdr *ui;
480 struct ether_header *eh;
481
482 pktgen_ref(pktg);
483
484 pb = kmalloc(sizeof(*pb), M_PKTGEN, M_WAITOK | M_ZERO);
485 pb->pb_ifp = ifp;
486 pb->pb_ifsq = ifsq;
487 pb->pb_inuse = 1;
488 pb->pb_buf = kmalloc(PKTGEN_BUFSZ, M_PKTGEN, M_WAITOK);
489 pb->pb_len = len;
490 pb->pb_cpuid = cpuid;
491 pb->pb_pktg = pktg;
492 netmsg_init(&pb->pb_nmsg, NULL, &netisr_adone_rport, 0,
493 pktgen_buf_send);
494 LIST_INSERT_HEAD(&p->pktg_buflist, pb, pb_link);
495
496 dst = &pktg->pktg_dst[i % pktg->pktg_ndst];
497 ++i;
498
499 m->m_ext.ext_arg = pb;
500 m->m_ext.ext_buf = pb->pb_buf;
501 m->m_ext.ext_free = pktgen_buf_free;
502 m->m_ext.ext_ref = pktgen_buf_ref;
503 m->m_ext.ext_size = PKTGEN_BUFSZ;
504
505 m->m_data = m->m_ext.ext_buf;
506 m->m_flags |= M_EXT;
507 m->m_len = m->m_pkthdr.len = len;
508
509 m->m_data += ETHER_HDR_LEN;
510 m->m_len -= ETHER_HDR_LEN;
511 m->m_pkthdr.len -= ETHER_HDR_LEN;
512
513 ui = mtod(m, struct udpiphdr *);
514 ui->ui_pr = IPPROTO_UDP;
515 if (pktg->pktg_flags & PKTG_F_SWITCH_SRCDST) {
516 ui->ui_src.s_addr = dst->sin_addr.s_addr;
517 ui->ui_dst.s_addr = pktg->pktg_src.sin_addr.s_addr;
518 ui->ui_sport = dst->sin_port;
519 ui->ui_dport = pktg->pktg_src.sin_port;
520 } else {
521 ui->ui_src.s_addr = pktg->pktg_src.sin_addr.s_addr;
522 ui->ui_dst.s_addr = dst->sin_addr.s_addr;
523 ui->ui_sport = pktg->pktg_src.sin_port;
524 ui->ui_dport = dst->sin_port;
525 }
526 ui->ui_ulen = ulen;
527 ui->ui_sum = in_pseudo(ui->ui_src.s_addr, ui->ui_dst.s_addr,
528 psum);
529 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
530
531 ip = (struct ip *)ui;
532 ip->ip_len = ip_len;
533 ip->ip_ttl = 64; /* XXX */
534 ip->ip_tos = 0; /* XXX */
535 ip->ip_vhl = IP_VHL_BORING;
536 ip->ip_off = 0;
537 ip->ip_sum = 0;
538 ip->ip_id = ip_newid();
539
540 in_delayed_cksum(m);
541
542 ip->ip_len = htons(ip->ip_len);
543 ip->ip_sum = in_cksum_hdr(ip);
544
545 m->m_data -= ETHER_HDR_LEN;
546 m->m_len += ETHER_HDR_LEN;
547 m->m_pkthdr.len += ETHER_HDR_LEN;
548
549 eh = mtod(m, struct ether_header *);
550 bcopy(pktg->pktg_dst_lladdr, eh->ether_dhost, ETHER_ADDR_LEN);
551 bcopy(IF_LLADDR(ifp), eh->ether_shost, ETHER_ADDR_LEN);
552 eh->ether_type = htons(ETHERTYPE_IP);
553
554 nextm = m->m_nextpkt;
555 m->m_nextpkt = NULL;
556
557 ifq_dispatch(ifp, m, NULL);
558
559 m = nextm;
560 }
561
562 callout_reset(&p->pktg_stop, pktg->pktg_duration * hz,
563 pktgen_pcpu_stop_cb, p);
564
565 pktgen_free(pktg);
566 }
567
568 static void
pktgen_mbuf(struct pktgen_buf * pb,struct mbuf * m)569 pktgen_mbuf(struct pktgen_buf *pb, struct mbuf *m)
570 {
571 m->m_ext.ext_arg = pb;
572 m->m_ext.ext_buf = pb->pb_buf;
573 m->m_ext.ext_free = pktgen_buf_free;
574 m->m_ext.ext_ref = pktgen_buf_ref;
575 m->m_ext.ext_size = PKTGEN_BUFSZ;
576
577 m->m_data = m->m_ext.ext_buf;
578 m->m_flags |= M_EXT;
579 m->m_len = m->m_pkthdr.len = pb->pb_len;
580 }
581
582 static void
pktgen_buf_send(netmsg_t msg)583 pktgen_buf_send(netmsg_t msg)
584 {
585 struct pktgen_buf *pb = (struct pktgen_buf *)msg;
586 struct mbuf *m;
587
588 KKASSERT(&curthread->td_msgport == netisr_cpuport(pb->pb_cpuid));
589
590 crit_enter();
591 lwkt_replymsg(&pb->pb_nmsg.lmsg, 0);
592 crit_exit();
593
594 MGETHDR(m, M_WAITOK, MT_DATA);
595 pktgen_mbuf(pb, m);
596 ifq_dispatch(pb->pb_ifp, m, NULL);
597 }
598
599 static void
pktgen_buf_free(void * arg)600 pktgen_buf_free(void *arg)
601 {
602 struct pktgen_buf *pb = arg;
603 struct mbuf *m;
604
605 KKASSERT(pb->pb_inuse > 0);
606 if (pb->pb_done) {
607 if (atomic_fetchadd_int(&pb->pb_inuse, -1) == 1) {
608 struct pktgen *pktg;
609
610 pktg = pb->pb_pktg;
611 crit_enter();
612 LIST_REMOVE(pb, pb_link);
613 crit_exit();
614 kfree(pb->pb_buf, M_PKTGEN);
615 kfree(pb, M_PKTGEN);
616
617 pktgen_free(pktg);
618 }
619 return;
620 }
621
622 if (&curthread->td_msgport != netisr_cpuport(pb->pb_cpuid)) {
623 KKASSERT(pb->pb_cpuid == mycpuid);
624 crit_enter();
625 KKASSERT(pb->pb_nmsg.lmsg.ms_flags & MSGF_DONE);
626 lwkt_sendmsg(netisr_cpuport(pb->pb_cpuid), &pb->pb_nmsg.lmsg);
627 crit_exit();
628 return;
629 }
630
631 MGETHDR(m, M_WAITOK, MT_DATA);
632 pktgen_mbuf(pb, m);
633 ifsq_enqueue(pb->pb_ifsq, m, NULL);
634 }
635
636 static void
pktgen_buf_ref(void * arg)637 pktgen_buf_ref(void *arg)
638 {
639 struct pktgen_buf *pb = arg;
640
641 panic("%s should never be called\n", __func__);
642
643 KKASSERT(pb->pb_inuse > 0);
644 atomic_add_int(&pb->pb_inuse, 1);
645 }
646
647 static void
pktgen_free(struct pktgen * pktg)648 pktgen_free(struct pktgen *pktg)
649 {
650 KKASSERT(pktg->pktg_refcnt > 0);
651 if (atomic_fetchadd_int(&pktg->pktg_refcnt, -1) == 1) {
652 int i;
653
654 if (pktg->pktg_dst != NULL)
655 kfree(pktg->pktg_dst, M_PKTGEN);
656
657 for (i = 0; i < ncpus; ++i)
658 KKASSERT(LIST_EMPTY(&pktg->pktg_pcpu[i].pktg_buflist));
659 kfree(pktg, M_PKTGEN);
660 }
661
662 KKASSERT(pktgen_refcnt > 0);
663 atomic_subtract_int(&pktgen_refcnt, 1);
664 }
665
666 static void
pktgen_pcpu_stop_cb(void * arg)667 pktgen_pcpu_stop_cb(void *arg)
668 {
669 struct pktgen_pcpu *p = arg;
670 struct pktgen_buf *pb;
671
672 crit_enter();
673 LIST_FOREACH(pb, &p->pktg_buflist, pb_link)
674 pb->pb_done = 1;
675 crit_exit();
676 }
677
678 static void
pktgen_ref(struct pktgen * pktg)679 pktgen_ref(struct pktgen *pktg)
680 {
681 atomic_add_int(&pktg->pktg_refcnt, 1);
682 atomic_add_int(&pktgen_refcnt, 1);
683 }
684