1 /* 2 * Copyright (c) 2012 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Sepherosa Ziehau <sepherosa@gmail.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #define _IP_VHL 36 37 #include <sys/param.h> 38 #include <sys/conf.h> 39 #include <sys/device.h> 40 #include <sys/in_cksum.h> 41 #include <sys/kernel.h> 42 #include <sys/malloc.h> 43 #include <sys/mbuf.h> 44 #include <sys/proc.h> 45 #include <sys/priv.h> 46 #include <sys/queue.h> 47 #include <sys/socket.h> 48 #include <sys/systm.h> 49 #include <sys/serialize.h> 50 51 #include <net/if.h> 52 #include <net/if_dl.h> 53 #include <net/if_var.h> 54 #include <net/ifq_var.h> 55 #include <net/ethernet.h> 56 #include <net/netmsg2.h> 57 #include <net/netisr2.h> 58 59 #include <netinet/in.h> 60 #include <netinet/in_pcb.h> 61 #include <netinet/ip.h> 62 #include <netinet/udp_var.h> 63 64 #include "pktgen.h" 65 66 #define CDEV_NAME "pktg" 67 68 #define PKTGEN_BUFSZ 2048 69 70 #ifndef PKTGEN_DEVCNT 71 #define PKTGEN_DEVCNT 4 72 #endif 73 74 struct pktgen; 75 76 struct netmsg_pktgen { 77 struct netmsg_base np_base; 78 struct pktgen *np_pktg; 79 struct ifaltq_subque *np_ifsq; 80 }; 81 82 struct pktgen_buf { 83 struct netmsg_base pb_nmsg; /* MUST BE THE FIRST */ 84 void *pb_buf; 85 volatile int pb_done; 86 int pb_inuse; 87 struct ifnet *pb_ifp; 88 struct ifaltq_subque *pb_ifsq; 89 int pb_len; 90 int pb_cpuid; 91 struct pktgen *pb_pktg; 92 LIST_ENTRY(pktgen_buf) pb_link; 93 }; 94 95 struct pktgen_pcpu { 96 struct callout pktg_stop; 97 LIST_HEAD(, pktgen_buf) pktg_buflist; 98 }; 99 100 struct pktgen { 101 uint32_t pktg_flags; /* PKTG_F_ */ 102 int pktg_refcnt; 103 104 int pktg_duration; 105 106 int pktg_datalen; 107 struct ifnet *pktg_ifp; 108 109 int pktg_pktenq; 110 111 struct sockaddr_in pktg_src; 112 int pktg_ndst; 113 struct sockaddr_in *pktg_dst; 114 uint8_t pktg_dst_lladdr[ETHER_ADDR_LEN]; 115 116 struct pktgen_pcpu pktg_pcpu[MAXCPU]; 117 }; 118 119 #define PKTG_F_CONFIG 0x1 120 #define PKTG_F_RUNNING 0x4 121 #define PKTG_F_SWITCH_SRCDST 0x8 122 123 static int pktgen_modevent(module_t, int, void *); 124 125 static void pktgen_buf_free(void *); 126 static void pktgen_buf_ref(void *); 127 static void pktgen_buf_send(netmsg_t); 128 129 static int pktgen_config(struct pktgen *, 130 const struct pktgen_conf *); 131 static int pktgen_start(struct pktgen *, int); 132 static void pktgen_free(struct pktgen *); 133 static void pktgen_ref(struct pktgen *); 134 static void pktgen_pcpu_stop_cb(void *); 135 static void pktgen_mbuf(struct pktgen_buf *, struct mbuf *); 136 static void pktgen_start_ifsq(struct pktgen *, 137 struct ifaltq_subque *); 138 static void pktgen_start_ifsq_handler(netmsg_t); 139 140 static d_open_t pktgen_open; 141 static d_close_t pktgen_close; 142 static d_ioctl_t pktgen_ioctl; 143 144 static struct dev_ops pktgen_ops = { 145 { CDEV_NAME, 0, D_MPSAFE }, 146 .d_open = pktgen_open, 147 .d_close = pktgen_close, 148 .d_ioctl = pktgen_ioctl, 149 }; 150 151 static volatile int pktgen_refcnt; 152 static struct lwkt_token pktgen_tok = LWKT_TOKEN_INITIALIZER(pktgen_token); 153 154 MALLOC_DECLARE(M_PKTGEN); 155 MALLOC_DEFINE(M_PKTGEN, CDEV_NAME, "Packet generator"); 156 157 DEV_MODULE(pktgen, pktgen_modevent, NULL); 158 159 static int 160 pktgen_modevent(module_t mod, int type, void *data) 161 { 162 int error = 0, i; 163 164 switch (type) { 165 case MOD_LOAD: 166 for (i = 0; i < PKTGEN_DEVCNT; ++i) { 167 make_dev(&pktgen_ops, 0, UID_ROOT, GID_WHEEL, 0600, 168 CDEV_NAME"%d", i); 169 } 170 break; 171 172 case MOD_UNLOAD: 173 if (pktgen_refcnt > 0) 174 return EBUSY; 175 dev_ops_remove_all(&pktgen_ops); 176 break; 177 178 default: 179 error = EOPNOTSUPP; 180 break; 181 } 182 return error; 183 } 184 185 static int 186 pktgen_open(struct dev_open_args *ap) 187 { 188 cdev_t dev = ap->a_head.a_dev; 189 struct pktgen *pktg; 190 int error, i; 191 192 error = priv_check_cred(ap->a_cred, PRIV_ROOT, 0); 193 if (error) 194 return error; 195 196 lwkt_gettoken(&pktgen_tok); 197 198 if (dev->si_drv1 != NULL) { 199 lwkt_reltoken(&pktgen_tok); 200 return EBUSY; 201 } 202 203 pktg = kmalloc(sizeof(*pktg), M_PKTGEN, M_ZERO | M_WAITOK); 204 for (i = 0; i < ncpus; ++i) { 205 struct pktgen_pcpu *p = &pktg->pktg_pcpu[i]; 206 207 callout_init_mp(&p->pktg_stop); 208 LIST_INIT(&p->pktg_buflist); 209 } 210 211 dev->si_drv1 = pktg; 212 pktg->pktg_refcnt = 1; 213 214 atomic_add_int(&pktgen_refcnt, 1); 215 216 lwkt_reltoken(&pktgen_tok); 217 return 0; 218 } 219 220 static int 221 pktgen_close(struct dev_close_args *ap) 222 { 223 cdev_t dev = ap->a_head.a_dev; 224 struct pktgen *pktg = dev->si_drv1; 225 226 lwkt_gettoken(&pktgen_tok); 227 dev->si_drv1 = NULL; 228 lwkt_reltoken(&pktgen_tok); 229 230 pktgen_free(pktg); 231 232 return 0; 233 } 234 235 static int 236 pktgen_ioctl(struct dev_ioctl_args *ap __unused) 237 { 238 cdev_t dev = ap->a_head.a_dev; 239 caddr_t data = ap->a_data; 240 struct pktgen *pktg = dev->si_drv1; 241 int error; 242 243 lwkt_gettoken(&pktgen_tok); 244 245 switch (ap->a_cmd) { 246 case PKTGENSTART: 247 error = pktgen_start(pktg, 0); 248 break; 249 250 case PKTGENMQSTART: 251 error = pktgen_start(pktg, 1); 252 break; 253 254 case PKTGENSCONF: 255 error = pktgen_config(pktg, (const struct pktgen_conf *)data); 256 break; 257 258 default: 259 error = EOPNOTSUPP; 260 break; 261 } 262 263 lwkt_reltoken(&pktgen_tok); 264 return error; 265 } 266 267 static int 268 pktgen_config(struct pktgen *pktg, const struct pktgen_conf *conf) 269 { 270 const struct sockaddr_in *sin; 271 struct sockaddr_in *dst = NULL; 272 const struct sockaddr *sa; 273 struct ifnet *ifp; 274 size_t dst_size; 275 int i, error, pktenq; 276 277 if (pktg->pktg_flags & (PKTG_F_RUNNING | PKTG_F_CONFIG)) 278 return EBUSY; 279 280 if (conf->pc_datalen <= 0 || 281 conf->pc_datalen > ETHERMTU - sizeof(struct udpiphdr)) 282 return EINVAL; 283 if (conf->pc_duration <= 0) 284 return EINVAL; 285 286 sin = &conf->pc_src; 287 if (sin->sin_family != AF_INET) 288 return EPROTONOSUPPORT; 289 if (sin->sin_port == 0) 290 return EINVAL; 291 292 if (conf->pc_ndst <= 0) 293 return EINVAL; 294 dst_size = conf->pc_ndst * sizeof(struct sockaddr_in); 295 296 dst = kmalloc(dst_size, M_PKTGEN, M_WAITOK | M_NULLOK); 297 if (dst == NULL) 298 return ENOMEM; 299 300 error = copyin(conf->pc_dst, dst, dst_size); 301 if (error) 302 goto failed; 303 304 for (i = 0; i < conf->pc_ndst; ++i) { 305 sin = &dst[i]; 306 if (sin->sin_family != AF_INET) { 307 error = EPROTONOSUPPORT; 308 goto failed; 309 } 310 if (sin->sin_port == 0) { 311 error = EINVAL; 312 goto failed; 313 } 314 } 315 316 ifp = ifunit(conf->pc_ifname); 317 if (ifp == NULL) { 318 error = ENXIO; 319 goto failed; 320 } 321 322 pktenq = conf->pc_pktenq; 323 if (pktenq < 0 || pktenq > ifp->if_snd.altq_maxlen) { 324 error = ENOBUFS; 325 goto failed; 326 } else if (pktenq == 0) { 327 pktenq = (ifp->if_snd.altq_maxlen * 3) / 4; 328 } 329 330 sa = &conf->pc_dst_lladdr; 331 if (sa->sa_family != AF_LINK) { 332 error = EPROTONOSUPPORT; 333 goto failed; 334 } 335 if (sa->sa_len != ETHER_ADDR_LEN) { 336 error = EPROTONOSUPPORT; 337 goto failed; 338 } 339 if (ETHER_IS_MULTICAST(sa->sa_data) || 340 bcmp(sa->sa_data, ifp->if_broadcastaddr, ifp->if_addrlen) == 0) { 341 error = EADDRNOTAVAIL; 342 goto failed; 343 } 344 345 /* 346 * Accept the config 347 */ 348 pktg->pktg_flags |= PKTG_F_CONFIG; 349 350 if (conf->pc_flags & PKTGEN_FLAG_SWITCH_SRCDST) 351 pktg->pktg_flags |= PKTG_F_SWITCH_SRCDST; 352 pktg->pktg_duration = conf->pc_duration; 353 pktg->pktg_datalen = conf->pc_datalen; 354 pktg->pktg_pktenq = pktenq; 355 pktg->pktg_ifp = ifp; 356 pktg->pktg_src = conf->pc_src; 357 pktg->pktg_ndst = conf->pc_ndst; 358 KKASSERT(pktg->pktg_dst == NULL); 359 pktg->pktg_dst = dst; 360 bcopy(sa->sa_data, pktg->pktg_dst_lladdr, ETHER_ADDR_LEN); 361 362 return 0; 363 364 failed: 365 if (dst != NULL) 366 kfree(dst, M_PKTGEN); 367 return error; 368 } 369 370 static void 371 pktgen_start_ifsq(struct pktgen *pktg, struct ifaltq_subque *ifsq) 372 { 373 struct netmsg_pktgen *np; 374 375 np = kmalloc(sizeof(*np), M_LWKTMSG, M_WAITOK); 376 netmsg_init(&np->np_base, NULL, &netisr_afree_rport, 0, 377 pktgen_start_ifsq_handler); 378 np->np_pktg = pktg; 379 np->np_ifsq = ifsq; 380 381 lwkt_sendmsg(netisr_cpuport(ifsq_get_cpuid(ifsq)), &np->np_base.lmsg); 382 } 383 384 static int 385 pktgen_start(struct pktgen *pktg, int mq) 386 { 387 struct ifaltq *ifq; 388 389 if ((pktg->pktg_flags & PKTG_F_CONFIG) == 0) 390 return EINVAL; 391 if (pktg->pktg_flags & PKTG_F_RUNNING) 392 return EBUSY; 393 pktg->pktg_flags |= PKTG_F_RUNNING; 394 395 ifq = &pktg->pktg_ifp->if_snd; 396 if (!mq) { 397 pktgen_ref(pktg); 398 pktgen_start_ifsq(pktg, ifq_get_subq_default(ifq)); 399 } else { 400 int i; 401 402 for (i = 0; i < ifq->altq_subq_cnt; ++i) 403 pktgen_ref(pktg); 404 for (i = 0; i < ifq->altq_subq_cnt; ++i) 405 pktgen_start_ifsq(pktg, ifq_get_subq(ifq, i)); 406 } 407 return 0; 408 } 409 410 static void 411 pktgen_start_ifsq_handler(netmsg_t nmsg) 412 { 413 struct netmsg_pktgen *np = (struct netmsg_pktgen *)nmsg; 414 struct pktgen *pktg = np->np_pktg; 415 struct ifaltq_subque *ifsq = np->np_ifsq; 416 417 struct mbuf *m, *head = NULL, **next; 418 struct ifnet *ifp; 419 struct pktgen_pcpu *p; 420 int cpuid, i, alloc_cnt, keep_cnt; 421 422 u_short ulen, psum; 423 int len, ip_len; 424 425 /* Reply ASAP */ 426 lwkt_replymsg(&np->np_base.lmsg, 0); 427 428 ifp = pktg->pktg_ifp; 429 430 cpuid = ifsq_get_cpuid(ifsq); 431 KKASSERT(cpuid == mycpuid); 432 433 p = &pktg->pktg_pcpu[cpuid]; 434 435 keep_cnt = pktg->pktg_pktenq; 436 alloc_cnt = keep_cnt * 2; 437 438 /* 439 * Prefault enough mbuf into mbuf objcache 440 */ 441 next = &head; 442 for (i = 0; i < alloc_cnt; ++i) { 443 MGETHDR(m, M_WAITOK, MT_DATA); 444 *next = m; 445 next = &m->m_nextpkt; 446 } 447 448 for (i = 0; i < alloc_cnt - keep_cnt; ++i) { 449 m = head; 450 head = m->m_nextpkt; 451 m->m_nextpkt = NULL; 452 m_freem(m); 453 } 454 KKASSERT(head != NULL); 455 456 /* 457 * Setup the packets' data 458 */ 459 ip_len = pktg->pktg_datalen + sizeof(struct udpiphdr); 460 len = ip_len + ETHER_HDR_LEN; 461 462 psum = htons((u_short)pktg->pktg_datalen + sizeof(struct udphdr) + 463 IPPROTO_UDP); 464 ulen = htons(pktg->pktg_datalen + sizeof(struct udphdr)); 465 466 m = head; 467 i = 0; 468 while (m != NULL) { 469 struct mbuf *nextm; 470 const struct sockaddr_in *dst; 471 struct pktgen_buf *pb; 472 struct ip *ip; 473 struct udpiphdr *ui; 474 struct ether_header *eh; 475 476 pktgen_ref(pktg); 477 478 pb = kmalloc(sizeof(*pb), M_PKTGEN, M_WAITOK | M_ZERO); 479 pb->pb_ifp = ifp; 480 pb->pb_ifsq = ifsq; 481 pb->pb_inuse = 1; 482 pb->pb_buf = kmalloc(PKTGEN_BUFSZ, M_PKTGEN, M_WAITOK); 483 pb->pb_len = len; 484 pb->pb_cpuid = cpuid; 485 pb->pb_pktg = pktg; 486 netmsg_init(&pb->pb_nmsg, NULL, &netisr_adone_rport, 0, 487 pktgen_buf_send); 488 LIST_INSERT_HEAD(&p->pktg_buflist, pb, pb_link); 489 490 dst = &pktg->pktg_dst[i % pktg->pktg_ndst]; 491 ++i; 492 493 m->m_ext.ext_arg = pb; 494 m->m_ext.ext_buf = pb->pb_buf; 495 m->m_ext.ext_free = pktgen_buf_free; 496 m->m_ext.ext_ref = pktgen_buf_ref; 497 m->m_ext.ext_size = PKTGEN_BUFSZ; 498 499 m->m_data = m->m_ext.ext_buf; 500 m->m_flags |= M_EXT; 501 m->m_len = m->m_pkthdr.len = len; 502 503 m->m_data += ETHER_HDR_LEN; 504 m->m_len -= ETHER_HDR_LEN; 505 m->m_pkthdr.len -= ETHER_HDR_LEN; 506 507 ui = mtod(m, struct udpiphdr *); 508 ui->ui_pr = IPPROTO_UDP; 509 if (pktg->pktg_flags & PKTG_F_SWITCH_SRCDST) { 510 ui->ui_src.s_addr = dst->sin_addr.s_addr; 511 ui->ui_dst.s_addr = pktg->pktg_src.sin_addr.s_addr; 512 ui->ui_sport = dst->sin_port; 513 ui->ui_dport = pktg->pktg_src.sin_port; 514 } else { 515 ui->ui_src.s_addr = pktg->pktg_src.sin_addr.s_addr; 516 ui->ui_dst.s_addr = dst->sin_addr.s_addr; 517 ui->ui_sport = pktg->pktg_src.sin_port; 518 ui->ui_dport = dst->sin_port; 519 } 520 ui->ui_ulen = ulen; 521 ui->ui_sum = in_pseudo(ui->ui_src.s_addr, ui->ui_dst.s_addr, 522 psum); 523 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 524 525 ip = (struct ip *)ui; 526 ip->ip_len = ip_len; 527 ip->ip_ttl = 64; /* XXX */ 528 ip->ip_tos = 0; /* XXX */ 529 ip->ip_vhl = IP_VHL_BORING; 530 ip->ip_off = 0; 531 ip->ip_sum = 0; 532 ip->ip_id = ip_newid(); 533 534 in_delayed_cksum(m); 535 536 ip->ip_len = htons(ip->ip_len); 537 ip->ip_sum = in_cksum_hdr(ip); 538 539 m->m_data -= ETHER_HDR_LEN; 540 m->m_len += ETHER_HDR_LEN; 541 m->m_pkthdr.len += ETHER_HDR_LEN; 542 543 eh = mtod(m, struct ether_header *); 544 bcopy(pktg->pktg_dst_lladdr, eh->ether_dhost, ETHER_ADDR_LEN); 545 bcopy(IF_LLADDR(ifp), eh->ether_shost, ETHER_ADDR_LEN); 546 eh->ether_type = htons(ETHERTYPE_IP); 547 548 nextm = m->m_nextpkt; 549 m->m_nextpkt = NULL; 550 551 ifq_dispatch(ifp, m, NULL); 552 553 m = nextm; 554 } 555 556 callout_reset(&p->pktg_stop, pktg->pktg_duration * hz, 557 pktgen_pcpu_stop_cb, p); 558 559 pktgen_free(pktg); 560 } 561 562 static void 563 pktgen_mbuf(struct pktgen_buf *pb, struct mbuf *m) 564 { 565 m->m_ext.ext_arg = pb; 566 m->m_ext.ext_buf = pb->pb_buf; 567 m->m_ext.ext_free = pktgen_buf_free; 568 m->m_ext.ext_ref = pktgen_buf_ref; 569 m->m_ext.ext_size = PKTGEN_BUFSZ; 570 571 m->m_data = m->m_ext.ext_buf; 572 m->m_flags |= M_EXT; 573 m->m_len = m->m_pkthdr.len = pb->pb_len; 574 } 575 576 static void 577 pktgen_buf_send(netmsg_t msg) 578 { 579 struct pktgen_buf *pb = (struct pktgen_buf *)msg; 580 struct mbuf *m; 581 582 KKASSERT(&curthread->td_msgport == netisr_cpuport(pb->pb_cpuid)); 583 584 crit_enter(); 585 lwkt_replymsg(&pb->pb_nmsg.lmsg, 0); 586 crit_exit(); 587 588 MGETHDR(m, M_WAITOK, MT_DATA); 589 pktgen_mbuf(pb, m); 590 ifq_dispatch(pb->pb_ifp, m, NULL); 591 } 592 593 static void 594 pktgen_buf_free(void *arg) 595 { 596 struct pktgen_buf *pb = arg; 597 struct mbuf *m; 598 599 KKASSERT(pb->pb_inuse > 0); 600 if (pb->pb_done) { 601 if (atomic_fetchadd_int(&pb->pb_inuse, -1) == 1) { 602 struct pktgen *pktg; 603 604 pktg = pb->pb_pktg; 605 crit_enter(); 606 LIST_REMOVE(pb, pb_link); 607 crit_exit(); 608 kfree(pb->pb_buf, M_PKTGEN); 609 kfree(pb, M_PKTGEN); 610 611 pktgen_free(pktg); 612 } 613 return; 614 } 615 616 if (&curthread->td_msgport != netisr_cpuport(pb->pb_cpuid)) { 617 KKASSERT(pb->pb_cpuid == mycpuid); 618 crit_enter(); 619 KKASSERT(pb->pb_nmsg.lmsg.ms_flags & MSGF_DONE); 620 lwkt_sendmsg(netisr_cpuport(pb->pb_cpuid), &pb->pb_nmsg.lmsg); 621 crit_exit(); 622 return; 623 } 624 625 MGETHDR(m, M_WAITOK, MT_DATA); 626 pktgen_mbuf(pb, m); 627 ifsq_enqueue(pb->pb_ifsq, m, NULL); 628 } 629 630 static void 631 pktgen_buf_ref(void *arg) 632 { 633 struct pktgen_buf *pb = arg; 634 635 panic("%s should never be called\n", __func__); 636 637 KKASSERT(pb->pb_inuse > 0); 638 atomic_add_int(&pb->pb_inuse, 1); 639 } 640 641 static void 642 pktgen_free(struct pktgen *pktg) 643 { 644 KKASSERT(pktg->pktg_refcnt > 0); 645 if (atomic_fetchadd_int(&pktg->pktg_refcnt, -1) == 1) { 646 int i; 647 648 if (pktg->pktg_dst != NULL) 649 kfree(pktg->pktg_dst, M_PKTGEN); 650 651 for (i = 0; i < ncpus; ++i) 652 KKASSERT(LIST_EMPTY(&pktg->pktg_pcpu[i].pktg_buflist)); 653 kfree(pktg, M_PKTGEN); 654 } 655 656 KKASSERT(pktgen_refcnt > 0); 657 atomic_subtract_int(&pktgen_refcnt, 1); 658 } 659 660 static void 661 pktgen_pcpu_stop_cb(void *arg) 662 { 663 struct pktgen_pcpu *p = arg; 664 struct pktgen_buf *pb; 665 666 crit_enter(); 667 LIST_FOREACH(pb, &p->pktg_buflist, pb_link) 668 pb->pb_done = 1; 669 crit_exit(); 670 } 671 672 static void 673 pktgen_ref(struct pktgen *pktg) 674 { 675 atomic_add_int(&pktg->pktg_refcnt, 1); 676 atomic_add_int(&pktgen_refcnt, 1); 677 } 678