1 /* $NetBSD: bpf.c,v 1.251 2023/02/08 01:37:53 gutteridge Exp $ */
2
3 /*
4 * Copyright (c) 1990, 1991, 1993
5 * The Regents of the University of California. All rights reserved.
6 *
7 * This code is derived from the Stanford/CMU enet packet filter,
8 * (net/enet.c) distributed as part of 4.3BSD, and code contributed
9 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence
10 * Berkeley Laboratory.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. Neither the name of the University nor the names of its contributors
21 * may be used to endorse or promote products derived from this software
22 * without specific prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 *
36 * @(#)bpf.c 8.4 (Berkeley) 1/9/95
37 * static char rcsid[] =
38 * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp ";
39 */
40
41 #include <sys/cdefs.h>
42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.251 2023/02/08 01:37:53 gutteridge Exp $");
43
44 #if defined(_KERNEL_OPT)
45 #include "opt_bpf.h"
46 #include "sl.h"
47 #include "opt_net_mpsafe.h"
48 #endif
49
50 #include <sys/param.h>
51 #include <sys/systm.h>
52 #include <sys/mbuf.h>
53 #include <sys/buf.h>
54 #include <sys/time.h>
55 #include <sys/proc.h>
56 #include <sys/ioctl.h>
57 #include <sys/conf.h>
58 #include <sys/vnode.h>
59 #include <sys/queue.h>
60 #include <sys/stat.h>
61 #include <sys/module.h>
62 #include <sys/atomic.h>
63 #include <sys/cpu.h>
64
65 #include <sys/file.h>
66 #include <sys/filedesc.h>
67 #include <sys/tty.h>
68 #include <sys/uio.h>
69
70 #include <sys/protosw.h>
71 #include <sys/socket.h>
72 #include <sys/errno.h>
73 #include <sys/kernel.h>
74 #include <sys/poll.h>
75 #include <sys/sysctl.h>
76 #include <sys/kauth.h>
77 #include <sys/syslog.h>
78 #include <sys/percpu.h>
79 #include <sys/pserialize.h>
80 #include <sys/lwp.h>
81 #include <sys/xcall.h>
82
83 #include <net/if.h>
84 #include <net/slip.h>
85
86 #include <net/bpf.h>
87 #include <net/bpfdesc.h>
88 #include <net/bpfjit.h>
89
90 #include <net/if_arc.h>
91 #include <net/if_ether.h>
92 #include <net/if_types.h>
93
94 #include <netinet/in.h>
95 #include <netinet/if_inarp.h>
96
97
98 #include <compat/sys/sockio.h>
99
100 #ifndef BPF_BUFSIZE
101 /*
102 * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet
103 * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k).
104 */
105 # define BPF_BUFSIZE 32768
106 #endif
107
108 #define PRINET 26 /* interruptible */
109
110 /*
111 * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able.
112 * XXX the default values should be computed dynamically based
113 * on available memory size and available mbuf clusters.
114 */
115 static int bpf_bufsize = BPF_BUFSIZE;
116 static int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */
117 static bool bpf_jit = false;
118
119 struct bpfjit_ops bpfjit_module_ops = {
120 .bj_generate_code = NULL,
121 .bj_free_code = NULL
122 };
123
124 /*
125 * Global BPF statistics returned by net.bpf.stats sysctl.
126 */
127 static struct percpu *bpf_gstats_percpu; /* struct bpf_stat */
128
129 #define BPF_STATINC(id) \
130 { \
131 struct bpf_stat *__stats = \
132 percpu_getref(bpf_gstats_percpu); \
133 __stats->bs_##id++; \
134 percpu_putref(bpf_gstats_percpu); \
135 }
136
137 /*
138 * Locking notes:
139 * - bpf_mtx (adaptive mutex) protects:
140 * - Gobal lists: bpf_iflist and bpf_dlist
141 * - struct bpf_if
142 * - bpf_close
143 * - bpf_psz (pserialize)
144 * - struct bpf_d has two mutexes:
145 * - bd_buf_mtx (spin mutex) protects the buffers that can be accessed
146 * on packet tapping
147 * - bd_mtx (adaptive mutex) protects member variables other than the buffers
148 * - Locking order: bpf_mtx => bpf_d#bd_mtx => bpf_d#bd_buf_mtx
149 * - struct bpf_d obtained via fp->f_bpf in bpf_read and bpf_write is
150 * never freed because struct bpf_d is only freed in bpf_close and
151 * bpf_close never be called while executing bpf_read and bpf_write
152 * - A filter that is assigned to bpf_d can be replaced with another filter
153 * while tapping packets, so it needs to be done atomically
154 * - struct bpf_d is iterated on bpf_dlist with psz
155 * - struct bpf_if is iterated on bpf_iflist with psz or psref
156 */
157 /*
158 * Use a mutex to avoid a race condition between gathering the stats/peers
159 * and opening/closing the device.
160 */
161 static kmutex_t bpf_mtx;
162
163 static struct psref_class *bpf_psref_class __read_mostly;
164 static pserialize_t bpf_psz;
165
166 static inline void
bpf_if_acquire(struct bpf_if * bp,struct psref * psref)167 bpf_if_acquire(struct bpf_if *bp, struct psref *psref)
168 {
169
170 psref_acquire(psref, &bp->bif_psref, bpf_psref_class);
171 }
172
173 static inline void
bpf_if_release(struct bpf_if * bp,struct psref * psref)174 bpf_if_release(struct bpf_if *bp, struct psref *psref)
175 {
176
177 psref_release(psref, &bp->bif_psref, bpf_psref_class);
178 }
179
180 /*
181 * bpf_iflist is the list of interfaces; each corresponds to an ifnet
182 * bpf_dtab holds the descriptors, indexed by minor device #
183 */
184 static struct pslist_head bpf_iflist;
185 static struct pslist_head bpf_dlist;
186
187 /* Macros for bpf_d on bpf_dlist */
188 #define BPF_DLIST_WRITER_INSERT_HEAD(__d) \
189 PSLIST_WRITER_INSERT_HEAD(&bpf_dlist, (__d), bd_bpf_dlist_entry)
190 #define BPF_DLIST_READER_FOREACH(__d) \
191 PSLIST_READER_FOREACH((__d), &bpf_dlist, struct bpf_d, \
192 bd_bpf_dlist_entry)
193 #define BPF_DLIST_WRITER_FOREACH(__d) \
194 PSLIST_WRITER_FOREACH((__d), &bpf_dlist, struct bpf_d, \
195 bd_bpf_dlist_entry)
196 #define BPF_DLIST_ENTRY_INIT(__d) \
197 PSLIST_ENTRY_INIT((__d), bd_bpf_dlist_entry)
198 #define BPF_DLIST_WRITER_REMOVE(__d) \
199 PSLIST_WRITER_REMOVE((__d), bd_bpf_dlist_entry)
200 #define BPF_DLIST_ENTRY_DESTROY(__d) \
201 PSLIST_ENTRY_DESTROY((__d), bd_bpf_dlist_entry)
202
203 /* Macros for bpf_if on bpf_iflist */
204 #define BPF_IFLIST_WRITER_INSERT_HEAD(__bp) \
205 PSLIST_WRITER_INSERT_HEAD(&bpf_iflist, (__bp), bif_iflist_entry)
206 #define BPF_IFLIST_READER_FOREACH(__bp) \
207 PSLIST_READER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \
208 bif_iflist_entry)
209 #define BPF_IFLIST_WRITER_FOREACH(__bp) \
210 PSLIST_WRITER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \
211 bif_iflist_entry)
212 #define BPF_IFLIST_WRITER_REMOVE(__bp) \
213 PSLIST_WRITER_REMOVE((__bp), bif_iflist_entry)
214 #define BPF_IFLIST_ENTRY_INIT(__bp) \
215 PSLIST_ENTRY_INIT((__bp), bif_iflist_entry)
216 #define BPF_IFLIST_ENTRY_DESTROY(__bp) \
217 PSLIST_ENTRY_DESTROY((__bp), bif_iflist_entry)
218
219 /* Macros for bpf_d on bpf_if#bif_dlist_pslist */
220 #define BPFIF_DLIST_READER_FOREACH(__d, __bp) \
221 PSLIST_READER_FOREACH((__d), &(__bp)->bif_dlist_head, struct bpf_d, \
222 bd_bif_dlist_entry)
223 #define BPFIF_DLIST_WRITER_INSERT_HEAD(__bp, __d) \
224 PSLIST_WRITER_INSERT_HEAD(&(__bp)->bif_dlist_head, (__d), \
225 bd_bif_dlist_entry)
226 #define BPFIF_DLIST_WRITER_REMOVE(__d) \
227 PSLIST_WRITER_REMOVE((__d), bd_bif_dlist_entry)
228 #define BPFIF_DLIST_ENTRY_INIT(__d) \
229 PSLIST_ENTRY_INIT((__d), bd_bif_dlist_entry)
230 #define BPFIF_DLIST_READER_EMPTY(__bp) \
231 (PSLIST_READER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \
232 bd_bif_dlist_entry) == NULL)
233 #define BPFIF_DLIST_WRITER_EMPTY(__bp) \
234 (PSLIST_WRITER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \
235 bd_bif_dlist_entry) == NULL)
236 #define BPFIF_DLIST_ENTRY_DESTROY(__d) \
237 PSLIST_ENTRY_DESTROY((__d), bd_bif_dlist_entry)
238
239 static int bpf_allocbufs(struct bpf_d *);
240 static u_int bpf_xfilter(struct bpf_filter **, void *, u_int, u_int);
241 static void bpf_deliver(struct bpf_if *,
242 void *(*cpfn)(void *, const void *, size_t),
243 void *, u_int, u_int, const u_int);
244 static void bpf_freed(struct bpf_d *);
245 static void bpf_free_filter(struct bpf_filter *);
246 static void bpf_ifname(struct ifnet *, struct ifreq *);
247 static void *bpf_mcpy(void *, const void *, size_t);
248 static int bpf_movein(struct ifnet *, struct uio *, int, uint64_t,
249 struct mbuf **, struct sockaddr *,
250 struct bpf_filter **);
251 static void bpf_attachd(struct bpf_d *, struct bpf_if *);
252 static void bpf_detachd(struct bpf_d *);
253 static int bpf_setif(struct bpf_d *, struct ifreq *);
254 static int bpf_setf(struct bpf_d *, struct bpf_program *, u_long);
255 static void bpf_timed_out(void *);
256 static inline void
257 bpf_wakeup(struct bpf_d *);
258 static int bpf_hdrlen(struct bpf_d *);
259 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int,
260 void *(*)(void *, const void *, size_t), struct timespec *);
261 static void reset_d(struct bpf_d *);
262 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *);
263 static int bpf_setdlt(struct bpf_d *, u_int);
264
265 static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t,
266 int);
267 static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t,
268 int);
269 static int bpf_ioctl(struct file *, u_long, void *);
270 static int bpf_poll(struct file *, int);
271 static int bpf_stat(struct file *, struct stat *);
272 static int bpf_close(struct file *);
273 static int bpf_kqfilter(struct file *, struct knote *);
274
275 static const struct fileops bpf_fileops = {
276 .fo_name = "bpf",
277 .fo_read = bpf_read,
278 .fo_write = bpf_write,
279 .fo_ioctl = bpf_ioctl,
280 .fo_fcntl = fnullop_fcntl,
281 .fo_poll = bpf_poll,
282 .fo_stat = bpf_stat,
283 .fo_close = bpf_close,
284 .fo_kqfilter = bpf_kqfilter,
285 .fo_restart = fnullop_restart,
286 };
287
288 dev_type_open(bpfopen);
289
290 const struct cdevsw bpf_cdevsw = {
291 .d_open = bpfopen,
292 .d_close = noclose,
293 .d_read = noread,
294 .d_write = nowrite,
295 .d_ioctl = noioctl,
296 .d_stop = nostop,
297 .d_tty = notty,
298 .d_poll = nopoll,
299 .d_mmap = nommap,
300 .d_kqfilter = nokqfilter,
301 .d_discard = nodiscard,
302 .d_flag = D_OTHER | D_MPSAFE
303 };
304
305 bpfjit_func_t
bpf_jit_generate(bpf_ctx_t * bc,void * code,size_t size)306 bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size)
307 {
308 struct bpfjit_ops *ops = &bpfjit_module_ops;
309 bpfjit_func_t (*generate_code)(const bpf_ctx_t *,
310 const struct bpf_insn *, size_t);
311
312 generate_code = atomic_load_acquire(&ops->bj_generate_code);
313 if (generate_code != NULL) {
314 return generate_code(bc, code, size);
315 }
316 return NULL;
317 }
318
319 void
bpf_jit_freecode(bpfjit_func_t jcode)320 bpf_jit_freecode(bpfjit_func_t jcode)
321 {
322 KASSERT(bpfjit_module_ops.bj_free_code != NULL);
323 bpfjit_module_ops.bj_free_code(jcode);
324 }
325
326 static int
bpf_movein(struct ifnet * ifp,struct uio * uio,int linktype,uint64_t mtu,struct mbuf ** mp,struct sockaddr * sockp,struct bpf_filter ** wfilter)327 bpf_movein(struct ifnet *ifp, struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp,
328 struct sockaddr *sockp, struct bpf_filter **wfilter)
329 {
330 struct mbuf *m, *m0, *n;
331 int error;
332 size_t len;
333 size_t hlen;
334 size_t align;
335 u_int slen;
336
337 /*
338 * Build a sockaddr based on the data link layer type.
339 * We do this at this level because the ethernet header
340 * is copied directly into the data field of the sockaddr.
341 * In the case of SLIP, there is no header and the packet
342 * is forwarded as is.
343 * Also, we are careful to leave room at the front of the mbuf
344 * for the link level header.
345 */
346 switch (linktype) {
347
348 case DLT_SLIP:
349 sockp->sa_family = AF_INET;
350 hlen = 0;
351 align = 0;
352 break;
353
354 case DLT_PPP:
355 sockp->sa_family = AF_UNSPEC;
356 hlen = 0;
357 align = 0;
358 break;
359
360 case DLT_EN10MB:
361 sockp->sa_family = AF_UNSPEC;
362 /* XXX Would MAXLINKHDR be better? */
363 /* 6(dst)+6(src)+2(type) */
364 hlen = sizeof(struct ether_header);
365 align = 2;
366 break;
367
368 case DLT_ARCNET:
369 sockp->sa_family = AF_UNSPEC;
370 hlen = ARC_HDRLEN;
371 align = 5;
372 break;
373
374 case DLT_FDDI:
375 sockp->sa_family = AF_LINK;
376 /* XXX 4(FORMAC)+6(dst)+6(src) */
377 hlen = 16;
378 align = 0;
379 break;
380
381 case DLT_ECONET:
382 sockp->sa_family = AF_UNSPEC;
383 hlen = 6;
384 align = 2;
385 break;
386
387 case DLT_NULL:
388 sockp->sa_family = AF_UNSPEC;
389 if (ifp->if_type == IFT_LOOP) {
390 /* Set here to apply the following validations */
391 hlen = sizeof(uint32_t);
392 } else
393 hlen = 0;
394 align = 0;
395 break;
396
397 default:
398 return (EIO);
399 }
400
401 len = uio->uio_resid;
402 /*
403 * If there aren't enough bytes for a link level header or the
404 * packet length exceeds the interface mtu, return an error.
405 */
406 if (len - hlen > mtu)
407 return (EMSGSIZE);
408
409 m0 = m = m_gethdr(M_WAIT, MT_DATA);
410 m_reset_rcvif(m);
411 m->m_pkthdr.len = (int)(len - hlen);
412 if (len + align > MHLEN) {
413 m_clget(m, M_WAIT);
414 if ((m->m_flags & M_EXT) == 0) {
415 error = ENOBUFS;
416 goto bad;
417 }
418 }
419
420 /* Ensure the data is properly aligned */
421 if (align > 0)
422 m->m_data += align;
423
424 for (;;) {
425 len = M_TRAILINGSPACE(m);
426 if (len > uio->uio_resid)
427 len = uio->uio_resid;
428 error = uiomove(mtod(m, void *), len, uio);
429 if (error)
430 goto bad;
431 m->m_len = len;
432
433 if (uio->uio_resid == 0)
434 break;
435
436 n = m_get(M_WAIT, MT_DATA);
437 m_clget(n, M_WAIT); /* if fails, there is no problem */
438 m->m_next = n;
439 m = n;
440 }
441
442 slen = bpf_xfilter(wfilter, mtod(m, u_char *), len, len);
443 if (slen == 0) {
444 error = EPERM;
445 goto bad;
446 }
447
448 if (hlen != 0) {
449 if (linktype == DLT_NULL && ifp->if_type == IFT_LOOP) {
450 uint32_t af;
451 /* the link header indicates the address family */
452 memcpy(&af, mtod(m0, void *), sizeof(af));
453 sockp->sa_family = af;
454 } else {
455 /* move link level header in the top of mbuf to sa_data */
456 memcpy(sockp->sa_data, mtod(m0, void *), hlen);
457 }
458 m0->m_data += hlen;
459 m0->m_len -= hlen;
460 }
461
462 *mp = m0;
463 return (0);
464
465 bad:
466 m_freem(m0);
467 return (error);
468 }
469
470 /*
471 * Attach file to the bpf interface, i.e. make d listen on bp.
472 */
473 static void
bpf_attachd(struct bpf_d * d,struct bpf_if * bp)474 bpf_attachd(struct bpf_d *d, struct bpf_if *bp)
475 {
476 struct bpf_event_tracker *t;
477
478 KASSERT(mutex_owned(&bpf_mtx));
479 KASSERT(mutex_owned(d->bd_mtx));
480 /*
481 * Point d at bp, and add d to the interface's list of listeners.
482 * Finally, point the driver's bpf cookie at the interface so
483 * it will divert packets to bpf.
484 */
485 d->bd_bif = bp;
486 BPFIF_DLIST_WRITER_INSERT_HEAD(bp, d);
487
488 *bp->bif_driverp = bp;
489
490 SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) {
491 t->bet_notify(bp, bp->bif_ifp, bp->bif_dlt,
492 BPF_TRACK_EVENT_ATTACH);
493 }
494 }
495
496 /*
497 * Detach a file from its interface.
498 */
499 static void
bpf_detachd(struct bpf_d * d)500 bpf_detachd(struct bpf_d *d)
501 {
502 struct bpf_if *bp;
503 struct bpf_event_tracker *t;
504
505 KASSERT(mutex_owned(&bpf_mtx));
506 KASSERT(mutex_owned(d->bd_mtx));
507
508 bp = d->bd_bif;
509 /*
510 * Check if this descriptor had requested promiscuous mode.
511 * If so, turn it off.
512 */
513 if (d->bd_promisc) {
514 int error __diagused;
515
516 d->bd_promisc = 0;
517 /*
518 * Take device out of promiscuous mode. Since we were
519 * able to enter promiscuous mode, we should be able
520 * to turn it off. But we can get an error if
521 * the interface was configured down, so only panic
522 * if we don't get an unexpected error.
523 */
524 KERNEL_LOCK_UNLESS_NET_MPSAFE();
525 error = ifpromisc(bp->bif_ifp, 0);
526 KERNEL_UNLOCK_UNLESS_NET_MPSAFE();
527 #ifdef DIAGNOSTIC
528 if (error)
529 printf("%s: ifpromisc failed: %d", __func__, error);
530 #endif
531 }
532
533 /* Remove d from the interface's descriptor list. */
534 BPFIF_DLIST_WRITER_REMOVE(d);
535
536 pserialize_perform(bpf_psz);
537
538 if (BPFIF_DLIST_WRITER_EMPTY(bp)) {
539 /*
540 * Let the driver know that there are no more listeners.
541 */
542 *d->bd_bif->bif_driverp = NULL;
543 }
544
545 d->bd_bif = NULL;
546
547 SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) {
548 t->bet_notify(bp, bp->bif_ifp, bp->bif_dlt,
549 BPF_TRACK_EVENT_DETACH);
550 }
551 }
552
553 static void
bpf_init(void)554 bpf_init(void)
555 {
556
557 mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE);
558 bpf_psz = pserialize_create();
559 bpf_psref_class = psref_class_create("bpf", IPL_SOFTNET);
560
561 PSLIST_INIT(&bpf_iflist);
562 PSLIST_INIT(&bpf_dlist);
563
564 bpf_gstats_percpu = percpu_alloc(sizeof(struct bpf_stat));
565
566 return;
567 }
568
569 /*
570 * bpfilterattach() is called at boot time. We don't need to do anything
571 * here, since any initialization will happen as part of module init code.
572 */
573 /* ARGSUSED */
574 void
bpfilterattach(int n)575 bpfilterattach(int n)
576 {
577
578 }
579
580 /*
581 * Open ethernet device. Clones.
582 */
583 /* ARGSUSED */
584 int
bpfopen(dev_t dev,int flag,int mode,struct lwp * l)585 bpfopen(dev_t dev, int flag, int mode, struct lwp *l)
586 {
587 struct bpf_d *d;
588 struct file *fp;
589 int error, fd;
590
591 /* falloc() will fill in the descriptor for us. */
592 if ((error = fd_allocfile(&fp, &fd)) != 0)
593 return error;
594
595 d = kmem_zalloc(sizeof(*d), KM_SLEEP);
596 d->bd_bufsize = bpf_bufsize;
597 d->bd_direction = BPF_D_INOUT;
598 d->bd_feedback = 0;
599 d->bd_pid = l->l_proc->p_pid;
600 #ifdef _LP64
601 if (curproc->p_flag & PK_32)
602 d->bd_compat32 = 1;
603 #endif
604 getnanotime(&d->bd_btime);
605 d->bd_atime = d->bd_mtime = d->bd_btime;
606 callout_init(&d->bd_callout, CALLOUT_MPSAFE);
607 selinit(&d->bd_sel);
608 d->bd_jitcode = NULL;
609 d->bd_rfilter = NULL;
610 d->bd_wfilter = NULL;
611 d->bd_locked = 0;
612 BPF_DLIST_ENTRY_INIT(d);
613 BPFIF_DLIST_ENTRY_INIT(d);
614 d->bd_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET);
615 d->bd_buf_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
616 cv_init(&d->bd_cv, "bpf");
617
618 mutex_enter(&bpf_mtx);
619 BPF_DLIST_WRITER_INSERT_HEAD(d);
620 mutex_exit(&bpf_mtx);
621
622 return fd_clone(fp, fd, flag, &bpf_fileops, d);
623 }
624
625 /*
626 * Close the descriptor by detaching it from its interface,
627 * deallocating its buffers, and marking it free.
628 */
629 /* ARGSUSED */
630 static int
bpf_close(struct file * fp)631 bpf_close(struct file *fp)
632 {
633 struct bpf_d *d;
634
635 mutex_enter(&bpf_mtx);
636
637 if ((d = fp->f_bpf) == NULL) {
638 mutex_exit(&bpf_mtx);
639 return 0;
640 }
641
642 /*
643 * Refresh the PID associated with this bpf file.
644 */
645 d->bd_pid = curproc->p_pid;
646
647 mutex_enter(d->bd_mtx);
648 if (d->bd_state == BPF_WAITING)
649 callout_halt(&d->bd_callout, d->bd_mtx);
650 d->bd_state = BPF_IDLE;
651 if (d->bd_bif)
652 bpf_detachd(d);
653 mutex_exit(d->bd_mtx);
654
655 BPF_DLIST_WRITER_REMOVE(d);
656
657 pserialize_perform(bpf_psz);
658 mutex_exit(&bpf_mtx);
659
660 BPFIF_DLIST_ENTRY_DESTROY(d);
661 BPF_DLIST_ENTRY_DESTROY(d);
662 fp->f_bpf = NULL;
663 bpf_freed(d);
664 callout_destroy(&d->bd_callout);
665 seldestroy(&d->bd_sel);
666 mutex_obj_free(d->bd_mtx);
667 mutex_obj_free(d->bd_buf_mtx);
668 cv_destroy(&d->bd_cv);
669
670 kmem_free(d, sizeof(*d));
671
672 return (0);
673 }
674
675 /*
676 * Rotate the packet buffers in descriptor d. Move the store buffer
677 * into the hold slot, and the free buffer into the store slot.
678 * Zero the length of the new store buffer.
679 */
680 #define ROTATE_BUFFERS(d) \
681 (d)->bd_hbuf = (d)->bd_sbuf; \
682 (d)->bd_hlen = (d)->bd_slen; \
683 (d)->bd_sbuf = (d)->bd_fbuf; \
684 (d)->bd_slen = 0; \
685 (d)->bd_fbuf = NULL;
686 /*
687 * bpfread - read next chunk of packets from buffers
688 */
689 static int
bpf_read(struct file * fp,off_t * offp,struct uio * uio,kauth_cred_t cred,int flags)690 bpf_read(struct file *fp, off_t *offp, struct uio *uio,
691 kauth_cred_t cred, int flags)
692 {
693 struct bpf_d *d = fp->f_bpf;
694 int timed_out;
695 int error;
696
697 /*
698 * Refresh the PID associated with this bpf file.
699 */
700 d->bd_pid = curproc->p_pid;
701
702 getnanotime(&d->bd_atime);
703 /*
704 * Restrict application to use a buffer the same size as
705 * the kernel buffers.
706 */
707 if (uio->uio_resid != d->bd_bufsize)
708 return (EINVAL);
709
710 mutex_enter(d->bd_mtx);
711 if (d->bd_state == BPF_WAITING)
712 callout_halt(&d->bd_callout, d->bd_mtx);
713 timed_out = (d->bd_state == BPF_TIMED_OUT);
714 d->bd_state = BPF_IDLE;
715 mutex_exit(d->bd_mtx);
716 /*
717 * If the hold buffer is empty, then do a timed sleep, which
718 * ends when the timeout expires or when enough packets
719 * have arrived to fill the store buffer.
720 */
721 mutex_enter(d->bd_buf_mtx);
722 while (d->bd_hbuf == NULL) {
723 if (fp->f_flag & FNONBLOCK) {
724 if (d->bd_slen == 0) {
725 error = EWOULDBLOCK;
726 goto out;
727 }
728 ROTATE_BUFFERS(d);
729 break;
730 }
731
732 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) {
733 /*
734 * A packet(s) either arrived since the previous
735 * read or arrived while we were asleep.
736 * Rotate the buffers and return what's here.
737 */
738 ROTATE_BUFFERS(d);
739 break;
740 }
741
742 error = cv_timedwait_sig(&d->bd_cv, d->bd_buf_mtx, d->bd_rtout);
743
744 if (error == EINTR || error == ERESTART)
745 goto out;
746
747 if (error == EWOULDBLOCK) {
748 /*
749 * On a timeout, return what's in the buffer,
750 * which may be nothing. If there is something
751 * in the store buffer, we can rotate the buffers.
752 */
753 if (d->bd_hbuf)
754 /*
755 * We filled up the buffer in between
756 * getting the timeout and arriving
757 * here, so we don't need to rotate.
758 */
759 break;
760
761 if (d->bd_slen == 0) {
762 error = 0;
763 goto out;
764 }
765 ROTATE_BUFFERS(d);
766 break;
767 }
768 if (error != 0)
769 goto out;
770 }
771 /*
772 * At this point, we know we have something in the hold slot.
773 */
774 mutex_exit(d->bd_buf_mtx);
775
776 /*
777 * Move data from hold buffer into user space.
778 * We know the entire buffer is transferred since
779 * we checked above that the read buffer is bpf_bufsize bytes.
780 */
781 error = uiomove(d->bd_hbuf, d->bd_hlen, uio);
782
783 mutex_enter(d->bd_buf_mtx);
784 d->bd_fbuf = d->bd_hbuf;
785 d->bd_hbuf = NULL;
786 d->bd_hlen = 0;
787 out:
788 mutex_exit(d->bd_buf_mtx);
789 return (error);
790 }
791
792
793 /*
794 * If there are processes sleeping on this descriptor, wake them up.
795 */
796 static inline void
bpf_wakeup(struct bpf_d * d)797 bpf_wakeup(struct bpf_d *d)
798 {
799
800 mutex_enter(d->bd_buf_mtx);
801 cv_broadcast(&d->bd_cv);
802 mutex_exit(d->bd_buf_mtx);
803
804 if (d->bd_async)
805 fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL);
806 selnotify(&d->bd_sel, 0, 0);
807 }
808
809 static void
bpf_timed_out(void * arg)810 bpf_timed_out(void *arg)
811 {
812 struct bpf_d *d = arg;
813
814 mutex_enter(d->bd_mtx);
815 if (d->bd_state == BPF_WAITING) {
816 d->bd_state = BPF_TIMED_OUT;
817 if (d->bd_slen != 0)
818 bpf_wakeup(d);
819 }
820 mutex_exit(d->bd_mtx);
821 }
822
823
824 static int
bpf_write(struct file * fp,off_t * offp,struct uio * uio,kauth_cred_t cred,int flags)825 bpf_write(struct file *fp, off_t *offp, struct uio *uio,
826 kauth_cred_t cred, int flags)
827 {
828 struct bpf_d *d = fp->f_bpf;
829 struct bpf_if *bp;
830 struct ifnet *ifp;
831 struct mbuf *m, *mc;
832 int error;
833 static struct sockaddr_storage dst;
834 struct psref psref;
835 int bound;
836
837 /*
838 * Refresh the PID associated with this bpf file.
839 */
840 d->bd_pid = curproc->p_pid;
841
842 m = NULL; /* XXX gcc */
843
844 bound = curlwp_bind();
845 mutex_enter(d->bd_mtx);
846 bp = d->bd_bif;
847 if (bp == NULL) {
848 mutex_exit(d->bd_mtx);
849 error = ENXIO;
850 goto out_bindx;
851 }
852 bpf_if_acquire(bp, &psref);
853 mutex_exit(d->bd_mtx);
854
855 getnanotime(&d->bd_mtime);
856
857 ifp = bp->bif_ifp;
858 if (if_is_deactivated(ifp)) {
859 error = ENXIO;
860 goto out;
861 }
862
863 if (uio->uio_resid == 0) {
864 error = 0;
865 goto out;
866 }
867
868 error = bpf_movein(ifp, uio, (int)bp->bif_dlt, ifp->if_mtu, &m,
869 (struct sockaddr *) &dst, &d->bd_wfilter);
870 if (error)
871 goto out;
872
873 if (m->m_pkthdr.len > ifp->if_mtu) {
874 m_freem(m);
875 error = EMSGSIZE;
876 goto out;
877 }
878
879 /*
880 * If writing to a loopback interface, the address family has
881 * already been specially computed in bpf_movein(), so don't
882 * clobber it, or the loopback will reject it in looutput().
883 */
884 if (d->bd_hdrcmplt && ifp->if_type != IFT_LOOP)
885 dst.ss_family = pseudo_AF_HDRCMPLT;
886
887 if (d->bd_feedback) {
888 mc = m_dup(m, 0, M_COPYALL, M_NOWAIT);
889 if (mc != NULL)
890 m_set_rcvif(mc, ifp);
891 /* Set M_PROMISC for outgoing packets to be discarded. */
892 if (1 /*d->bd_direction == BPF_D_INOUT*/)
893 m->m_flags |= M_PROMISC;
894 } else
895 mc = NULL;
896
897 error = if_output_lock(ifp, ifp, m, (struct sockaddr *) &dst, NULL);
898
899 if (mc != NULL) {
900 if (error == 0) {
901 int s = splsoftnet();
902 KERNEL_LOCK_UNLESS_IFP_MPSAFE(ifp);
903 ifp->_if_input(ifp, mc);
904 KERNEL_UNLOCK_UNLESS_IFP_MPSAFE(ifp);
905 splx(s);
906 } else
907 m_freem(mc);
908 }
909 /*
910 * The driver frees the mbuf.
911 */
912 out:
913 bpf_if_release(bp, &psref);
914 out_bindx:
915 curlwp_bindx(bound);
916 return error;
917 }
918
919 /*
920 * Reset a descriptor by flushing its packet buffer and clearing the
921 * receive and drop counts.
922 */
923 static void
reset_d(struct bpf_d * d)924 reset_d(struct bpf_d *d)
925 {
926
927 KASSERT(mutex_owned(d->bd_mtx));
928
929 mutex_enter(d->bd_buf_mtx);
930 if (d->bd_hbuf) {
931 /* Free the hold buffer. */
932 d->bd_fbuf = d->bd_hbuf;
933 d->bd_hbuf = NULL;
934 }
935 d->bd_slen = 0;
936 d->bd_hlen = 0;
937 d->bd_rcount = 0;
938 d->bd_dcount = 0;
939 d->bd_ccount = 0;
940 mutex_exit(d->bd_buf_mtx);
941 }
942
943 /*
944 * FIONREAD Check for read packet available.
945 * BIOCGBLEN Get buffer len [for read()].
946 * BIOCSETF Set ethernet read filter.
947 * BIOCFLUSH Flush read packet buffer.
948 * BIOCPROMISC Put interface into promiscuous mode.
949 * BIOCGDLT Get link layer type.
950 * BIOCGETIF Get interface name.
951 * BIOCSETIF Set interface.
952 * BIOCSRTIMEOUT Set read timeout.
953 * BIOCGRTIMEOUT Get read timeout.
954 * BIOCGSTATS Get packet stats.
955 * BIOCIMMEDIATE Set immediate mode.
956 * BIOCVERSION Get filter language version.
957 * BIOCGHDRCMPLT Get "header already complete" flag.
958 * BIOCSHDRCMPLT Set "header already complete" flag.
959 * BIOCSFEEDBACK Set packet feedback mode.
960 * BIOCGFEEDBACK Get packet feedback mode.
961 * BIOCGDIRECTION Get packet direction flag
962 * BIOCSDIRECTION Set packet direction flag
963 */
964 /* ARGSUSED */
965 static int
bpf_ioctl(struct file * fp,u_long cmd,void * addr)966 bpf_ioctl(struct file *fp, u_long cmd, void *addr)
967 {
968 struct bpf_d *d = fp->f_bpf;
969 int error = 0;
970
971 /*
972 * Refresh the PID associated with this bpf file.
973 */
974 d->bd_pid = curproc->p_pid;
975 #ifdef _LP64
976 if (curproc->p_flag & PK_32)
977 d->bd_compat32 = 1;
978 else
979 d->bd_compat32 = 0;
980 #endif
981
982 mutex_enter(d->bd_mtx);
983 if (d->bd_state == BPF_WAITING)
984 callout_halt(&d->bd_callout, d->bd_mtx);
985 d->bd_state = BPF_IDLE;
986 mutex_exit(d->bd_mtx);
987
988 if (d->bd_locked) {
989 switch (cmd) {
990 case BIOCGBLEN: /* FALLTHROUGH */
991 case BIOCFLUSH: /* FALLTHROUGH */
992 case BIOCGDLT: /* FALLTHROUGH */
993 case BIOCGDLTLIST: /* FALLTHROUGH */
994 case BIOCGETIF: /* FALLTHROUGH */
995 case BIOCGRTIMEOUT: /* FALLTHROUGH */
996 case BIOCGSTATS: /* FALLTHROUGH */
997 case BIOCVERSION: /* FALLTHROUGH */
998 case BIOCGHDRCMPLT: /* FALLTHROUGH */
999 case FIONREAD: /* FALLTHROUGH */
1000 case BIOCLOCK: /* FALLTHROUGH */
1001 case BIOCSRTIMEOUT: /* FALLTHROUGH */
1002 case BIOCIMMEDIATE: /* FALLTHROUGH */
1003 case TIOCGPGRP:
1004 break;
1005 default:
1006 return EPERM;
1007 }
1008 }
1009
1010 switch (cmd) {
1011
1012 default:
1013 error = EINVAL;
1014 break;
1015
1016 /*
1017 * Check for read packet available.
1018 */
1019 case FIONREAD:
1020 {
1021 int n;
1022
1023 mutex_enter(d->bd_buf_mtx);
1024 n = d->bd_slen;
1025 if (d->bd_hbuf)
1026 n += d->bd_hlen;
1027 mutex_exit(d->bd_buf_mtx);
1028
1029 *(int *)addr = n;
1030 break;
1031 }
1032
1033 /*
1034 * Get buffer len [for read()].
1035 */
1036 case BIOCGBLEN:
1037 *(u_int *)addr = d->bd_bufsize;
1038 break;
1039
1040 /*
1041 * Set buffer length.
1042 */
1043 case BIOCSBLEN:
1044 /*
1045 * Forbid to change the buffer length if buffers are already
1046 * allocated.
1047 */
1048 mutex_enter(d->bd_mtx);
1049 mutex_enter(d->bd_buf_mtx);
1050 if (d->bd_bif != NULL || d->bd_sbuf != NULL)
1051 error = EINVAL;
1052 else {
1053 u_int size = *(u_int *)addr;
1054
1055 if (size > bpf_maxbufsize)
1056 *(u_int *)addr = size = bpf_maxbufsize;
1057 else if (size < BPF_MINBUFSIZE)
1058 *(u_int *)addr = size = BPF_MINBUFSIZE;
1059 d->bd_bufsize = size;
1060 }
1061 mutex_exit(d->bd_buf_mtx);
1062 mutex_exit(d->bd_mtx);
1063 break;
1064
1065 /*
1066 * Set link layer read filter.
1067 */
1068 case BIOCSETF: /* FALLTHROUGH */
1069 case BIOCSETWF:
1070 error = bpf_setf(d, addr, cmd);
1071 break;
1072
1073 case BIOCLOCK:
1074 d->bd_locked = 1;
1075 break;
1076
1077 /*
1078 * Flush read packet buffer.
1079 */
1080 case BIOCFLUSH:
1081 mutex_enter(d->bd_mtx);
1082 reset_d(d);
1083 mutex_exit(d->bd_mtx);
1084 break;
1085
1086 /*
1087 * Put interface into promiscuous mode.
1088 */
1089 case BIOCPROMISC:
1090 mutex_enter(d->bd_mtx);
1091 if (d->bd_bif == NULL) {
1092 mutex_exit(d->bd_mtx);
1093 /*
1094 * No interface attached yet.
1095 */
1096 error = EINVAL;
1097 break;
1098 }
1099 if (d->bd_promisc == 0) {
1100 KERNEL_LOCK_UNLESS_NET_MPSAFE();
1101 error = ifpromisc(d->bd_bif->bif_ifp, 1);
1102 KERNEL_UNLOCK_UNLESS_NET_MPSAFE();
1103 if (error == 0)
1104 d->bd_promisc = 1;
1105 }
1106 mutex_exit(d->bd_mtx);
1107 break;
1108
1109 /*
1110 * Get device parameters.
1111 */
1112 case BIOCGDLT:
1113 mutex_enter(d->bd_mtx);
1114 if (d->bd_bif == NULL)
1115 error = EINVAL;
1116 else
1117 *(u_int *)addr = d->bd_bif->bif_dlt;
1118 mutex_exit(d->bd_mtx);
1119 break;
1120
1121 /*
1122 * Get a list of supported device parameters.
1123 */
1124 case BIOCGDLTLIST:
1125 mutex_enter(d->bd_mtx);
1126 if (d->bd_bif == NULL)
1127 error = EINVAL;
1128 else
1129 error = bpf_getdltlist(d, addr);
1130 mutex_exit(d->bd_mtx);
1131 break;
1132
1133 /*
1134 * Set device parameters.
1135 */
1136 case BIOCSDLT:
1137 mutex_enter(&bpf_mtx);
1138 mutex_enter(d->bd_mtx);
1139 if (d->bd_bif == NULL)
1140 error = EINVAL;
1141 else
1142 error = bpf_setdlt(d, *(u_int *)addr);
1143 mutex_exit(d->bd_mtx);
1144 mutex_exit(&bpf_mtx);
1145 break;
1146
1147 /*
1148 * Set interface name.
1149 */
1150 #ifdef OBIOCGETIF
1151 case OBIOCGETIF:
1152 #endif
1153 case BIOCGETIF:
1154 mutex_enter(d->bd_mtx);
1155 if (d->bd_bif == NULL)
1156 error = EINVAL;
1157 else
1158 bpf_ifname(d->bd_bif->bif_ifp, addr);
1159 mutex_exit(d->bd_mtx);
1160 break;
1161
1162 /*
1163 * Set interface.
1164 */
1165 #ifdef OBIOCSETIF
1166 case OBIOCSETIF:
1167 #endif
1168 case BIOCSETIF:
1169 mutex_enter(&bpf_mtx);
1170 error = bpf_setif(d, addr);
1171 mutex_exit(&bpf_mtx);
1172 break;
1173
1174 /*
1175 * Set read timeout.
1176 */
1177 case BIOCSRTIMEOUT:
1178 {
1179 struct timeval *tv = addr;
1180
1181 /* Compute number of ticks. */
1182 if (tv->tv_sec < 0 ||
1183 tv->tv_usec < 0 || tv->tv_usec >= 1000000) {
1184 error = EINVAL;
1185 break;
1186 } else if (tv->tv_sec > INT_MAX/hz - 1) {
1187 d->bd_rtout = INT_MAX;
1188 } else {
1189 d->bd_rtout = tv->tv_sec * hz
1190 + tv->tv_usec / tick;
1191 }
1192 if ((d->bd_rtout == 0) && (tv->tv_usec != 0))
1193 d->bd_rtout = 1;
1194 break;
1195 }
1196
1197 #ifdef BIOCGORTIMEOUT
1198 /*
1199 * Get read timeout.
1200 */
1201 case BIOCGORTIMEOUT:
1202 {
1203 struct timeval50 *tv = addr;
1204
1205 tv->tv_sec = d->bd_rtout / hz;
1206 tv->tv_usec = (d->bd_rtout % hz) * tick;
1207 break;
1208 }
1209 #endif
1210
1211 #ifdef BIOCSORTIMEOUT
1212 /*
1213 * Set read timeout.
1214 */
1215 case BIOCSORTIMEOUT:
1216 {
1217 struct timeval50 *tv = addr;
1218
1219 /* Compute number of ticks. */
1220 if (tv->tv_sec < 0 ||
1221 tv->tv_usec < 0 || tv->tv_usec >= 1000000) {
1222 error = EINVAL;
1223 break;
1224 } else if (tv->tv_sec > INT_MAX/hz - 1) {
1225 d->bd_rtout = INT_MAX;
1226 } else {
1227 d->bd_rtout = tv->tv_sec * hz
1228 + tv->tv_usec / tick;
1229 }
1230 if ((d->bd_rtout == 0) && (tv->tv_usec != 0))
1231 d->bd_rtout = 1;
1232 break;
1233 }
1234 #endif
1235
1236 /*
1237 * Get read timeout.
1238 */
1239 case BIOCGRTIMEOUT:
1240 {
1241 struct timeval *tv = addr;
1242
1243 tv->tv_sec = d->bd_rtout / hz;
1244 tv->tv_usec = (d->bd_rtout % hz) * tick;
1245 break;
1246 }
1247 /*
1248 * Get packet stats.
1249 */
1250 case BIOCGSTATS:
1251 {
1252 struct bpf_stat *bs = addr;
1253
1254 bs->bs_recv = d->bd_rcount;
1255 bs->bs_drop = d->bd_dcount;
1256 bs->bs_capt = d->bd_ccount;
1257 break;
1258 }
1259
1260 case BIOCGSTATSOLD:
1261 {
1262 struct bpf_stat_old *bs = addr;
1263
1264 bs->bs_recv = d->bd_rcount;
1265 bs->bs_drop = d->bd_dcount;
1266 break;
1267 }
1268
1269 /*
1270 * Set immediate mode.
1271 */
1272 case BIOCIMMEDIATE:
1273 d->bd_immediate = *(u_int *)addr;
1274 break;
1275
1276 case BIOCVERSION:
1277 {
1278 struct bpf_version *bv = addr;
1279
1280 bv->bv_major = BPF_MAJOR_VERSION;
1281 bv->bv_minor = BPF_MINOR_VERSION;
1282 break;
1283 }
1284
1285 case BIOCGHDRCMPLT: /* get "header already complete" flag */
1286 *(u_int *)addr = d->bd_hdrcmplt;
1287 break;
1288
1289 case BIOCSHDRCMPLT: /* set "header already complete" flag */
1290 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0;
1291 break;
1292
1293 /*
1294 * Get packet direction flag
1295 */
1296 case BIOCGDIRECTION:
1297 *(u_int *)addr = d->bd_direction;
1298 break;
1299
1300 /*
1301 * Set packet direction flag
1302 */
1303 case BIOCSDIRECTION:
1304 {
1305 u_int direction;
1306
1307 direction = *(u_int *)addr;
1308 switch (direction) {
1309 case BPF_D_IN:
1310 case BPF_D_INOUT:
1311 case BPF_D_OUT:
1312 d->bd_direction = direction;
1313 break;
1314 default:
1315 error = EINVAL;
1316 }
1317 }
1318 break;
1319
1320 /*
1321 * Set "feed packets from bpf back to input" mode
1322 */
1323 case BIOCSFEEDBACK:
1324 d->bd_feedback = *(u_int *)addr;
1325 break;
1326
1327 /*
1328 * Get "feed packets from bpf back to input" mode
1329 */
1330 case BIOCGFEEDBACK:
1331 *(u_int *)addr = d->bd_feedback;
1332 break;
1333
1334 case FIONBIO: /* Non-blocking I/O */
1335 /*
1336 * No need to do anything special as we use IO_NDELAY in
1337 * bpfread() as an indication of whether or not to block
1338 * the read.
1339 */
1340 break;
1341
1342 case FIOASYNC: /* Send signal on receive packets */
1343 mutex_enter(d->bd_mtx);
1344 d->bd_async = *(int *)addr;
1345 mutex_exit(d->bd_mtx);
1346 break;
1347
1348 case TIOCSPGRP: /* Process or group to send signals to */
1349 case FIOSETOWN:
1350 error = fsetown(&d->bd_pgid, cmd, addr);
1351 break;
1352
1353 case TIOCGPGRP:
1354 case FIOGETOWN:
1355 error = fgetown(d->bd_pgid, cmd, addr);
1356 break;
1357 }
1358 return (error);
1359 }
1360
1361 /*
1362 * Set d's packet filter program to fp. If this file already has a filter,
1363 * free it and replace it. Returns EINVAL for bogus requests.
1364 */
1365 static int
bpf_setf(struct bpf_d * d,struct bpf_program * fp,u_long cmd)1366 bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd)
1367 {
1368 struct bpf_insn *fcode;
1369 bpfjit_func_t jcode;
1370 size_t flen, size = 0;
1371 struct bpf_filter *oldf, *newf, **storef;
1372
1373 jcode = NULL;
1374 flen = fp->bf_len;
1375
1376 if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) {
1377 return EINVAL;
1378 }
1379
1380 if (flen) {
1381 /*
1382 * Allocate the buffer, copy the byte-code from
1383 * userspace and validate it.
1384 */
1385 size = flen * sizeof(*fp->bf_insns);
1386 fcode = kmem_alloc(size, KM_SLEEP);
1387 if (copyin(fp->bf_insns, fcode, size) != 0 ||
1388 !bpf_validate(fcode, (int)flen)) {
1389 kmem_free(fcode, size);
1390 return EINVAL;
1391 }
1392 if (bpf_jit)
1393 jcode = bpf_jit_generate(NULL, fcode, flen);
1394 } else {
1395 fcode = NULL;
1396 }
1397
1398 newf = kmem_alloc(sizeof(*newf), KM_SLEEP);
1399 newf->bf_insn = fcode;
1400 newf->bf_size = size;
1401 newf->bf_jitcode = jcode;
1402 if (cmd == BIOCSETF)
1403 d->bd_jitcode = jcode; /* XXX just for kvm(3) users */
1404
1405 /* Need to hold bpf_mtx for pserialize_perform */
1406 mutex_enter(&bpf_mtx);
1407 mutex_enter(d->bd_mtx);
1408 if (cmd == BIOCSETWF) {
1409 oldf = d->bd_wfilter;
1410 storef = &d->bd_wfilter;
1411 } else {
1412 oldf = d->bd_rfilter;
1413 storef = &d->bd_rfilter;
1414 }
1415 atomic_store_release(storef, newf);
1416 reset_d(d);
1417 pserialize_perform(bpf_psz);
1418 mutex_exit(d->bd_mtx);
1419 mutex_exit(&bpf_mtx);
1420
1421 if (oldf != NULL)
1422 bpf_free_filter(oldf);
1423
1424 return 0;
1425 }
1426
1427 /*
1428 * Detach a file from its current interface (if attached at all) and attach
1429 * to the interface indicated by the name stored in ifr.
1430 * Return an errno or 0.
1431 */
1432 static int
bpf_setif(struct bpf_d * d,struct ifreq * ifr)1433 bpf_setif(struct bpf_d *d, struct ifreq *ifr)
1434 {
1435 struct bpf_if *bp;
1436 char *cp;
1437 int unit_seen, i, error;
1438
1439 KASSERT(mutex_owned(&bpf_mtx));
1440 /*
1441 * Make sure the provided name has a unit number, and default
1442 * it to '0' if not specified.
1443 * XXX This is ugly ... do this differently?
1444 */
1445 unit_seen = 0;
1446 cp = ifr->ifr_name;
1447 cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */
1448 while (*cp++)
1449 if (*cp >= '0' && *cp <= '9')
1450 unit_seen = 1;
1451 if (!unit_seen) {
1452 /* Make sure to leave room for the '\0'. */
1453 for (i = 0; i < (IFNAMSIZ - 1); ++i) {
1454 if ((ifr->ifr_name[i] >= 'a' &&
1455 ifr->ifr_name[i] <= 'z') ||
1456 (ifr->ifr_name[i] >= 'A' &&
1457 ifr->ifr_name[i] <= 'Z'))
1458 continue;
1459 ifr->ifr_name[i] = '0';
1460 }
1461 }
1462
1463 /*
1464 * Look through attached interfaces for the named one.
1465 */
1466 BPF_IFLIST_WRITER_FOREACH(bp) {
1467 struct ifnet *ifp = bp->bif_ifp;
1468
1469 if (ifp == NULL ||
1470 strcmp(ifp->if_xname, ifr->ifr_name) != 0)
1471 continue;
1472 /* skip additional entry */
1473 if (bp->bif_driverp != &ifp->if_bpf)
1474 continue;
1475 /*
1476 * We found the requested interface.
1477 * Allocate the packet buffers if we need to.
1478 * If we're already attached to requested interface,
1479 * just flush the buffer.
1480 */
1481 /*
1482 * bpf_allocbufs is called only here. bpf_mtx ensures that
1483 * no race condition happen on d->bd_sbuf.
1484 */
1485 if (d->bd_sbuf == NULL) {
1486 error = bpf_allocbufs(d);
1487 if (error != 0)
1488 return (error);
1489 }
1490 mutex_enter(d->bd_mtx);
1491 if (bp != d->bd_bif) {
1492 if (d->bd_bif) {
1493 /*
1494 * Detach if attached to something else.
1495 */
1496 bpf_detachd(d);
1497 BPFIF_DLIST_ENTRY_INIT(d);
1498 }
1499
1500 bpf_attachd(d, bp);
1501 }
1502 reset_d(d);
1503 mutex_exit(d->bd_mtx);
1504 return (0);
1505 }
1506 /* Not found. */
1507 return (ENXIO);
1508 }
1509
1510 /*
1511 * Copy the interface name to the ifreq.
1512 */
1513 static void
bpf_ifname(struct ifnet * ifp,struct ifreq * ifr)1514 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr)
1515 {
1516 memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ);
1517 }
1518
1519 static int
bpf_stat(struct file * fp,struct stat * st)1520 bpf_stat(struct file *fp, struct stat *st)
1521 {
1522 struct bpf_d *d = fp->f_bpf;
1523
1524 (void)memset(st, 0, sizeof(*st));
1525 mutex_enter(d->bd_mtx);
1526 st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid);
1527 st->st_atimespec = d->bd_atime;
1528 st->st_mtimespec = d->bd_mtime;
1529 st->st_ctimespec = st->st_birthtimespec = d->bd_btime;
1530 st->st_uid = kauth_cred_geteuid(fp->f_cred);
1531 st->st_gid = kauth_cred_getegid(fp->f_cred);
1532 st->st_mode = S_IFCHR;
1533 mutex_exit(d->bd_mtx);
1534 return 0;
1535 }
1536
1537 /*
1538 * Support for poll() system call
1539 *
1540 * Return true iff the specific operation will not block indefinitely - with
1541 * the assumption that it is safe to positively acknowledge a request for the
1542 * ability to write to the BPF device.
1543 * Otherwise, return false but make a note that a selnotify() must be done.
1544 */
1545 static int
bpf_poll(struct file * fp,int events)1546 bpf_poll(struct file *fp, int events)
1547 {
1548 struct bpf_d *d = fp->f_bpf;
1549 int revents;
1550
1551 /*
1552 * Refresh the PID associated with this bpf file.
1553 */
1554 mutex_enter(&bpf_mtx);
1555 d->bd_pid = curproc->p_pid;
1556
1557 revents = events & (POLLOUT | POLLWRNORM);
1558 if (events & (POLLIN | POLLRDNORM)) {
1559 /*
1560 * An imitation of the FIONREAD ioctl code.
1561 */
1562 mutex_enter(d->bd_mtx);
1563 if (d->bd_hlen != 0 ||
1564 ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) &&
1565 d->bd_slen != 0)) {
1566 revents |= events & (POLLIN | POLLRDNORM);
1567 } else {
1568 selrecord(curlwp, &d->bd_sel);
1569 /* Start the read timeout if necessary */
1570 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
1571 callout_reset(&d->bd_callout, d->bd_rtout,
1572 bpf_timed_out, d);
1573 d->bd_state = BPF_WAITING;
1574 }
1575 }
1576 mutex_exit(d->bd_mtx);
1577 }
1578
1579 mutex_exit(&bpf_mtx);
1580 return (revents);
1581 }
1582
1583 static void
filt_bpfrdetach(struct knote * kn)1584 filt_bpfrdetach(struct knote *kn)
1585 {
1586 struct bpf_d *d = kn->kn_hook;
1587
1588 mutex_enter(d->bd_buf_mtx);
1589 selremove_knote(&d->bd_sel, kn);
1590 mutex_exit(d->bd_buf_mtx);
1591 }
1592
1593 static int
filt_bpfread(struct knote * kn,long hint)1594 filt_bpfread(struct knote *kn, long hint)
1595 {
1596 struct bpf_d *d = kn->kn_hook;
1597 int rv;
1598
1599 /*
1600 * Refresh the PID associated with this bpf file.
1601 */
1602 d->bd_pid = curproc->p_pid;
1603
1604 mutex_enter(d->bd_buf_mtx);
1605 kn->kn_data = d->bd_hlen;
1606 if (d->bd_immediate)
1607 kn->kn_data += d->bd_slen;
1608 rv = (kn->kn_data > 0);
1609 mutex_exit(d->bd_buf_mtx);
1610 return rv;
1611 }
1612
1613 static const struct filterops bpfread_filtops = {
1614 .f_flags = FILTEROP_ISFD,
1615 .f_attach = NULL,
1616 .f_detach = filt_bpfrdetach,
1617 .f_event = filt_bpfread,
1618 };
1619
1620 static int
bpf_kqfilter(struct file * fp,struct knote * kn)1621 bpf_kqfilter(struct file *fp, struct knote *kn)
1622 {
1623 struct bpf_d *d = fp->f_bpf;
1624
1625 switch (kn->kn_filter) {
1626 case EVFILT_READ:
1627 kn->kn_fop = &bpfread_filtops;
1628 break;
1629
1630 default:
1631 return (EINVAL);
1632 }
1633
1634 kn->kn_hook = d;
1635
1636 mutex_enter(d->bd_buf_mtx);
1637 selrecord_knote(&d->bd_sel, kn);
1638 mutex_exit(d->bd_buf_mtx);
1639
1640 return (0);
1641 }
1642
1643 /*
1644 * Copy data from an mbuf chain into a buffer. This code is derived
1645 * from m_copydata in sys/uipc_mbuf.c.
1646 */
1647 static void *
bpf_mcpy(void * dst_arg,const void * src_arg,size_t len)1648 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len)
1649 {
1650 const struct mbuf *m;
1651 u_int count;
1652 u_char *dst;
1653
1654 m = src_arg;
1655 dst = dst_arg;
1656 while (len > 0) {
1657 if (m == NULL)
1658 panic("bpf_mcpy");
1659 count = uimin(m->m_len, len);
1660 memcpy(dst, mtod(m, const void *), count);
1661 m = m->m_next;
1662 dst += count;
1663 len -= count;
1664 }
1665 return dst_arg;
1666 }
1667
1668 static inline u_int
bpf_xfilter(struct bpf_filter ** filter,void * pkt,u_int pktlen,u_int buflen)1669 bpf_xfilter(struct bpf_filter **filter, void *pkt, u_int pktlen, u_int buflen)
1670 {
1671 struct bpf_filter *filt;
1672 uint32_t mem[BPF_MEMWORDS];
1673 bpf_args_t args = {
1674 .pkt = (const uint8_t *)pkt,
1675 .wirelen = pktlen,
1676 .buflen = buflen,
1677 .mem = mem,
1678 .arg = NULL
1679 };
1680 u_int slen;
1681
1682 filt = atomic_load_consume(filter);
1683 if (filt == NULL) /* No filter means accept all. */
1684 return (u_int)-1;
1685
1686 if (filt->bf_jitcode != NULL)
1687 slen = filt->bf_jitcode(NULL, &args);
1688 else
1689 slen = bpf_filter_ext(NULL, filt->bf_insn, &args);
1690 return slen;
1691 }
1692
1693 /*
1694 * Dispatch a packet to all the listeners on interface bp.
1695 *
1696 * pkt pointer to the packet, either a data buffer or an mbuf chain
1697 * buflen buffer length, if pkt is a data buffer
1698 * cpfn a function that can copy pkt into the listener's buffer
1699 * pktlen length of the packet
1700 * direction BPF_D_IN or BPF_D_OUT
1701 */
1702 static inline void
bpf_deliver(struct bpf_if * bp,void * (* cpfn)(void *,const void *,size_t),void * pkt,u_int pktlen,u_int buflen,const u_int direction)1703 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t),
1704 void *pkt, u_int pktlen, u_int buflen, const u_int direction)
1705 {
1706 bool gottime = false;
1707 struct timespec ts;
1708 struct bpf_d *d;
1709 int s;
1710 u_int slen;
1711
1712 KASSERT(!cpu_intr_p());
1713
1714 /*
1715 * Note that the IPL does not have to be raised at this point.
1716 * The only problem that could arise here is that if two different
1717 * interfaces shared any data. This is not the case.
1718 */
1719 s = pserialize_read_enter();
1720 BPFIF_DLIST_READER_FOREACH(d, bp) {
1721 if (direction == BPF_D_IN) {
1722 if (d->bd_direction == BPF_D_OUT)
1723 continue;
1724 } else { /* BPF_D_OUT */
1725 if (d->bd_direction == BPF_D_IN)
1726 continue;
1727 }
1728
1729 atomic_inc_ulong(&d->bd_rcount);
1730 BPF_STATINC(recv);
1731
1732 slen = bpf_xfilter(&d->bd_rfilter, pkt, pktlen, buflen);
1733 if (slen == 0)
1734 continue;
1735
1736 if (!gottime) {
1737 gottime = true;
1738 nanotime(&ts);
1739 }
1740 /* Assume catchpacket doesn't sleep */
1741 catchpacket(d, pkt, pktlen, slen, cpfn, &ts);
1742 }
1743 pserialize_read_exit(s);
1744 }
1745
1746 /*
1747 * Incoming linkage from device drivers, when the head of the packet is in
1748 * a buffer, and the tail is in an mbuf chain.
1749 */
1750 static void
_bpf_mtap2(struct bpf_if * bp,void * data,u_int dlen,struct mbuf * m,u_int direction)1751 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m,
1752 u_int direction)
1753 {
1754 u_int pktlen;
1755 struct mbuf mb;
1756
1757 /* Skip outgoing duplicate packets. */
1758 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) {
1759 m->m_flags &= ~M_PROMISC;
1760 return;
1761 }
1762
1763 pktlen = m_length(m) + dlen;
1764
1765 /*
1766 * Craft on-stack mbuf suitable for passing to bpf_filter.
1767 * Note that we cut corners here; we only set up what's
1768 * absolutely needed--this mbuf should never go anywhere else.
1769 */
1770 (void)memset(&mb, 0, sizeof(mb));
1771 mb.m_type = MT_DATA;
1772 mb.m_next = m;
1773 mb.m_data = data;
1774 mb.m_len = dlen;
1775
1776 bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, direction);
1777 }
1778
1779 /*
1780 * Incoming linkage from device drivers, when packet is in an mbuf chain.
1781 */
1782 static void
_bpf_mtap(struct bpf_if * bp,struct mbuf * m,u_int direction)1783 _bpf_mtap(struct bpf_if *bp, struct mbuf *m, u_int direction)
1784 {
1785 void *(*cpfn)(void *, const void *, size_t);
1786 u_int pktlen, buflen;
1787 void *marg;
1788
1789 /* Skip outgoing duplicate packets. */
1790 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) {
1791 m->m_flags &= ~M_PROMISC;
1792 return;
1793 }
1794
1795 pktlen = m_length(m);
1796
1797 /* Skip zero-sized packets. */
1798 if (__predict_false(pktlen == 0)) {
1799 return;
1800 }
1801
1802 if (pktlen == m->m_len) {
1803 cpfn = (void *)memcpy;
1804 marg = mtod(m, void *);
1805 buflen = pktlen;
1806 KASSERT(buflen != 0);
1807 } else {
1808 cpfn = bpf_mcpy;
1809 marg = m;
1810 buflen = 0;
1811 }
1812
1813 bpf_deliver(bp, cpfn, marg, pktlen, buflen, direction);
1814 }
1815
1816 /*
1817 * We need to prepend the address family as
1818 * a four byte field. Cons up a dummy header
1819 * to pacify bpf. This is safe because bpf
1820 * will only read from the mbuf (i.e., it won't
1821 * try to free it or keep a pointer a to it).
1822 */
1823 static void
_bpf_mtap_af(struct bpf_if * bp,uint32_t af,struct mbuf * m,u_int direction)1824 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m, u_int direction)
1825 {
1826 struct mbuf m0;
1827
1828 m0.m_type = MT_DATA;
1829 m0.m_flags = 0;
1830 m0.m_next = m;
1831 m0.m_nextpkt = NULL;
1832 m0.m_owner = NULL;
1833 m0.m_len = 4;
1834 m0.m_data = (char *)⁡
1835
1836 _bpf_mtap(bp, &m0, direction);
1837 }
1838
1839 /*
1840 * Put the SLIP pseudo-"link header" in place.
1841 * Note this M_PREPEND() should never fail,
1842 * since we know we always have enough space
1843 * in the input buffer.
1844 */
1845 static void
_bpf_mtap_sl_in(struct bpf_if * bp,u_char * chdr,struct mbuf ** m)1846 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m)
1847 {
1848 u_char *hp;
1849
1850 M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT);
1851 if (*m == NULL)
1852 return;
1853
1854 hp = mtod(*m, u_char *);
1855 hp[SLX_DIR] = SLIPDIR_IN;
1856 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN);
1857
1858 _bpf_mtap(bp, *m, BPF_D_IN);
1859
1860 m_adj(*m, SLIP_HDRLEN);
1861 }
1862
1863 /*
1864 * Put the SLIP pseudo-"link header" in
1865 * place. The compressed header is now
1866 * at the beginning of the mbuf.
1867 */
1868 static void
_bpf_mtap_sl_out(struct bpf_if * bp,u_char * chdr,struct mbuf * m)1869 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m)
1870 {
1871 struct mbuf m0;
1872 u_char *hp;
1873
1874 m0.m_type = MT_DATA;
1875 m0.m_flags = 0;
1876 m0.m_next = m;
1877 m0.m_nextpkt = NULL;
1878 m0.m_owner = NULL;
1879 m0.m_data = m0.m_dat;
1880 m0.m_len = SLIP_HDRLEN;
1881
1882 hp = mtod(&m0, u_char *);
1883
1884 hp[SLX_DIR] = SLIPDIR_OUT;
1885 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN);
1886
1887 _bpf_mtap(bp, &m0, BPF_D_OUT);
1888 m_freem(m);
1889 }
1890
1891 static struct mbuf *
bpf_mbuf_enqueue(struct bpf_if * bp,struct mbuf * m)1892 bpf_mbuf_enqueue(struct bpf_if *bp, struct mbuf *m)
1893 {
1894 struct mbuf *dup;
1895
1896 dup = m_dup(m, 0, M_COPYALL, M_NOWAIT);
1897 if (dup == NULL)
1898 return NULL;
1899
1900 if (bp->bif_mbuf_tail != NULL) {
1901 bp->bif_mbuf_tail->m_nextpkt = dup;
1902 } else {
1903 bp->bif_mbuf_head = dup;
1904 }
1905 bp->bif_mbuf_tail = dup;
1906 #ifdef BPF_MTAP_SOFTINT_DEBUG
1907 log(LOG_DEBUG, "%s: enqueued mbuf=%p to %s\n",
1908 __func__, dup, bp->bif_ifp->if_xname);
1909 #endif
1910
1911 return dup;
1912 }
1913
1914 static struct mbuf *
bpf_mbuf_dequeue(struct bpf_if * bp)1915 bpf_mbuf_dequeue(struct bpf_if *bp)
1916 {
1917 struct mbuf *m;
1918 int s;
1919
1920 /* XXX NOMPSAFE: assumed running on one CPU */
1921 s = splnet();
1922 m = bp->bif_mbuf_head;
1923 if (m != NULL) {
1924 bp->bif_mbuf_head = m->m_nextpkt;
1925 m->m_nextpkt = NULL;
1926
1927 if (bp->bif_mbuf_head == NULL)
1928 bp->bif_mbuf_tail = NULL;
1929 #ifdef BPF_MTAP_SOFTINT_DEBUG
1930 log(LOG_DEBUG, "%s: dequeued mbuf=%p from %s\n",
1931 __func__, m, bp->bif_ifp->if_xname);
1932 #endif
1933 }
1934 splx(s);
1935
1936 return m;
1937 }
1938
1939 static void
bpf_mtap_si(void * arg)1940 bpf_mtap_si(void *arg)
1941 {
1942 struct bpf_if *bp = arg;
1943 struct mbuf *m;
1944
1945 while ((m = bpf_mbuf_dequeue(bp)) != NULL) {
1946 #ifdef BPF_MTAP_SOFTINT_DEBUG
1947 log(LOG_DEBUG, "%s: tapping mbuf=%p on %s\n",
1948 __func__, m, bp->bif_ifp->if_xname);
1949 #endif
1950 bpf_ops->bpf_mtap(bp, m, BPF_D_IN);
1951 m_freem(m);
1952 }
1953 }
1954
1955 static void
_bpf_mtap_softint(struct ifnet * ifp,struct mbuf * m)1956 _bpf_mtap_softint(struct ifnet *ifp, struct mbuf *m)
1957 {
1958 struct bpf_if *bp = ifp->if_bpf;
1959 struct mbuf *dup;
1960
1961 KASSERT(cpu_intr_p());
1962
1963 /* To avoid extra invocations of the softint */
1964 if (BPFIF_DLIST_READER_EMPTY(bp))
1965 return;
1966 KASSERT(bp->bif_si != NULL);
1967
1968 dup = bpf_mbuf_enqueue(bp, m);
1969 if (dup != NULL)
1970 softint_schedule(bp->bif_si);
1971 }
1972
1973 static int
bpf_hdrlen(struct bpf_d * d)1974 bpf_hdrlen(struct bpf_d *d)
1975 {
1976 int hdrlen = d->bd_bif->bif_hdrlen;
1977 /*
1978 * Compute the length of the bpf header. This is not necessarily
1979 * equal to SIZEOF_BPF_HDR because we want to insert spacing such
1980 * that the network layer header begins on a longword boundary (for
1981 * performance reasons and to alleviate alignment restrictions).
1982 */
1983 #ifdef _LP64
1984 if (d->bd_compat32)
1985 return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen);
1986 else
1987 #endif
1988 return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen);
1989 }
1990
1991 /*
1992 * Move the packet data from interface memory (pkt) into the
1993 * store buffer. Call the wakeup functions if it's time to wake up
1994 * a listener (buffer full), "cpfn" is the routine called to do the
1995 * actual data transfer. memcpy is passed in to copy contiguous chunks,
1996 * while bpf_mcpy is passed in to copy mbuf chains. In the latter case,
1997 * pkt is really an mbuf.
1998 */
1999 static void
catchpacket(struct bpf_d * d,u_char * pkt,u_int pktlen,u_int snaplen,void * (* cpfn)(void *,const void *,size_t),struct timespec * ts)2000 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen,
2001 void *(*cpfn)(void *, const void *, size_t), struct timespec *ts)
2002 {
2003 char *h;
2004 int totlen, curlen, caplen;
2005 int hdrlen = bpf_hdrlen(d);
2006 int do_wakeup = 0;
2007
2008 atomic_inc_ulong(&d->bd_ccount);
2009 BPF_STATINC(capt);
2010 /*
2011 * Figure out how many bytes to move. If the packet is
2012 * greater or equal to the snapshot length, transfer that
2013 * much. Otherwise, transfer the whole packet (unless
2014 * we hit the buffer size limit).
2015 */
2016 totlen = hdrlen + uimin(snaplen, pktlen);
2017 if (totlen > d->bd_bufsize)
2018 totlen = d->bd_bufsize;
2019 /*
2020 * If we adjusted totlen to fit the bufsize, it could be that
2021 * totlen is smaller than hdrlen because of the link layer header.
2022 */
2023 caplen = totlen - hdrlen;
2024 if (caplen < 0)
2025 caplen = 0;
2026
2027 mutex_enter(d->bd_buf_mtx);
2028 /*
2029 * Round up the end of the previous packet to the next longword.
2030 */
2031 #ifdef _LP64
2032 if (d->bd_compat32)
2033 curlen = BPF_WORDALIGN32(d->bd_slen);
2034 else
2035 #endif
2036 curlen = BPF_WORDALIGN(d->bd_slen);
2037 if (curlen + totlen > d->bd_bufsize) {
2038 /*
2039 * This packet will overflow the storage buffer.
2040 * Rotate the buffers if we can, then wakeup any
2041 * pending reads.
2042 */
2043 if (d->bd_fbuf == NULL) {
2044 mutex_exit(d->bd_buf_mtx);
2045 /*
2046 * We haven't completed the previous read yet,
2047 * so drop the packet.
2048 */
2049 atomic_inc_ulong(&d->bd_dcount);
2050 BPF_STATINC(drop);
2051 return;
2052 }
2053 ROTATE_BUFFERS(d);
2054 do_wakeup = 1;
2055 curlen = 0;
2056 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) {
2057 /*
2058 * Immediate mode is set, or the read timeout has
2059 * already expired during a select call. A packet
2060 * arrived, so the reader should be woken up.
2061 */
2062 do_wakeup = 1;
2063 }
2064
2065 /*
2066 * Append the bpf header.
2067 */
2068 h = (char *)d->bd_sbuf + curlen;
2069 #ifdef _LP64
2070 if (d->bd_compat32) {
2071 struct bpf_hdr32 *hp32;
2072
2073 hp32 = (struct bpf_hdr32 *)h;
2074 hp32->bh_tstamp.tv_sec = ts->tv_sec;
2075 hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000;
2076 hp32->bh_datalen = pktlen;
2077 hp32->bh_hdrlen = hdrlen;
2078 hp32->bh_caplen = caplen;
2079 } else
2080 #endif
2081 {
2082 struct bpf_hdr *hp;
2083
2084 hp = (struct bpf_hdr *)h;
2085 hp->bh_tstamp.tv_sec = ts->tv_sec;
2086 hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000;
2087 hp->bh_datalen = pktlen;
2088 hp->bh_hdrlen = hdrlen;
2089 hp->bh_caplen = caplen;
2090 }
2091
2092 /*
2093 * Copy the packet data into the store buffer and update its length.
2094 */
2095 (*cpfn)(h + hdrlen, pkt, caplen);
2096 d->bd_slen = curlen + totlen;
2097 mutex_exit(d->bd_buf_mtx);
2098
2099 /*
2100 * Call bpf_wakeup after bd_slen has been updated so that kevent(2)
2101 * will cause filt_bpfread() to be called with it adjusted.
2102 */
2103 if (do_wakeup)
2104 bpf_wakeup(d);
2105 }
2106
2107 /*
2108 * Initialize all nonzero fields of a descriptor.
2109 */
2110 static int
bpf_allocbufs(struct bpf_d * d)2111 bpf_allocbufs(struct bpf_d *d)
2112 {
2113
2114 d->bd_fbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP);
2115 if (!d->bd_fbuf)
2116 return (ENOBUFS);
2117 d->bd_sbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP);
2118 if (!d->bd_sbuf) {
2119 kmem_free(d->bd_fbuf, d->bd_bufsize);
2120 return (ENOBUFS);
2121 }
2122 d->bd_slen = 0;
2123 d->bd_hlen = 0;
2124 return (0);
2125 }
2126
2127 static void
bpf_free_filter(struct bpf_filter * filter)2128 bpf_free_filter(struct bpf_filter *filter)
2129 {
2130
2131 KASSERT(filter != NULL);
2132
2133 if (filter->bf_insn != NULL)
2134 kmem_free(filter->bf_insn, filter->bf_size);
2135 if (filter->bf_jitcode != NULL)
2136 bpf_jit_freecode(filter->bf_jitcode);
2137 kmem_free(filter, sizeof(*filter));
2138 }
2139
2140 /*
2141 * Free buffers currently in use by a descriptor.
2142 * Called on close.
2143 */
2144 static void
bpf_freed(struct bpf_d * d)2145 bpf_freed(struct bpf_d *d)
2146 {
2147 /*
2148 * We don't need to lock out interrupts since this descriptor has
2149 * been detached from its interface and it yet hasn't been marked
2150 * free.
2151 */
2152 if (d->bd_sbuf != NULL) {
2153 kmem_free(d->bd_sbuf, d->bd_bufsize);
2154 if (d->bd_hbuf != NULL)
2155 kmem_free(d->bd_hbuf, d->bd_bufsize);
2156 if (d->bd_fbuf != NULL)
2157 kmem_free(d->bd_fbuf, d->bd_bufsize);
2158 }
2159 if (d->bd_rfilter != NULL) {
2160 bpf_free_filter(d->bd_rfilter);
2161 d->bd_rfilter = NULL;
2162 }
2163 if (d->bd_wfilter != NULL) {
2164 bpf_free_filter(d->bd_wfilter);
2165 d->bd_wfilter = NULL;
2166 }
2167 d->bd_jitcode = NULL;
2168 }
2169
2170 /*
2171 * Attach an interface to bpf. dlt is the link layer type;
2172 * hdrlen is the fixed size of the link header for the specified dlt
2173 * (variable length headers not yet supported).
2174 */
2175 static void
_bpfattach(struct ifnet * ifp,u_int dlt,u_int hdrlen,struct bpf_if ** driverp)2176 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp)
2177 {
2178 struct bpf_if *bp;
2179
2180 bp = kmem_alloc(sizeof(*bp), KM_SLEEP);
2181
2182 mutex_enter(&bpf_mtx);
2183 bp->bif_driverp = driverp;
2184 bp->bif_ifp = ifp;
2185 bp->bif_dlt = dlt;
2186 bp->bif_si = NULL;
2187 BPF_IFLIST_ENTRY_INIT(bp);
2188 PSLIST_INIT(&bp->bif_dlist_head);
2189 psref_target_init(&bp->bif_psref, bpf_psref_class);
2190 SLIST_INIT(&bp->bif_trackers);
2191
2192 BPF_IFLIST_WRITER_INSERT_HEAD(bp);
2193
2194 *bp->bif_driverp = NULL;
2195
2196 bp->bif_hdrlen = hdrlen;
2197 mutex_exit(&bpf_mtx);
2198 #if 0
2199 printf("bpf: %s attached with dlt %x\n", ifp->if_xname, dlt);
2200 #endif
2201 }
2202
2203 static void
_bpf_mtap_softint_init(struct ifnet * ifp)2204 _bpf_mtap_softint_init(struct ifnet *ifp)
2205 {
2206 struct bpf_if *bp;
2207
2208 mutex_enter(&bpf_mtx);
2209 BPF_IFLIST_WRITER_FOREACH(bp) {
2210 if (bp->bif_ifp != ifp)
2211 continue;
2212
2213 bp->bif_mbuf_head = NULL;
2214 bp->bif_mbuf_tail = NULL;
2215 bp->bif_si = softint_establish(SOFTINT_NET, bpf_mtap_si, bp);
2216 if (bp->bif_si == NULL)
2217 panic("%s: softint_establish() failed", __func__);
2218 break;
2219 }
2220 mutex_exit(&bpf_mtx);
2221
2222 if (bp == NULL)
2223 panic("%s: no bpf_if found for %s", __func__, ifp->if_xname);
2224 }
2225
2226 /*
2227 * Remove an interface from bpf.
2228 */
2229 static void
_bpfdetach(struct ifnet * ifp)2230 _bpfdetach(struct ifnet *ifp)
2231 {
2232 struct bpf_if *bp;
2233 struct bpf_d *d;
2234 int s;
2235
2236 mutex_enter(&bpf_mtx);
2237 /* Nuke the vnodes for any open instances */
2238 again_d:
2239 BPF_DLIST_WRITER_FOREACH(d) {
2240 mutex_enter(d->bd_mtx);
2241 if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) {
2242 /*
2243 * Detach the descriptor from an interface now.
2244 * It will be free'ed later by close routine.
2245 */
2246 bpf_detachd(d);
2247 mutex_exit(d->bd_mtx);
2248 goto again_d;
2249 }
2250 mutex_exit(d->bd_mtx);
2251 }
2252
2253 again:
2254 BPF_IFLIST_WRITER_FOREACH(bp) {
2255 if (bp->bif_ifp == ifp) {
2256 BPF_IFLIST_WRITER_REMOVE(bp);
2257
2258 pserialize_perform(bpf_psz);
2259 psref_target_destroy(&bp->bif_psref, bpf_psref_class);
2260
2261 while (!SLIST_EMPTY(&bp->bif_trackers)) {
2262 struct bpf_event_tracker *t =
2263 SLIST_FIRST(&bp->bif_trackers);
2264 SLIST_REMOVE_HEAD(&bp->bif_trackers,
2265 bet_entries);
2266 kmem_free(t, sizeof(*t));
2267 }
2268
2269 BPF_IFLIST_ENTRY_DESTROY(bp);
2270 if (bp->bif_si != NULL) {
2271 /* XXX NOMPSAFE: assumed running on one CPU */
2272 s = splnet();
2273 while (bp->bif_mbuf_head != NULL) {
2274 struct mbuf *m = bp->bif_mbuf_head;
2275 bp->bif_mbuf_head = m->m_nextpkt;
2276 m_freem(m);
2277 }
2278 splx(s);
2279 softint_disestablish(bp->bif_si);
2280 }
2281 kmem_free(bp, sizeof(*bp));
2282 goto again;
2283 }
2284 }
2285 mutex_exit(&bpf_mtx);
2286 }
2287
2288 /*
2289 * Change the data link type of a interface.
2290 */
2291 static void
_bpf_change_type(struct ifnet * ifp,u_int dlt,u_int hdrlen)2292 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen)
2293 {
2294 struct bpf_if *bp;
2295
2296 mutex_enter(&bpf_mtx);
2297 BPF_IFLIST_WRITER_FOREACH(bp) {
2298 if (bp->bif_driverp == &ifp->if_bpf)
2299 break;
2300 }
2301 if (bp == NULL)
2302 panic("bpf_change_type");
2303
2304 bp->bif_dlt = dlt;
2305
2306 bp->bif_hdrlen = hdrlen;
2307 mutex_exit(&bpf_mtx);
2308 }
2309
2310 /*
2311 * Get a list of available data link type of the interface.
2312 */
2313 static int
bpf_getdltlist(struct bpf_d * d,struct bpf_dltlist * bfl)2314 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl)
2315 {
2316 int n, error;
2317 struct ifnet *ifp;
2318 struct bpf_if *bp;
2319 int s, bound;
2320
2321 KASSERT(mutex_owned(d->bd_mtx));
2322
2323 ifp = d->bd_bif->bif_ifp;
2324 n = 0;
2325 error = 0;
2326
2327 bound = curlwp_bind();
2328 s = pserialize_read_enter();
2329 BPF_IFLIST_READER_FOREACH(bp) {
2330 if (bp->bif_ifp != ifp)
2331 continue;
2332 if (bfl->bfl_list != NULL) {
2333 struct psref psref;
2334
2335 if (n >= bfl->bfl_len) {
2336 pserialize_read_exit(s);
2337 return ENOMEM;
2338 }
2339
2340 bpf_if_acquire(bp, &psref);
2341 pserialize_read_exit(s);
2342
2343 error = copyout(&bp->bif_dlt,
2344 bfl->bfl_list + n, sizeof(u_int));
2345
2346 s = pserialize_read_enter();
2347 bpf_if_release(bp, &psref);
2348 }
2349 n++;
2350 }
2351 pserialize_read_exit(s);
2352 curlwp_bindx(bound);
2353
2354 bfl->bfl_len = n;
2355 return error;
2356 }
2357
2358 /*
2359 * Set the data link type of a BPF instance.
2360 */
2361 static int
bpf_setdlt(struct bpf_d * d,u_int dlt)2362 bpf_setdlt(struct bpf_d *d, u_int dlt)
2363 {
2364 int error, opromisc;
2365 struct ifnet *ifp;
2366 struct bpf_if *bp;
2367
2368 KASSERT(mutex_owned(&bpf_mtx));
2369 KASSERT(mutex_owned(d->bd_mtx));
2370
2371 if (d->bd_bif->bif_dlt == dlt)
2372 return 0;
2373 ifp = d->bd_bif->bif_ifp;
2374 BPF_IFLIST_WRITER_FOREACH(bp) {
2375 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt)
2376 break;
2377 }
2378 if (bp == NULL)
2379 return EINVAL;
2380 opromisc = d->bd_promisc;
2381 bpf_detachd(d);
2382 BPFIF_DLIST_ENTRY_INIT(d);
2383 bpf_attachd(d, bp);
2384 reset_d(d);
2385 if (opromisc) {
2386 KERNEL_LOCK_UNLESS_NET_MPSAFE();
2387 error = ifpromisc(bp->bif_ifp, 1);
2388 KERNEL_UNLOCK_UNLESS_NET_MPSAFE();
2389 if (error)
2390 printf("%s: bpf_setdlt: ifpromisc failed (%d)\n",
2391 bp->bif_ifp->if_xname, error);
2392 else
2393 d->bd_promisc = 1;
2394 }
2395 return 0;
2396 }
2397
2398 static int
sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS)2399 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS)
2400 {
2401 int newsize, error;
2402 struct sysctlnode node;
2403
2404 node = *rnode;
2405 node.sysctl_data = &newsize;
2406 newsize = bpf_maxbufsize;
2407 error = sysctl_lookup(SYSCTLFN_CALL(&node));
2408 if (error || newp == NULL)
2409 return (error);
2410
2411 if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE)
2412 return (EINVAL);
2413
2414 bpf_maxbufsize = newsize;
2415
2416 return (0);
2417 }
2418
2419 #if defined(MODULAR) || defined(BPFJIT)
2420 static int
sysctl_net_bpf_jit(SYSCTLFN_ARGS)2421 sysctl_net_bpf_jit(SYSCTLFN_ARGS)
2422 {
2423 bool newval;
2424 int error;
2425 struct sysctlnode node;
2426
2427 node = *rnode;
2428 node.sysctl_data = &newval;
2429 newval = bpf_jit;
2430 error = sysctl_lookup(SYSCTLFN_CALL(&node));
2431 if (error != 0 || newp == NULL)
2432 return error;
2433
2434 bpf_jit = newval;
2435 if (newval && bpfjit_module_ops.bj_generate_code == NULL) {
2436 printf("JIT compilation is postponed "
2437 "until after bpfjit module is loaded\n");
2438 }
2439
2440 return 0;
2441 }
2442 #endif
2443
2444 static int
sysctl_net_bpf_peers(SYSCTLFN_ARGS)2445 sysctl_net_bpf_peers(SYSCTLFN_ARGS)
2446 {
2447 int error, elem_count;
2448 struct bpf_d *dp;
2449 struct bpf_d_ext dpe;
2450 size_t len, needed, elem_size, out_size;
2451 char *sp;
2452
2453 if (namelen == 1 && name[0] == CTL_QUERY)
2454 return (sysctl_query(SYSCTLFN_CALL(rnode)));
2455
2456 if (namelen != 2)
2457 return (EINVAL);
2458
2459 /* BPF peers is privileged information. */
2460 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE,
2461 KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL);
2462 if (error)
2463 return (EPERM);
2464
2465 len = (oldp != NULL) ? *oldlenp : 0;
2466 sp = oldp;
2467 elem_size = name[0];
2468 elem_count = name[1];
2469 out_size = MIN(sizeof(dpe), elem_size);
2470 needed = 0;
2471
2472 if (elem_size < 1 || elem_count < 0)
2473 return (EINVAL);
2474
2475 mutex_enter(&bpf_mtx);
2476 BPF_DLIST_WRITER_FOREACH(dp) {
2477 if (len >= elem_size && elem_count > 0) {
2478 #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field
2479 BPF_EXT(bufsize);
2480 BPF_EXT(promisc);
2481 BPF_EXT(state);
2482 BPF_EXT(immediate);
2483 BPF_EXT(hdrcmplt);
2484 BPF_EXT(direction);
2485 BPF_EXT(pid);
2486 BPF_EXT(rcount);
2487 BPF_EXT(dcount);
2488 BPF_EXT(ccount);
2489 #undef BPF_EXT
2490 mutex_enter(dp->bd_mtx);
2491 if (dp->bd_bif)
2492 (void)strlcpy(dpe.bde_ifname,
2493 dp->bd_bif->bif_ifp->if_xname,
2494 IFNAMSIZ - 1);
2495 else
2496 dpe.bde_ifname[0] = '\0';
2497 dpe.bde_locked = dp->bd_locked;
2498 mutex_exit(dp->bd_mtx);
2499
2500 error = copyout(&dpe, sp, out_size);
2501 if (error)
2502 break;
2503 sp += elem_size;
2504 len -= elem_size;
2505 }
2506 needed += elem_size;
2507 if (elem_count > 0 && elem_count != INT_MAX)
2508 elem_count--;
2509 }
2510 mutex_exit(&bpf_mtx);
2511
2512 *oldlenp = needed;
2513
2514 return (error);
2515 }
2516
2517 static void
bpf_stats(void * p,void * arg,struct cpu_info * ci __unused)2518 bpf_stats(void *p, void *arg, struct cpu_info *ci __unused)
2519 {
2520 struct bpf_stat *const stats = p;
2521 struct bpf_stat *sum = arg;
2522
2523 int s = splnet();
2524
2525 sum->bs_recv += stats->bs_recv;
2526 sum->bs_drop += stats->bs_drop;
2527 sum->bs_capt += stats->bs_capt;
2528
2529 splx(s);
2530 }
2531
2532 static int
bpf_sysctl_gstats_handler(SYSCTLFN_ARGS)2533 bpf_sysctl_gstats_handler(SYSCTLFN_ARGS)
2534 {
2535 struct sysctlnode node;
2536 int error;
2537 struct bpf_stat sum;
2538
2539 memset(&sum, 0, sizeof(sum));
2540 node = *rnode;
2541
2542 percpu_foreach_xcall(bpf_gstats_percpu, XC_HIGHPRI_IPL(IPL_SOFTNET),
2543 bpf_stats, &sum);
2544
2545 node.sysctl_data = ∑
2546 node.sysctl_size = sizeof(sum);
2547 error = sysctl_lookup(SYSCTLFN_CALL(&node));
2548 if (error != 0 || newp == NULL)
2549 return error;
2550
2551 return 0;
2552 }
2553
2554 SYSCTL_SETUP(sysctl_net_bpf_setup, "bpf sysctls")
2555 {
2556 const struct sysctlnode *node;
2557
2558 node = NULL;
2559 sysctl_createv(clog, 0, NULL, &node,
2560 CTLFLAG_PERMANENT,
2561 CTLTYPE_NODE, "bpf",
2562 SYSCTL_DESCR("BPF options"),
2563 NULL, 0, NULL, 0,
2564 CTL_NET, CTL_CREATE, CTL_EOL);
2565 if (node != NULL) {
2566 #if defined(MODULAR) || defined(BPFJIT)
2567 sysctl_createv(clog, 0, NULL, NULL,
2568 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
2569 CTLTYPE_BOOL, "jit",
2570 SYSCTL_DESCR("Toggle Just-In-Time compilation"),
2571 sysctl_net_bpf_jit, 0, &bpf_jit, 0,
2572 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
2573 #endif
2574 sysctl_createv(clog, 0, NULL, NULL,
2575 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
2576 CTLTYPE_INT, "maxbufsize",
2577 SYSCTL_DESCR("Maximum size for data capture buffer"),
2578 sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0,
2579 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
2580 sysctl_createv(clog, 0, NULL, NULL,
2581 CTLFLAG_PERMANENT,
2582 CTLTYPE_STRUCT, "stats",
2583 SYSCTL_DESCR("BPF stats"),
2584 bpf_sysctl_gstats_handler, 0, NULL, 0,
2585 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
2586 sysctl_createv(clog, 0, NULL, NULL,
2587 CTLFLAG_PERMANENT,
2588 CTLTYPE_STRUCT, "peers",
2589 SYSCTL_DESCR("BPF peers"),
2590 sysctl_net_bpf_peers, 0, NULL, 0,
2591 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL);
2592 }
2593
2594 }
2595
2596 static int
_bpf_register_track_event(struct bpf_if ** driverp,void (* _fun)(struct bpf_if *,struct ifnet *,int,int))2597 _bpf_register_track_event(struct bpf_if **driverp,
2598 void (*_fun)(struct bpf_if *, struct ifnet *, int, int))
2599 {
2600 struct bpf_if *bp;
2601 struct bpf_event_tracker *t;
2602 int ret = ENOENT;
2603
2604 t = kmem_zalloc(sizeof(*t), KM_SLEEP);
2605 if (!t)
2606 return ENOMEM;
2607 t->bet_notify = _fun;
2608
2609 mutex_enter(&bpf_mtx);
2610 BPF_IFLIST_WRITER_FOREACH(bp) {
2611 if (bp->bif_driverp != driverp)
2612 continue;
2613 SLIST_INSERT_HEAD(&bp->bif_trackers, t, bet_entries);
2614 ret = 0;
2615 break;
2616 }
2617 mutex_exit(&bpf_mtx);
2618
2619 return ret;
2620 }
2621
2622 static int
_bpf_deregister_track_event(struct bpf_if ** driverp,void (* _fun)(struct bpf_if *,struct ifnet *,int,int))2623 _bpf_deregister_track_event(struct bpf_if **driverp,
2624 void (*_fun)(struct bpf_if *, struct ifnet *, int, int))
2625 {
2626 struct bpf_if *bp;
2627 struct bpf_event_tracker *t = NULL;
2628 int ret = ENOENT;
2629
2630 mutex_enter(&bpf_mtx);
2631 BPF_IFLIST_WRITER_FOREACH(bp) {
2632 if (bp->bif_driverp != driverp)
2633 continue;
2634 SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) {
2635 if (t->bet_notify == _fun) {
2636 ret = 0;
2637 break;
2638 }
2639 }
2640 if (ret == 0)
2641 break;
2642 }
2643 if (ret == 0 && t && t->bet_notify == _fun) {
2644 SLIST_REMOVE(&bp->bif_trackers, t, bpf_event_tracker,
2645 bet_entries);
2646 }
2647 mutex_exit(&bpf_mtx);
2648 if (ret == 0)
2649 kmem_free(t, sizeof(*t));
2650 return ret;
2651 }
2652
2653 struct bpf_ops bpf_ops_kernel = {
2654 .bpf_attach = _bpfattach,
2655 .bpf_detach = _bpfdetach,
2656 .bpf_change_type = _bpf_change_type,
2657 .bpf_register_track_event = _bpf_register_track_event,
2658 .bpf_deregister_track_event = _bpf_deregister_track_event,
2659
2660 .bpf_mtap = _bpf_mtap,
2661 .bpf_mtap2 = _bpf_mtap2,
2662 .bpf_mtap_af = _bpf_mtap_af,
2663 .bpf_mtap_sl_in = _bpf_mtap_sl_in,
2664 .bpf_mtap_sl_out = _bpf_mtap_sl_out,
2665
2666 .bpf_mtap_softint = _bpf_mtap_softint,
2667 .bpf_mtap_softint_init = _bpf_mtap_softint_init,
2668 };
2669
2670 MODULE(MODULE_CLASS_DRIVER, bpf, "bpf_filter");
2671
2672 static int
bpf_modcmd(modcmd_t cmd,void * arg)2673 bpf_modcmd(modcmd_t cmd, void *arg)
2674 {
2675 #ifdef _MODULE
2676 devmajor_t bmajor, cmajor;
2677 #endif
2678 int error = 0;
2679
2680 switch (cmd) {
2681 case MODULE_CMD_INIT:
2682 bpf_init();
2683 #ifdef _MODULE
2684 bmajor = cmajor = NODEVMAJOR;
2685 error = devsw_attach("bpf", NULL, &bmajor,
2686 &bpf_cdevsw, &cmajor);
2687 if (error)
2688 break;
2689 #endif
2690
2691 bpf_ops_handover_enter(&bpf_ops_kernel);
2692 atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel);
2693 bpf_ops_handover_exit();
2694 break;
2695
2696 case MODULE_CMD_FINI:
2697 /*
2698 * While there is no reference counting for bpf callers,
2699 * unload could at least in theory be done similarly to
2700 * system call disestablishment. This should even be
2701 * a little simpler:
2702 *
2703 * 1) replace op vector with stubs
2704 * 2) post update to all cpus with xc
2705 * 3) check that nobody is in bpf anymore
2706 * (it's doubtful we'd want something like l_sysent,
2707 * but we could do something like *signed* percpu
2708 * counters. if the sum is 0, we're good).
2709 * 4) if fail, unroll changes
2710 *
2711 * NOTE: change won't be atomic to the outside. some
2712 * packets may be not captured even if unload is
2713 * not successful. I think packet capture not working
2714 * is a perfectly logical consequence of trying to
2715 * disable packet capture.
2716 */
2717 error = EOPNOTSUPP;
2718 break;
2719
2720 default:
2721 error = ENOTTY;
2722 break;
2723 }
2724
2725 return error;
2726 }
2727