15b497af4SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2546ac1ffSJohn Fastabend /* Copyright (c) 2017 Covalent IO, Inc. http://covalent.io
3546ac1ffSJohn Fastabend */
4546ac1ffSJohn Fastabend
5546ac1ffSJohn Fastabend /* Devmaps primary use is as a backend map for XDP BPF helper call
6546ac1ffSJohn Fastabend * bpf_redirect_map(). Because XDP is mostly concerned with performance we
7546ac1ffSJohn Fastabend * spent some effort to ensure the datapath with redirect maps does not use
8546ac1ffSJohn Fastabend * any locking. This is a quick note on the details.
9546ac1ffSJohn Fastabend *
10546ac1ffSJohn Fastabend * We have three possible paths to get into the devmap control plane bpf
11546ac1ffSJohn Fastabend * syscalls, bpf programs, and driver side xmit/flush operations. A bpf syscall
12546ac1ffSJohn Fastabend * will invoke an update, delete, or lookup operation. To ensure updates and
13546ac1ffSJohn Fastabend * deletes appear atomic from the datapath side xchg() is used to modify the
14546ac1ffSJohn Fastabend * netdev_map array. Then because the datapath does a lookup into the netdev_map
15546ac1ffSJohn Fastabend * array (read-only) from an RCU critical section we use call_rcu() to wait for
16546ac1ffSJohn Fastabend * an rcu grace period before free'ing the old data structures. This ensures the
17546ac1ffSJohn Fastabend * datapath always has a valid copy. However, the datapath does a "flush"
18546ac1ffSJohn Fastabend * operation that pushes any pending packets in the driver outside the RCU
19546ac1ffSJohn Fastabend * critical section. Each bpf_dtab_netdev tracks these pending operations using
20d5df2830SToke Høiland-Jørgensen * a per-cpu flush list. The bpf_dtab_netdev object will not be destroyed until
21d5df2830SToke Høiland-Jørgensen * this list is empty, indicating outstanding flush operations have completed.
22546ac1ffSJohn Fastabend *
23546ac1ffSJohn Fastabend * BPF syscalls may race with BPF program calls on any of the update, delete
24546ac1ffSJohn Fastabend * or lookup operations. As noted above the xchg() operation also keep the
25546ac1ffSJohn Fastabend * netdev_map consistent in this case. From the devmap side BPF programs
26546ac1ffSJohn Fastabend * calling into these operations are the same as multiple user space threads
27546ac1ffSJohn Fastabend * making system calls.
282ddf71e2SJohn Fastabend *
292ddf71e2SJohn Fastabend * Finally, any of the above may race with a netdev_unregister notifier. The
302ddf71e2SJohn Fastabend * unregister notifier must search for net devices in the map structure that
312ddf71e2SJohn Fastabend * contain a reference to the net device and remove them. This is a two step
322ddf71e2SJohn Fastabend * process (a) dereference the bpf_dtab_netdev object in netdev_map and (b)
332ddf71e2SJohn Fastabend * check to see if the ifindex is the same as the net_device being removed.
344cc7b954SJohn Fastabend * When removing the dev a cmpxchg() is used to ensure the correct dev is
354cc7b954SJohn Fastabend * removed, in the case of a concurrent update or delete operation it is
364cc7b954SJohn Fastabend * possible that the initially referenced dev is no longer in the map. As the
374cc7b954SJohn Fastabend * notifier hook walks the map we know that new dev references can not be
384cc7b954SJohn Fastabend * added by the user because core infrastructure ensures dev_get_by_index()
394cc7b954SJohn Fastabend * calls will fail at this point.
406f9d451aSToke Høiland-Jørgensen *
416f9d451aSToke Høiland-Jørgensen * The devmap_hash type is a map type which interprets keys as ifindexes and
426f9d451aSToke Høiland-Jørgensen * indexes these using a hashmap. This allows maps that use ifindex as key to be
436f9d451aSToke Høiland-Jørgensen * densely packed instead of having holes in the lookup array for unused
446f9d451aSToke Høiland-Jørgensen * ifindexes. The setup and packet enqueue/send code is shared between the two
456f9d451aSToke Høiland-Jørgensen * types of devmap; only the lookup and insertion is different.
46546ac1ffSJohn Fastabend */
47546ac1ffSJohn Fastabend #include <linux/bpf.h>
4867f29e07SJesper Dangaard Brouer #include <net/xdp.h>
49546ac1ffSJohn Fastabend #include <linux/filter.h>
5067f29e07SJesper Dangaard Brouer #include <trace/events/xdp.h>
51c317ab71SMenglong Dong #include <linux/btf_ids.h>
52546ac1ffSJohn Fastabend
536e71b04aSChenbo Feng #define DEV_CREATE_FLAG_MASK \
546e71b04aSChenbo Feng (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)
556e71b04aSChenbo Feng
5675ccae62SToke Høiland-Jørgensen struct xdp_dev_bulk_queue {
575d053f9dSJesper Dangaard Brouer struct xdp_frame *q[DEV_MAP_BULK_SIZE];
58d5df2830SToke Høiland-Jørgensen struct list_head flush_node;
5975ccae62SToke Høiland-Jørgensen struct net_device *dev;
6038edddb8SJesper Dangaard Brouer struct net_device *dev_rx;
61cb261b59SJesper Dangaard Brouer struct bpf_prog *xdp_prog;
625d053f9dSJesper Dangaard Brouer unsigned int count;
635d053f9dSJesper Dangaard Brouer };
645d053f9dSJesper Dangaard Brouer
65546ac1ffSJohn Fastabend struct bpf_dtab_netdev {
6667f29e07SJesper Dangaard Brouer struct net_device *dev; /* must be first member, due to tracepoint */
676f9d451aSToke Høiland-Jørgensen struct hlist_node index_hlist;
68fbee97feSDavid Ahern struct bpf_prog *xdp_prog;
69af4d045cSDaniel Borkmann struct rcu_head rcu;
7075ccae62SToke Høiland-Jørgensen unsigned int idx;
717f1c0426SDavid Ahern struct bpf_devmap_val val;
72546ac1ffSJohn Fastabend };
73546ac1ffSJohn Fastabend
74546ac1ffSJohn Fastabend struct bpf_dtab {
75546ac1ffSJohn Fastabend struct bpf_map map;
76782347b6SToke Høiland-Jørgensen struct bpf_dtab_netdev __rcu **netdev_map; /* DEVMAP type only */
772ddf71e2SJohn Fastabend struct list_head list;
786f9d451aSToke Høiland-Jørgensen
796f9d451aSToke Høiland-Jørgensen /* these are only used for DEVMAP_HASH type maps */
806f9d451aSToke Høiland-Jørgensen struct hlist_head *dev_index_head;
816f9d451aSToke Høiland-Jørgensen spinlock_t index_lock;
826f9d451aSToke Høiland-Jørgensen unsigned int items;
836f9d451aSToke Høiland-Jørgensen u32 n_buckets;
84546ac1ffSJohn Fastabend };
85546ac1ffSJohn Fastabend
861d233886SToke Høiland-Jørgensen static DEFINE_SPINLOCK(dev_map_lock);
874cc7b954SJohn Fastabend static LIST_HEAD(dev_map_list);
882ddf71e2SJohn Fastabend
dev_map_create_hash(unsigned int entries,int numa_node)892ddf71e2SJohn Fastabend static struct hlist_head *dev_map_create_hash(unsigned int entries,
9099c51064SToke Høiland-Jørgensen int numa_node)
9199c51064SToke Høiland-Jørgensen {
926f9d451aSToke Høiland-Jørgensen int i;
936f9d451aSToke Høiland-Jørgensen struct hlist_head *hash;
946f9d451aSToke Høiland-Jørgensen
956f9d451aSToke Høiland-Jørgensen hash = bpf_map_area_alloc((u64) entries * sizeof(*hash), numa_node);
967dd5d437SBui Quang Minh if (hash != NULL)
976f9d451aSToke Høiland-Jørgensen for (i = 0; i < entries; i++)
986f9d451aSToke Høiland-Jørgensen INIT_HLIST_HEAD(&hash[i]);
996f9d451aSToke Høiland-Jørgensen
1006f9d451aSToke Høiland-Jørgensen return hash;
1016f9d451aSToke Høiland-Jørgensen }
1026f9d451aSToke Høiland-Jørgensen
dev_map_index_hash(struct bpf_dtab * dtab,int idx)1036f9d451aSToke Høiland-Jørgensen static inline struct hlist_head *dev_map_index_hash(struct bpf_dtab *dtab,
104071cdeceSToke Høiland-Jørgensen int idx)
105071cdeceSToke Høiland-Jørgensen {
106071cdeceSToke Høiland-Jørgensen return &dtab->dev_index_head[idx & (dtab->n_buckets - 1)];
107071cdeceSToke Høiland-Jørgensen }
108071cdeceSToke Høiland-Jørgensen
dev_map_alloc_check(union bpf_attr * attr)109071cdeceSToke Høiland-Jørgensen static int dev_map_alloc_check(union bpf_attr *attr)
110*fd8db077SFlorian Lehner {
111546ac1ffSJohn Fastabend u32 valsize = attr->value_size;
112fbee97feSDavid Ahern
113546ac1ffSJohn Fastabend /* check sanity of attributes. 2 value sizes supported:
114fbee97feSDavid Ahern * 4 bytes: ifindex
115fbee97feSDavid Ahern * 8 bytes: ifindex + prog fd
116fbee97feSDavid Ahern */
117fbee97feSDavid Ahern if (attr->max_entries == 0 || attr->key_size != 4 ||
118546ac1ffSJohn Fastabend (valsize != offsetofend(struct bpf_devmap_val, ifindex) &&
119fbee97feSDavid Ahern valsize != offsetofend(struct bpf_devmap_val, bpf_prog.fd)) ||
120fbee97feSDavid Ahern attr->map_flags & ~DEV_CREATE_FLAG_MASK)
121fbee97feSDavid Ahern return -EINVAL;
122fca16e51SToke Høiland-Jørgensen
123546ac1ffSJohn Fastabend if (attr->map_type == BPF_MAP_TYPE_DEVMAP_HASH) {
124*fd8db077SFlorian Lehner /* Hash table size must be power of 2; roundup_pow_of_two()
125*fd8db077SFlorian Lehner * can overflow into UB on 32-bit arches
126*fd8db077SFlorian Lehner */
127*fd8db077SFlorian Lehner if (attr->max_entries > 1UL << 31)
128*fd8db077SFlorian Lehner return -EINVAL;
129*fd8db077SFlorian Lehner }
130*fd8db077SFlorian Lehner
131*fd8db077SFlorian Lehner return 0;
132*fd8db077SFlorian Lehner }
133*fd8db077SFlorian Lehner
dev_map_init_map(struct bpf_dtab * dtab,union bpf_attr * attr)134*fd8db077SFlorian Lehner static int dev_map_init_map(struct bpf_dtab *dtab, union bpf_attr *attr)
135*fd8db077SFlorian Lehner {
136*fd8db077SFlorian Lehner /* Lookup returns a pointer straight to dev->ifindex, so make sure the
1370cdbb4b0SToke Høiland-Jørgensen * verifier prevents writes from the BPF side
1380cdbb4b0SToke Høiland-Jørgensen */
1390cdbb4b0SToke Høiland-Jørgensen attr->map_flags |= BPF_F_RDONLY_PROG;
1400cdbb4b0SToke Høiland-Jørgensen bpf_map_init_from_attr(&dtab->map, attr);
141bd475643SJakub Kicinski
142546ac1ffSJohn Fastabend if (attr->map_type == BPF_MAP_TYPE_DEVMAP_HASH) {
1436f9d451aSToke Høiland-Jørgensen /* Hash table size must be power of 2 */
144*fd8db077SFlorian Lehner dtab->n_buckets = roundup_pow_of_two(dtab->map.max_entries);
1456f9d451aSToke Høiland-Jørgensen dtab->dev_index_head = dev_map_create_hash(dtab->n_buckets,
14699c51064SToke Høiland-Jørgensen dtab->map.numa_node);
14799c51064SToke Høiland-Jørgensen if (!dtab->dev_index_head)
148071cdeceSToke Høiland-Jørgensen return -ENOMEM;
149844f157fSRoman Gushchin
150071cdeceSToke Høiland-Jørgensen spin_lock_init(&dtab->index_lock);
151071cdeceSToke Høiland-Jørgensen } else {
152071cdeceSToke Høiland-Jørgensen dtab->netdev_map = bpf_map_area_alloc((u64) dtab->map.max_entries *
1537dd5d437SBui Quang Minh sizeof(struct bpf_dtab_netdev *),
15496eabe7aSMartin KaFai Lau dtab->map.numa_node);
15596eabe7aSMartin KaFai Lau if (!dtab->netdev_map)
156546ac1ffSJohn Fastabend return -ENOMEM;
157844f157fSRoman Gushchin }
1586f9d451aSToke Høiland-Jørgensen
1596f9d451aSToke Høiland-Jørgensen return 0;
160fca16e51SToke Høiland-Jørgensen }
161fca16e51SToke Høiland-Jørgensen
dev_map_alloc(union bpf_attr * attr)162fca16e51SToke Høiland-Jørgensen static struct bpf_map *dev_map_alloc(union bpf_attr *attr)
163fca16e51SToke Høiland-Jørgensen {
164fca16e51SToke Høiland-Jørgensen struct bpf_dtab *dtab;
165fca16e51SToke Høiland-Jørgensen int err;
166fca16e51SToke Høiland-Jørgensen
167fca16e51SToke Høiland-Jørgensen dtab = bpf_map_area_alloc(sizeof(*dtab), NUMA_NO_NODE);
16873cf09a3SYafang Shao if (!dtab)
169fca16e51SToke Høiland-Jørgensen return ERR_PTR(-ENOMEM);
170fca16e51SToke Høiland-Jørgensen
171fca16e51SToke Høiland-Jørgensen err = dev_map_init_map(dtab, attr);
172fca16e51SToke Høiland-Jørgensen if (err) {
173fca16e51SToke Høiland-Jørgensen bpf_map_area_free(dtab);
17473cf09a3SYafang Shao return ERR_PTR(err);
175582db7e0STobias Klauser }
176546ac1ffSJohn Fastabend
177546ac1ffSJohn Fastabend spin_lock(&dev_map_lock);
178fca16e51SToke Høiland-Jørgensen list_add_tail_rcu(&dtab->list, &dev_map_list);
179fca16e51SToke Høiland-Jørgensen spin_unlock(&dev_map_lock);
180fca16e51SToke Høiland-Jørgensen
181fca16e51SToke Høiland-Jørgensen return &dtab->map;
182fca16e51SToke Høiland-Jørgensen }
183fca16e51SToke Høiland-Jørgensen
dev_map_free(struct bpf_map * map)184fca16e51SToke Høiland-Jørgensen static void dev_map_free(struct bpf_map *map)
185546ac1ffSJohn Fastabend {
186546ac1ffSJohn Fastabend struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
187546ac1ffSJohn Fastabend int i;
1880536b852SBjörn Töpel
189546ac1ffSJohn Fastabend /* At this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0,
190546ac1ffSJohn Fastabend * so the programs (can be more than one that used this map) were
191546ac1ffSJohn Fastabend * disconnected from events. The following synchronize_rcu() guarantees
19242a84a8cSJohn Fastabend * both rcu read critical sections complete and waits for
19342a84a8cSJohn Fastabend * preempt-disable regions (NAPI being the relevant context here) so we
19442a84a8cSJohn Fastabend * are certain there will be no further reads against the netdev_map and
19542a84a8cSJohn Fastabend * all flush operations are complete. Flush operations can only be done
19642a84a8cSJohn Fastabend * from NAPI context for this reason.
19742a84a8cSJohn Fastabend */
198546ac1ffSJohn Fastabend
199274043c6SDaniel Borkmann spin_lock(&dev_map_lock);
200274043c6SDaniel Borkmann list_del_rcu(&dtab->list);
201274043c6SDaniel Borkmann spin_unlock(&dev_map_lock);
202274043c6SDaniel Borkmann
203274043c6SDaniel Borkmann /* bpf_redirect_info->map is assigned in __bpf_xdp_redirect_map()
204e624d4edSHangbin Liu * during NAPI callback and cleared after the XDP redirect. There is no
205546ac1ffSJohn Fastabend * explicit RCU read section which protects bpf_redirect_info->map but
206546ac1ffSJohn Fastabend * local_bh_disable() also marks the beginning an RCU section. This
2072baae354SEric Dumazet * makes the complete softirq callback RCU protected. Thus after
2082baae354SEric Dumazet * following synchronize_rcu() there no bpf_redirect_info->map == map
2092baae354SEric Dumazet * assignment.
210071cdeceSToke Høiland-Jørgensen */
211071cdeceSToke Høiland-Jørgensen synchronize_rcu();
212071cdeceSToke Høiland-Jørgensen
213071cdeceSToke Høiland-Jørgensen /* Make sure prior __dev_map_entry_free() have completed. */
214071cdeceSToke Høiland-Jørgensen rcu_barrier();
215071cdeceSToke Høiland-Jørgensen
216071cdeceSToke Høiland-Jørgensen if (dtab->map.map_type == BPF_MAP_TYPE_DEVMAP_HASH) {
217071cdeceSToke Høiland-Jørgensen for (i = 0; i < dtab->n_buckets; i++) {
218071cdeceSToke Høiland-Jørgensen struct bpf_dtab_netdev *dev;
219071cdeceSToke Høiland-Jørgensen struct hlist_head *head;
220fbee97feSDavid Ahern struct hlist_node *next;
221fbee97feSDavid Ahern
222071cdeceSToke Høiland-Jørgensen head = dev_map_index_hash(dtab, i);
223071cdeceSToke Høiland-Jørgensen
224071cdeceSToke Høiland-Jørgensen hlist_for_each_entry_safe(dev, next, head, index_hlist) {
225071cdeceSToke Høiland-Jørgensen hlist_del_rcu(&dev->index_hlist);
226071cdeceSToke Høiland-Jørgensen if (dev->xdp_prog)
22799c51064SToke Høiland-Jørgensen bpf_prog_put(dev->xdp_prog);
228071cdeceSToke Høiland-Jørgensen dev_put(dev->dev);
229546ac1ffSJohn Fastabend kfree(dev);
230546ac1ffSJohn Fastabend }
231546ac1ffSJohn Fastabend }
232782347b6SToke Høiland-Jørgensen
233546ac1ffSJohn Fastabend bpf_map_area_free(dtab->dev_index_head);
234546ac1ffSJohn Fastabend } else {
235546ac1ffSJohn Fastabend for (i = 0; i < dtab->map.max_entries; i++) {
236fbee97feSDavid Ahern struct bpf_dtab_netdev *dev;
237fbee97feSDavid Ahern
238546ac1ffSJohn Fastabend dev = rcu_dereference_raw(dtab->netdev_map[i]);
239546ac1ffSJohn Fastabend if (!dev)
240546ac1ffSJohn Fastabend continue;
241546ac1ffSJohn Fastabend
242546ac1ffSJohn Fastabend if (dev->xdp_prog)
243071cdeceSToke Høiland-Jørgensen bpf_prog_put(dev->xdp_prog);
244071cdeceSToke Høiland-Jørgensen dev_put(dev->dev);
24573cf09a3SYafang Shao kfree(dev);
246546ac1ffSJohn Fastabend }
247546ac1ffSJohn Fastabend
248546ac1ffSJohn Fastabend bpf_map_area_free(dtab->netdev_map);
249546ac1ffSJohn Fastabend }
250546ac1ffSJohn Fastabend
251546ac1ffSJohn Fastabend bpf_map_area_free(dtab);
252af4d045cSDaniel Borkmann }
253546ac1ffSJohn Fastabend
dev_map_get_next_key(struct bpf_map * map,void * key,void * next_key)254546ac1ffSJohn Fastabend static int dev_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
255546ac1ffSJohn Fastabend {
256546ac1ffSJohn Fastabend struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
257546ac1ffSJohn Fastabend u32 index = key ? *(u32 *)key : U32_MAX;
258546ac1ffSJohn Fastabend u32 *next = next_key;
259546ac1ffSJohn Fastabend
260546ac1ffSJohn Fastabend if (index >= dtab->map.max_entries) {
261546ac1ffSJohn Fastabend *next = 0;
262546ac1ffSJohn Fastabend return 0;
263546ac1ffSJohn Fastabend }
264546ac1ffSJohn Fastabend
265782347b6SToke Høiland-Jørgensen if (index == dtab->map.max_entries - 1)
266782347b6SToke Høiland-Jørgensen return -ENOENT;
267782347b6SToke Høiland-Jørgensen *next = index + 1;
268782347b6SToke Høiland-Jørgensen return 0;
269e6a4750fSBjörn Töpel }
2706f9d451aSToke Høiland-Jørgensen
2716f9d451aSToke Høiland-Jørgensen /* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
2726f9d451aSToke Høiland-Jørgensen * by local_bh_disable() (from XDP calls inside NAPI). The
2736f9d451aSToke Høiland-Jørgensen * rcu_read_lock_bh_held() below makes lockdep accept both.
2746f9d451aSToke Høiland-Jørgensen */
__dev_map_hash_lookup_elem(struct bpf_map * map,u32 key)275485ec2eaSAmol Grover static void *__dev_map_hash_lookup_elem(struct bpf_map *map, u32 key)
276485ec2eaSAmol Grover {
2776f9d451aSToke Høiland-Jørgensen struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
2786f9d451aSToke Høiland-Jørgensen struct hlist_head *head = dev_map_index_hash(dtab, key);
2796f9d451aSToke Høiland-Jørgensen struct bpf_dtab_netdev *dev;
2806f9d451aSToke Høiland-Jørgensen
2816f9d451aSToke Høiland-Jørgensen hlist_for_each_entry_rcu(dev, head, index_hlist,
2826f9d451aSToke Høiland-Jørgensen lockdep_is_held(&dtab->index_lock))
2836f9d451aSToke Høiland-Jørgensen if (dev->idx == key)
2846f9d451aSToke Høiland-Jørgensen return dev;
2856f9d451aSToke Høiland-Jørgensen
2866f9d451aSToke Høiland-Jørgensen return NULL;
2876f9d451aSToke Høiland-Jørgensen }
2886f9d451aSToke Høiland-Jørgensen
dev_map_hash_get_next_key(struct bpf_map * map,void * key,void * next_key)2896f9d451aSToke Høiland-Jørgensen static int dev_map_hash_get_next_key(struct bpf_map *map, void *key,
2906f9d451aSToke Høiland-Jørgensen void *next_key)
2916f9d451aSToke Høiland-Jørgensen {
2926f9d451aSToke Høiland-Jørgensen struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
2936f9d451aSToke Høiland-Jørgensen u32 idx, *next = next_key;
2946f9d451aSToke Høiland-Jørgensen struct bpf_dtab_netdev *dev, *next_dev;
2956f9d451aSToke Høiland-Jørgensen struct hlist_head *head;
2966f9d451aSToke Høiland-Jørgensen int i = 0;
2976f9d451aSToke Høiland-Jørgensen
2986f9d451aSToke Høiland-Jørgensen if (!key)
2996f9d451aSToke Høiland-Jørgensen goto find_first;
3006f9d451aSToke Høiland-Jørgensen
3016f9d451aSToke Høiland-Jørgensen idx = *(u32 *)key;
3026f9d451aSToke Høiland-Jørgensen
3036f9d451aSToke Høiland-Jørgensen dev = __dev_map_hash_lookup_elem(map, idx);
3046f9d451aSToke Høiland-Jørgensen if (!dev)
3056f9d451aSToke Høiland-Jørgensen goto find_first;
3066f9d451aSToke Høiland-Jørgensen
3076f9d451aSToke Høiland-Jørgensen next_dev = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(&dev->index_hlist)),
3086f9d451aSToke Høiland-Jørgensen struct bpf_dtab_netdev, index_hlist);
3096f9d451aSToke Høiland-Jørgensen
3106f9d451aSToke Høiland-Jørgensen if (next_dev) {
3116f9d451aSToke Høiland-Jørgensen *next = next_dev->idx;
3126f9d451aSToke Høiland-Jørgensen return 0;
3136f9d451aSToke Høiland-Jørgensen }
3146f9d451aSToke Høiland-Jørgensen
3156f9d451aSToke Høiland-Jørgensen i = idx & (dtab->n_buckets - 1);
3166f9d451aSToke Høiland-Jørgensen i++;
3176f9d451aSToke Høiland-Jørgensen
3186f9d451aSToke Høiland-Jørgensen find_first:
3196f9d451aSToke Høiland-Jørgensen for (; i < dtab->n_buckets; i++) {
3206f9d451aSToke Høiland-Jørgensen head = dev_map_index_hash(dtab, i);
3216f9d451aSToke Høiland-Jørgensen
3226f9d451aSToke Høiland-Jørgensen next_dev = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),
3236f9d451aSToke Høiland-Jørgensen struct bpf_dtab_netdev,
3246f9d451aSToke Høiland-Jørgensen index_hlist);
3256f9d451aSToke Høiland-Jørgensen if (next_dev) {
3266f9d451aSToke Høiland-Jørgensen *next = next_dev->idx;
3276f9d451aSToke Høiland-Jørgensen return 0;
328cb261b59SJesper Dangaard Brouer }
329cb261b59SJesper Dangaard Brouer }
330cb261b59SJesper Dangaard Brouer
331cb261b59SJesper Dangaard Brouer return -ENOENT;
332cb261b59SJesper Dangaard Brouer }
333cb261b59SJesper Dangaard Brouer
dev_map_bpf_prog_run(struct bpf_prog * xdp_prog,struct xdp_frame ** frames,int n,struct net_device * dev)334cb261b59SJesper Dangaard Brouer static int dev_map_bpf_prog_run(struct bpf_prog *xdp_prog,
335cb261b59SJesper Dangaard Brouer struct xdp_frame **frames, int n,
336cb261b59SJesper Dangaard Brouer struct net_device *dev)
337cb261b59SJesper Dangaard Brouer {
338cb261b59SJesper Dangaard Brouer struct xdp_txq_info txq = { .dev = dev };
339cb261b59SJesper Dangaard Brouer struct xdp_buff xdp;
340cb261b59SJesper Dangaard Brouer int i, nframes = 0;
341cb261b59SJesper Dangaard Brouer
342cb261b59SJesper Dangaard Brouer for (i = 0; i < n; i++) {
343cb261b59SJesper Dangaard Brouer struct xdp_frame *xdpf = frames[i];
344cb261b59SJesper Dangaard Brouer u32 act;
345cb261b59SJesper Dangaard Brouer int err;
346cb261b59SJesper Dangaard Brouer
347cb261b59SJesper Dangaard Brouer xdp_convert_frame_to_buff(xdpf, &xdp);
348cb261b59SJesper Dangaard Brouer xdp.txq = &txq;
349cb261b59SJesper Dangaard Brouer
350cb261b59SJesper Dangaard Brouer act = bpf_prog_run_xdp(xdp_prog, &xdp);
351cb261b59SJesper Dangaard Brouer switch (act) {
352cb261b59SJesper Dangaard Brouer case XDP_PASS:
353cb261b59SJesper Dangaard Brouer err = xdp_update_frame_from_buff(&xdp, xdpf);
354c8064e5bSPaolo Abeni if (unlikely(err < 0))
355cb261b59SJesper Dangaard Brouer xdp_return_frame_rx_napi(xdpf);
356cb261b59SJesper Dangaard Brouer else
357cb261b59SJesper Dangaard Brouer frames[nframes++] = xdpf;
358cb261b59SJesper Dangaard Brouer break;
359cb261b59SJesper Dangaard Brouer default:
360cb261b59SJesper Dangaard Brouer bpf_warn_invalid_xdp_action(NULL, xdp_prog, act);
361cb261b59SJesper Dangaard Brouer fallthrough;
362cb261b59SJesper Dangaard Brouer case XDP_ABORTED:
363cb261b59SJesper Dangaard Brouer trace_xdp_exception(dev, xdp_prog, act);
364cb261b59SJesper Dangaard Brouer fallthrough;
365cb261b59SJesper Dangaard Brouer case XDP_DROP:
366cb261b59SJesper Dangaard Brouer xdp_return_frame_rx_napi(xdpf);
367ebc4ecd4SBjörn Töpel break;
3685d053f9dSJesper Dangaard Brouer }
36975ccae62SToke Høiland-Jørgensen }
370cb261b59SJesper Dangaard Brouer return nframes; /* sent frames count */
371e8e0f0f4SHangbin Liu }
372cb261b59SJesper Dangaard Brouer
bq_xmit_all(struct xdp_dev_bulk_queue * bq,u32 flags)3735d053f9dSJesper Dangaard Brouer static void bq_xmit_all(struct xdp_dev_bulk_queue *bq, u32 flags)
3745d053f9dSJesper Dangaard Brouer {
375cb261b59SJesper Dangaard Brouer struct net_device *dev = bq->dev;
376ebc4ecd4SBjörn Töpel unsigned int cnt = bq->count;
3775d053f9dSJesper Dangaard Brouer int sent = 0, err = 0;
378cb261b59SJesper Dangaard Brouer int to_send = cnt;
3795d053f9dSJesper Dangaard Brouer int i;
3805d053f9dSJesper Dangaard Brouer
3815d053f9dSJesper Dangaard Brouer if (unlikely(!cnt))
3825d053f9dSJesper Dangaard Brouer return;
3835d053f9dSJesper Dangaard Brouer
384cb261b59SJesper Dangaard Brouer for (i = 0; i < cnt; i++) {
385cb261b59SJesper Dangaard Brouer struct xdp_frame *xdpf = bq->q[i];
386cb261b59SJesper Dangaard Brouer
387cb261b59SJesper Dangaard Brouer prefetch(xdpf);
388cb261b59SJesper Dangaard Brouer }
389cb261b59SJesper Dangaard Brouer
390cb261b59SJesper Dangaard Brouer if (bq->xdp_prog) {
391735fc405SJesper Dangaard Brouer to_send = dev_map_bpf_prog_run(bq->xdp_prog, bq->q, cnt, dev);
392fdc13979SLorenzo Bianconi if (!to_send)
393fdc13979SLorenzo Bianconi goto out;
394fdc13979SLorenzo Bianconi }
395e74de52eSJesper Dangaard Brouer
396735fc405SJesper Dangaard Brouer sent = dev->netdev_ops->ndo_xdp_xmit(dev, to_send, bq->q, flags);
39738edddb8SJesper Dangaard Brouer if (sent < 0) {
3985d053f9dSJesper Dangaard Brouer /* If ndo_xdp_xmit fails with an errno, no frames have
399fdc13979SLorenzo Bianconi * been xmit'ed.
400fdc13979SLorenzo Bianconi */
401735fc405SJesper Dangaard Brouer err = sent;
402cb261b59SJesper Dangaard Brouer sent = 0;
403fdc13979SLorenzo Bianconi }
404735fc405SJesper Dangaard Brouer
405cb261b59SJesper Dangaard Brouer /* If not all frames have been transmitted, it is our
406fdc13979SLorenzo Bianconi * responsibility to free them
407e8e0f0f4SHangbin Liu */
4085d053f9dSJesper Dangaard Brouer for (i = sent; unlikely(i < to_send); i++)
4095d053f9dSJesper Dangaard Brouer xdp_return_frame_rx_napi(bq->q[i]);
410782347b6SToke Høiland-Jørgensen
411782347b6SToke Høiland-Jørgensen out:
412782347b6SToke Høiland-Jørgensen bq->count = 0;
41311393cc9SJohn Fastabend trace_xdp_devmap_xmit(bq->dev_rx, dev, sent, cnt - sent, err);
4141d233886SToke Høiland-Jørgensen }
41511393cc9SJohn Fastabend
4161d233886SToke Høiland-Jørgensen /* __dev_flush is called from xdp_do_flush() which _must_ be signalled from the
41775ccae62SToke Høiland-Jørgensen * driver before returning from its napi->poll() routine. See the comment above
41811393cc9SJohn Fastabend * xdp_do_flush() in filter.c.
419cb261b59SJesper Dangaard Brouer */
__dev_flush(struct list_head * flush_list)4200536b852SBjörn Töpel void __dev_flush(struct list_head *flush_list)
421cb261b59SJesper Dangaard Brouer {
422cb261b59SJesper Dangaard Brouer struct xdp_dev_bulk_queue *bq, *tmp;
423cb261b59SJesper Dangaard Brouer
424cb261b59SJesper Dangaard Brouer list_for_each_entry_safe(bq, tmp, flush_list, flush_node) {
42511393cc9SJohn Fastabend bq_xmit_all(bq, XDP_XMIT_FLUSH);
42611393cc9SJohn Fastabend bq->dev_rx = NULL;
4279a675ba5SSebastian Andrzej Siewior bq->xdp_prog = NULL;
4289a675ba5SSebastian Andrzej Siewior __list_del_clearprev(&bq->flush_node);
4299a675ba5SSebastian Andrzej Siewior }
4309a675ba5SSebastian Andrzej Siewior }
4319a675ba5SSebastian Andrzej Siewior
4329a675ba5SSebastian Andrzej Siewior /* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
4339a675ba5SSebastian Andrzej Siewior * by local_bh_disable() (from XDP calls inside NAPI). The
4349a675ba5SSebastian Andrzej Siewior * rcu_read_lock_bh_held() below makes lockdep accept both.
4359a675ba5SSebastian Andrzej Siewior */
__dev_map_lookup_elem(struct bpf_map * map,u32 key)4369a675ba5SSebastian Andrzej Siewior static void *__dev_map_lookup_elem(struct bpf_map *map, u32 key)
437782347b6SToke Høiland-Jørgensen {
438782347b6SToke Høiland-Jørgensen struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
439782347b6SToke Høiland-Jørgensen struct bpf_dtab_netdev *obj;
440546ac1ffSJohn Fastabend
441e6a4750fSBjörn Töpel if (key >= map->max_entries)
442546ac1ffSJohn Fastabend return NULL;
443546ac1ffSJohn Fastabend
44467f29e07SJesper Dangaard Brouer obj = rcu_dereference_check(dtab->netdev_map[key],
445546ac1ffSJohn Fastabend rcu_read_lock_bh_held());
446af4d045cSDaniel Borkmann return obj;
447546ac1ffSJohn Fastabend }
448546ac1ffSJohn Fastabend
449782347b6SToke Høiland-Jørgensen /* Runs in NAPI, i.e., softirq under local_bh_disable(). Thus, safe percpu
450782347b6SToke Høiland-Jørgensen * variable access, and map elements stick around. See comment above
45167f29e07SJesper Dangaard Brouer * xdp_do_flush() in filter.c.
45267f29e07SJesper Dangaard Brouer */
bq_enqueue(struct net_device * dev,struct xdp_frame * xdpf,struct net_device * dev_rx,struct bpf_prog * xdp_prog)45367f29e07SJesper Dangaard Brouer static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
454782347b6SToke Høiland-Jørgensen struct net_device *dev_rx, struct bpf_prog *xdp_prog)
455782347b6SToke Høiland-Jørgensen {
456782347b6SToke Høiland-Jørgensen struct xdp_dev_bulk_queue *bq = this_cpu_ptr(dev->xdp_bulkq);
4575d053f9dSJesper Dangaard Brouer
458ebc4ecd4SBjörn Töpel if (unlikely(bq->count == DEV_MAP_BULK_SIZE))
459cb261b59SJesper Dangaard Brouer bq_xmit_all(bq, 0);
4605d053f9dSJesper Dangaard Brouer
4611d233886SToke Høiland-Jørgensen /* Ingress dev_rx will be the same for all xdp_frame's in
46275ccae62SToke Høiland-Jørgensen * bulk_queue, because bq stored per-CPU and must be flushed
4635d053f9dSJesper Dangaard Brouer * from net_device drivers NAPI func end.
4645d053f9dSJesper Dangaard Brouer *
4650536b852SBjörn Töpel * Do the same with xdp_prog and flush_list since these fields
4665d053f9dSJesper Dangaard Brouer * are only ever modified together.
46738edddb8SJesper Dangaard Brouer */
46838edddb8SJesper Dangaard Brouer if (!bq->dev_rx) {
46938edddb8SJesper Dangaard Brouer struct list_head *flush_list = bpf_net_ctx_get_dev_flush_list();
470cb261b59SJesper Dangaard Brouer
471cb261b59SJesper Dangaard Brouer bq->dev_rx = dev_rx;
472cb261b59SJesper Dangaard Brouer bq->xdp_prog = xdp_prog;
47338edddb8SJesper Dangaard Brouer list_add(&bq->flush_node, flush_list);
474cb261b59SJesper Dangaard Brouer }
47538edddb8SJesper Dangaard Brouer
476cb261b59SJesper Dangaard Brouer bq->q[bq->count++] = xdpf;
477d5df2830SToke Høiland-Jørgensen }
4785d053f9dSJesper Dangaard Brouer
__xdp_enqueue(struct net_device * dev,struct xdp_frame * xdpf,struct net_device * dev_rx,struct bpf_prog * xdp_prog)4795d053f9dSJesper Dangaard Brouer static inline int __xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
480cb261b59SJesper Dangaard Brouer struct net_device *dev_rx,
481cb261b59SJesper Dangaard Brouer struct bpf_prog *xdp_prog)
482cb261b59SJesper Dangaard Brouer {
483d53ad5d8SToke Høiland-Jørgensen int err;
484cb261b59SJesper Dangaard Brouer
485cb261b59SJesper Dangaard Brouer if (!(dev->xdp_features & NETDEV_XDP_ACT_NDO_XMIT))
48667f29e07SJesper Dangaard Brouer return -EOPNOTSUPP;
487d8d7218aSToshiaki Makita
48867f29e07SJesper Dangaard Brouer if (unlikely(!(dev->xdp_features & NETDEV_XDP_ACT_NDO_XMIT_SG) &&
489b9d460c9SLorenzo Bianconi xdp_frame_has_frags(xdpf)))
490b9d460c9SLorenzo Bianconi return -EOPNOTSUPP;
491b9d460c9SLorenzo Bianconi
492b9d460c9SLorenzo Bianconi err = xdp_ok_fwd_dev(dev, xdp_get_frame_len(xdpf));
493b9d460c9SLorenzo Bianconi if (unlikely(err))
49467f29e07SJesper Dangaard Brouer return err;
49567f29e07SJesper Dangaard Brouer
496bd82ea52SLorenzo Bianconi bq_enqueue(dev, xdpf, dev_rx, xdp_prog);
497d8d7218aSToshiaki Makita return 0;
498d8d7218aSToshiaki Makita }
499d8d7218aSToshiaki Makita
dev_map_bpf_prog_run_skb(struct sk_buff * skb,struct bpf_dtab_netdev * dst)500cb261b59SJesper Dangaard Brouer static u32 dev_map_bpf_prog_run_skb(struct sk_buff *skb, struct bpf_dtab_netdev *dst)
501ebc4ecd4SBjörn Töpel {
502546ac1ffSJohn Fastabend struct xdp_txq_info txq = { .dev = dst->dev };
503546ac1ffSJohn Fastabend struct xdp_buff xdp;
5042ea5eabaSKumar Kartikeya Dwivedi u32 act;
5052ea5eabaSKumar Kartikeya Dwivedi
5062ea5eabaSKumar Kartikeya Dwivedi if (!dst->xdp_prog)
5072ea5eabaSKumar Kartikeya Dwivedi return XDP_PASS;
5082ea5eabaSKumar Kartikeya Dwivedi
5092ea5eabaSKumar Kartikeya Dwivedi __skb_pull(skb, skb->mac_len);
5102ea5eabaSKumar Kartikeya Dwivedi xdp.txq = &txq;
5112ea5eabaSKumar Kartikeya Dwivedi
5122ea5eabaSKumar Kartikeya Dwivedi act = bpf_prog_run_generic_xdp(skb, &xdp, dst->xdp_prog);
5132ea5eabaSKumar Kartikeya Dwivedi switch (act) {
5142ea5eabaSKumar Kartikeya Dwivedi case XDP_PASS:
5152ea5eabaSKumar Kartikeya Dwivedi __skb_push(skb, skb->mac_len);
5162ea5eabaSKumar Kartikeya Dwivedi break;
5172ea5eabaSKumar Kartikeya Dwivedi default:
5182ea5eabaSKumar Kartikeya Dwivedi bpf_warn_invalid_xdp_action(NULL, dst->xdp_prog, act);
5192ea5eabaSKumar Kartikeya Dwivedi fallthrough;
5202ea5eabaSKumar Kartikeya Dwivedi case XDP_ABORTED:
5212ea5eabaSKumar Kartikeya Dwivedi trace_xdp_exception(dst->dev, dst->xdp_prog, act);
522c8064e5bSPaolo Abeni fallthrough;
5232ea5eabaSKumar Kartikeya Dwivedi case XDP_DROP:
5242ea5eabaSKumar Kartikeya Dwivedi kfree_skb(skb);
5252ea5eabaSKumar Kartikeya Dwivedi break;
5262ea5eabaSKumar Kartikeya Dwivedi }
5272ea5eabaSKumar Kartikeya Dwivedi
5282ea5eabaSKumar Kartikeya Dwivedi return act;
5292ea5eabaSKumar Kartikeya Dwivedi }
5302ea5eabaSKumar Kartikeya Dwivedi
dev_xdp_enqueue(struct net_device * dev,struct xdp_frame * xdpf,struct net_device * dev_rx)5312ea5eabaSKumar Kartikeya Dwivedi int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
5322ea5eabaSKumar Kartikeya Dwivedi struct net_device *dev_rx)
5332ea5eabaSKumar Kartikeya Dwivedi {
5342ea5eabaSKumar Kartikeya Dwivedi return __xdp_enqueue(dev, xdpf, dev_rx, NULL);
535d53ad5d8SToke Høiland-Jørgensen }
5361d233886SToke Høiland-Jørgensen
dev_map_enqueue(struct bpf_dtab_netdev * dst,struct xdp_frame * xdpf,struct net_device * dev_rx)5371d233886SToke Høiland-Jørgensen int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
538d53ad5d8SToke Høiland-Jørgensen struct net_device *dev_rx)
5391d233886SToke Høiland-Jørgensen {
5401d233886SToke Høiland-Jørgensen struct net_device *dev = dst->dev;
541d53ad5d8SToke Høiland-Jørgensen
5421d233886SToke Høiland-Jørgensen return __xdp_enqueue(dev, xdpf, dev_rx, dst->xdp_prog);
5431d233886SToke Høiland-Jørgensen }
5441d233886SToke Høiland-Jørgensen
is_valid_dst(struct bpf_dtab_netdev * obj,struct xdp_frame * xdpf)5451d233886SToke Høiland-Jørgensen static bool is_valid_dst(struct bpf_dtab_netdev *obj, struct xdp_frame *xdpf)
546d53ad5d8SToke Høiland-Jørgensen {
5471d233886SToke Høiland-Jørgensen if (!obj)
5481d233886SToke Høiland-Jørgensen return false;
549d53ad5d8SToke Høiland-Jørgensen
550e624d4edSHangbin Liu if (!(obj->dev->xdp_features & NETDEV_XDP_ACT_NDO_XMIT))
551b9d460c9SLorenzo Bianconi return false;
552b9d460c9SLorenzo Bianconi
553b9d460c9SLorenzo Bianconi if (unlikely(!(obj->dev->xdp_features & NETDEV_XDP_ACT_NDO_XMIT_SG) &&
554b9d460c9SLorenzo Bianconi xdp_frame_has_frags(xdpf)))
555b9d460c9SLorenzo Bianconi return false;
556b9d460c9SLorenzo Bianconi
557b9d460c9SLorenzo Bianconi if (xdp_ok_fwd_dev(obj->dev, xdp_get_frame_len(xdpf)))
558b9d460c9SLorenzo Bianconi return false;
559e624d4edSHangbin Liu
560e624d4edSHangbin Liu return true;
561bd82ea52SLorenzo Bianconi }
562e624d4edSHangbin Liu
dev_map_enqueue_clone(struct bpf_dtab_netdev * obj,struct net_device * dev_rx,struct xdp_frame * xdpf)563e624d4edSHangbin Liu static int dev_map_enqueue_clone(struct bpf_dtab_netdev *obj,
564e624d4edSHangbin Liu struct net_device *dev_rx,
565e624d4edSHangbin Liu struct xdp_frame *xdpf)
566e624d4edSHangbin Liu {
567e624d4edSHangbin Liu struct xdp_frame *nxdpf;
568e624d4edSHangbin Liu
569e624d4edSHangbin Liu nxdpf = xdpf_clone(xdpf);
570e624d4edSHangbin Liu if (!nxdpf)
571e624d4edSHangbin Liu return -ENOMEM;
572e624d4edSHangbin Liu
573e624d4edSHangbin Liu bq_enqueue(obj->dev, nxdpf, dev_rx, obj->xdp_prog);
574e624d4edSHangbin Liu
575e624d4edSHangbin Liu return 0;
576e624d4edSHangbin Liu }
577e624d4edSHangbin Liu
is_ifindex_excluded(int * excluded,int num_excluded,int ifindex)578e624d4edSHangbin Liu static inline bool is_ifindex_excluded(int *excluded, int num_excluded, int ifindex)
579e624d4edSHangbin Liu {
580e624d4edSHangbin Liu while (num_excluded--) {
581e624d4edSHangbin Liu if (ifindex == excluded[num_excluded])
582aeea1b86SJussi Maki return true;
583aeea1b86SJussi Maki }
584aeea1b86SJussi Maki return false;
585aeea1b86SJussi Maki }
586aeea1b86SJussi Maki
587aeea1b86SJussi Maki /* Get ifindex of each upper device. 'indexes' must be able to hold at
588aeea1b86SJussi Maki * least MAX_NEST_DEV elements.
589aeea1b86SJussi Maki * Returns the number of ifindexes added.
590aeea1b86SJussi Maki */
get_upper_ifindexes(struct net_device * dev,int * indexes)591aeea1b86SJussi Maki static int get_upper_ifindexes(struct net_device *dev, int *indexes)
592aeea1b86SJussi Maki {
593aeea1b86SJussi Maki struct net_device *upper;
594aeea1b86SJussi Maki struct list_head *iter;
595aeea1b86SJussi Maki int n = 0;
596aeea1b86SJussi Maki
597aeea1b86SJussi Maki netdev_for_each_upper_dev_rcu(dev, upper, iter) {
598aeea1b86SJussi Maki indexes[n++] = upper->ifindex;
599aeea1b86SJussi Maki }
600aeea1b86SJussi Maki return n;
601aeea1b86SJussi Maki }
602aeea1b86SJussi Maki
dev_map_enqueue_multi(struct xdp_frame * xdpf,struct net_device * dev_rx,struct bpf_map * map,bool exclude_ingress)603aeea1b86SJussi Maki int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,
604aeea1b86SJussi Maki struct bpf_map *map, bool exclude_ingress)
605aeea1b86SJussi Maki {
606aeea1b86SJussi Maki struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
607d53ad5d8SToke Høiland-Jørgensen struct bpf_dtab_netdev *dst, *last_dst = NULL;
608e624d4edSHangbin Liu int excluded_devices[1+MAX_NEST_DEV];
609e624d4edSHangbin Liu struct hlist_head *head;
610e624d4edSHangbin Liu int num_excluded = 0;
611e624d4edSHangbin Liu unsigned int i;
612aeea1b86SJussi Maki int err;
613e624d4edSHangbin Liu
614aeea1b86SJussi Maki if (exclude_ingress) {
615e624d4edSHangbin Liu num_excluded = get_upper_ifindexes(dev_rx, excluded_devices);
616e624d4edSHangbin Liu excluded_devices[num_excluded++] = dev_rx->ifindex;
617e624d4edSHangbin Liu }
618aeea1b86SJussi Maki
619aeea1b86SJussi Maki if (map->map_type == BPF_MAP_TYPE_DEVMAP) {
620aeea1b86SJussi Maki for (i = 0; i < map->max_entries; i++) {
621aeea1b86SJussi Maki dst = rcu_dereference_check(dtab->netdev_map[i],
622aeea1b86SJussi Maki rcu_read_lock_bh_held());
623e624d4edSHangbin Liu if (!is_valid_dst(dst, xdpf))
624e624d4edSHangbin Liu continue;
6250fc4dcc1SToke Høiland-Jørgensen
6260fc4dcc1SToke Høiland-Jørgensen if (is_ifindex_excluded(excluded_devices, num_excluded, dst->dev->ifindex))
627d53ad5d8SToke Høiland-Jørgensen continue;
628aeea1b86SJussi Maki
629aeea1b86SJussi Maki /* we only need n-1 clones; last_dst enqueued below */
630aeea1b86SJussi Maki if (!last_dst) {
631e624d4edSHangbin Liu last_dst = dst;
632e624d4edSHangbin Liu continue;
633e624d4edSHangbin Liu }
634e624d4edSHangbin Liu
635e624d4edSHangbin Liu err = dev_map_enqueue_clone(last_dst, dev_rx, xdpf);
636e624d4edSHangbin Liu if (err)
637e624d4edSHangbin Liu return err;
638e624d4edSHangbin Liu
639e624d4edSHangbin Liu last_dst = dst;
640e624d4edSHangbin Liu }
641e624d4edSHangbin Liu } else { /* BPF_MAP_TYPE_DEVMAP_HASH */
642e624d4edSHangbin Liu for (i = 0; i < dtab->n_buckets; i++) {
643e624d4edSHangbin Liu head = dev_map_index_hash(dtab, i);
644e624d4edSHangbin Liu hlist_for_each_entry_rcu(dst, head, index_hlist,
645e624d4edSHangbin Liu lockdep_is_held(&dtab->index_lock)) {
646e624d4edSHangbin Liu if (!is_valid_dst(dst, xdpf))
647e624d4edSHangbin Liu continue;
648e624d4edSHangbin Liu
649e624d4edSHangbin Liu if (is_ifindex_excluded(excluded_devices, num_excluded,
650d53ad5d8SToke Høiland-Jørgensen dst->dev->ifindex))
651aeea1b86SJussi Maki continue;
652aeea1b86SJussi Maki
653aeea1b86SJussi Maki /* we only need n-1 clones; last_dst enqueued below */
654aeea1b86SJussi Maki if (!last_dst) {
655e624d4edSHangbin Liu last_dst = dst;
656e624d4edSHangbin Liu continue;
657e624d4edSHangbin Liu }
658e624d4edSHangbin Liu
659e624d4edSHangbin Liu err = dev_map_enqueue_clone(last_dst, dev_rx, xdpf);
660e624d4edSHangbin Liu if (err)
661e624d4edSHangbin Liu return err;
662e624d4edSHangbin Liu
663e624d4edSHangbin Liu last_dst = dst;
664e624d4edSHangbin Liu }
665e624d4edSHangbin Liu }
666e624d4edSHangbin Liu }
667e624d4edSHangbin Liu
668e624d4edSHangbin Liu /* consume the last copy of the frame */
669e624d4edSHangbin Liu if (last_dst)
670e624d4edSHangbin Liu bq_enqueue(last_dst->dev, xdpf, dev_rx, last_dst->xdp_prog);
671e624d4edSHangbin Liu else
672e624d4edSHangbin Liu xdp_return_frame_rx_napi(xdpf); /* dtab is empty */
673e624d4edSHangbin Liu
674e624d4edSHangbin Liu return 0;
675e624d4edSHangbin Liu }
676e624d4edSHangbin Liu
dev_map_generic_redirect(struct bpf_dtab_netdev * dst,struct sk_buff * skb,struct bpf_prog * xdp_prog)677e624d4edSHangbin Liu int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,
678e624d4edSHangbin Liu struct bpf_prog *xdp_prog)
679e624d4edSHangbin Liu {
680e624d4edSHangbin Liu int err;
6816d5fc195SToshiaki Makita
6826d5fc195SToshiaki Makita err = xdp_ok_fwd_dev(dst->dev, skb->len);
6836d5fc195SToshiaki Makita if (unlikely(err))
6846d5fc195SToshiaki Makita return err;
6856d5fc195SToshiaki Makita
686d8d7218aSToshiaki Makita /* Redirect has already succeeded semantically at this point, so we just
6876d5fc195SToshiaki Makita * return 0 even if packet is dropped. Helper below takes care of
6886d5fc195SToshiaki Makita * freeing skb.
6892ea5eabaSKumar Kartikeya Dwivedi */
6902ea5eabaSKumar Kartikeya Dwivedi if (dev_map_bpf_prog_run_skb(skb, dst) != XDP_PASS)
6912ea5eabaSKumar Kartikeya Dwivedi return 0;
6922ea5eabaSKumar Kartikeya Dwivedi
6932ea5eabaSKumar Kartikeya Dwivedi skb->dev = dst->dev;
6942ea5eabaSKumar Kartikeya Dwivedi generic_xdp_tx(skb, xdp_prog);
6952ea5eabaSKumar Kartikeya Dwivedi
6962ea5eabaSKumar Kartikeya Dwivedi return 0;
6976d5fc195SToshiaki Makita }
6986d5fc195SToshiaki Makita
dev_map_redirect_clone(struct bpf_dtab_netdev * dst,struct sk_buff * skb,struct bpf_prog * xdp_prog)6996d5fc195SToshiaki Makita static int dev_map_redirect_clone(struct bpf_dtab_netdev *dst,
7006d5fc195SToshiaki Makita struct sk_buff *skb,
7016d5fc195SToshiaki Makita struct bpf_prog *xdp_prog)
7026d5fc195SToshiaki Makita {
703e624d4edSHangbin Liu struct sk_buff *nskb;
704e624d4edSHangbin Liu int err;
705e624d4edSHangbin Liu
706e624d4edSHangbin Liu nskb = skb_clone(skb, GFP_ATOMIC);
707e624d4edSHangbin Liu if (!nskb)
708e624d4edSHangbin Liu return -ENOMEM;
709e624d4edSHangbin Liu
710e624d4edSHangbin Liu err = dev_map_generic_redirect(dst, nskb, xdp_prog);
711e624d4edSHangbin Liu if (unlikely(err)) {
712e624d4edSHangbin Liu consume_skb(nskb);
713e624d4edSHangbin Liu return err;
714e624d4edSHangbin Liu }
715e624d4edSHangbin Liu
716e624d4edSHangbin Liu return 0;
717e624d4edSHangbin Liu }
718e624d4edSHangbin Liu
dev_map_redirect_multi(struct net_device * dev,struct sk_buff * skb,struct bpf_prog * xdp_prog,struct bpf_map * map,bool exclude_ingress)719e624d4edSHangbin Liu int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,
720e624d4edSHangbin Liu struct bpf_prog *xdp_prog, struct bpf_map *map,
721e624d4edSHangbin Liu bool exclude_ingress)
722e624d4edSHangbin Liu {
723e624d4edSHangbin Liu struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
724e624d4edSHangbin Liu struct bpf_dtab_netdev *dst, *last_dst = NULL;
725e624d4edSHangbin Liu int excluded_devices[1+MAX_NEST_DEV];
726e624d4edSHangbin Liu struct hlist_head *head;
727e624d4edSHangbin Liu struct hlist_node *next;
728e624d4edSHangbin Liu int num_excluded = 0;
729aeea1b86SJussi Maki unsigned int i;
730e624d4edSHangbin Liu int err;
731e624d4edSHangbin Liu
732aeea1b86SJussi Maki if (exclude_ingress) {
733e624d4edSHangbin Liu num_excluded = get_upper_ifindexes(dev, excluded_devices);
734e624d4edSHangbin Liu excluded_devices[num_excluded++] = dev->ifindex;
735e624d4edSHangbin Liu }
736aeea1b86SJussi Maki
737aeea1b86SJussi Maki if (map->map_type == BPF_MAP_TYPE_DEVMAP) {
738aeea1b86SJussi Maki for (i = 0; i < map->max_entries; i++) {
739aeea1b86SJussi Maki dst = rcu_dereference_check(dtab->netdev_map[i],
740aeea1b86SJussi Maki rcu_read_lock_bh_held());
741e624d4edSHangbin Liu if (!dst)
742e624d4edSHangbin Liu continue;
7430fc4dcc1SToke Høiland-Jørgensen
7440fc4dcc1SToke Høiland-Jørgensen if (is_ifindex_excluded(excluded_devices, num_excluded, dst->dev->ifindex))
745aeea1b86SJussi Maki continue;
746aeea1b86SJussi Maki
747aeea1b86SJussi Maki /* we only need n-1 clones; last_dst enqueued below */
748aeea1b86SJussi Maki if (!last_dst) {
749e624d4edSHangbin Liu last_dst = dst;
750e624d4edSHangbin Liu continue;
751e624d4edSHangbin Liu }
752e624d4edSHangbin Liu
753e624d4edSHangbin Liu err = dev_map_redirect_clone(last_dst, skb, xdp_prog);
754e624d4edSHangbin Liu if (err)
755e624d4edSHangbin Liu return err;
756e624d4edSHangbin Liu
757e624d4edSHangbin Liu last_dst = dst;
758e624d4edSHangbin Liu
759e624d4edSHangbin Liu }
760e624d4edSHangbin Liu } else { /* BPF_MAP_TYPE_DEVMAP_HASH */
761e624d4edSHangbin Liu for (i = 0; i < dtab->n_buckets; i++) {
762aeea1b86SJussi Maki head = dev_map_index_hash(dtab, i);
763e624d4edSHangbin Liu hlist_for_each_entry_safe(dst, next, head, index_hlist) {
764e624d4edSHangbin Liu if (is_ifindex_excluded(excluded_devices, num_excluded,
765e624d4edSHangbin Liu dst->dev->ifindex))
766e624d4edSHangbin Liu continue;
767e624d4edSHangbin Liu
768aeea1b86SJussi Maki /* we only need n-1 clones; last_dst enqueued below */
769aeea1b86SJussi Maki if (!last_dst) {
770e624d4edSHangbin Liu last_dst = dst;
771e624d4edSHangbin Liu continue;
772e624d4edSHangbin Liu }
773e624d4edSHangbin Liu
774e624d4edSHangbin Liu err = dev_map_redirect_clone(last_dst, skb, xdp_prog);
775e624d4edSHangbin Liu if (err)
776e624d4edSHangbin Liu return err;
777e624d4edSHangbin Liu
778e624d4edSHangbin Liu last_dst = dst;
779e624d4edSHangbin Liu }
780e624d4edSHangbin Liu }
781e624d4edSHangbin Liu }
782e624d4edSHangbin Liu
783e624d4edSHangbin Liu /* consume the first skb and return */
784e624d4edSHangbin Liu if (last_dst)
785e624d4edSHangbin Liu return dev_map_generic_redirect(last_dst, skb, xdp_prog);
786e624d4edSHangbin Liu
787e624d4edSHangbin Liu /* dtab is empty */
788e624d4edSHangbin Liu consume_skb(skb);
789e624d4edSHangbin Liu return 0;
790e624d4edSHangbin Liu }
791e624d4edSHangbin Liu
dev_map_lookup_elem(struct bpf_map * map,void * key)792e624d4edSHangbin Liu static void *dev_map_lookup_elem(struct bpf_map *map, void *key)
793e624d4edSHangbin Liu {
794e624d4edSHangbin Liu struct bpf_dtab_netdev *obj = __dev_map_lookup_elem(map, *(u32 *)key);
795e624d4edSHangbin Liu
796af4d045cSDaniel Borkmann return obj ? &obj->val : NULL;
79711393cc9SJohn Fastabend }
79867f29e07SJesper Dangaard Brouer
dev_map_hash_lookup_elem(struct bpf_map * map,void * key)799af4d045cSDaniel Borkmann static void *dev_map_hash_lookup_elem(struct bpf_map *map, void *key)
8007f1c0426SDavid Ahern {
801af4d045cSDaniel Borkmann struct bpf_dtab_netdev *obj = __dev_map_hash_lookup_elem(map,
802af4d045cSDaniel Borkmann *(u32 *)key);
8036f9d451aSToke Høiland-Jørgensen return obj ? &obj->val : NULL;
8046f9d451aSToke Høiland-Jørgensen }
8056f9d451aSToke Høiland-Jørgensen
__dev_map_entry_free(struct rcu_head * rcu)8066f9d451aSToke Høiland-Jørgensen static void __dev_map_entry_free(struct rcu_head *rcu)
8077f1c0426SDavid Ahern {
8086f9d451aSToke Høiland-Jørgensen struct bpf_dtab_netdev *dev;
8096f9d451aSToke Høiland-Jørgensen
810546ac1ffSJohn Fastabend dev = container_of(rcu, struct bpf_dtab_netdev, rcu);
811546ac1ffSJohn Fastabend if (dev->xdp_prog)
812af4d045cSDaniel Borkmann bpf_prog_put(dev->xdp_prog);
813546ac1ffSJohn Fastabend dev_put(dev->dev);
814af4d045cSDaniel Borkmann kfree(dev);
815fbee97feSDavid Ahern }
816fbee97feSDavid Ahern
dev_map_delete_elem(struct bpf_map * map,void * key)817af4d045cSDaniel Borkmann static long dev_map_delete_elem(struct bpf_map *map, void *key)
818af4d045cSDaniel Borkmann {
819546ac1ffSJohn Fastabend struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
820546ac1ffSJohn Fastabend struct bpf_dtab_netdev *old_dev;
821d7ba4cc9SJP Kobryn int k = *(u32 *)key;
822546ac1ffSJohn Fastabend
823546ac1ffSJohn Fastabend if (k >= map->max_entries)
824546ac1ffSJohn Fastabend return -EINVAL;
825546ac1ffSJohn Fastabend
826546ac1ffSJohn Fastabend old_dev = unrcu_pointer(xchg(&dtab->netdev_map[k], NULL));
827546ac1ffSJohn Fastabend if (old_dev) {
828546ac1ffSJohn Fastabend call_rcu(&old_dev->rcu, __dev_map_entry_free);
829546ac1ffSJohn Fastabend atomic_dec((atomic_t *)&dtab->items);
830782347b6SToke Høiland-Jørgensen }
831fa5e83dfSYafang Shao return 0;
832546ac1ffSJohn Fastabend }
833fa5e83dfSYafang Shao
dev_map_hash_delete_elem(struct bpf_map * map,void * key)834fa5e83dfSYafang Shao static long dev_map_hash_delete_elem(struct bpf_map *map, void *key)
835546ac1ffSJohn Fastabend {
836546ac1ffSJohn Fastabend struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
837546ac1ffSJohn Fastabend struct bpf_dtab_netdev *old_dev;
838d7ba4cc9SJP Kobryn int k = *(u32 *)key;
8396f9d451aSToke Høiland-Jørgensen unsigned long flags;
8406f9d451aSToke Høiland-Jørgensen int ret = -ENOENT;
8416f9d451aSToke Høiland-Jørgensen
8426f9d451aSToke Høiland-Jørgensen spin_lock_irqsave(&dtab->index_lock, flags);
8436f9d451aSToke Høiland-Jørgensen
8446f9d451aSToke Høiland-Jørgensen old_dev = __dev_map_hash_lookup_elem(map, k);
8456f9d451aSToke Høiland-Jørgensen if (old_dev) {
8466f9d451aSToke Høiland-Jørgensen dtab->items--;
8476f9d451aSToke Høiland-Jørgensen hlist_del_init_rcu(&old_dev->index_hlist);
8486f9d451aSToke Høiland-Jørgensen call_rcu(&old_dev->rcu, __dev_map_entry_free);
8496f9d451aSToke Høiland-Jørgensen ret = 0;
8506f9d451aSToke Høiland-Jørgensen }
8516f9d451aSToke Høiland-Jørgensen spin_unlock_irqrestore(&dtab->index_lock, flags);
8526f9d451aSToke Høiland-Jørgensen
8536f9d451aSToke Høiland-Jørgensen return ret;
8546f9d451aSToke Høiland-Jørgensen }
8556f9d451aSToke Høiland-Jørgensen
__dev_map_alloc_node(struct net * net,struct bpf_dtab * dtab,struct bpf_devmap_val * val,unsigned int idx)8566f9d451aSToke Høiland-Jørgensen static struct bpf_dtab_netdev *__dev_map_alloc_node(struct net *net,
8576f9d451aSToke Høiland-Jørgensen struct bpf_dtab *dtab,
8586f9d451aSToke Høiland-Jørgensen struct bpf_devmap_val *val,
8596f9d451aSToke Høiland-Jørgensen unsigned int idx)
860fca16e51SToke Høiland-Jørgensen {
861fca16e51SToke Høiland-Jørgensen struct bpf_prog *prog = NULL;
8627f1c0426SDavid Ahern struct bpf_dtab_netdev *dev;
863fca16e51SToke Høiland-Jørgensen
864546ac1ffSJohn Fastabend dev = bpf_map_kmalloc_node(&dtab->map, sizeof(*dev),
865fbee97feSDavid Ahern GFP_NOWAIT | __GFP_NOWARN,
866fca16e51SToke Høiland-Jørgensen dtab->map.numa_node);
867546ac1ffSJohn Fastabend if (!dev)
8681440290aSRoman Gushchin return ERR_PTR(-ENOMEM);
869ace2bee8SYafang Shao
87075ccae62SToke Høiland-Jørgensen dev->dev = dev_get_by_index(net, val->ifindex);
871546ac1ffSJohn Fastabend if (!dev->dev)
872fca16e51SToke Høiland-Jørgensen goto err_out;
873546ac1ffSJohn Fastabend
8747f1c0426SDavid Ahern if (val->bpf_prog.fd > 0) {
8757f1c0426SDavid Ahern prog = bpf_prog_get_type_dev(val->bpf_prog.fd,
8767f1c0426SDavid Ahern BPF_PROG_TYPE_XDP, false);
877546ac1ffSJohn Fastabend if (IS_ERR(prog))
878281920b7SJesper Dangaard Brouer goto err_put_dev;
879fbee97feSDavid Ahern if (prog->expected_attach_type != BPF_XDP_DEVMAP ||
880fbee97feSDavid Ahern !bpf_prog_map_compatible(&dtab->map, prog))
881fbee97feSDavid Ahern goto err_put_prog;
882fbee97feSDavid Ahern }
883f45d5b6cSToke Hoiland-Jorgensen
884f45d5b6cSToke Hoiland-Jorgensen dev->idx = idx;
885fbee97feSDavid Ahern if (prog) {
886fbee97feSDavid Ahern dev->xdp_prog = prog;
887fbee97feSDavid Ahern dev->val.bpf_prog.id = prog->aux->id;
888fca16e51SToke Høiland-Jørgensen } else {
889fbee97feSDavid Ahern dev->xdp_prog = NULL;
890fbee97feSDavid Ahern dev->val.bpf_prog.id = 0;
891fbee97feSDavid Ahern }
892fbee97feSDavid Ahern dev->val.ifindex = val->ifindex;
893fbee97feSDavid Ahern
894fbee97feSDavid Ahern return dev;
895fbee97feSDavid Ahern err_put_prog:
8967f1c0426SDavid Ahern bpf_prog_put(prog);
897fca16e51SToke Høiland-Jørgensen err_put_dev:
898fca16e51SToke Høiland-Jørgensen dev_put(dev->dev);
899fbee97feSDavid Ahern err_out:
900fbee97feSDavid Ahern kfree(dev);
901fbee97feSDavid Ahern return ERR_PTR(-EINVAL);
902fbee97feSDavid Ahern }
9037f1c0426SDavid Ahern
__dev_map_update_elem(struct net * net,struct bpf_map * map,void * key,void * value,u64 map_flags)9047f1c0426SDavid Ahern static long __dev_map_update_elem(struct net *net, struct bpf_map *map,
9057f1c0426SDavid Ahern void *key, void *value, u64 map_flags)
906fca16e51SToke Høiland-Jørgensen {
907fca16e51SToke Høiland-Jørgensen struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
908d7ba4cc9SJP Kobryn struct bpf_dtab_netdev *dev, *old_dev;
909fca16e51SToke Høiland-Jørgensen struct bpf_devmap_val val = {};
910fca16e51SToke Høiland-Jørgensen u32 i = *(u32 *)key;
911fca16e51SToke Høiland-Jørgensen
912fca16e51SToke Høiland-Jørgensen if (unlikely(map_flags > BPF_EXIST))
913281920b7SJesper Dangaard Brouer return -EINVAL;
914fca16e51SToke Høiland-Jørgensen if (unlikely(i >= dtab->map.max_entries))
915fca16e51SToke Høiland-Jørgensen return -E2BIG;
916fca16e51SToke Høiland-Jørgensen if (unlikely(map_flags == BPF_NOEXIST))
917fca16e51SToke Høiland-Jørgensen return -EEXIST;
918fca16e51SToke Høiland-Jørgensen
919fca16e51SToke Høiland-Jørgensen /* already verified value_size <= sizeof val */
920fca16e51SToke Høiland-Jørgensen memcpy(&val, value, map->value_size);
921fca16e51SToke Høiland-Jørgensen
922fca16e51SToke Høiland-Jørgensen if (!val.ifindex) {
9237f1c0426SDavid Ahern dev = NULL;
9247f1c0426SDavid Ahern /* can not specify fd if ifindex is 0 */
9257f1c0426SDavid Ahern if (val.bpf_prog.fd > 0)
9267f1c0426SDavid Ahern return -EINVAL;
927fca16e51SToke Høiland-Jørgensen } else {
928fbee97feSDavid Ahern dev = __dev_map_alloc_node(net, dtab, &val, i);
929281920b7SJesper Dangaard Brouer if (IS_ERR(dev))
930fbee97feSDavid Ahern return PTR_ERR(dev);
931fca16e51SToke Høiland-Jørgensen }
9327f1c0426SDavid Ahern
933fca16e51SToke Høiland-Jørgensen /* Use call_rcu() here to ensure rcu critical sections have completed
934fca16e51SToke Høiland-Jørgensen * Remembering the driver side flush operation will happen before the
935546ac1ffSJohn Fastabend * net device is removed.
936546ac1ffSJohn Fastabend */
937546ac1ffSJohn Fastabend old_dev = unrcu_pointer(xchg(&dtab->netdev_map[i], RCU_INITIALIZER(dev)));
938546ac1ffSJohn Fastabend if (old_dev)
939546ac1ffSJohn Fastabend call_rcu(&old_dev->rcu, __dev_map_entry_free);
940546ac1ffSJohn Fastabend else
941782347b6SToke Høiland-Jørgensen atomic_inc((atomic_t *)&dtab->items);
942546ac1ffSJohn Fastabend
943546ac1ffSJohn Fastabend return 0;
944fa5e83dfSYafang Shao }
945fa5e83dfSYafang Shao
dev_map_update_elem(struct bpf_map * map,void * key,void * value,u64 map_flags)946546ac1ffSJohn Fastabend static long dev_map_update_elem(struct bpf_map *map, void *key, void *value,
947546ac1ffSJohn Fastabend u64 map_flags)
948546ac1ffSJohn Fastabend {
949546ac1ffSJohn Fastabend return __dev_map_update_elem(current->nsproxy->net_ns,
950d7ba4cc9SJP Kobryn map, key, value, map_flags);
951fca16e51SToke Høiland-Jørgensen }
952fca16e51SToke Høiland-Jørgensen
__dev_map_hash_update_elem(struct net * net,struct bpf_map * map,void * key,void * value,u64 map_flags)953fca16e51SToke Høiland-Jørgensen static long __dev_map_hash_update_elem(struct net *net, struct bpf_map *map,
954fca16e51SToke Høiland-Jørgensen void *key, void *value, u64 map_flags)
955fca16e51SToke Høiland-Jørgensen {
956fca16e51SToke Høiland-Jørgensen struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
957d7ba4cc9SJP Kobryn struct bpf_dtab_netdev *dev, *old_dev;
9586f9d451aSToke Høiland-Jørgensen struct bpf_devmap_val val = {};
9596f9d451aSToke Høiland-Jørgensen u32 idx = *(u32 *)key;
9606f9d451aSToke Høiland-Jørgensen unsigned long flags;
9616f9d451aSToke Høiland-Jørgensen int err = -EEXIST;
962281920b7SJesper Dangaard Brouer
9636f9d451aSToke Høiland-Jørgensen /* already verified value_size <= sizeof val */
9646f9d451aSToke Høiland-Jørgensen memcpy(&val, value, map->value_size);
965af58e7eeSToke Høiland-Jørgensen
9666f9d451aSToke Høiland-Jørgensen if (unlikely(map_flags > BPF_EXIST || !val.ifindex))
9677f1c0426SDavid Ahern return -EINVAL;
9687f1c0426SDavid Ahern
9697f1c0426SDavid Ahern spin_lock_irqsave(&dtab->index_lock, flags);
9707f1c0426SDavid Ahern
9716f9d451aSToke Høiland-Jørgensen old_dev = __dev_map_hash_lookup_elem(map, idx);
9726f9d451aSToke Høiland-Jørgensen if (old_dev && (map_flags & BPF_NOEXIST))
973af58e7eeSToke Høiland-Jørgensen goto out_err;
974af58e7eeSToke Høiland-Jørgensen
9756f9d451aSToke Høiland-Jørgensen dev = __dev_map_alloc_node(net, dtab, &val, idx);
9766f9d451aSToke Høiland-Jørgensen if (IS_ERR(dev)) {
977af58e7eeSToke Høiland-Jørgensen err = PTR_ERR(dev);
9786f9d451aSToke Høiland-Jørgensen goto out_err;
9797f1c0426SDavid Ahern }
980af58e7eeSToke Høiland-Jørgensen
981af58e7eeSToke Høiland-Jørgensen if (old_dev) {
982af58e7eeSToke Høiland-Jørgensen hlist_del_rcu(&old_dev->index_hlist);
983af58e7eeSToke Høiland-Jørgensen } else {
9846f9d451aSToke Høiland-Jørgensen if (dtab->items >= dtab->map.max_entries) {
9856f9d451aSToke Høiland-Jørgensen spin_unlock_irqrestore(&dtab->index_lock, flags);
9866f9d451aSToke Høiland-Jørgensen call_rcu(&dev->rcu, __dev_map_entry_free);
9876f9d451aSToke Høiland-Jørgensen return -E2BIG;
9886f9d451aSToke Høiland-Jørgensen }
9896f9d451aSToke Høiland-Jørgensen dtab->items++;
9906f9d451aSToke Høiland-Jørgensen }
9916f9d451aSToke Høiland-Jørgensen
9926f9d451aSToke Høiland-Jørgensen hlist_add_head_rcu(&dev->index_hlist,
9936f9d451aSToke Høiland-Jørgensen dev_map_index_hash(dtab, idx));
9946f9d451aSToke Høiland-Jørgensen spin_unlock_irqrestore(&dtab->index_lock, flags);
9956f9d451aSToke Høiland-Jørgensen
9966f9d451aSToke Høiland-Jørgensen if (old_dev)
9976f9d451aSToke Høiland-Jørgensen call_rcu(&old_dev->rcu, __dev_map_entry_free);
9986f9d451aSToke Høiland-Jørgensen
9996f9d451aSToke Høiland-Jørgensen return 0;
10006f9d451aSToke Høiland-Jørgensen
10016f9d451aSToke Høiland-Jørgensen out_err:
10026f9d451aSToke Høiland-Jørgensen spin_unlock_irqrestore(&dtab->index_lock, flags);
10036f9d451aSToke Høiland-Jørgensen return err;
1004af58e7eeSToke Høiland-Jørgensen }
1005af58e7eeSToke Høiland-Jørgensen
dev_map_hash_update_elem(struct bpf_map * map,void * key,void * value,u64 map_flags)1006af58e7eeSToke Høiland-Jørgensen static long dev_map_hash_update_elem(struct bpf_map *map, void *key, void *value,
1007af58e7eeSToke Høiland-Jørgensen u64 map_flags)
10086f9d451aSToke Høiland-Jørgensen {
10096f9d451aSToke Høiland-Jørgensen return __dev_map_hash_update_elem(current->nsproxy->net_ns,
1010d7ba4cc9SJP Kobryn map, key, value, map_flags);
10116f9d451aSToke Høiland-Jørgensen }
10126f9d451aSToke Høiland-Jørgensen
dev_map_redirect(struct bpf_map * map,u64 ifindex,u64 flags)10136f9d451aSToke Høiland-Jørgensen static long dev_map_redirect(struct bpf_map *map, u64 ifindex, u64 flags)
10146f9d451aSToke Høiland-Jørgensen {
10156f9d451aSToke Høiland-Jørgensen return __bpf_xdp_redirect_map(map, ifindex, flags,
10166f9d451aSToke Høiland-Jørgensen BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS,
1017d7ba4cc9SJP Kobryn __dev_map_lookup_elem);
1018e6a4750fSBjörn Töpel }
1019e624d4edSHangbin Liu
dev_hash_map_redirect(struct bpf_map * map,u64 ifindex,u64 flags)1020e624d4edSHangbin Liu static long dev_hash_map_redirect(struct bpf_map *map, u64 ifindex, u64 flags)
1021e624d4edSHangbin Liu {
1022e6a4750fSBjörn Töpel return __bpf_xdp_redirect_map(map, ifindex, flags,
1023e6a4750fSBjörn Töpel BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS,
1024d7ba4cc9SJP Kobryn __dev_map_hash_lookup_elem);
1025e6a4750fSBjörn Töpel }
1026e624d4edSHangbin Liu
dev_map_mem_usage(const struct bpf_map * map)1027e624d4edSHangbin Liu static u64 dev_map_mem_usage(const struct bpf_map *map)
1028e624d4edSHangbin Liu {
1029e6a4750fSBjörn Töpel struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
1030e6a4750fSBjörn Töpel u64 usage = sizeof(struct bpf_dtab);
1031fa5e83dfSYafang Shao
1032fa5e83dfSYafang Shao if (map->map_type == BPF_MAP_TYPE_DEVMAP_HASH)
1033fa5e83dfSYafang Shao usage += (u64)dtab->n_buckets * sizeof(struct hlist_head);
1034fa5e83dfSYafang Shao else
1035fa5e83dfSYafang Shao usage += (u64)map->max_entries * sizeof(struct bpf_dtab_netdev *);
1036fa5e83dfSYafang Shao usage += atomic_read((atomic_t *)&dtab->items) *
1037fa5e83dfSYafang Shao (u64)sizeof(struct bpf_dtab_netdev);
1038fa5e83dfSYafang Shao return usage;
1039fa5e83dfSYafang Shao }
1040fa5e83dfSYafang Shao
1041fa5e83dfSYafang Shao BTF_ID_LIST_SINGLE(dev_map_btf_ids, struct, bpf_dtab)
1042fa5e83dfSYafang Shao const struct bpf_map_ops dev_map_ops = {
1043fa5e83dfSYafang Shao .map_meta_equal = bpf_map_meta_equal,
1044fa5e83dfSYafang Shao .map_alloc_check = dev_map_alloc_check,
1045c317ab71SMenglong Dong .map_alloc = dev_map_alloc,
1046546ac1ffSJohn Fastabend .map_free = dev_map_free,
1047f4d05259SMartin KaFai Lau .map_get_next_key = dev_map_get_next_key,
1048*fd8db077SFlorian Lehner .map_lookup_elem = dev_map_lookup_elem,
1049546ac1ffSJohn Fastabend .map_update_elem = dev_map_update_elem,
1050546ac1ffSJohn Fastabend .map_delete_elem = dev_map_delete_elem,
1051546ac1ffSJohn Fastabend .map_check_btf = map_check_no_btf,
1052546ac1ffSJohn Fastabend .map_mem_usage = dev_map_mem_usage,
1053546ac1ffSJohn Fastabend .map_btf_id = &dev_map_btf_ids[0],
1054546ac1ffSJohn Fastabend .map_redirect = dev_map_redirect,
1055e8d2bec0SDaniel Borkmann };
1056fa5e83dfSYafang Shao
1057c317ab71SMenglong Dong const struct bpf_map_ops dev_map_hash_ops = {
1058e6a4750fSBjörn Töpel .map_meta_equal = bpf_map_meta_equal,
1059546ac1ffSJohn Fastabend .map_alloc_check = dev_map_alloc_check,
10602ddf71e2SJohn Fastabend .map_alloc = dev_map_alloc,
10616f9d451aSToke Høiland-Jørgensen .map_free = dev_map_free,
1062f4d05259SMartin KaFai Lau .map_get_next_key = dev_map_hash_get_next_key,
1063*fd8db077SFlorian Lehner .map_lookup_elem = dev_map_hash_lookup_elem,
10646f9d451aSToke Høiland-Jørgensen .map_update_elem = dev_map_hash_update_elem,
10656f9d451aSToke Høiland-Jørgensen .map_delete_elem = dev_map_hash_delete_elem,
10666f9d451aSToke Høiland-Jørgensen .map_check_btf = map_check_no_btf,
10676f9d451aSToke Høiland-Jørgensen .map_mem_usage = dev_map_mem_usage,
10686f9d451aSToke Høiland-Jørgensen .map_btf_id = &dev_map_btf_ids[0],
10696f9d451aSToke Høiland-Jørgensen .map_redirect = dev_hash_map_redirect,
10706f9d451aSToke Høiland-Jørgensen };
1071fa5e83dfSYafang Shao
dev_map_hash_remove_netdev(struct bpf_dtab * dtab,struct net_device * netdev)1072c317ab71SMenglong Dong static void dev_map_hash_remove_netdev(struct bpf_dtab *dtab,
1073e6a4750fSBjörn Töpel struct net_device *netdev)
10746f9d451aSToke Høiland-Jørgensen {
10756f9d451aSToke Høiland-Jørgensen unsigned long flags;
1076ce197d83SToke Høiland-Jørgensen u32 i;
1077ce197d83SToke Høiland-Jørgensen
1078ce197d83SToke Høiland-Jørgensen spin_lock_irqsave(&dtab->index_lock, flags);
1079ce197d83SToke Høiland-Jørgensen for (i = 0; i < dtab->n_buckets; i++) {
1080ce197d83SToke Høiland-Jørgensen struct bpf_dtab_netdev *dev;
1081ce197d83SToke Høiland-Jørgensen struct hlist_head *head;
1082ce197d83SToke Høiland-Jørgensen struct hlist_node *next;
1083ce197d83SToke Høiland-Jørgensen
1084ce197d83SToke Høiland-Jørgensen head = dev_map_index_hash(dtab, i);
1085ce197d83SToke Høiland-Jørgensen
1086ce197d83SToke Høiland-Jørgensen hlist_for_each_entry_safe(dev, next, head, index_hlist) {
1087ce197d83SToke Høiland-Jørgensen if (netdev != dev->dev)
1088ce197d83SToke Høiland-Jørgensen continue;
1089ce197d83SToke Høiland-Jørgensen
1090ce197d83SToke Høiland-Jørgensen dtab->items--;
1091ce197d83SToke Høiland-Jørgensen hlist_del_rcu(&dev->index_hlist);
1092ce197d83SToke Høiland-Jørgensen call_rcu(&dev->rcu, __dev_map_entry_free);
1093ce197d83SToke Høiland-Jørgensen }
1094ce197d83SToke Høiland-Jørgensen }
1095ce197d83SToke Høiland-Jørgensen spin_unlock_irqrestore(&dtab->index_lock, flags);
1096ce197d83SToke Høiland-Jørgensen }
1097ce197d83SToke Høiland-Jørgensen
dev_map_notification(struct notifier_block * notifier,ulong event,void * ptr)1098ce197d83SToke Høiland-Jørgensen static int dev_map_notification(struct notifier_block *notifier,
1099ce197d83SToke Høiland-Jørgensen ulong event, void *ptr)
1100ce197d83SToke Høiland-Jørgensen {
1101ce197d83SToke Høiland-Jørgensen struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
11022ddf71e2SJohn Fastabend struct bpf_dtab *dtab;
11032ddf71e2SJohn Fastabend int i, cpu;
11042ddf71e2SJohn Fastabend
11052ddf71e2SJohn Fastabend switch (event) {
11062ddf71e2SJohn Fastabend case NETDEV_REGISTER:
110775ccae62SToke Høiland-Jørgensen if (!netdev->netdev_ops->ndo_xdp_xmit || netdev->xdp_bulkq)
11082ddf71e2SJohn Fastabend break;
11092ddf71e2SJohn Fastabend
111075ccae62SToke Høiland-Jørgensen /* will be freed in free_netdev() */
111175ccae62SToke Høiland-Jørgensen netdev->xdp_bulkq = alloc_percpu(struct xdp_dev_bulk_queue);
111275ccae62SToke Høiland-Jørgensen if (!netdev->xdp_bulkq)
111375ccae62SToke Høiland-Jørgensen return NOTIFY_BAD;
111475ccae62SToke Høiland-Jørgensen
11157d4553b6SJun'ichi Nomura for_each_possible_cpu(cpu)
111675ccae62SToke Høiland-Jørgensen per_cpu_ptr(netdev->xdp_bulkq, cpu)->dev = netdev;
111775ccae62SToke Høiland-Jørgensen break;
111875ccae62SToke Høiland-Jørgensen case NETDEV_UNREGISTER:
111975ccae62SToke Høiland-Jørgensen /* This rcu_read_lock/unlock pair is needed because
112075ccae62SToke Høiland-Jørgensen * dev_map_list is an RCU list AND to ensure a delete
112175ccae62SToke Høiland-Jørgensen * operation does not free a netdev_map entry while we
11222ddf71e2SJohn Fastabend * are comparing it against the netdev being unregistered.
11234cc7b954SJohn Fastabend */
11244cc7b954SJohn Fastabend rcu_read_lock();
11254cc7b954SJohn Fastabend list_for_each_entry_rcu(dtab, &dev_map_list, list) {
11264cc7b954SJohn Fastabend if (dtab->map.map_type == BPF_MAP_TYPE_DEVMAP_HASH) {
11274cc7b954SJohn Fastabend dev_map_hash_remove_netdev(dtab, netdev);
11284cc7b954SJohn Fastabend continue;
11294cc7b954SJohn Fastabend }
1130ce197d83SToke Høiland-Jørgensen
1131ce197d83SToke Høiland-Jørgensen for (i = 0; i < dtab->map.max_entries; i++) {
1132ce197d83SToke Høiland-Jørgensen struct bpf_dtab_netdev *dev, *odev;
1133ce197d83SToke Høiland-Jørgensen
1134ce197d83SToke Høiland-Jørgensen dev = rcu_dereference(dtab->netdev_map[i]);
11352ddf71e2SJohn Fastabend if (!dev || netdev != dev->dev)
11364cc7b954SJohn Fastabend continue;
11372ddf71e2SJohn Fastabend odev = unrcu_pointer(cmpxchg(&dtab->netdev_map[i], RCU_INITIALIZER(dev), NULL));
1138782347b6SToke Høiland-Jørgensen if (dev == odev) {
1139f592f804STaehee Yoo call_rcu(&dev->rcu,
11402ddf71e2SJohn Fastabend __dev_map_entry_free);
1141782347b6SToke Høiland-Jørgensen atomic_dec((atomic_t *)&dtab->items);
1142fa5e83dfSYafang Shao }
11432ddf71e2SJohn Fastabend }
11442ddf71e2SJohn Fastabend }
1145fa5e83dfSYafang Shao rcu_read_unlock();
1146fa5e83dfSYafang Shao break;
11472ddf71e2SJohn Fastabend default:
11482ddf71e2SJohn Fastabend break;
11494cc7b954SJohn Fastabend }
11502ddf71e2SJohn Fastabend return NOTIFY_OK;
11512ddf71e2SJohn Fastabend }
11522ddf71e2SJohn Fastabend
11532ddf71e2SJohn Fastabend static struct notifier_block dev_map_notifier = {
11542ddf71e2SJohn Fastabend .notifier_call = dev_map_notification,
11552ddf71e2SJohn Fastabend };
11562ddf71e2SJohn Fastabend
dev_map_init(void)11572ddf71e2SJohn Fastabend static int __init dev_map_init(void)
11582ddf71e2SJohn Fastabend {
11592ddf71e2SJohn Fastabend /* Assure tracepoint shadow struct _bpf_dtab_netdev is in sync */
11602ddf71e2SJohn Fastabend BUILD_BUG_ON(offsetof(struct bpf_dtab_netdev, dev) !=
11612ddf71e2SJohn Fastabend offsetof(struct _bpf_dtab_netdev, dev));
11622ddf71e2SJohn Fastabend register_netdevice_notifier(&dev_map_notifier);
116396360004SBjörn Töpel
116496360004SBjörn Töpel return 0;
116567f29e07SJesper Dangaard Brouer }
116667f29e07SJesper Dangaard Brouer
116767f29e07SJesper Dangaard Brouer subsys_initcall(dev_map_init);
11682ddf71e2SJohn Fastabend