1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3 */
4 #ifndef _LINUX_BPF_H
5 #define _LINUX_BPF_H 1
6
7 #include <uapi/linux/bpf.h>
8 #include <uapi/linux/filter.h>
9
10 #include <linux/workqueue.h>
11 #include <linux/file.h>
12 #include <linux/percpu.h>
13 #include <linux/err.h>
14 #include <linux/rbtree_latch.h>
15 #include <linux/numa.h>
16 #include <linux/mm_types.h>
17 #include <linux/wait.h>
18 #include <linux/refcount.h>
19 #include <linux/mutex.h>
20 #include <linux/module.h>
21 #include <linux/kallsyms.h>
22 #include <linux/capability.h>
23 #include <linux/sched/mm.h>
24 #include <linux/slab.h>
25 #include <linux/percpu-refcount.h>
26 #include <linux/stddef.h>
27 #include <linux/bpfptr.h>
28 #include <linux/btf.h>
29 #include <linux/rcupdate_trace.h>
30 #include <linux/static_call.h>
31 #include <linux/memcontrol.h>
32 #include <linux/cfi.h>
33
34 struct bpf_verifier_env;
35 struct bpf_verifier_log;
36 struct perf_event;
37 struct bpf_prog;
38 struct bpf_prog_aux;
39 struct bpf_map;
40 struct bpf_arena;
41 struct sock;
42 struct seq_file;
43 struct btf;
44 struct btf_type;
45 struct exception_table_entry;
46 struct seq_operations;
47 struct bpf_iter_aux_info;
48 struct bpf_local_storage;
49 struct bpf_local_storage_map;
50 struct kobject;
51 struct mem_cgroup;
52 struct module;
53 struct bpf_func_state;
54 struct ftrace_ops;
55 struct cgroup;
56 struct bpf_token;
57 struct user_namespace;
58 struct super_block;
59 struct inode;
60
61 extern struct idr btf_idr;
62 extern spinlock_t btf_idr_lock;
63 extern struct kobject *btf_kobj;
64 extern struct bpf_mem_alloc bpf_global_ma, bpf_global_percpu_ma;
65 extern bool bpf_global_ma_set;
66
67 typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64);
68 typedef int (*bpf_iter_init_seq_priv_t)(void *private_data,
69 struct bpf_iter_aux_info *aux);
70 typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data);
71 typedef unsigned int (*bpf_func_t)(const void *,
72 const struct bpf_insn *);
73 struct bpf_iter_seq_info {
74 const struct seq_operations *seq_ops;
75 bpf_iter_init_seq_priv_t init_seq_private;
76 bpf_iter_fini_seq_priv_t fini_seq_private;
77 u32 seq_priv_size;
78 };
79
80 /* map is generic key/value storage optionally accessible by eBPF programs */
81 struct bpf_map_ops {
82 /* funcs callable from userspace (via syscall) */
83 int (*map_alloc_check)(union bpf_attr *attr);
84 struct bpf_map *(*map_alloc)(union bpf_attr *attr);
85 void (*map_release)(struct bpf_map *map, struct file *map_file);
86 void (*map_free)(struct bpf_map *map);
87 int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key);
88 void (*map_release_uref)(struct bpf_map *map);
89 void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key);
90 int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr,
91 union bpf_attr __user *uattr);
92 int (*map_lookup_and_delete_elem)(struct bpf_map *map, void *key,
93 void *value, u64 flags);
94 int (*map_lookup_and_delete_batch)(struct bpf_map *map,
95 const union bpf_attr *attr,
96 union bpf_attr __user *uattr);
97 int (*map_update_batch)(struct bpf_map *map, struct file *map_file,
98 const union bpf_attr *attr,
99 union bpf_attr __user *uattr);
100 int (*map_delete_batch)(struct bpf_map *map, const union bpf_attr *attr,
101 union bpf_attr __user *uattr);
102
103 /* funcs callable from userspace and from eBPF programs */
104 void *(*map_lookup_elem)(struct bpf_map *map, void *key);
105 long (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags);
106 long (*map_delete_elem)(struct bpf_map *map, void *key);
107 long (*map_push_elem)(struct bpf_map *map, void *value, u64 flags);
108 long (*map_pop_elem)(struct bpf_map *map, void *value);
109 long (*map_peek_elem)(struct bpf_map *map, void *value);
110 void *(*map_lookup_percpu_elem)(struct bpf_map *map, void *key, u32 cpu);
111
112 /* funcs called by prog_array and perf_event_array map */
113 void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,
114 int fd);
115 /* If need_defer is true, the implementation should guarantee that
116 * the to-be-put element is still alive before the bpf program, which
117 * may manipulate it, exists.
118 */
119 void (*map_fd_put_ptr)(struct bpf_map *map, void *ptr, bool need_defer);
120 int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf);
121 u32 (*map_fd_sys_lookup_elem)(void *ptr);
122 void (*map_seq_show_elem)(struct bpf_map *map, void *key,
123 struct seq_file *m);
124 int (*map_check_btf)(const struct bpf_map *map,
125 const struct btf *btf,
126 const struct btf_type *key_type,
127 const struct btf_type *value_type);
128
129 /* Prog poke tracking helpers. */
130 int (*map_poke_track)(struct bpf_map *map, struct bpf_prog_aux *aux);
131 void (*map_poke_untrack)(struct bpf_map *map, struct bpf_prog_aux *aux);
132 void (*map_poke_run)(struct bpf_map *map, u32 key, struct bpf_prog *old,
133 struct bpf_prog *new);
134
135 /* Direct value access helpers. */
136 int (*map_direct_value_addr)(const struct bpf_map *map,
137 u64 *imm, u32 off);
138 int (*map_direct_value_meta)(const struct bpf_map *map,
139 u64 imm, u32 *off);
140 int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma);
141 __poll_t (*map_poll)(struct bpf_map *map, struct file *filp,
142 struct poll_table_struct *pts);
143 unsigned long (*map_get_unmapped_area)(struct file *filep, unsigned long addr,
144 unsigned long len, unsigned long pgoff,
145 unsigned long flags);
146
147 /* Functions called by bpf_local_storage maps */
148 int (*map_local_storage_charge)(struct bpf_local_storage_map *smap,
149 void *owner, u32 size);
150 void (*map_local_storage_uncharge)(struct bpf_local_storage_map *smap,
151 void *owner, u32 size);
152 struct bpf_local_storage __rcu ** (*map_owner_storage_ptr)(void *owner);
153
154 /* Misc helpers.*/
155 long (*map_redirect)(struct bpf_map *map, u64 key, u64 flags);
156
157 /* map_meta_equal must be implemented for maps that can be
158 * used as an inner map. It is a runtime check to ensure
159 * an inner map can be inserted to an outer map.
160 *
161 * Some properties of the inner map has been used during the
162 * verification time. When inserting an inner map at the runtime,
163 * map_meta_equal has to ensure the inserting map has the same
164 * properties that the verifier has used earlier.
165 */
166 bool (*map_meta_equal)(const struct bpf_map *meta0,
167 const struct bpf_map *meta1);
168
169
170 int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env,
171 struct bpf_func_state *caller,
172 struct bpf_func_state *callee);
173 long (*map_for_each_callback)(struct bpf_map *map,
174 bpf_callback_t callback_fn,
175 void *callback_ctx, u64 flags);
176
177 u64 (*map_mem_usage)(const struct bpf_map *map);
178
179 /* BTF id of struct allocated by map_alloc */
180 int *map_btf_id;
181
182 /* bpf_iter info used to open a seq_file */
183 const struct bpf_iter_seq_info *iter_seq_info;
184 };
185
186 enum {
187 /* Support at most 11 fields in a BTF type */
188 BTF_FIELDS_MAX = 11,
189 };
190
191 enum btf_field_type {
192 BPF_SPIN_LOCK = (1 << 0),
193 BPF_TIMER = (1 << 1),
194 BPF_KPTR_UNREF = (1 << 2),
195 BPF_KPTR_REF = (1 << 3),
196 BPF_KPTR_PERCPU = (1 << 4),
197 BPF_KPTR = BPF_KPTR_UNREF | BPF_KPTR_REF | BPF_KPTR_PERCPU,
198 BPF_LIST_HEAD = (1 << 5),
199 BPF_LIST_NODE = (1 << 6),
200 BPF_RB_ROOT = (1 << 7),
201 BPF_RB_NODE = (1 << 8),
202 BPF_GRAPH_NODE = BPF_RB_NODE | BPF_LIST_NODE,
203 BPF_GRAPH_ROOT = BPF_RB_ROOT | BPF_LIST_HEAD,
204 BPF_REFCOUNT = (1 << 9),
205 BPF_WORKQUEUE = (1 << 10),
206 };
207
208 typedef void (*btf_dtor_kfunc_t)(void *);
209
210 struct btf_field_kptr {
211 struct btf *btf;
212 struct module *module;
213 /* dtor used if btf_is_kernel(btf), otherwise the type is
214 * program-allocated, dtor is NULL, and __bpf_obj_drop_impl is used
215 */
216 btf_dtor_kfunc_t dtor;
217 u32 btf_id;
218 };
219
220 struct btf_field_graph_root {
221 struct btf *btf;
222 u32 value_btf_id;
223 u32 node_offset;
224 struct btf_record *value_rec;
225 };
226
227 struct btf_field {
228 u32 offset;
229 u32 size;
230 enum btf_field_type type;
231 union {
232 struct btf_field_kptr kptr;
233 struct btf_field_graph_root graph_root;
234 };
235 };
236
237 struct btf_record {
238 u32 cnt;
239 u32 field_mask;
240 int spin_lock_off;
241 int timer_off;
242 int wq_off;
243 int refcount_off;
244 struct btf_field fields[];
245 };
246
247 /* Non-opaque version of bpf_rb_node in uapi/linux/bpf.h */
248 struct bpf_rb_node_kern {
249 struct rb_node rb_node;
250 void *owner;
251 } __attribute__((aligned(8)));
252
253 /* Non-opaque version of bpf_list_node in uapi/linux/bpf.h */
254 struct bpf_list_node_kern {
255 struct list_head list_head;
256 void *owner;
257 } __attribute__((aligned(8)));
258
259 struct bpf_map {
260 const struct bpf_map_ops *ops;
261 struct bpf_map *inner_map_meta;
262 #ifdef CONFIG_SECURITY
263 void *security;
264 #endif
265 enum bpf_map_type map_type;
266 u32 key_size;
267 u32 value_size;
268 u32 max_entries;
269 u64 map_extra; /* any per-map-type extra fields */
270 u32 map_flags;
271 u32 id;
272 struct btf_record *record;
273 int numa_node;
274 u32 btf_key_type_id;
275 u32 btf_value_type_id;
276 u32 btf_vmlinux_value_type_id;
277 struct btf *btf;
278 #ifdef CONFIG_MEMCG
279 struct obj_cgroup *objcg;
280 #endif
281 char name[BPF_OBJ_NAME_LEN];
282 struct mutex freeze_mutex;
283 atomic64_t refcnt;
284 atomic64_t usercnt;
285 /* rcu is used before freeing and work is only used during freeing */
286 union {
287 struct work_struct work;
288 struct rcu_head rcu;
289 };
290 atomic64_t writecnt;
291 /* 'Ownership' of program-containing map is claimed by the first program
292 * that is going to use this map or by the first program which FD is
293 * stored in the map to make sure that all callers and callees have the
294 * same prog type, JITed flag and xdp_has_frags flag.
295 */
296 struct {
297 const struct btf_type *attach_func_proto;
298 spinlock_t lock;
299 enum bpf_prog_type type;
300 bool jited;
301 bool xdp_has_frags;
302 } owner;
303 bool bypass_spec_v1;
304 bool frozen; /* write-once; write-protected by freeze_mutex */
305 bool free_after_mult_rcu_gp;
306 bool free_after_rcu_gp;
307 atomic64_t sleepable_refcnt;
308 s64 __percpu *elem_count;
309 };
310
btf_field_type_name(enum btf_field_type type)311 static inline const char *btf_field_type_name(enum btf_field_type type)
312 {
313 switch (type) {
314 case BPF_SPIN_LOCK:
315 return "bpf_spin_lock";
316 case BPF_TIMER:
317 return "bpf_timer";
318 case BPF_WORKQUEUE:
319 return "bpf_wq";
320 case BPF_KPTR_UNREF:
321 case BPF_KPTR_REF:
322 return "kptr";
323 case BPF_KPTR_PERCPU:
324 return "percpu_kptr";
325 case BPF_LIST_HEAD:
326 return "bpf_list_head";
327 case BPF_LIST_NODE:
328 return "bpf_list_node";
329 case BPF_RB_ROOT:
330 return "bpf_rb_root";
331 case BPF_RB_NODE:
332 return "bpf_rb_node";
333 case BPF_REFCOUNT:
334 return "bpf_refcount";
335 default:
336 WARN_ON_ONCE(1);
337 return "unknown";
338 }
339 }
340
btf_field_type_size(enum btf_field_type type)341 static inline u32 btf_field_type_size(enum btf_field_type type)
342 {
343 switch (type) {
344 case BPF_SPIN_LOCK:
345 return sizeof(struct bpf_spin_lock);
346 case BPF_TIMER:
347 return sizeof(struct bpf_timer);
348 case BPF_WORKQUEUE:
349 return sizeof(struct bpf_wq);
350 case BPF_KPTR_UNREF:
351 case BPF_KPTR_REF:
352 case BPF_KPTR_PERCPU:
353 return sizeof(u64);
354 case BPF_LIST_HEAD:
355 return sizeof(struct bpf_list_head);
356 case BPF_LIST_NODE:
357 return sizeof(struct bpf_list_node);
358 case BPF_RB_ROOT:
359 return sizeof(struct bpf_rb_root);
360 case BPF_RB_NODE:
361 return sizeof(struct bpf_rb_node);
362 case BPF_REFCOUNT:
363 return sizeof(struct bpf_refcount);
364 default:
365 WARN_ON_ONCE(1);
366 return 0;
367 }
368 }
369
btf_field_type_align(enum btf_field_type type)370 static inline u32 btf_field_type_align(enum btf_field_type type)
371 {
372 switch (type) {
373 case BPF_SPIN_LOCK:
374 return __alignof__(struct bpf_spin_lock);
375 case BPF_TIMER:
376 return __alignof__(struct bpf_timer);
377 case BPF_WORKQUEUE:
378 return __alignof__(struct bpf_wq);
379 case BPF_KPTR_UNREF:
380 case BPF_KPTR_REF:
381 case BPF_KPTR_PERCPU:
382 return __alignof__(u64);
383 case BPF_LIST_HEAD:
384 return __alignof__(struct bpf_list_head);
385 case BPF_LIST_NODE:
386 return __alignof__(struct bpf_list_node);
387 case BPF_RB_ROOT:
388 return __alignof__(struct bpf_rb_root);
389 case BPF_RB_NODE:
390 return __alignof__(struct bpf_rb_node);
391 case BPF_REFCOUNT:
392 return __alignof__(struct bpf_refcount);
393 default:
394 WARN_ON_ONCE(1);
395 return 0;
396 }
397 }
398
bpf_obj_init_field(const struct btf_field * field,void * addr)399 static inline void bpf_obj_init_field(const struct btf_field *field, void *addr)
400 {
401 memset(addr, 0, field->size);
402
403 switch (field->type) {
404 case BPF_REFCOUNT:
405 refcount_set((refcount_t *)addr, 1);
406 break;
407 case BPF_RB_NODE:
408 RB_CLEAR_NODE((struct rb_node *)addr);
409 break;
410 case BPF_LIST_HEAD:
411 case BPF_LIST_NODE:
412 INIT_LIST_HEAD((struct list_head *)addr);
413 break;
414 case BPF_RB_ROOT:
415 /* RB_ROOT_CACHED 0-inits, no need to do anything after memset */
416 case BPF_SPIN_LOCK:
417 case BPF_TIMER:
418 case BPF_WORKQUEUE:
419 case BPF_KPTR_UNREF:
420 case BPF_KPTR_REF:
421 case BPF_KPTR_PERCPU:
422 break;
423 default:
424 WARN_ON_ONCE(1);
425 return;
426 }
427 }
428
btf_record_has_field(const struct btf_record * rec,enum btf_field_type type)429 static inline bool btf_record_has_field(const struct btf_record *rec, enum btf_field_type type)
430 {
431 if (IS_ERR_OR_NULL(rec))
432 return false;
433 return rec->field_mask & type;
434 }
435
bpf_obj_init(const struct btf_record * rec,void * obj)436 static inline void bpf_obj_init(const struct btf_record *rec, void *obj)
437 {
438 int i;
439
440 if (IS_ERR_OR_NULL(rec))
441 return;
442 for (i = 0; i < rec->cnt; i++)
443 bpf_obj_init_field(&rec->fields[i], obj + rec->fields[i].offset);
444 }
445
446 /* 'dst' must be a temporary buffer and should not point to memory that is being
447 * used in parallel by a bpf program or bpf syscall, otherwise the access from
448 * the bpf program or bpf syscall may be corrupted by the reinitialization,
449 * leading to weird problems. Even 'dst' is newly-allocated from bpf memory
450 * allocator, it is still possible for 'dst' to be used in parallel by a bpf
451 * program or bpf syscall.
452 */
check_and_init_map_value(struct bpf_map * map,void * dst)453 static inline void check_and_init_map_value(struct bpf_map *map, void *dst)
454 {
455 bpf_obj_init(map->record, dst);
456 }
457
458 /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and
459 * forced to use 'long' read/writes to try to atomically copy long counters.
460 * Best-effort only. No barriers here, since it _will_ race with concurrent
461 * updates from BPF programs. Called from bpf syscall and mostly used with
462 * size 8 or 16 bytes, so ask compiler to inline it.
463 */
bpf_long_memcpy(void * dst,const void * src,u32 size)464 static inline void bpf_long_memcpy(void *dst, const void *src, u32 size)
465 {
466 const long *lsrc = src;
467 long *ldst = dst;
468
469 size /= sizeof(long);
470 while (size--)
471 data_race(*ldst++ = *lsrc++);
472 }
473
474 /* copy everything but bpf_spin_lock, bpf_timer, and kptrs. There could be one of each. */
bpf_obj_memcpy(struct btf_record * rec,void * dst,void * src,u32 size,bool long_memcpy)475 static inline void bpf_obj_memcpy(struct btf_record *rec,
476 void *dst, void *src, u32 size,
477 bool long_memcpy)
478 {
479 u32 curr_off = 0;
480 int i;
481
482 if (IS_ERR_OR_NULL(rec)) {
483 if (long_memcpy)
484 bpf_long_memcpy(dst, src, round_up(size, 8));
485 else
486 memcpy(dst, src, size);
487 return;
488 }
489
490 for (i = 0; i < rec->cnt; i++) {
491 u32 next_off = rec->fields[i].offset;
492 u32 sz = next_off - curr_off;
493
494 memcpy(dst + curr_off, src + curr_off, sz);
495 curr_off += rec->fields[i].size + sz;
496 }
497 memcpy(dst + curr_off, src + curr_off, size - curr_off);
498 }
499
copy_map_value(struct bpf_map * map,void * dst,void * src)500 static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)
501 {
502 bpf_obj_memcpy(map->record, dst, src, map->value_size, false);
503 }
504
copy_map_value_long(struct bpf_map * map,void * dst,void * src)505 static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src)
506 {
507 bpf_obj_memcpy(map->record, dst, src, map->value_size, true);
508 }
509
bpf_obj_memzero(struct btf_record * rec,void * dst,u32 size)510 static inline void bpf_obj_memzero(struct btf_record *rec, void *dst, u32 size)
511 {
512 u32 curr_off = 0;
513 int i;
514
515 if (IS_ERR_OR_NULL(rec)) {
516 memset(dst, 0, size);
517 return;
518 }
519
520 for (i = 0; i < rec->cnt; i++) {
521 u32 next_off = rec->fields[i].offset;
522 u32 sz = next_off - curr_off;
523
524 memset(dst + curr_off, 0, sz);
525 curr_off += rec->fields[i].size + sz;
526 }
527 memset(dst + curr_off, 0, size - curr_off);
528 }
529
zero_map_value(struct bpf_map * map,void * dst)530 static inline void zero_map_value(struct bpf_map *map, void *dst)
531 {
532 bpf_obj_memzero(map->record, dst, map->value_size);
533 }
534
535 void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
536 bool lock_src);
537 void bpf_timer_cancel_and_free(void *timer);
538 void bpf_wq_cancel_and_free(void *timer);
539 void bpf_list_head_free(const struct btf_field *field, void *list_head,
540 struct bpf_spin_lock *spin_lock);
541 void bpf_rb_root_free(const struct btf_field *field, void *rb_root,
542 struct bpf_spin_lock *spin_lock);
543 u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena);
544 u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena);
545 int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size);
546
547 struct bpf_offload_dev;
548 struct bpf_offloaded_map;
549
550 struct bpf_map_dev_ops {
551 int (*map_get_next_key)(struct bpf_offloaded_map *map,
552 void *key, void *next_key);
553 int (*map_lookup_elem)(struct bpf_offloaded_map *map,
554 void *key, void *value);
555 int (*map_update_elem)(struct bpf_offloaded_map *map,
556 void *key, void *value, u64 flags);
557 int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key);
558 };
559
560 struct bpf_offloaded_map {
561 struct bpf_map map;
562 struct net_device *netdev;
563 const struct bpf_map_dev_ops *dev_ops;
564 void *dev_priv;
565 struct list_head offloads;
566 };
567
map_to_offmap(struct bpf_map * map)568 static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map)
569 {
570 return container_of(map, struct bpf_offloaded_map, map);
571 }
572
bpf_map_offload_neutral(const struct bpf_map * map)573 static inline bool bpf_map_offload_neutral(const struct bpf_map *map)
574 {
575 return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
576 }
577
bpf_map_support_seq_show(const struct bpf_map * map)578 static inline bool bpf_map_support_seq_show(const struct bpf_map *map)
579 {
580 return (map->btf_value_type_id || map->btf_vmlinux_value_type_id) &&
581 map->ops->map_seq_show_elem;
582 }
583
584 int map_check_no_btf(const struct bpf_map *map,
585 const struct btf *btf,
586 const struct btf_type *key_type,
587 const struct btf_type *value_type);
588
589 bool bpf_map_meta_equal(const struct bpf_map *meta0,
590 const struct bpf_map *meta1);
591
592 extern const struct bpf_map_ops bpf_map_offload_ops;
593
594 /* bpf_type_flag contains a set of flags that are applicable to the values of
595 * arg_type, ret_type and reg_type. For example, a pointer value may be null,
596 * or a memory is read-only. We classify types into two categories: base types
597 * and extended types. Extended types are base types combined with a type flag.
598 *
599 * Currently there are no more than 32 base types in arg_type, ret_type and
600 * reg_types.
601 */
602 #define BPF_BASE_TYPE_BITS 8
603
604 enum bpf_type_flag {
605 /* PTR may be NULL. */
606 PTR_MAYBE_NULL = BIT(0 + BPF_BASE_TYPE_BITS),
607
608 /* MEM is read-only. When applied on bpf_arg, it indicates the arg is
609 * compatible with both mutable and immutable memory.
610 */
611 MEM_RDONLY = BIT(1 + BPF_BASE_TYPE_BITS),
612
613 /* MEM points to BPF ring buffer reservation. */
614 MEM_RINGBUF = BIT(2 + BPF_BASE_TYPE_BITS),
615
616 /* MEM is in user address space. */
617 MEM_USER = BIT(3 + BPF_BASE_TYPE_BITS),
618
619 /* MEM is a percpu memory. MEM_PERCPU tags PTR_TO_BTF_ID. When tagged
620 * with MEM_PERCPU, PTR_TO_BTF_ID _cannot_ be directly accessed. In
621 * order to drop this tag, it must be passed into bpf_per_cpu_ptr()
622 * or bpf_this_cpu_ptr(), which will return the pointer corresponding
623 * to the specified cpu.
624 */
625 MEM_PERCPU = BIT(4 + BPF_BASE_TYPE_BITS),
626
627 /* Indicates that the argument will be released. */
628 OBJ_RELEASE = BIT(5 + BPF_BASE_TYPE_BITS),
629
630 /* PTR is not trusted. This is only used with PTR_TO_BTF_ID, to mark
631 * unreferenced and referenced kptr loaded from map value using a load
632 * instruction, so that they can only be dereferenced but not escape the
633 * BPF program into the kernel (i.e. cannot be passed as arguments to
634 * kfunc or bpf helpers).
635 */
636 PTR_UNTRUSTED = BIT(6 + BPF_BASE_TYPE_BITS),
637
638 MEM_UNINIT = BIT(7 + BPF_BASE_TYPE_BITS),
639
640 /* DYNPTR points to memory local to the bpf program. */
641 DYNPTR_TYPE_LOCAL = BIT(8 + BPF_BASE_TYPE_BITS),
642
643 /* DYNPTR points to a kernel-produced ringbuf record. */
644 DYNPTR_TYPE_RINGBUF = BIT(9 + BPF_BASE_TYPE_BITS),
645
646 /* Size is known at compile time. */
647 MEM_FIXED_SIZE = BIT(10 + BPF_BASE_TYPE_BITS),
648
649 /* MEM is of an allocated object of type in program BTF. This is used to
650 * tag PTR_TO_BTF_ID allocated using bpf_obj_new.
651 */
652 MEM_ALLOC = BIT(11 + BPF_BASE_TYPE_BITS),
653
654 /* PTR was passed from the kernel in a trusted context, and may be
655 * passed to KF_TRUSTED_ARGS kfuncs or BPF helper functions.
656 * Confusingly, this is _not_ the opposite of PTR_UNTRUSTED above.
657 * PTR_UNTRUSTED refers to a kptr that was read directly from a map
658 * without invoking bpf_kptr_xchg(). What we really need to know is
659 * whether a pointer is safe to pass to a kfunc or BPF helper function.
660 * While PTR_UNTRUSTED pointers are unsafe to pass to kfuncs and BPF
661 * helpers, they do not cover all possible instances of unsafe
662 * pointers. For example, a pointer that was obtained from walking a
663 * struct will _not_ get the PTR_UNTRUSTED type modifier, despite the
664 * fact that it may be NULL, invalid, etc. This is due to backwards
665 * compatibility requirements, as this was the behavior that was first
666 * introduced when kptrs were added. The behavior is now considered
667 * deprecated, and PTR_UNTRUSTED will eventually be removed.
668 *
669 * PTR_TRUSTED, on the other hand, is a pointer that the kernel
670 * guarantees to be valid and safe to pass to kfuncs and BPF helpers.
671 * For example, pointers passed to tracepoint arguments are considered
672 * PTR_TRUSTED, as are pointers that are passed to struct_ops
673 * callbacks. As alluded to above, pointers that are obtained from
674 * walking PTR_TRUSTED pointers are _not_ trusted. For example, if a
675 * struct task_struct *task is PTR_TRUSTED, then accessing
676 * task->last_wakee will lose the PTR_TRUSTED modifier when it's stored
677 * in a BPF register. Similarly, pointers passed to certain programs
678 * types such as kretprobes are not guaranteed to be valid, as they may
679 * for example contain an object that was recently freed.
680 */
681 PTR_TRUSTED = BIT(12 + BPF_BASE_TYPE_BITS),
682
683 /* MEM is tagged with rcu and memory access needs rcu_read_lock protection. */
684 MEM_RCU = BIT(13 + BPF_BASE_TYPE_BITS),
685
686 /* Used to tag PTR_TO_BTF_ID | MEM_ALLOC references which are non-owning.
687 * Currently only valid for linked-list and rbtree nodes. If the nodes
688 * have a bpf_refcount_field, they must be tagged MEM_RCU as well.
689 */
690 NON_OWN_REF = BIT(14 + BPF_BASE_TYPE_BITS),
691
692 /* DYNPTR points to sk_buff */
693 DYNPTR_TYPE_SKB = BIT(15 + BPF_BASE_TYPE_BITS),
694
695 /* DYNPTR points to xdp_buff */
696 DYNPTR_TYPE_XDP = BIT(16 + BPF_BASE_TYPE_BITS),
697
698 /* Memory must be aligned on some architectures, used in combination with
699 * MEM_FIXED_SIZE.
700 */
701 MEM_ALIGNED = BIT(17 + BPF_BASE_TYPE_BITS),
702
703 __BPF_TYPE_FLAG_MAX,
704 __BPF_TYPE_LAST_FLAG = __BPF_TYPE_FLAG_MAX - 1,
705 };
706
707 #define DYNPTR_TYPE_FLAG_MASK (DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF | DYNPTR_TYPE_SKB \
708 | DYNPTR_TYPE_XDP)
709
710 /* Max number of base types. */
711 #define BPF_BASE_TYPE_LIMIT (1UL << BPF_BASE_TYPE_BITS)
712
713 /* Max number of all types. */
714 #define BPF_TYPE_LIMIT (__BPF_TYPE_LAST_FLAG | (__BPF_TYPE_LAST_FLAG - 1))
715
716 /* function argument constraints */
717 enum bpf_arg_type {
718 ARG_DONTCARE = 0, /* unused argument in helper function */
719
720 /* the following constraints used to prototype
721 * bpf_map_lookup/update/delete_elem() functions
722 */
723 ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */
724 ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */
725 ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */
726
727 /* Used to prototype bpf_memcmp() and other functions that access data
728 * on eBPF program stack
729 */
730 ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */
731 ARG_PTR_TO_ARENA,
732
733 ARG_CONST_SIZE, /* number of bytes accessed from memory */
734 ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */
735
736 ARG_PTR_TO_CTX, /* pointer to context */
737 ARG_ANYTHING, /* any (initialized) argument is ok */
738 ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */
739 ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */
740 ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */
741 ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */
742 ARG_PTR_TO_RINGBUF_MEM, /* pointer to dynamically reserved ringbuf memory */
743 ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */
744 ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */
745 ARG_PTR_TO_PERCPU_BTF_ID, /* pointer to in-kernel percpu type */
746 ARG_PTR_TO_FUNC, /* pointer to a bpf program function */
747 ARG_PTR_TO_STACK, /* pointer to stack */
748 ARG_PTR_TO_CONST_STR, /* pointer to a null terminated read-only string */
749 ARG_PTR_TO_TIMER, /* pointer to bpf_timer */
750 ARG_KPTR_XCHG_DEST, /* pointer to destination that kptrs are bpf_kptr_xchg'd into */
751 ARG_PTR_TO_DYNPTR, /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */
752 __BPF_ARG_TYPE_MAX,
753
754 /* Extended arg_types. */
755 ARG_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MAP_VALUE,
756 ARG_PTR_TO_MEM_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MEM,
757 ARG_PTR_TO_CTX_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_CTX,
758 ARG_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_SOCKET,
759 ARG_PTR_TO_STACK_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_STACK,
760 ARG_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_BTF_ID,
761 /* pointer to memory does not need to be initialized, helper function must fill
762 * all bytes or clear them in error case.
763 */
764 ARG_PTR_TO_UNINIT_MEM = MEM_UNINIT | ARG_PTR_TO_MEM,
765 /* Pointer to valid memory of size known at compile time. */
766 ARG_PTR_TO_FIXED_SIZE_MEM = MEM_FIXED_SIZE | ARG_PTR_TO_MEM,
767
768 /* This must be the last entry. Its purpose is to ensure the enum is
769 * wide enough to hold the higher bits reserved for bpf_type_flag.
770 */
771 __BPF_ARG_TYPE_LIMIT = BPF_TYPE_LIMIT,
772 };
773 static_assert(__BPF_ARG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
774
775 /* type of values returned from helper functions */
776 enum bpf_return_type {
777 RET_INTEGER, /* function returns integer */
778 RET_VOID, /* function doesn't return anything */
779 RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */
780 RET_PTR_TO_SOCKET, /* returns a pointer to a socket */
781 RET_PTR_TO_TCP_SOCK, /* returns a pointer to a tcp_sock */
782 RET_PTR_TO_SOCK_COMMON, /* returns a pointer to a sock_common */
783 RET_PTR_TO_MEM, /* returns a pointer to memory */
784 RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */
785 RET_PTR_TO_BTF_ID, /* returns a pointer to a btf_id */
786 __BPF_RET_TYPE_MAX,
787
788 /* Extended ret_types. */
789 RET_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_MAP_VALUE,
790 RET_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCKET,
791 RET_PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_TCP_SOCK,
792 RET_PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCK_COMMON,
793 RET_PTR_TO_RINGBUF_MEM_OR_NULL = PTR_MAYBE_NULL | MEM_RINGBUF | RET_PTR_TO_MEM,
794 RET_PTR_TO_DYNPTR_MEM_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_MEM,
795 RET_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_BTF_ID,
796 RET_PTR_TO_BTF_ID_TRUSTED = PTR_TRUSTED | RET_PTR_TO_BTF_ID,
797
798 /* This must be the last entry. Its purpose is to ensure the enum is
799 * wide enough to hold the higher bits reserved for bpf_type_flag.
800 */
801 __BPF_RET_TYPE_LIMIT = BPF_TYPE_LIMIT,
802 };
803 static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
804
805 /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
806 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL
807 * instructions after verifying
808 */
809 struct bpf_func_proto {
810 u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
811 bool gpl_only;
812 bool pkt_access;
813 bool might_sleep;
814 /* set to true if helper follows contract for llvm
815 * attribute bpf_fastcall:
816 * - void functions do not scratch r0
817 * - functions taking N arguments scratch only registers r1-rN
818 */
819 bool allow_fastcall;
820 enum bpf_return_type ret_type;
821 union {
822 struct {
823 enum bpf_arg_type arg1_type;
824 enum bpf_arg_type arg2_type;
825 enum bpf_arg_type arg3_type;
826 enum bpf_arg_type arg4_type;
827 enum bpf_arg_type arg5_type;
828 };
829 enum bpf_arg_type arg_type[5];
830 };
831 union {
832 struct {
833 u32 *arg1_btf_id;
834 u32 *arg2_btf_id;
835 u32 *arg3_btf_id;
836 u32 *arg4_btf_id;
837 u32 *arg5_btf_id;
838 };
839 u32 *arg_btf_id[5];
840 struct {
841 size_t arg1_size;
842 size_t arg2_size;
843 size_t arg3_size;
844 size_t arg4_size;
845 size_t arg5_size;
846 };
847 size_t arg_size[5];
848 };
849 int *ret_btf_id; /* return value btf_id */
850 bool (*allowed)(const struct bpf_prog *prog);
851 };
852
853 /* bpf_context is intentionally undefined structure. Pointer to bpf_context is
854 * the first argument to eBPF programs.
855 * For socket filters: 'struct bpf_context *' == 'struct sk_buff *'
856 */
857 struct bpf_context;
858
859 enum bpf_access_type {
860 BPF_READ = 1,
861 BPF_WRITE = 2
862 };
863
864 /* types of values stored in eBPF registers */
865 /* Pointer types represent:
866 * pointer
867 * pointer + imm
868 * pointer + (u16) var
869 * pointer + (u16) var + imm
870 * if (range > 0) then [ptr, ptr + range - off) is safe to access
871 * if (id > 0) means that some 'var' was added
872 * if (off > 0) means that 'imm' was added
873 */
874 enum bpf_reg_type {
875 NOT_INIT = 0, /* nothing was written into register */
876 SCALAR_VALUE, /* reg doesn't contain a valid pointer */
877 PTR_TO_CTX, /* reg points to bpf_context */
878 CONST_PTR_TO_MAP, /* reg points to struct bpf_map */
879 PTR_TO_MAP_VALUE, /* reg points to map element value */
880 PTR_TO_MAP_KEY, /* reg points to a map element key */
881 PTR_TO_STACK, /* reg == frame_pointer + offset */
882 PTR_TO_PACKET_META, /* skb->data - meta_len */
883 PTR_TO_PACKET, /* reg points to skb->data */
884 PTR_TO_PACKET_END, /* skb->data + headlen */
885 PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */
886 PTR_TO_SOCKET, /* reg points to struct bpf_sock */
887 PTR_TO_SOCK_COMMON, /* reg points to sock_common */
888 PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */
889 PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */
890 PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */
891 /* PTR_TO_BTF_ID points to a kernel struct that does not need
892 * to be null checked by the BPF program. This does not imply the
893 * pointer is _not_ null and in practice this can easily be a null
894 * pointer when reading pointer chains. The assumption is program
895 * context will handle null pointer dereference typically via fault
896 * handling. The verifier must keep this in mind and can make no
897 * assumptions about null or non-null when doing branch analysis.
898 * Further, when passed into helpers the helpers can not, without
899 * additional context, assume the value is non-null.
900 */
901 PTR_TO_BTF_ID,
902 /* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not
903 * been checked for null. Used primarily to inform the verifier
904 * an explicit null check is required for this struct.
905 */
906 PTR_TO_MEM, /* reg points to valid memory region */
907 PTR_TO_ARENA,
908 PTR_TO_BUF, /* reg points to a read/write buffer */
909 PTR_TO_FUNC, /* reg points to a bpf program function */
910 CONST_PTR_TO_DYNPTR, /* reg points to a const struct bpf_dynptr */
911 __BPF_REG_TYPE_MAX,
912
913 /* Extended reg_types. */
914 PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | PTR_TO_MAP_VALUE,
915 PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCKET,
916 PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON,
917 PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | PTR_TO_TCP_SOCK,
918 PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | PTR_TO_BTF_ID,
919
920 /* This must be the last entry. Its purpose is to ensure the enum is
921 * wide enough to hold the higher bits reserved for bpf_type_flag.
922 */
923 __BPF_REG_TYPE_LIMIT = BPF_TYPE_LIMIT,
924 };
925 static_assert(__BPF_REG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
926
927 /* The information passed from prog-specific *_is_valid_access
928 * back to the verifier.
929 */
930 struct bpf_insn_access_aux {
931 enum bpf_reg_type reg_type;
932 bool is_ldsx;
933 union {
934 int ctx_field_size;
935 struct {
936 struct btf *btf;
937 u32 btf_id;
938 };
939 };
940 struct bpf_verifier_log *log; /* for verbose logs */
941 bool is_retval; /* is accessing function return value ? */
942 };
943
944 static inline void
bpf_ctx_record_field_size(struct bpf_insn_access_aux * aux,u32 size)945 bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size)
946 {
947 aux->ctx_field_size = size;
948 }
949
bpf_is_ldimm64(const struct bpf_insn * insn)950 static bool bpf_is_ldimm64(const struct bpf_insn *insn)
951 {
952 return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
953 }
954
bpf_pseudo_func(const struct bpf_insn * insn)955 static inline bool bpf_pseudo_func(const struct bpf_insn *insn)
956 {
957 return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
958 }
959
960 struct bpf_prog_ops {
961 int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,
962 union bpf_attr __user *uattr);
963 };
964
965 struct bpf_reg_state;
966 struct bpf_verifier_ops {
967 /* return eBPF function prototype for verification */
968 const struct bpf_func_proto *
969 (*get_func_proto)(enum bpf_func_id func_id,
970 const struct bpf_prog *prog);
971
972 /* return true if 'size' wide access at offset 'off' within bpf_context
973 * with 'type' (read or write) is allowed
974 */
975 bool (*is_valid_access)(int off, int size, enum bpf_access_type type,
976 const struct bpf_prog *prog,
977 struct bpf_insn_access_aux *info);
978 int (*gen_prologue)(struct bpf_insn *insn, bool direct_write,
979 const struct bpf_prog *prog);
980 int (*gen_epilogue)(struct bpf_insn *insn, const struct bpf_prog *prog,
981 s16 ctx_stack_off);
982 int (*gen_ld_abs)(const struct bpf_insn *orig,
983 struct bpf_insn *insn_buf);
984 u32 (*convert_ctx_access)(enum bpf_access_type type,
985 const struct bpf_insn *src,
986 struct bpf_insn *dst,
987 struct bpf_prog *prog, u32 *target_size);
988 int (*btf_struct_access)(struct bpf_verifier_log *log,
989 const struct bpf_reg_state *reg,
990 int off, int size);
991 };
992
993 struct bpf_prog_offload_ops {
994 /* verifier basic callbacks */
995 int (*insn_hook)(struct bpf_verifier_env *env,
996 int insn_idx, int prev_insn_idx);
997 int (*finalize)(struct bpf_verifier_env *env);
998 /* verifier optimization callbacks (called after .finalize) */
999 int (*replace_insn)(struct bpf_verifier_env *env, u32 off,
1000 struct bpf_insn *insn);
1001 int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt);
1002 /* program management callbacks */
1003 int (*prepare)(struct bpf_prog *prog);
1004 int (*translate)(struct bpf_prog *prog);
1005 void (*destroy)(struct bpf_prog *prog);
1006 };
1007
1008 struct bpf_prog_offload {
1009 struct bpf_prog *prog;
1010 struct net_device *netdev;
1011 struct bpf_offload_dev *offdev;
1012 void *dev_priv;
1013 struct list_head offloads;
1014 bool dev_state;
1015 bool opt_failed;
1016 void *jited_image;
1017 u32 jited_len;
1018 };
1019
1020 enum bpf_cgroup_storage_type {
1021 BPF_CGROUP_STORAGE_SHARED,
1022 BPF_CGROUP_STORAGE_PERCPU,
1023 __BPF_CGROUP_STORAGE_MAX
1024 };
1025
1026 #define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX
1027
1028 /* The longest tracepoint has 12 args.
1029 * See include/trace/bpf_probe.h
1030 */
1031 #define MAX_BPF_FUNC_ARGS 12
1032
1033 /* The maximum number of arguments passed through registers
1034 * a single function may have.
1035 */
1036 #define MAX_BPF_FUNC_REG_ARGS 5
1037
1038 /* The argument is a structure. */
1039 #define BTF_FMODEL_STRUCT_ARG BIT(0)
1040
1041 /* The argument is signed. */
1042 #define BTF_FMODEL_SIGNED_ARG BIT(1)
1043
1044 struct btf_func_model {
1045 u8 ret_size;
1046 u8 ret_flags;
1047 u8 nr_args;
1048 u8 arg_size[MAX_BPF_FUNC_ARGS];
1049 u8 arg_flags[MAX_BPF_FUNC_ARGS];
1050 };
1051
1052 /* Restore arguments before returning from trampoline to let original function
1053 * continue executing. This flag is used for fentry progs when there are no
1054 * fexit progs.
1055 */
1056 #define BPF_TRAMP_F_RESTORE_REGS BIT(0)
1057 /* Call original function after fentry progs, but before fexit progs.
1058 * Makes sense for fentry/fexit, normal calls and indirect calls.
1059 */
1060 #define BPF_TRAMP_F_CALL_ORIG BIT(1)
1061 /* Skip current frame and return to parent. Makes sense for fentry/fexit
1062 * programs only. Should not be used with normal calls and indirect calls.
1063 */
1064 #define BPF_TRAMP_F_SKIP_FRAME BIT(2)
1065 /* Store IP address of the caller on the trampoline stack,
1066 * so it's available for trampoline's programs.
1067 */
1068 #define BPF_TRAMP_F_IP_ARG BIT(3)
1069 /* Return the return value of fentry prog. Only used by bpf_struct_ops. */
1070 #define BPF_TRAMP_F_RET_FENTRY_RET BIT(4)
1071
1072 /* Get original function from stack instead of from provided direct address.
1073 * Makes sense for trampolines with fexit or fmod_ret programs.
1074 */
1075 #define BPF_TRAMP_F_ORIG_STACK BIT(5)
1076
1077 /* This trampoline is on a function with another ftrace_ops with IPMODIFY,
1078 * e.g., a live patch. This flag is set and cleared by ftrace call backs,
1079 */
1080 #define BPF_TRAMP_F_SHARE_IPMODIFY BIT(6)
1081
1082 /* Indicate that current trampoline is in a tail call context. Then, it has to
1083 * cache and restore tail_call_cnt to avoid infinite tail call loop.
1084 */
1085 #define BPF_TRAMP_F_TAIL_CALL_CTX BIT(7)
1086
1087 /*
1088 * Indicate the trampoline should be suitable to receive indirect calls;
1089 * without this indirectly calling the generated code can result in #UD/#CP,
1090 * depending on the CFI options.
1091 *
1092 * Used by bpf_struct_ops.
1093 *
1094 * Incompatible with FENTRY usage, overloads @func_addr argument.
1095 */
1096 #define BPF_TRAMP_F_INDIRECT BIT(8)
1097
1098 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
1099 * bytes on x86.
1100 */
1101 enum {
1102 #if defined(__s390x__)
1103 BPF_MAX_TRAMP_LINKS = 27,
1104 #else
1105 BPF_MAX_TRAMP_LINKS = 38,
1106 #endif
1107 };
1108
1109 struct bpf_tramp_links {
1110 struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS];
1111 int nr_links;
1112 };
1113
1114 struct bpf_tramp_run_ctx;
1115
1116 /* Different use cases for BPF trampoline:
1117 * 1. replace nop at the function entry (kprobe equivalent)
1118 * flags = BPF_TRAMP_F_RESTORE_REGS
1119 * fentry = a set of programs to run before returning from trampoline
1120 *
1121 * 2. replace nop at the function entry (kprobe + kretprobe equivalent)
1122 * flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME
1123 * orig_call = fentry_ip + MCOUNT_INSN_SIZE
1124 * fentry = a set of program to run before calling original function
1125 * fexit = a set of program to run after original function
1126 *
1127 * 3. replace direct call instruction anywhere in the function body
1128 * or assign a function pointer for indirect call (like tcp_congestion_ops->cong_avoid)
1129 * With flags = 0
1130 * fentry = a set of programs to run before returning from trampoline
1131 * With flags = BPF_TRAMP_F_CALL_ORIG
1132 * orig_call = original callback addr or direct function addr
1133 * fentry = a set of program to run before calling original function
1134 * fexit = a set of program to run after original function
1135 */
1136 struct bpf_tramp_image;
1137 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
1138 const struct btf_func_model *m, u32 flags,
1139 struct bpf_tramp_links *tlinks,
1140 void *func_addr);
1141 void *arch_alloc_bpf_trampoline(unsigned int size);
1142 void arch_free_bpf_trampoline(void *image, unsigned int size);
1143 int __must_check arch_protect_bpf_trampoline(void *image, unsigned int size);
1144 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
1145 struct bpf_tramp_links *tlinks, void *func_addr);
1146
1147 u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
1148 struct bpf_tramp_run_ctx *run_ctx);
1149 void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,
1150 struct bpf_tramp_run_ctx *run_ctx);
1151 void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr);
1152 void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);
1153 typedef u64 (*bpf_trampoline_enter_t)(struct bpf_prog *prog,
1154 struct bpf_tramp_run_ctx *run_ctx);
1155 typedef void (*bpf_trampoline_exit_t)(struct bpf_prog *prog, u64 start,
1156 struct bpf_tramp_run_ctx *run_ctx);
1157 bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog);
1158 bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog);
1159
1160 struct bpf_ksym {
1161 unsigned long start;
1162 unsigned long end;
1163 char name[KSYM_NAME_LEN];
1164 struct list_head lnode;
1165 struct latch_tree_node tnode;
1166 bool prog;
1167 };
1168
1169 enum bpf_tramp_prog_type {
1170 BPF_TRAMP_FENTRY,
1171 BPF_TRAMP_FEXIT,
1172 BPF_TRAMP_MODIFY_RETURN,
1173 BPF_TRAMP_MAX,
1174 BPF_TRAMP_REPLACE, /* more than MAX */
1175 };
1176
1177 struct bpf_tramp_image {
1178 void *image;
1179 int size;
1180 struct bpf_ksym ksym;
1181 struct percpu_ref pcref;
1182 void *ip_after_call;
1183 void *ip_epilogue;
1184 union {
1185 struct rcu_head rcu;
1186 struct work_struct work;
1187 };
1188 };
1189
1190 struct bpf_trampoline {
1191 /* hlist for trampoline_table */
1192 struct hlist_node hlist;
1193 struct ftrace_ops *fops;
1194 /* serializes access to fields of this trampoline */
1195 struct mutex mutex;
1196 refcount_t refcnt;
1197 u32 flags;
1198 u64 key;
1199 struct {
1200 struct btf_func_model model;
1201 void *addr;
1202 bool ftrace_managed;
1203 } func;
1204 /* if !NULL this is BPF_PROG_TYPE_EXT program that extends another BPF
1205 * program by replacing one of its functions. func.addr is the address
1206 * of the function it replaced.
1207 */
1208 struct bpf_prog *extension_prog;
1209 /* list of BPF programs using this trampoline */
1210 struct hlist_head progs_hlist[BPF_TRAMP_MAX];
1211 /* Number of attached programs. A counter per kind. */
1212 int progs_cnt[BPF_TRAMP_MAX];
1213 /* Executable image of trampoline */
1214 struct bpf_tramp_image *cur_image;
1215 };
1216
1217 struct bpf_attach_target_info {
1218 struct btf_func_model fmodel;
1219 long tgt_addr;
1220 struct module *tgt_mod;
1221 const char *tgt_name;
1222 const struct btf_type *tgt_type;
1223 };
1224
1225 #define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */
1226
1227 struct bpf_dispatcher_prog {
1228 struct bpf_prog *prog;
1229 refcount_t users;
1230 };
1231
1232 struct bpf_dispatcher {
1233 /* dispatcher mutex */
1234 struct mutex mutex;
1235 void *func;
1236 struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX];
1237 int num_progs;
1238 void *image;
1239 void *rw_image;
1240 u32 image_off;
1241 struct bpf_ksym ksym;
1242 #ifdef CONFIG_HAVE_STATIC_CALL
1243 struct static_call_key *sc_key;
1244 void *sc_tramp;
1245 #endif
1246 };
1247
1248 #ifndef __bpfcall
1249 #define __bpfcall __nocfi
1250 #endif
1251
bpf_dispatcher_nop_func(const void * ctx,const struct bpf_insn * insnsi,bpf_func_t bpf_func)1252 static __always_inline __bpfcall unsigned int bpf_dispatcher_nop_func(
1253 const void *ctx,
1254 const struct bpf_insn *insnsi,
1255 bpf_func_t bpf_func)
1256 {
1257 return bpf_func(ctx, insnsi);
1258 }
1259
1260 /* the implementation of the opaque uapi struct bpf_dynptr */
1261 struct bpf_dynptr_kern {
1262 void *data;
1263 /* Size represents the number of usable bytes of dynptr data.
1264 * If for example the offset is at 4 for a local dynptr whose data is
1265 * of type u64, the number of usable bytes is 4.
1266 *
1267 * The upper 8 bits are reserved. It is as follows:
1268 * Bits 0 - 23 = size
1269 * Bits 24 - 30 = dynptr type
1270 * Bit 31 = whether dynptr is read-only
1271 */
1272 u32 size;
1273 u32 offset;
1274 } __aligned(8);
1275
1276 enum bpf_dynptr_type {
1277 BPF_DYNPTR_TYPE_INVALID,
1278 /* Points to memory that is local to the bpf program */
1279 BPF_DYNPTR_TYPE_LOCAL,
1280 /* Underlying data is a ringbuf record */
1281 BPF_DYNPTR_TYPE_RINGBUF,
1282 /* Underlying data is a sk_buff */
1283 BPF_DYNPTR_TYPE_SKB,
1284 /* Underlying data is a xdp_buff */
1285 BPF_DYNPTR_TYPE_XDP,
1286 };
1287
1288 int bpf_dynptr_check_size(u32 size);
1289 u32 __bpf_dynptr_size(const struct bpf_dynptr_kern *ptr);
1290 const void *__bpf_dynptr_data(const struct bpf_dynptr_kern *ptr, u32 len);
1291 void *__bpf_dynptr_data_rw(const struct bpf_dynptr_kern *ptr, u32 len);
1292 bool __bpf_dynptr_is_rdonly(const struct bpf_dynptr_kern *ptr);
1293
1294 #ifdef CONFIG_BPF_JIT
1295 int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
1296 int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
1297 struct bpf_trampoline *bpf_trampoline_get(u64 key,
1298 struct bpf_attach_target_info *tgt_info);
1299 void bpf_trampoline_put(struct bpf_trampoline *tr);
1300 int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs);
1301
1302 /*
1303 * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn
1304 * indirection with a direct call to the bpf program. If the architecture does
1305 * not have STATIC_CALL, avoid a double-indirection.
1306 */
1307 #ifdef CONFIG_HAVE_STATIC_CALL
1308
1309 #define __BPF_DISPATCHER_SC_INIT(_name) \
1310 .sc_key = &STATIC_CALL_KEY(_name), \
1311 .sc_tramp = STATIC_CALL_TRAMP_ADDR(_name),
1312
1313 #define __BPF_DISPATCHER_SC(name) \
1314 DEFINE_STATIC_CALL(bpf_dispatcher_##name##_call, bpf_dispatcher_nop_func)
1315
1316 #define __BPF_DISPATCHER_CALL(name) \
1317 static_call(bpf_dispatcher_##name##_call)(ctx, insnsi, bpf_func)
1318
1319 #define __BPF_DISPATCHER_UPDATE(_d, _new) \
1320 __static_call_update((_d)->sc_key, (_d)->sc_tramp, (_new))
1321
1322 #else
1323 #define __BPF_DISPATCHER_SC_INIT(name)
1324 #define __BPF_DISPATCHER_SC(name)
1325 #define __BPF_DISPATCHER_CALL(name) bpf_func(ctx, insnsi)
1326 #define __BPF_DISPATCHER_UPDATE(_d, _new)
1327 #endif
1328
1329 #define BPF_DISPATCHER_INIT(_name) { \
1330 .mutex = __MUTEX_INITIALIZER(_name.mutex), \
1331 .func = &_name##_func, \
1332 .progs = {}, \
1333 .num_progs = 0, \
1334 .image = NULL, \
1335 .image_off = 0, \
1336 .ksym = { \
1337 .name = #_name, \
1338 .lnode = LIST_HEAD_INIT(_name.ksym.lnode), \
1339 }, \
1340 __BPF_DISPATCHER_SC_INIT(_name##_call) \
1341 }
1342
1343 #define DEFINE_BPF_DISPATCHER(name) \
1344 __BPF_DISPATCHER_SC(name); \
1345 noinline __bpfcall unsigned int bpf_dispatcher_##name##_func( \
1346 const void *ctx, \
1347 const struct bpf_insn *insnsi, \
1348 bpf_func_t bpf_func) \
1349 { \
1350 return __BPF_DISPATCHER_CALL(name); \
1351 } \
1352 EXPORT_SYMBOL(bpf_dispatcher_##name##_func); \
1353 struct bpf_dispatcher bpf_dispatcher_##name = \
1354 BPF_DISPATCHER_INIT(bpf_dispatcher_##name);
1355
1356 #define DECLARE_BPF_DISPATCHER(name) \
1357 unsigned int bpf_dispatcher_##name##_func( \
1358 const void *ctx, \
1359 const struct bpf_insn *insnsi, \
1360 bpf_func_t bpf_func); \
1361 extern struct bpf_dispatcher bpf_dispatcher_##name;
1362
1363 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func
1364 #define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name)
1365 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
1366 struct bpf_prog *to);
1367 /* Called only from JIT-enabled code, so there's no need for stubs. */
1368 void bpf_image_ksym_add(void *data, unsigned int size, struct bpf_ksym *ksym);
1369 void bpf_image_ksym_del(struct bpf_ksym *ksym);
1370 void bpf_ksym_add(struct bpf_ksym *ksym);
1371 void bpf_ksym_del(struct bpf_ksym *ksym);
1372 int bpf_jit_charge_modmem(u32 size);
1373 void bpf_jit_uncharge_modmem(u32 size);
1374 bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
1375 #else
bpf_trampoline_link_prog(struct bpf_tramp_link * link,struct bpf_trampoline * tr)1376 static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
1377 struct bpf_trampoline *tr)
1378 {
1379 return -ENOTSUPP;
1380 }
bpf_trampoline_unlink_prog(struct bpf_tramp_link * link,struct bpf_trampoline * tr)1381 static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
1382 struct bpf_trampoline *tr)
1383 {
1384 return -ENOTSUPP;
1385 }
bpf_trampoline_get(u64 key,struct bpf_attach_target_info * tgt_info)1386 static inline struct bpf_trampoline *bpf_trampoline_get(u64 key,
1387 struct bpf_attach_target_info *tgt_info)
1388 {
1389 return NULL;
1390 }
bpf_trampoline_put(struct bpf_trampoline * tr)1391 static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {}
1392 #define DEFINE_BPF_DISPATCHER(name)
1393 #define DECLARE_BPF_DISPATCHER(name)
1394 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nop_func
1395 #define BPF_DISPATCHER_PTR(name) NULL
bpf_dispatcher_change_prog(struct bpf_dispatcher * d,struct bpf_prog * from,struct bpf_prog * to)1396 static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher *d,
1397 struct bpf_prog *from,
1398 struct bpf_prog *to) {}
is_bpf_image_address(unsigned long address)1399 static inline bool is_bpf_image_address(unsigned long address)
1400 {
1401 return false;
1402 }
bpf_prog_has_trampoline(const struct bpf_prog * prog)1403 static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
1404 {
1405 return false;
1406 }
1407 #endif
1408
1409 struct bpf_func_info_aux {
1410 u16 linkage;
1411 bool unreliable;
1412 bool called : 1;
1413 bool verified : 1;
1414 };
1415
1416 enum bpf_jit_poke_reason {
1417 BPF_POKE_REASON_TAIL_CALL,
1418 };
1419
1420 /* Descriptor of pokes pointing /into/ the JITed image. */
1421 struct bpf_jit_poke_descriptor {
1422 void *tailcall_target;
1423 void *tailcall_bypass;
1424 void *bypass_addr;
1425 void *aux;
1426 union {
1427 struct {
1428 struct bpf_map *map;
1429 u32 key;
1430 } tail_call;
1431 };
1432 bool tailcall_target_stable;
1433 u8 adj_off;
1434 u16 reason;
1435 u32 insn_idx;
1436 };
1437
1438 /* reg_type info for ctx arguments */
1439 struct bpf_ctx_arg_aux {
1440 u32 offset;
1441 enum bpf_reg_type reg_type;
1442 struct btf *btf;
1443 u32 btf_id;
1444 };
1445
1446 struct btf_mod_pair {
1447 struct btf *btf;
1448 struct module *module;
1449 };
1450
1451 struct bpf_kfunc_desc_tab;
1452
1453 struct bpf_prog_aux {
1454 atomic64_t refcnt;
1455 u32 used_map_cnt;
1456 u32 used_btf_cnt;
1457 u32 max_ctx_offset;
1458 u32 max_pkt_offset;
1459 u32 max_tp_access;
1460 u32 stack_depth;
1461 u32 id;
1462 u32 func_cnt; /* used by non-func prog as the number of func progs */
1463 u32 real_func_cnt; /* includes hidden progs, only used for JIT and freeing progs */
1464 u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */
1465 u32 attach_btf_id; /* in-kernel BTF type id to attach to */
1466 u32 ctx_arg_info_size;
1467 u32 max_rdonly_access;
1468 u32 max_rdwr_access;
1469 struct btf *attach_btf;
1470 const struct bpf_ctx_arg_aux *ctx_arg_info;
1471 struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */
1472 struct bpf_prog *dst_prog;
1473 struct bpf_trampoline *dst_trampoline;
1474 enum bpf_prog_type saved_dst_prog_type;
1475 enum bpf_attach_type saved_dst_attach_type;
1476 bool verifier_zext; /* Zero extensions has been inserted by verifier. */
1477 bool dev_bound; /* Program is bound to the netdev. */
1478 bool offload_requested; /* Program is bound and offloaded to the netdev. */
1479 bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */
1480 bool attach_tracing_prog; /* true if tracing another tracing program */
1481 bool func_proto_unreliable;
1482 bool tail_call_reachable;
1483 bool xdp_has_frags;
1484 bool exception_cb;
1485 bool exception_boundary;
1486 struct bpf_arena *arena;
1487 /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
1488 const struct btf_type *attach_func_proto;
1489 /* function name for valid attach_btf_id */
1490 const char *attach_func_name;
1491 struct bpf_prog **func;
1492 void *jit_data; /* JIT specific data. arch dependent */
1493 struct bpf_jit_poke_descriptor *poke_tab;
1494 struct bpf_kfunc_desc_tab *kfunc_tab;
1495 struct bpf_kfunc_btf_tab *kfunc_btf_tab;
1496 u32 size_poke_tab;
1497 #ifdef CONFIG_FINEIBT
1498 struct bpf_ksym ksym_prefix;
1499 #endif
1500 struct bpf_ksym ksym;
1501 const struct bpf_prog_ops *ops;
1502 struct bpf_map **used_maps;
1503 struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */
1504 struct btf_mod_pair *used_btfs;
1505 struct bpf_prog *prog;
1506 struct user_struct *user;
1507 u64 load_time; /* ns since boottime */
1508 u32 verified_insns;
1509 int cgroup_atype; /* enum cgroup_bpf_attach_type */
1510 struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
1511 char name[BPF_OBJ_NAME_LEN];
1512 u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64);
1513 #ifdef CONFIG_SECURITY
1514 void *security;
1515 #endif
1516 struct bpf_token *token;
1517 struct bpf_prog_offload *offload;
1518 struct btf *btf;
1519 struct bpf_func_info *func_info;
1520 struct bpf_func_info_aux *func_info_aux;
1521 /* bpf_line_info loaded from userspace. linfo->insn_off
1522 * has the xlated insn offset.
1523 * Both the main and sub prog share the same linfo.
1524 * The subprog can access its first linfo by
1525 * using the linfo_idx.
1526 */
1527 struct bpf_line_info *linfo;
1528 /* jited_linfo is the jited addr of the linfo. It has a
1529 * one to one mapping to linfo:
1530 * jited_linfo[i] is the jited addr for the linfo[i]->insn_off.
1531 * Both the main and sub prog share the same jited_linfo.
1532 * The subprog can access its first jited_linfo by
1533 * using the linfo_idx.
1534 */
1535 void **jited_linfo;
1536 u32 func_info_cnt;
1537 u32 nr_linfo;
1538 /* subprog can use linfo_idx to access its first linfo and
1539 * jited_linfo.
1540 * main prog always has linfo_idx == 0
1541 */
1542 u32 linfo_idx;
1543 struct module *mod;
1544 u32 num_exentries;
1545 struct exception_table_entry *extable;
1546 union {
1547 struct work_struct work;
1548 struct rcu_head rcu;
1549 };
1550 };
1551
1552 struct bpf_prog {
1553 u16 pages; /* Number of allocated pages */
1554 u16 jited:1, /* Is our filter JIT'ed? */
1555 jit_requested:1,/* archs need to JIT the prog */
1556 gpl_compatible:1, /* Is filter GPL compatible? */
1557 cb_access:1, /* Is control block accessed? */
1558 dst_needed:1, /* Do we need dst entry? */
1559 blinding_requested:1, /* needs constant blinding */
1560 blinded:1, /* Was blinded */
1561 is_func:1, /* program is a bpf function */
1562 kprobe_override:1, /* Do we override a kprobe? */
1563 has_callchain_buf:1, /* callchain buffer allocated? */
1564 enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */
1565 call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */
1566 call_get_func_ip:1, /* Do we call get_func_ip() */
1567 tstamp_type_access:1, /* Accessed __sk_buff->tstamp_type */
1568 sleepable:1; /* BPF program is sleepable */
1569 enum bpf_prog_type type; /* Type of BPF program */
1570 enum bpf_attach_type expected_attach_type; /* For some prog types */
1571 u32 len; /* Number of filter blocks */
1572 u32 jited_len; /* Size of jited insns in bytes */
1573 u8 tag[BPF_TAG_SIZE];
1574 struct bpf_prog_stats __percpu *stats;
1575 int __percpu *active;
1576 unsigned int (*bpf_func)(const void *ctx,
1577 const struct bpf_insn *insn);
1578 struct bpf_prog_aux *aux; /* Auxiliary fields */
1579 struct sock_fprog_kern *orig_prog; /* Original BPF program */
1580 /* Instructions for interpreter */
1581 union {
1582 DECLARE_FLEX_ARRAY(struct sock_filter, insns);
1583 DECLARE_FLEX_ARRAY(struct bpf_insn, insnsi);
1584 };
1585 };
1586
1587 struct bpf_array_aux {
1588 /* Programs with direct jumps into programs part of this array. */
1589 struct list_head poke_progs;
1590 struct bpf_map *map;
1591 struct mutex poke_mutex;
1592 struct work_struct work;
1593 };
1594
1595 struct bpf_link {
1596 atomic64_t refcnt;
1597 u32 id;
1598 enum bpf_link_type type;
1599 const struct bpf_link_ops *ops;
1600 struct bpf_prog *prog;
1601 /* rcu is used before freeing, work can be used to schedule that
1602 * RCU-based freeing before that, so they never overlap
1603 */
1604 union {
1605 struct rcu_head rcu;
1606 struct work_struct work;
1607 };
1608 };
1609
1610 struct bpf_link_ops {
1611 void (*release)(struct bpf_link *link);
1612 /* deallocate link resources callback, called without RCU grace period
1613 * waiting
1614 */
1615 void (*dealloc)(struct bpf_link *link);
1616 /* deallocate link resources callback, called after RCU grace period;
1617 * if underlying BPF program is sleepable we go through tasks trace
1618 * RCU GP and then "classic" RCU GP
1619 */
1620 void (*dealloc_deferred)(struct bpf_link *link);
1621 int (*detach)(struct bpf_link *link);
1622 int (*update_prog)(struct bpf_link *link, struct bpf_prog *new_prog,
1623 struct bpf_prog *old_prog);
1624 void (*show_fdinfo)(const struct bpf_link *link, struct seq_file *seq);
1625 int (*fill_link_info)(const struct bpf_link *link,
1626 struct bpf_link_info *info);
1627 int (*update_map)(struct bpf_link *link, struct bpf_map *new_map,
1628 struct bpf_map *old_map);
1629 __poll_t (*poll)(struct file *file, struct poll_table_struct *pts);
1630 };
1631
1632 struct bpf_tramp_link {
1633 struct bpf_link link;
1634 struct hlist_node tramp_hlist;
1635 u64 cookie;
1636 };
1637
1638 struct bpf_shim_tramp_link {
1639 struct bpf_tramp_link link;
1640 struct bpf_trampoline *trampoline;
1641 };
1642
1643 struct bpf_tracing_link {
1644 struct bpf_tramp_link link;
1645 enum bpf_attach_type attach_type;
1646 struct bpf_trampoline *trampoline;
1647 struct bpf_prog *tgt_prog;
1648 };
1649
1650 struct bpf_raw_tp_link {
1651 struct bpf_link link;
1652 struct bpf_raw_event_map *btp;
1653 u64 cookie;
1654 };
1655
1656 struct bpf_link_primer {
1657 struct bpf_link *link;
1658 struct file *file;
1659 int fd;
1660 u32 id;
1661 };
1662
1663 struct bpf_mount_opts {
1664 kuid_t uid;
1665 kgid_t gid;
1666 umode_t mode;
1667
1668 /* BPF token-related delegation options */
1669 u64 delegate_cmds;
1670 u64 delegate_maps;
1671 u64 delegate_progs;
1672 u64 delegate_attachs;
1673 };
1674
1675 struct bpf_token {
1676 struct work_struct work;
1677 atomic64_t refcnt;
1678 struct user_namespace *userns;
1679 u64 allowed_cmds;
1680 u64 allowed_maps;
1681 u64 allowed_progs;
1682 u64 allowed_attachs;
1683 #ifdef CONFIG_SECURITY
1684 void *security;
1685 #endif
1686 };
1687
1688 struct bpf_struct_ops_value;
1689 struct btf_member;
1690
1691 #define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64
1692 /**
1693 * struct bpf_struct_ops - A structure of callbacks allowing a subsystem to
1694 * define a BPF_MAP_TYPE_STRUCT_OPS map type composed
1695 * of BPF_PROG_TYPE_STRUCT_OPS progs.
1696 * @verifier_ops: A structure of callbacks that are invoked by the verifier
1697 * when determining whether the struct_ops progs in the
1698 * struct_ops map are valid.
1699 * @init: A callback that is invoked a single time, and before any other
1700 * callback, to initialize the structure. A nonzero return value means
1701 * the subsystem could not be initialized.
1702 * @check_member: When defined, a callback invoked by the verifier to allow
1703 * the subsystem to determine if an entry in the struct_ops map
1704 * is valid. A nonzero return value means that the map is
1705 * invalid and should be rejected by the verifier.
1706 * @init_member: A callback that is invoked for each member of the struct_ops
1707 * map to allow the subsystem to initialize the member. A nonzero
1708 * value means the member could not be initialized. This callback
1709 * is exclusive with the @type, @type_id, @value_type, and
1710 * @value_id fields.
1711 * @reg: A callback that is invoked when the struct_ops map has been
1712 * initialized and is being attached to. Zero means the struct_ops map
1713 * has been successfully registered and is live. A nonzero return value
1714 * means the struct_ops map could not be registered.
1715 * @unreg: A callback that is invoked when the struct_ops map should be
1716 * unregistered.
1717 * @update: A callback that is invoked when the live struct_ops map is being
1718 * updated to contain new values. This callback is only invoked when
1719 * the struct_ops map is loaded with BPF_F_LINK. If not defined, the
1720 * it is assumed that the struct_ops map cannot be updated.
1721 * @validate: A callback that is invoked after all of the members have been
1722 * initialized. This callback should perform static checks on the
1723 * map, meaning that it should either fail or succeed
1724 * deterministically. A struct_ops map that has been validated may
1725 * not necessarily succeed in being registered if the call to @reg
1726 * fails. For example, a valid struct_ops map may be loaded, but
1727 * then fail to be registered due to there being another active
1728 * struct_ops map on the system in the subsystem already. For this
1729 * reason, if this callback is not defined, the check is skipped as
1730 * the struct_ops map will have final verification performed in
1731 * @reg.
1732 * @type: BTF type.
1733 * @value_type: Value type.
1734 * @name: The name of the struct bpf_struct_ops object.
1735 * @func_models: Func models
1736 * @type_id: BTF type id.
1737 * @value_id: BTF value id.
1738 */
1739 struct bpf_struct_ops {
1740 const struct bpf_verifier_ops *verifier_ops;
1741 int (*init)(struct btf *btf);
1742 int (*check_member)(const struct btf_type *t,
1743 const struct btf_member *member,
1744 const struct bpf_prog *prog);
1745 int (*init_member)(const struct btf_type *t,
1746 const struct btf_member *member,
1747 void *kdata, const void *udata);
1748 int (*reg)(void *kdata, struct bpf_link *link);
1749 void (*unreg)(void *kdata, struct bpf_link *link);
1750 int (*update)(void *kdata, void *old_kdata, struct bpf_link *link);
1751 int (*validate)(void *kdata);
1752 void *cfi_stubs;
1753 struct module *owner;
1754 const char *name;
1755 struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS];
1756 };
1757
1758 /* Every member of a struct_ops type has an instance even a member is not
1759 * an operator (function pointer). The "info" field will be assigned to
1760 * prog->aux->ctx_arg_info of BPF struct_ops programs to provide the
1761 * argument information required by the verifier to verify the program.
1762 *
1763 * btf_ctx_access() will lookup prog->aux->ctx_arg_info to find the
1764 * corresponding entry for an given argument.
1765 */
1766 struct bpf_struct_ops_arg_info {
1767 struct bpf_ctx_arg_aux *info;
1768 u32 cnt;
1769 };
1770
1771 struct bpf_struct_ops_desc {
1772 struct bpf_struct_ops *st_ops;
1773
1774 const struct btf_type *type;
1775 const struct btf_type *value_type;
1776 u32 type_id;
1777 u32 value_id;
1778
1779 /* Collection of argument information for each member */
1780 struct bpf_struct_ops_arg_info *arg_info;
1781 };
1782
1783 enum bpf_struct_ops_state {
1784 BPF_STRUCT_OPS_STATE_INIT,
1785 BPF_STRUCT_OPS_STATE_INUSE,
1786 BPF_STRUCT_OPS_STATE_TOBEFREE,
1787 BPF_STRUCT_OPS_STATE_READY,
1788 };
1789
1790 struct bpf_struct_ops_common_value {
1791 refcount_t refcnt;
1792 enum bpf_struct_ops_state state;
1793 };
1794
1795 #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)
1796 /* This macro helps developer to register a struct_ops type and generate
1797 * type information correctly. Developers should use this macro to register
1798 * a struct_ops type instead of calling __register_bpf_struct_ops() directly.
1799 */
1800 #define register_bpf_struct_ops(st_ops, type) \
1801 ({ \
1802 struct bpf_struct_ops_##type { \
1803 struct bpf_struct_ops_common_value common; \
1804 struct type data ____cacheline_aligned_in_smp; \
1805 }; \
1806 BTF_TYPE_EMIT(struct bpf_struct_ops_##type); \
1807 __register_bpf_struct_ops(st_ops); \
1808 })
1809 #define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA))
1810 bool bpf_struct_ops_get(const void *kdata);
1811 void bpf_struct_ops_put(const void *kdata);
1812 int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff);
1813 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
1814 void *value);
1815 int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
1816 struct bpf_tramp_link *link,
1817 const struct btf_func_model *model,
1818 void *stub_func,
1819 void **image, u32 *image_off,
1820 bool allow_alloc);
1821 void bpf_struct_ops_image_free(void *image);
bpf_try_module_get(const void * data,struct module * owner)1822 static inline bool bpf_try_module_get(const void *data, struct module *owner)
1823 {
1824 if (owner == BPF_MODULE_OWNER)
1825 return bpf_struct_ops_get(data);
1826 else
1827 return try_module_get(owner);
1828 }
bpf_module_put(const void * data,struct module * owner)1829 static inline void bpf_module_put(const void *data, struct module *owner)
1830 {
1831 if (owner == BPF_MODULE_OWNER)
1832 bpf_struct_ops_put(data);
1833 else
1834 module_put(owner);
1835 }
1836 int bpf_struct_ops_link_create(union bpf_attr *attr);
1837
1838 #ifdef CONFIG_NET
1839 /* Define it here to avoid the use of forward declaration */
1840 struct bpf_dummy_ops_state {
1841 int val;
1842 };
1843
1844 struct bpf_dummy_ops {
1845 int (*test_1)(struct bpf_dummy_ops_state *cb);
1846 int (*test_2)(struct bpf_dummy_ops_state *cb, int a1, unsigned short a2,
1847 char a3, unsigned long a4);
1848 int (*test_sleepable)(struct bpf_dummy_ops_state *cb);
1849 };
1850
1851 int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
1852 union bpf_attr __user *uattr);
1853 #endif
1854 int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,
1855 struct btf *btf,
1856 struct bpf_verifier_log *log);
1857 void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map);
1858 void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc);
1859 #else
1860 #define register_bpf_struct_ops(st_ops, type) ({ (void *)(st_ops); 0; })
bpf_try_module_get(const void * data,struct module * owner)1861 static inline bool bpf_try_module_get(const void *data, struct module *owner)
1862 {
1863 return try_module_get(owner);
1864 }
bpf_module_put(const void * data,struct module * owner)1865 static inline void bpf_module_put(const void *data, struct module *owner)
1866 {
1867 module_put(owner);
1868 }
bpf_struct_ops_supported(const struct bpf_struct_ops * st_ops,u32 moff)1869 static inline int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff)
1870 {
1871 return -ENOTSUPP;
1872 }
bpf_struct_ops_map_sys_lookup_elem(struct bpf_map * map,void * key,void * value)1873 static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map,
1874 void *key,
1875 void *value)
1876 {
1877 return -EINVAL;
1878 }
bpf_struct_ops_link_create(union bpf_attr * attr)1879 static inline int bpf_struct_ops_link_create(union bpf_attr *attr)
1880 {
1881 return -EOPNOTSUPP;
1882 }
bpf_map_struct_ops_info_fill(struct bpf_map_info * info,struct bpf_map * map)1883 static inline void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map)
1884 {
1885 }
1886
bpf_struct_ops_desc_release(struct bpf_struct_ops_desc * st_ops_desc)1887 static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc)
1888 {
1889 }
1890
1891 #endif
1892
1893 #if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)
1894 int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
1895 int cgroup_atype);
1896 void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog);
1897 #else
bpf_trampoline_link_cgroup_shim(struct bpf_prog * prog,int cgroup_atype)1898 static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
1899 int cgroup_atype)
1900 {
1901 return -EOPNOTSUPP;
1902 }
bpf_trampoline_unlink_cgroup_shim(struct bpf_prog * prog)1903 static inline void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)
1904 {
1905 }
1906 #endif
1907
1908 struct bpf_array {
1909 struct bpf_map map;
1910 u32 elem_size;
1911 u32 index_mask;
1912 struct bpf_array_aux *aux;
1913 union {
1914 DECLARE_FLEX_ARRAY(char, value) __aligned(8);
1915 DECLARE_FLEX_ARRAY(void *, ptrs) __aligned(8);
1916 DECLARE_FLEX_ARRAY(void __percpu *, pptrs) __aligned(8);
1917 };
1918 };
1919
1920 #define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */
1921 #define MAX_TAIL_CALL_CNT 33
1922
1923 /* Maximum number of loops for bpf_loop and bpf_iter_num.
1924 * It's enum to expose it (and thus make it discoverable) through BTF.
1925 */
1926 enum {
1927 BPF_MAX_LOOPS = 8 * 1024 * 1024,
1928 };
1929
1930 #define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \
1931 BPF_F_RDONLY_PROG | \
1932 BPF_F_WRONLY | \
1933 BPF_F_WRONLY_PROG)
1934
1935 #define BPF_MAP_CAN_READ BIT(0)
1936 #define BPF_MAP_CAN_WRITE BIT(1)
1937
1938 /* Maximum number of user-producer ring buffer samples that can be drained in
1939 * a call to bpf_user_ringbuf_drain().
1940 */
1941 #define BPF_MAX_USER_RINGBUF_SAMPLES (128 * 1024)
1942
bpf_map_flags_to_cap(struct bpf_map * map)1943 static inline u32 bpf_map_flags_to_cap(struct bpf_map *map)
1944 {
1945 u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
1946
1947 /* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is
1948 * not possible.
1949 */
1950 if (access_flags & BPF_F_RDONLY_PROG)
1951 return BPF_MAP_CAN_READ;
1952 else if (access_flags & BPF_F_WRONLY_PROG)
1953 return BPF_MAP_CAN_WRITE;
1954 else
1955 return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE;
1956 }
1957
bpf_map_flags_access_ok(u32 access_flags)1958 static inline bool bpf_map_flags_access_ok(u32 access_flags)
1959 {
1960 return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) !=
1961 (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
1962 }
1963
1964 struct bpf_event_entry {
1965 struct perf_event *event;
1966 struct file *perf_file;
1967 struct file *map_file;
1968 struct rcu_head rcu;
1969 };
1970
map_type_contains_progs(struct bpf_map * map)1971 static inline bool map_type_contains_progs(struct bpf_map *map)
1972 {
1973 return map->map_type == BPF_MAP_TYPE_PROG_ARRAY ||
1974 map->map_type == BPF_MAP_TYPE_DEVMAP ||
1975 map->map_type == BPF_MAP_TYPE_CPUMAP;
1976 }
1977
1978 bool bpf_prog_map_compatible(struct bpf_map *map, const struct bpf_prog *fp);
1979 int bpf_prog_calc_tag(struct bpf_prog *fp);
1980
1981 const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
1982 const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void);
1983
1984 typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src,
1985 unsigned long off, unsigned long len);
1986 typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type,
1987 const struct bpf_insn *src,
1988 struct bpf_insn *dst,
1989 struct bpf_prog *prog,
1990 u32 *target_size);
1991
1992 u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
1993 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy);
1994
1995 /* an array of programs to be executed under rcu_lock.
1996 *
1997 * Typical usage:
1998 * ret = bpf_prog_run_array(rcu_dereference(&bpf_prog_array), ctx, bpf_prog_run);
1999 *
2000 * the structure returned by bpf_prog_array_alloc() should be populated
2001 * with program pointers and the last pointer must be NULL.
2002 * The user has to keep refcnt on the program and make sure the program
2003 * is removed from the array before bpf_prog_put().
2004 * The 'struct bpf_prog_array *' should only be replaced with xchg()
2005 * since other cpus are walking the array of pointers in parallel.
2006 */
2007 struct bpf_prog_array_item {
2008 struct bpf_prog *prog;
2009 union {
2010 struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
2011 u64 bpf_cookie;
2012 };
2013 };
2014
2015 struct bpf_prog_array {
2016 struct rcu_head rcu;
2017 struct bpf_prog_array_item items[];
2018 };
2019
2020 struct bpf_empty_prog_array {
2021 struct bpf_prog_array hdr;
2022 struct bpf_prog *null_prog;
2023 };
2024
2025 /* to avoid allocating empty bpf_prog_array for cgroups that
2026 * don't have bpf program attached use one global 'bpf_empty_prog_array'
2027 * It will not be modified the caller of bpf_prog_array_alloc()
2028 * (since caller requested prog_cnt == 0)
2029 * that pointer should be 'freed' by bpf_prog_array_free()
2030 */
2031 extern struct bpf_empty_prog_array bpf_empty_prog_array;
2032
2033 struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
2034 void bpf_prog_array_free(struct bpf_prog_array *progs);
2035 /* Use when traversal over the bpf_prog_array uses tasks_trace rcu */
2036 void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs);
2037 int bpf_prog_array_length(struct bpf_prog_array *progs);
2038 bool bpf_prog_array_is_empty(struct bpf_prog_array *array);
2039 int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs,
2040 __u32 __user *prog_ids, u32 cnt);
2041
2042 void bpf_prog_array_delete_safe(struct bpf_prog_array *progs,
2043 struct bpf_prog *old_prog);
2044 int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index);
2045 int bpf_prog_array_update_at(struct bpf_prog_array *array, int index,
2046 struct bpf_prog *prog);
2047 int bpf_prog_array_copy_info(struct bpf_prog_array *array,
2048 u32 *prog_ids, u32 request_cnt,
2049 u32 *prog_cnt);
2050 int bpf_prog_array_copy(struct bpf_prog_array *old_array,
2051 struct bpf_prog *exclude_prog,
2052 struct bpf_prog *include_prog,
2053 u64 bpf_cookie,
2054 struct bpf_prog_array **new_array);
2055
2056 struct bpf_run_ctx {};
2057
2058 struct bpf_cg_run_ctx {
2059 struct bpf_run_ctx run_ctx;
2060 const struct bpf_prog_array_item *prog_item;
2061 int retval;
2062 };
2063
2064 struct bpf_trace_run_ctx {
2065 struct bpf_run_ctx run_ctx;
2066 u64 bpf_cookie;
2067 bool is_uprobe;
2068 };
2069
2070 struct bpf_tramp_run_ctx {
2071 struct bpf_run_ctx run_ctx;
2072 u64 bpf_cookie;
2073 struct bpf_run_ctx *saved_run_ctx;
2074 };
2075
bpf_set_run_ctx(struct bpf_run_ctx * new_ctx)2076 static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx)
2077 {
2078 struct bpf_run_ctx *old_ctx = NULL;
2079
2080 #ifdef CONFIG_BPF_SYSCALL
2081 old_ctx = current->bpf_ctx;
2082 current->bpf_ctx = new_ctx;
2083 #endif
2084 return old_ctx;
2085 }
2086
bpf_reset_run_ctx(struct bpf_run_ctx * old_ctx)2087 static inline void bpf_reset_run_ctx(struct bpf_run_ctx *old_ctx)
2088 {
2089 #ifdef CONFIG_BPF_SYSCALL
2090 current->bpf_ctx = old_ctx;
2091 #endif
2092 }
2093
2094 /* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */
2095 #define BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE (1 << 0)
2096 /* BPF program asks to set CN on the packet. */
2097 #define BPF_RET_SET_CN (1 << 0)
2098
2099 typedef u32 (*bpf_prog_run_fn)(const struct bpf_prog *prog, const void *ctx);
2100
2101 static __always_inline u32
bpf_prog_run_array(const struct bpf_prog_array * array,const void * ctx,bpf_prog_run_fn run_prog)2102 bpf_prog_run_array(const struct bpf_prog_array *array,
2103 const void *ctx, bpf_prog_run_fn run_prog)
2104 {
2105 const struct bpf_prog_array_item *item;
2106 const struct bpf_prog *prog;
2107 struct bpf_run_ctx *old_run_ctx;
2108 struct bpf_trace_run_ctx run_ctx;
2109 u32 ret = 1;
2110
2111 RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "no rcu lock held");
2112
2113 if (unlikely(!array))
2114 return ret;
2115
2116 run_ctx.is_uprobe = false;
2117
2118 migrate_disable();
2119 old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
2120 item = &array->items[0];
2121 while ((prog = READ_ONCE(item->prog))) {
2122 run_ctx.bpf_cookie = item->bpf_cookie;
2123 ret &= run_prog(prog, ctx);
2124 item++;
2125 }
2126 bpf_reset_run_ctx(old_run_ctx);
2127 migrate_enable();
2128 return ret;
2129 }
2130
2131 /* Notes on RCU design for bpf_prog_arrays containing sleepable programs:
2132 *
2133 * We use the tasks_trace rcu flavor read section to protect the bpf_prog_array
2134 * overall. As a result, we must use the bpf_prog_array_free_sleepable
2135 * in order to use the tasks_trace rcu grace period.
2136 *
2137 * When a non-sleepable program is inside the array, we take the rcu read
2138 * section and disable preemption for that program alone, so it can access
2139 * rcu-protected dynamically sized maps.
2140 */
2141 static __always_inline u32
bpf_prog_run_array_uprobe(const struct bpf_prog_array __rcu * array_rcu,const void * ctx,bpf_prog_run_fn run_prog)2142 bpf_prog_run_array_uprobe(const struct bpf_prog_array __rcu *array_rcu,
2143 const void *ctx, bpf_prog_run_fn run_prog)
2144 {
2145 const struct bpf_prog_array_item *item;
2146 const struct bpf_prog *prog;
2147 const struct bpf_prog_array *array;
2148 struct bpf_run_ctx *old_run_ctx;
2149 struct bpf_trace_run_ctx run_ctx;
2150 u32 ret = 1;
2151
2152 might_fault();
2153
2154 rcu_read_lock_trace();
2155 migrate_disable();
2156
2157 run_ctx.is_uprobe = true;
2158
2159 array = rcu_dereference_check(array_rcu, rcu_read_lock_trace_held());
2160 if (unlikely(!array))
2161 goto out;
2162 old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
2163 item = &array->items[0];
2164 while ((prog = READ_ONCE(item->prog))) {
2165 if (!prog->sleepable)
2166 rcu_read_lock();
2167
2168 run_ctx.bpf_cookie = item->bpf_cookie;
2169 ret &= run_prog(prog, ctx);
2170 item++;
2171
2172 if (!prog->sleepable)
2173 rcu_read_unlock();
2174 }
2175 bpf_reset_run_ctx(old_run_ctx);
2176 out:
2177 migrate_enable();
2178 rcu_read_unlock_trace();
2179 return ret;
2180 }
2181
2182 #ifdef CONFIG_BPF_SYSCALL
2183 DECLARE_PER_CPU(int, bpf_prog_active);
2184 extern struct mutex bpf_stats_enabled_mutex;
2185
2186 /*
2187 * Block execution of BPF programs attached to instrumentation (perf,
2188 * kprobes, tracepoints) to prevent deadlocks on map operations as any of
2189 * these events can happen inside a region which holds a map bucket lock
2190 * and can deadlock on it.
2191 */
bpf_disable_instrumentation(void)2192 static inline void bpf_disable_instrumentation(void)
2193 {
2194 migrate_disable();
2195 this_cpu_inc(bpf_prog_active);
2196 }
2197
bpf_enable_instrumentation(void)2198 static inline void bpf_enable_instrumentation(void)
2199 {
2200 this_cpu_dec(bpf_prog_active);
2201 migrate_enable();
2202 }
2203
2204 extern const struct super_operations bpf_super_ops;
2205 extern const struct file_operations bpf_map_fops;
2206 extern const struct file_operations bpf_prog_fops;
2207 extern const struct file_operations bpf_iter_fops;
2208
2209 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
2210 extern const struct bpf_prog_ops _name ## _prog_ops; \
2211 extern const struct bpf_verifier_ops _name ## _verifier_ops;
2212 #define BPF_MAP_TYPE(_id, _ops) \
2213 extern const struct bpf_map_ops _ops;
2214 #define BPF_LINK_TYPE(_id, _name)
2215 #include <linux/bpf_types.h>
2216 #undef BPF_PROG_TYPE
2217 #undef BPF_MAP_TYPE
2218 #undef BPF_LINK_TYPE
2219
2220 extern const struct bpf_prog_ops bpf_offload_prog_ops;
2221 extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops;
2222 extern const struct bpf_verifier_ops xdp_analyzer_ops;
2223
2224 struct bpf_prog *bpf_prog_get(u32 ufd);
2225 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
2226 bool attach_drv);
2227 void bpf_prog_add(struct bpf_prog *prog, int i);
2228 void bpf_prog_sub(struct bpf_prog *prog, int i);
2229 void bpf_prog_inc(struct bpf_prog *prog);
2230 struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog);
2231 void bpf_prog_put(struct bpf_prog *prog);
2232
2233 void bpf_prog_free_id(struct bpf_prog *prog);
2234 void bpf_map_free_id(struct bpf_map *map);
2235
2236 struct btf_field *btf_record_find(const struct btf_record *rec,
2237 u32 offset, u32 field_mask);
2238 void btf_record_free(struct btf_record *rec);
2239 void bpf_map_free_record(struct bpf_map *map);
2240 struct btf_record *btf_record_dup(const struct btf_record *rec);
2241 bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b);
2242 void bpf_obj_free_timer(const struct btf_record *rec, void *obj);
2243 void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj);
2244 void bpf_obj_free_fields(const struct btf_record *rec, void *obj);
2245 void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu);
2246
2247 struct bpf_map *bpf_map_get(u32 ufd);
2248 struct bpf_map *bpf_map_get_with_uref(u32 ufd);
2249
__bpf_map_get(struct fd f)2250 static inline struct bpf_map *__bpf_map_get(struct fd f)
2251 {
2252 if (fd_empty(f))
2253 return ERR_PTR(-EBADF);
2254 if (unlikely(fd_file(f)->f_op != &bpf_map_fops))
2255 return ERR_PTR(-EINVAL);
2256 return fd_file(f)->private_data;
2257 }
2258
2259 void bpf_map_inc(struct bpf_map *map);
2260 void bpf_map_inc_with_uref(struct bpf_map *map);
2261 struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref);
2262 struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map);
2263 void bpf_map_put_with_uref(struct bpf_map *map);
2264 void bpf_map_put(struct bpf_map *map);
2265 void *bpf_map_area_alloc(u64 size, int numa_node);
2266 void *bpf_map_area_mmapable_alloc(u64 size, int numa_node);
2267 void bpf_map_area_free(void *base);
2268 bool bpf_map_write_active(const struct bpf_map *map);
2269 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);
2270 int generic_map_lookup_batch(struct bpf_map *map,
2271 const union bpf_attr *attr,
2272 union bpf_attr __user *uattr);
2273 int generic_map_update_batch(struct bpf_map *map, struct file *map_file,
2274 const union bpf_attr *attr,
2275 union bpf_attr __user *uattr);
2276 int generic_map_delete_batch(struct bpf_map *map,
2277 const union bpf_attr *attr,
2278 union bpf_attr __user *uattr);
2279 struct bpf_map *bpf_map_get_curr_or_next(u32 *id);
2280 struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id);
2281
2282 int bpf_map_alloc_pages(const struct bpf_map *map, gfp_t gfp, int nid,
2283 unsigned long nr_pages, struct page **page_array);
2284 #ifdef CONFIG_MEMCG
2285 void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
2286 int node);
2287 void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags);
2288 void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size,
2289 gfp_t flags);
2290 void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,
2291 size_t align, gfp_t flags);
2292 #else
2293 /*
2294 * These specialized allocators have to be macros for their allocations to be
2295 * accounted separately (to have separate alloc_tag).
2296 */
2297 #define bpf_map_kmalloc_node(_map, _size, _flags, _node) \
2298 kmalloc_node(_size, _flags, _node)
2299 #define bpf_map_kzalloc(_map, _size, _flags) \
2300 kzalloc(_size, _flags)
2301 #define bpf_map_kvcalloc(_map, _n, _size, _flags) \
2302 kvcalloc(_n, _size, _flags)
2303 #define bpf_map_alloc_percpu(_map, _size, _align, _flags) \
2304 __alloc_percpu_gfp(_size, _align, _flags)
2305 #endif
2306
2307 static inline int
bpf_map_init_elem_count(struct bpf_map * map)2308 bpf_map_init_elem_count(struct bpf_map *map)
2309 {
2310 size_t size = sizeof(*map->elem_count), align = size;
2311 gfp_t flags = GFP_USER | __GFP_NOWARN;
2312
2313 map->elem_count = bpf_map_alloc_percpu(map, size, align, flags);
2314 if (!map->elem_count)
2315 return -ENOMEM;
2316
2317 return 0;
2318 }
2319
2320 static inline void
bpf_map_free_elem_count(struct bpf_map * map)2321 bpf_map_free_elem_count(struct bpf_map *map)
2322 {
2323 free_percpu(map->elem_count);
2324 }
2325
bpf_map_inc_elem_count(struct bpf_map * map)2326 static inline void bpf_map_inc_elem_count(struct bpf_map *map)
2327 {
2328 this_cpu_inc(*map->elem_count);
2329 }
2330
bpf_map_dec_elem_count(struct bpf_map * map)2331 static inline void bpf_map_dec_elem_count(struct bpf_map *map)
2332 {
2333 this_cpu_dec(*map->elem_count);
2334 }
2335
2336 extern int sysctl_unprivileged_bpf_disabled;
2337
2338 bool bpf_token_capable(const struct bpf_token *token, int cap);
2339
bpf_allow_ptr_leaks(const struct bpf_token * token)2340 static inline bool bpf_allow_ptr_leaks(const struct bpf_token *token)
2341 {
2342 return bpf_token_capable(token, CAP_PERFMON);
2343 }
2344
bpf_allow_uninit_stack(const struct bpf_token * token)2345 static inline bool bpf_allow_uninit_stack(const struct bpf_token *token)
2346 {
2347 return bpf_token_capable(token, CAP_PERFMON);
2348 }
2349
bpf_bypass_spec_v1(const struct bpf_token * token)2350 static inline bool bpf_bypass_spec_v1(const struct bpf_token *token)
2351 {
2352 return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON);
2353 }
2354
bpf_bypass_spec_v4(const struct bpf_token * token)2355 static inline bool bpf_bypass_spec_v4(const struct bpf_token *token)
2356 {
2357 return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON);
2358 }
2359
2360 int bpf_map_new_fd(struct bpf_map *map, int flags);
2361 int bpf_prog_new_fd(struct bpf_prog *prog);
2362
2363 void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
2364 const struct bpf_link_ops *ops, struct bpf_prog *prog);
2365 int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer);
2366 int bpf_link_settle(struct bpf_link_primer *primer);
2367 void bpf_link_cleanup(struct bpf_link_primer *primer);
2368 void bpf_link_inc(struct bpf_link *link);
2369 struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link);
2370 void bpf_link_put(struct bpf_link *link);
2371 int bpf_link_new_fd(struct bpf_link *link);
2372 struct bpf_link *bpf_link_get_from_fd(u32 ufd);
2373 struct bpf_link *bpf_link_get_curr_or_next(u32 *id);
2374
2375 void bpf_token_inc(struct bpf_token *token);
2376 void bpf_token_put(struct bpf_token *token);
2377 int bpf_token_create(union bpf_attr *attr);
2378 struct bpf_token *bpf_token_get_from_fd(u32 ufd);
2379
2380 bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd);
2381 bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type);
2382 bool bpf_token_allow_prog_type(const struct bpf_token *token,
2383 enum bpf_prog_type prog_type,
2384 enum bpf_attach_type attach_type);
2385
2386 int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname);
2387 int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags);
2388 struct inode *bpf_get_inode(struct super_block *sb, const struct inode *dir,
2389 umode_t mode);
2390
2391 #define BPF_ITER_FUNC_PREFIX "bpf_iter_"
2392 #define DEFINE_BPF_ITER_FUNC(target, args...) \
2393 extern int bpf_iter_ ## target(args); \
2394 int __init bpf_iter_ ## target(args) { return 0; }
2395
2396 /*
2397 * The task type of iterators.
2398 *
2399 * For BPF task iterators, they can be parameterized with various
2400 * parameters to visit only some of tasks.
2401 *
2402 * BPF_TASK_ITER_ALL (default)
2403 * Iterate over resources of every task.
2404 *
2405 * BPF_TASK_ITER_TID
2406 * Iterate over resources of a task/tid.
2407 *
2408 * BPF_TASK_ITER_TGID
2409 * Iterate over resources of every task of a process / task group.
2410 */
2411 enum bpf_iter_task_type {
2412 BPF_TASK_ITER_ALL = 0,
2413 BPF_TASK_ITER_TID,
2414 BPF_TASK_ITER_TGID,
2415 };
2416
2417 struct bpf_iter_aux_info {
2418 /* for map_elem iter */
2419 struct bpf_map *map;
2420
2421 /* for cgroup iter */
2422 struct {
2423 struct cgroup *start; /* starting cgroup */
2424 enum bpf_cgroup_iter_order order;
2425 } cgroup;
2426 struct {
2427 enum bpf_iter_task_type type;
2428 u32 pid;
2429 } task;
2430 };
2431
2432 typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog,
2433 union bpf_iter_link_info *linfo,
2434 struct bpf_iter_aux_info *aux);
2435 typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux);
2436 typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux,
2437 struct seq_file *seq);
2438 typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux,
2439 struct bpf_link_info *info);
2440 typedef const struct bpf_func_proto *
2441 (*bpf_iter_get_func_proto_t)(enum bpf_func_id func_id,
2442 const struct bpf_prog *prog);
2443
2444 enum bpf_iter_feature {
2445 BPF_ITER_RESCHED = BIT(0),
2446 };
2447
2448 #define BPF_ITER_CTX_ARG_MAX 2
2449 struct bpf_iter_reg {
2450 const char *target;
2451 bpf_iter_attach_target_t attach_target;
2452 bpf_iter_detach_target_t detach_target;
2453 bpf_iter_show_fdinfo_t show_fdinfo;
2454 bpf_iter_fill_link_info_t fill_link_info;
2455 bpf_iter_get_func_proto_t get_func_proto;
2456 u32 ctx_arg_info_size;
2457 u32 feature;
2458 struct bpf_ctx_arg_aux ctx_arg_info[BPF_ITER_CTX_ARG_MAX];
2459 const struct bpf_iter_seq_info *seq_info;
2460 };
2461
2462 struct bpf_iter_meta {
2463 __bpf_md_ptr(struct seq_file *, seq);
2464 u64 session_id;
2465 u64 seq_num;
2466 };
2467
2468 struct bpf_iter__bpf_map_elem {
2469 __bpf_md_ptr(struct bpf_iter_meta *, meta);
2470 __bpf_md_ptr(struct bpf_map *, map);
2471 __bpf_md_ptr(void *, key);
2472 __bpf_md_ptr(void *, value);
2473 };
2474
2475 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info);
2476 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info);
2477 bool bpf_iter_prog_supported(struct bpf_prog *prog);
2478 const struct bpf_func_proto *
2479 bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
2480 int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_prog *prog);
2481 int bpf_iter_new_fd(struct bpf_link *link);
2482 bool bpf_link_is_iter(struct bpf_link *link);
2483 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop);
2484 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx);
2485 void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux,
2486 struct seq_file *seq);
2487 int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux,
2488 struct bpf_link_info *info);
2489
2490 int map_set_for_each_callback_args(struct bpf_verifier_env *env,
2491 struct bpf_func_state *caller,
2492 struct bpf_func_state *callee);
2493
2494 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value);
2495 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value);
2496 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
2497 u64 flags);
2498 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
2499 u64 flags);
2500
2501 int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value);
2502
2503 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
2504 void *key, void *value, u64 map_flags);
2505 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
2506 int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file,
2507 void *key, void *value, u64 map_flags);
2508 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
2509
2510 int bpf_get_file_flag(int flags);
2511 int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size,
2512 size_t actual_size);
2513
2514 /* verify correctness of eBPF program */
2515 int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size);
2516
2517 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
2518 void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
2519 #endif
2520
2521 struct btf *bpf_get_btf_vmlinux(void);
2522
2523 /* Map specifics */
2524 struct xdp_frame;
2525 struct sk_buff;
2526 struct bpf_dtab_netdev;
2527 struct bpf_cpu_map_entry;
2528
2529 void __dev_flush(struct list_head *flush_list);
2530 int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
2531 struct net_device *dev_rx);
2532 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
2533 struct net_device *dev_rx);
2534 int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,
2535 struct bpf_map *map, bool exclude_ingress);
2536 int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,
2537 struct bpf_prog *xdp_prog);
2538 int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,
2539 struct bpf_prog *xdp_prog, struct bpf_map *map,
2540 bool exclude_ingress);
2541
2542 void __cpu_map_flush(struct list_head *flush_list);
2543 int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf,
2544 struct net_device *dev_rx);
2545 int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,
2546 struct sk_buff *skb);
2547
2548 /* Return map's numa specified by userspace */
bpf_map_attr_numa_node(const union bpf_attr * attr)2549 static inline int bpf_map_attr_numa_node(const union bpf_attr *attr)
2550 {
2551 return (attr->map_flags & BPF_F_NUMA_NODE) ?
2552 attr->numa_node : NUMA_NO_NODE;
2553 }
2554
2555 struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type);
2556 int array_map_alloc_check(union bpf_attr *attr);
2557
2558 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
2559 union bpf_attr __user *uattr);
2560 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
2561 union bpf_attr __user *uattr);
2562 int bpf_prog_test_run_tracing(struct bpf_prog *prog,
2563 const union bpf_attr *kattr,
2564 union bpf_attr __user *uattr);
2565 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
2566 const union bpf_attr *kattr,
2567 union bpf_attr __user *uattr);
2568 int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
2569 const union bpf_attr *kattr,
2570 union bpf_attr __user *uattr);
2571 int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
2572 const union bpf_attr *kattr,
2573 union bpf_attr __user *uattr);
2574 int bpf_prog_test_run_nf(struct bpf_prog *prog,
2575 const union bpf_attr *kattr,
2576 union bpf_attr __user *uattr);
2577 bool btf_ctx_access(int off, int size, enum bpf_access_type type,
2578 const struct bpf_prog *prog,
2579 struct bpf_insn_access_aux *info);
2580
bpf_tracing_ctx_access(int off,int size,enum bpf_access_type type)2581 static inline bool bpf_tracing_ctx_access(int off, int size,
2582 enum bpf_access_type type)
2583 {
2584 if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
2585 return false;
2586 if (type != BPF_READ)
2587 return false;
2588 if (off % size != 0)
2589 return false;
2590 return true;
2591 }
2592
bpf_tracing_btf_ctx_access(int off,int size,enum bpf_access_type type,const struct bpf_prog * prog,struct bpf_insn_access_aux * info)2593 static inline bool bpf_tracing_btf_ctx_access(int off, int size,
2594 enum bpf_access_type type,
2595 const struct bpf_prog *prog,
2596 struct bpf_insn_access_aux *info)
2597 {
2598 if (!bpf_tracing_ctx_access(off, size, type))
2599 return false;
2600 return btf_ctx_access(off, size, type, prog, info);
2601 }
2602
2603 int btf_struct_access(struct bpf_verifier_log *log,
2604 const struct bpf_reg_state *reg,
2605 int off, int size, enum bpf_access_type atype,
2606 u32 *next_btf_id, enum bpf_type_flag *flag, const char **field_name);
2607 bool btf_struct_ids_match(struct bpf_verifier_log *log,
2608 const struct btf *btf, u32 id, int off,
2609 const struct btf *need_btf, u32 need_type_id,
2610 bool strict);
2611
2612 int btf_distill_func_proto(struct bpf_verifier_log *log,
2613 struct btf *btf,
2614 const struct btf_type *func_proto,
2615 const char *func_name,
2616 struct btf_func_model *m);
2617
2618 struct bpf_reg_state;
2619 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog);
2620 int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog,
2621 struct btf *btf, const struct btf_type *t);
2622 const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt,
2623 int comp_idx, const char *tag_key);
2624 int btf_find_next_decl_tag(const struct btf *btf, const struct btf_type *pt,
2625 int comp_idx, const char *tag_key, int last_id);
2626
2627 struct bpf_prog *bpf_prog_by_id(u32 id);
2628 struct bpf_link *bpf_link_by_id(u32 id);
2629
2630 const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id,
2631 const struct bpf_prog *prog);
2632 void bpf_task_storage_free(struct task_struct *task);
2633 void bpf_cgrp_storage_free(struct cgroup *cgroup);
2634 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);
2635 const struct btf_func_model *
2636 bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
2637 const struct bpf_insn *insn);
2638 int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
2639 u16 btf_fd_idx, u8 **func_addr);
2640
2641 struct bpf_core_ctx {
2642 struct bpf_verifier_log *log;
2643 const struct btf *btf;
2644 };
2645
2646 bool btf_nested_type_is_trusted(struct bpf_verifier_log *log,
2647 const struct bpf_reg_state *reg,
2648 const char *field_name, u32 btf_id, const char *suffix);
2649
2650 bool btf_type_ids_nocast_alias(struct bpf_verifier_log *log,
2651 const struct btf *reg_btf, u32 reg_id,
2652 const struct btf *arg_btf, u32 arg_id);
2653
2654 int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo,
2655 int relo_idx, void *insn);
2656
unprivileged_ebpf_enabled(void)2657 static inline bool unprivileged_ebpf_enabled(void)
2658 {
2659 return !sysctl_unprivileged_bpf_disabled;
2660 }
2661
2662 /* Not all bpf prog type has the bpf_ctx.
2663 * For the bpf prog type that has initialized the bpf_ctx,
2664 * this function can be used to decide if a kernel function
2665 * is called by a bpf program.
2666 */
has_current_bpf_ctx(void)2667 static inline bool has_current_bpf_ctx(void)
2668 {
2669 return !!current->bpf_ctx;
2670 }
2671
2672 void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog);
2673
2674 void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
2675 enum bpf_dynptr_type type, u32 offset, u32 size);
2676 void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
2677 void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);
2678
2679 #else /* !CONFIG_BPF_SYSCALL */
bpf_prog_get(u32 ufd)2680 static inline struct bpf_prog *bpf_prog_get(u32 ufd)
2681 {
2682 return ERR_PTR(-EOPNOTSUPP);
2683 }
2684
bpf_prog_get_type_dev(u32 ufd,enum bpf_prog_type type,bool attach_drv)2685 static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd,
2686 enum bpf_prog_type type,
2687 bool attach_drv)
2688 {
2689 return ERR_PTR(-EOPNOTSUPP);
2690 }
2691
bpf_prog_add(struct bpf_prog * prog,int i)2692 static inline void bpf_prog_add(struct bpf_prog *prog, int i)
2693 {
2694 }
2695
bpf_prog_sub(struct bpf_prog * prog,int i)2696 static inline void bpf_prog_sub(struct bpf_prog *prog, int i)
2697 {
2698 }
2699
bpf_prog_put(struct bpf_prog * prog)2700 static inline void bpf_prog_put(struct bpf_prog *prog)
2701 {
2702 }
2703
bpf_prog_inc(struct bpf_prog * prog)2704 static inline void bpf_prog_inc(struct bpf_prog *prog)
2705 {
2706 }
2707
2708 static inline struct bpf_prog *__must_check
bpf_prog_inc_not_zero(struct bpf_prog * prog)2709 bpf_prog_inc_not_zero(struct bpf_prog *prog)
2710 {
2711 return ERR_PTR(-EOPNOTSUPP);
2712 }
2713
bpf_link_init(struct bpf_link * link,enum bpf_link_type type,const struct bpf_link_ops * ops,struct bpf_prog * prog)2714 static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
2715 const struct bpf_link_ops *ops,
2716 struct bpf_prog *prog)
2717 {
2718 }
2719
bpf_link_prime(struct bpf_link * link,struct bpf_link_primer * primer)2720 static inline int bpf_link_prime(struct bpf_link *link,
2721 struct bpf_link_primer *primer)
2722 {
2723 return -EOPNOTSUPP;
2724 }
2725
bpf_link_settle(struct bpf_link_primer * primer)2726 static inline int bpf_link_settle(struct bpf_link_primer *primer)
2727 {
2728 return -EOPNOTSUPP;
2729 }
2730
bpf_link_cleanup(struct bpf_link_primer * primer)2731 static inline void bpf_link_cleanup(struct bpf_link_primer *primer)
2732 {
2733 }
2734
bpf_link_inc(struct bpf_link * link)2735 static inline void bpf_link_inc(struct bpf_link *link)
2736 {
2737 }
2738
bpf_link_inc_not_zero(struct bpf_link * link)2739 static inline struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link)
2740 {
2741 return NULL;
2742 }
2743
bpf_link_put(struct bpf_link * link)2744 static inline void bpf_link_put(struct bpf_link *link)
2745 {
2746 }
2747
bpf_obj_get_user(const char __user * pathname,int flags)2748 static inline int bpf_obj_get_user(const char __user *pathname, int flags)
2749 {
2750 return -EOPNOTSUPP;
2751 }
2752
bpf_token_capable(const struct bpf_token * token,int cap)2753 static inline bool bpf_token_capable(const struct bpf_token *token, int cap)
2754 {
2755 return capable(cap) || (cap != CAP_SYS_ADMIN && capable(CAP_SYS_ADMIN));
2756 }
2757
bpf_token_inc(struct bpf_token * token)2758 static inline void bpf_token_inc(struct bpf_token *token)
2759 {
2760 }
2761
bpf_token_put(struct bpf_token * token)2762 static inline void bpf_token_put(struct bpf_token *token)
2763 {
2764 }
2765
bpf_token_get_from_fd(u32 ufd)2766 static inline struct bpf_token *bpf_token_get_from_fd(u32 ufd)
2767 {
2768 return ERR_PTR(-EOPNOTSUPP);
2769 }
2770
__dev_flush(struct list_head * flush_list)2771 static inline void __dev_flush(struct list_head *flush_list)
2772 {
2773 }
2774
2775 struct xdp_frame;
2776 struct bpf_dtab_netdev;
2777 struct bpf_cpu_map_entry;
2778
2779 static inline
dev_xdp_enqueue(struct net_device * dev,struct xdp_frame * xdpf,struct net_device * dev_rx)2780 int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
2781 struct net_device *dev_rx)
2782 {
2783 return 0;
2784 }
2785
2786 static inline
dev_map_enqueue(struct bpf_dtab_netdev * dst,struct xdp_frame * xdpf,struct net_device * dev_rx)2787 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
2788 struct net_device *dev_rx)
2789 {
2790 return 0;
2791 }
2792
2793 static inline
dev_map_enqueue_multi(struct xdp_frame * xdpf,struct net_device * dev_rx,struct bpf_map * map,bool exclude_ingress)2794 int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,
2795 struct bpf_map *map, bool exclude_ingress)
2796 {
2797 return 0;
2798 }
2799
2800 struct sk_buff;
2801
dev_map_generic_redirect(struct bpf_dtab_netdev * dst,struct sk_buff * skb,struct bpf_prog * xdp_prog)2802 static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst,
2803 struct sk_buff *skb,
2804 struct bpf_prog *xdp_prog)
2805 {
2806 return 0;
2807 }
2808
2809 static inline
dev_map_redirect_multi(struct net_device * dev,struct sk_buff * skb,struct bpf_prog * xdp_prog,struct bpf_map * map,bool exclude_ingress)2810 int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,
2811 struct bpf_prog *xdp_prog, struct bpf_map *map,
2812 bool exclude_ingress)
2813 {
2814 return 0;
2815 }
2816
__cpu_map_flush(struct list_head * flush_list)2817 static inline void __cpu_map_flush(struct list_head *flush_list)
2818 {
2819 }
2820
cpu_map_enqueue(struct bpf_cpu_map_entry * rcpu,struct xdp_frame * xdpf,struct net_device * dev_rx)2821 static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu,
2822 struct xdp_frame *xdpf,
2823 struct net_device *dev_rx)
2824 {
2825 return 0;
2826 }
2827
cpu_map_generic_redirect(struct bpf_cpu_map_entry * rcpu,struct sk_buff * skb)2828 static inline int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,
2829 struct sk_buff *skb)
2830 {
2831 return -EOPNOTSUPP;
2832 }
2833
bpf_prog_get_type_path(const char * name,enum bpf_prog_type type)2834 static inline struct bpf_prog *bpf_prog_get_type_path(const char *name,
2835 enum bpf_prog_type type)
2836 {
2837 return ERR_PTR(-EOPNOTSUPP);
2838 }
2839
bpf_prog_test_run_xdp(struct bpf_prog * prog,const union bpf_attr * kattr,union bpf_attr __user * uattr)2840 static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog,
2841 const union bpf_attr *kattr,
2842 union bpf_attr __user *uattr)
2843 {
2844 return -ENOTSUPP;
2845 }
2846
bpf_prog_test_run_skb(struct bpf_prog * prog,const union bpf_attr * kattr,union bpf_attr __user * uattr)2847 static inline int bpf_prog_test_run_skb(struct bpf_prog *prog,
2848 const union bpf_attr *kattr,
2849 union bpf_attr __user *uattr)
2850 {
2851 return -ENOTSUPP;
2852 }
2853
bpf_prog_test_run_tracing(struct bpf_prog * prog,const union bpf_attr * kattr,union bpf_attr __user * uattr)2854 static inline int bpf_prog_test_run_tracing(struct bpf_prog *prog,
2855 const union bpf_attr *kattr,
2856 union bpf_attr __user *uattr)
2857 {
2858 return -ENOTSUPP;
2859 }
2860
bpf_prog_test_run_flow_dissector(struct bpf_prog * prog,const union bpf_attr * kattr,union bpf_attr __user * uattr)2861 static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
2862 const union bpf_attr *kattr,
2863 union bpf_attr __user *uattr)
2864 {
2865 return -ENOTSUPP;
2866 }
2867
bpf_prog_test_run_sk_lookup(struct bpf_prog * prog,const union bpf_attr * kattr,union bpf_attr __user * uattr)2868 static inline int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
2869 const union bpf_attr *kattr,
2870 union bpf_attr __user *uattr)
2871 {
2872 return -ENOTSUPP;
2873 }
2874
bpf_map_put(struct bpf_map * map)2875 static inline void bpf_map_put(struct bpf_map *map)
2876 {
2877 }
2878
bpf_prog_by_id(u32 id)2879 static inline struct bpf_prog *bpf_prog_by_id(u32 id)
2880 {
2881 return ERR_PTR(-ENOTSUPP);
2882 }
2883
btf_struct_access(struct bpf_verifier_log * log,const struct bpf_reg_state * reg,int off,int size,enum bpf_access_type atype,u32 * next_btf_id,enum bpf_type_flag * flag,const char ** field_name)2884 static inline int btf_struct_access(struct bpf_verifier_log *log,
2885 const struct bpf_reg_state *reg,
2886 int off, int size, enum bpf_access_type atype,
2887 u32 *next_btf_id, enum bpf_type_flag *flag,
2888 const char **field_name)
2889 {
2890 return -EACCES;
2891 }
2892
2893 static inline const struct bpf_func_proto *
bpf_base_func_proto(enum bpf_func_id func_id,const struct bpf_prog * prog)2894 bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
2895 {
2896 return NULL;
2897 }
2898
bpf_task_storage_free(struct task_struct * task)2899 static inline void bpf_task_storage_free(struct task_struct *task)
2900 {
2901 }
2902
bpf_prog_has_kfunc_call(const struct bpf_prog * prog)2903 static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
2904 {
2905 return false;
2906 }
2907
2908 static inline const struct btf_func_model *
bpf_jit_find_kfunc_model(const struct bpf_prog * prog,const struct bpf_insn * insn)2909 bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
2910 const struct bpf_insn *insn)
2911 {
2912 return NULL;
2913 }
2914
2915 static inline int
bpf_get_kfunc_addr(const struct bpf_prog * prog,u32 func_id,u16 btf_fd_idx,u8 ** func_addr)2916 bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
2917 u16 btf_fd_idx, u8 **func_addr)
2918 {
2919 return -ENOTSUPP;
2920 }
2921
unprivileged_ebpf_enabled(void)2922 static inline bool unprivileged_ebpf_enabled(void)
2923 {
2924 return false;
2925 }
2926
has_current_bpf_ctx(void)2927 static inline bool has_current_bpf_ctx(void)
2928 {
2929 return false;
2930 }
2931
bpf_prog_inc_misses_counter(struct bpf_prog * prog)2932 static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog)
2933 {
2934 }
2935
bpf_cgrp_storage_free(struct cgroup * cgroup)2936 static inline void bpf_cgrp_storage_free(struct cgroup *cgroup)
2937 {
2938 }
2939
bpf_dynptr_init(struct bpf_dynptr_kern * ptr,void * data,enum bpf_dynptr_type type,u32 offset,u32 size)2940 static inline void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
2941 enum bpf_dynptr_type type, u32 offset, u32 size)
2942 {
2943 }
2944
bpf_dynptr_set_null(struct bpf_dynptr_kern * ptr)2945 static inline void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr)
2946 {
2947 }
2948
bpf_dynptr_set_rdonly(struct bpf_dynptr_kern * ptr)2949 static inline void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr)
2950 {
2951 }
2952 #endif /* CONFIG_BPF_SYSCALL */
2953
2954 static __always_inline int
bpf_probe_read_kernel_common(void * dst,u32 size,const void * unsafe_ptr)2955 bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr)
2956 {
2957 int ret = -EFAULT;
2958
2959 if (IS_ENABLED(CONFIG_BPF_EVENTS))
2960 ret = copy_from_kernel_nofault(dst, unsafe_ptr, size);
2961 if (unlikely(ret < 0))
2962 memset(dst, 0, size);
2963 return ret;
2964 }
2965
2966 void __bpf_free_used_btfs(struct btf_mod_pair *used_btfs, u32 len);
2967
bpf_prog_get_type(u32 ufd,enum bpf_prog_type type)2968 static inline struct bpf_prog *bpf_prog_get_type(u32 ufd,
2969 enum bpf_prog_type type)
2970 {
2971 return bpf_prog_get_type_dev(ufd, type, false);
2972 }
2973
2974 void __bpf_free_used_maps(struct bpf_prog_aux *aux,
2975 struct bpf_map **used_maps, u32 len);
2976
2977 bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool);
2978
2979 int bpf_prog_offload_compile(struct bpf_prog *prog);
2980 void bpf_prog_dev_bound_destroy(struct bpf_prog *prog);
2981 int bpf_prog_offload_info_fill(struct bpf_prog_info *info,
2982 struct bpf_prog *prog);
2983
2984 int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map);
2985
2986 int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value);
2987 int bpf_map_offload_update_elem(struct bpf_map *map,
2988 void *key, void *value, u64 flags);
2989 int bpf_map_offload_delete_elem(struct bpf_map *map, void *key);
2990 int bpf_map_offload_get_next_key(struct bpf_map *map,
2991 void *key, void *next_key);
2992
2993 bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map);
2994
2995 struct bpf_offload_dev *
2996 bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv);
2997 void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev);
2998 void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev);
2999 int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
3000 struct net_device *netdev);
3001 void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
3002 struct net_device *netdev);
3003 bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev);
3004
3005 void unpriv_ebpf_notify(int new_state);
3006
3007 #if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL)
3008 int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,
3009 struct bpf_prog_aux *prog_aux);
3010 void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id);
3011 int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr);
3012 int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog);
3013 void bpf_dev_bound_netdev_unregister(struct net_device *dev);
3014
bpf_prog_is_dev_bound(const struct bpf_prog_aux * aux)3015 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
3016 {
3017 return aux->dev_bound;
3018 }
3019
bpf_prog_is_offloaded(const struct bpf_prog_aux * aux)3020 static inline bool bpf_prog_is_offloaded(const struct bpf_prog_aux *aux)
3021 {
3022 return aux->offload_requested;
3023 }
3024
3025 bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs);
3026
bpf_map_is_offloaded(struct bpf_map * map)3027 static inline bool bpf_map_is_offloaded(struct bpf_map *map)
3028 {
3029 return unlikely(map->ops == &bpf_map_offload_ops);
3030 }
3031
3032 struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr);
3033 void bpf_map_offload_map_free(struct bpf_map *map);
3034 u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map);
3035 int bpf_prog_test_run_syscall(struct bpf_prog *prog,
3036 const union bpf_attr *kattr,
3037 union bpf_attr __user *uattr);
3038
3039 int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog);
3040 int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype);
3041 int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, u64 flags);
3042 int sock_map_bpf_prog_query(const union bpf_attr *attr,
3043 union bpf_attr __user *uattr);
3044 int sock_map_link_create(const union bpf_attr *attr, struct bpf_prog *prog);
3045
3046 void sock_map_unhash(struct sock *sk);
3047 void sock_map_destroy(struct sock *sk);
3048 void sock_map_close(struct sock *sk, long timeout);
3049 #else
bpf_dev_bound_kfunc_check(struct bpf_verifier_log * log,struct bpf_prog_aux * prog_aux)3050 static inline int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,
3051 struct bpf_prog_aux *prog_aux)
3052 {
3053 return -EOPNOTSUPP;
3054 }
3055
bpf_dev_bound_resolve_kfunc(struct bpf_prog * prog,u32 func_id)3056 static inline void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog,
3057 u32 func_id)
3058 {
3059 return NULL;
3060 }
3061
bpf_prog_dev_bound_init(struct bpf_prog * prog,union bpf_attr * attr)3062 static inline int bpf_prog_dev_bound_init(struct bpf_prog *prog,
3063 union bpf_attr *attr)
3064 {
3065 return -EOPNOTSUPP;
3066 }
3067
bpf_prog_dev_bound_inherit(struct bpf_prog * new_prog,struct bpf_prog * old_prog)3068 static inline int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog,
3069 struct bpf_prog *old_prog)
3070 {
3071 return -EOPNOTSUPP;
3072 }
3073
bpf_dev_bound_netdev_unregister(struct net_device * dev)3074 static inline void bpf_dev_bound_netdev_unregister(struct net_device *dev)
3075 {
3076 }
3077
bpf_prog_is_dev_bound(const struct bpf_prog_aux * aux)3078 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
3079 {
3080 return false;
3081 }
3082
bpf_prog_is_offloaded(struct bpf_prog_aux * aux)3083 static inline bool bpf_prog_is_offloaded(struct bpf_prog_aux *aux)
3084 {
3085 return false;
3086 }
3087
bpf_prog_dev_bound_match(const struct bpf_prog * lhs,const struct bpf_prog * rhs)3088 static inline bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs)
3089 {
3090 return false;
3091 }
3092
bpf_map_is_offloaded(struct bpf_map * map)3093 static inline bool bpf_map_is_offloaded(struct bpf_map *map)
3094 {
3095 return false;
3096 }
3097
bpf_map_offload_map_alloc(union bpf_attr * attr)3098 static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
3099 {
3100 return ERR_PTR(-EOPNOTSUPP);
3101 }
3102
bpf_map_offload_map_free(struct bpf_map * map)3103 static inline void bpf_map_offload_map_free(struct bpf_map *map)
3104 {
3105 }
3106
bpf_map_offload_map_mem_usage(const struct bpf_map * map)3107 static inline u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map)
3108 {
3109 return 0;
3110 }
3111
bpf_prog_test_run_syscall(struct bpf_prog * prog,const union bpf_attr * kattr,union bpf_attr __user * uattr)3112 static inline int bpf_prog_test_run_syscall(struct bpf_prog *prog,
3113 const union bpf_attr *kattr,
3114 union bpf_attr __user *uattr)
3115 {
3116 return -ENOTSUPP;
3117 }
3118
3119 #ifdef CONFIG_BPF_SYSCALL
sock_map_get_from_fd(const union bpf_attr * attr,struct bpf_prog * prog)3120 static inline int sock_map_get_from_fd(const union bpf_attr *attr,
3121 struct bpf_prog *prog)
3122 {
3123 return -EINVAL;
3124 }
3125
sock_map_prog_detach(const union bpf_attr * attr,enum bpf_prog_type ptype)3126 static inline int sock_map_prog_detach(const union bpf_attr *attr,
3127 enum bpf_prog_type ptype)
3128 {
3129 return -EOPNOTSUPP;
3130 }
3131
sock_map_update_elem_sys(struct bpf_map * map,void * key,void * value,u64 flags)3132 static inline int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value,
3133 u64 flags)
3134 {
3135 return -EOPNOTSUPP;
3136 }
3137
sock_map_bpf_prog_query(const union bpf_attr * attr,union bpf_attr __user * uattr)3138 static inline int sock_map_bpf_prog_query(const union bpf_attr *attr,
3139 union bpf_attr __user *uattr)
3140 {
3141 return -EINVAL;
3142 }
3143
sock_map_link_create(const union bpf_attr * attr,struct bpf_prog * prog)3144 static inline int sock_map_link_create(const union bpf_attr *attr, struct bpf_prog *prog)
3145 {
3146 return -EOPNOTSUPP;
3147 }
3148 #endif /* CONFIG_BPF_SYSCALL */
3149 #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */
3150
3151 static __always_inline void
bpf_prog_inc_misses_counters(const struct bpf_prog_array * array)3152 bpf_prog_inc_misses_counters(const struct bpf_prog_array *array)
3153 {
3154 const struct bpf_prog_array_item *item;
3155 struct bpf_prog *prog;
3156
3157 if (unlikely(!array))
3158 return;
3159
3160 item = &array->items[0];
3161 while ((prog = READ_ONCE(item->prog))) {
3162 bpf_prog_inc_misses_counter(prog);
3163 item++;
3164 }
3165 }
3166
3167 #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL)
3168 void bpf_sk_reuseport_detach(struct sock *sk);
3169 int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key,
3170 void *value);
3171 int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key,
3172 void *value, u64 map_flags);
3173 #else
bpf_sk_reuseport_detach(struct sock * sk)3174 static inline void bpf_sk_reuseport_detach(struct sock *sk)
3175 {
3176 }
3177
3178 #ifdef CONFIG_BPF_SYSCALL
bpf_fd_reuseport_array_lookup_elem(struct bpf_map * map,void * key,void * value)3179 static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map,
3180 void *key, void *value)
3181 {
3182 return -EOPNOTSUPP;
3183 }
3184
bpf_fd_reuseport_array_update_elem(struct bpf_map * map,void * key,void * value,u64 map_flags)3185 static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map,
3186 void *key, void *value,
3187 u64 map_flags)
3188 {
3189 return -EOPNOTSUPP;
3190 }
3191 #endif /* CONFIG_BPF_SYSCALL */
3192 #endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */
3193
3194 /* verifier prototypes for helper functions called from eBPF programs */
3195 extern const struct bpf_func_proto bpf_map_lookup_elem_proto;
3196 extern const struct bpf_func_proto bpf_map_update_elem_proto;
3197 extern const struct bpf_func_proto bpf_map_delete_elem_proto;
3198 extern const struct bpf_func_proto bpf_map_push_elem_proto;
3199 extern const struct bpf_func_proto bpf_map_pop_elem_proto;
3200 extern const struct bpf_func_proto bpf_map_peek_elem_proto;
3201 extern const struct bpf_func_proto bpf_map_lookup_percpu_elem_proto;
3202
3203 extern const struct bpf_func_proto bpf_get_prandom_u32_proto;
3204 extern const struct bpf_func_proto bpf_get_smp_processor_id_proto;
3205 extern const struct bpf_func_proto bpf_get_numa_node_id_proto;
3206 extern const struct bpf_func_proto bpf_tail_call_proto;
3207 extern const struct bpf_func_proto bpf_ktime_get_ns_proto;
3208 extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto;
3209 extern const struct bpf_func_proto bpf_ktime_get_tai_ns_proto;
3210 extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto;
3211 extern const struct bpf_func_proto bpf_get_current_uid_gid_proto;
3212 extern const struct bpf_func_proto bpf_get_current_comm_proto;
3213 extern const struct bpf_func_proto bpf_get_stackid_proto;
3214 extern const struct bpf_func_proto bpf_get_stack_proto;
3215 extern const struct bpf_func_proto bpf_get_stack_sleepable_proto;
3216 extern const struct bpf_func_proto bpf_get_task_stack_proto;
3217 extern const struct bpf_func_proto bpf_get_task_stack_sleepable_proto;
3218 extern const struct bpf_func_proto bpf_get_stackid_proto_pe;
3219 extern const struct bpf_func_proto bpf_get_stack_proto_pe;
3220 extern const struct bpf_func_proto bpf_sock_map_update_proto;
3221 extern const struct bpf_func_proto bpf_sock_hash_update_proto;
3222 extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto;
3223 extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto;
3224 extern const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto;
3225 extern const struct bpf_func_proto bpf_current_task_under_cgroup_proto;
3226 extern const struct bpf_func_proto bpf_msg_redirect_hash_proto;
3227 extern const struct bpf_func_proto bpf_msg_redirect_map_proto;
3228 extern const struct bpf_func_proto bpf_sk_redirect_hash_proto;
3229 extern const struct bpf_func_proto bpf_sk_redirect_map_proto;
3230 extern const struct bpf_func_proto bpf_spin_lock_proto;
3231 extern const struct bpf_func_proto bpf_spin_unlock_proto;
3232 extern const struct bpf_func_proto bpf_get_local_storage_proto;
3233 extern const struct bpf_func_proto bpf_strtol_proto;
3234 extern const struct bpf_func_proto bpf_strtoul_proto;
3235 extern const struct bpf_func_proto bpf_tcp_sock_proto;
3236 extern const struct bpf_func_proto bpf_jiffies64_proto;
3237 extern const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto;
3238 extern const struct bpf_func_proto bpf_event_output_data_proto;
3239 extern const struct bpf_func_proto bpf_ringbuf_output_proto;
3240 extern const struct bpf_func_proto bpf_ringbuf_reserve_proto;
3241 extern const struct bpf_func_proto bpf_ringbuf_submit_proto;
3242 extern const struct bpf_func_proto bpf_ringbuf_discard_proto;
3243 extern const struct bpf_func_proto bpf_ringbuf_query_proto;
3244 extern const struct bpf_func_proto bpf_ringbuf_reserve_dynptr_proto;
3245 extern const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto;
3246 extern const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto;
3247 extern const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto;
3248 extern const struct bpf_func_proto bpf_skc_to_tcp_sock_proto;
3249 extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto;
3250 extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto;
3251 extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto;
3252 extern const struct bpf_func_proto bpf_skc_to_unix_sock_proto;
3253 extern const struct bpf_func_proto bpf_skc_to_mptcp_sock_proto;
3254 extern const struct bpf_func_proto bpf_copy_from_user_proto;
3255 extern const struct bpf_func_proto bpf_snprintf_btf_proto;
3256 extern const struct bpf_func_proto bpf_snprintf_proto;
3257 extern const struct bpf_func_proto bpf_per_cpu_ptr_proto;
3258 extern const struct bpf_func_proto bpf_this_cpu_ptr_proto;
3259 extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto;
3260 extern const struct bpf_func_proto bpf_sock_from_file_proto;
3261 extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto;
3262 extern const struct bpf_func_proto bpf_task_storage_get_recur_proto;
3263 extern const struct bpf_func_proto bpf_task_storage_get_proto;
3264 extern const struct bpf_func_proto bpf_task_storage_delete_recur_proto;
3265 extern const struct bpf_func_proto bpf_task_storage_delete_proto;
3266 extern const struct bpf_func_proto bpf_for_each_map_elem_proto;
3267 extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto;
3268 extern const struct bpf_func_proto bpf_sk_setsockopt_proto;
3269 extern const struct bpf_func_proto bpf_sk_getsockopt_proto;
3270 extern const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto;
3271 extern const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto;
3272 extern const struct bpf_func_proto bpf_find_vma_proto;
3273 extern const struct bpf_func_proto bpf_loop_proto;
3274 extern const struct bpf_func_proto bpf_copy_from_user_task_proto;
3275 extern const struct bpf_func_proto bpf_set_retval_proto;
3276 extern const struct bpf_func_proto bpf_get_retval_proto;
3277 extern const struct bpf_func_proto bpf_user_ringbuf_drain_proto;
3278 extern const struct bpf_func_proto bpf_cgrp_storage_get_proto;
3279 extern const struct bpf_func_proto bpf_cgrp_storage_delete_proto;
3280
3281 const struct bpf_func_proto *tracing_prog_func_proto(
3282 enum bpf_func_id func_id, const struct bpf_prog *prog);
3283
3284 /* Shared helpers among cBPF and eBPF. */
3285 void bpf_user_rnd_init_once(void);
3286 u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
3287 u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
3288
3289 #if defined(CONFIG_NET)
3290 bool bpf_sock_common_is_valid_access(int off, int size,
3291 enum bpf_access_type type,
3292 struct bpf_insn_access_aux *info);
3293 bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type,
3294 struct bpf_insn_access_aux *info);
3295 u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
3296 const struct bpf_insn *si,
3297 struct bpf_insn *insn_buf,
3298 struct bpf_prog *prog,
3299 u32 *target_size);
3300 int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags,
3301 struct bpf_dynptr *ptr);
3302 #else
bpf_sock_common_is_valid_access(int off,int size,enum bpf_access_type type,struct bpf_insn_access_aux * info)3303 static inline bool bpf_sock_common_is_valid_access(int off, int size,
3304 enum bpf_access_type type,
3305 struct bpf_insn_access_aux *info)
3306 {
3307 return false;
3308 }
bpf_sock_is_valid_access(int off,int size,enum bpf_access_type type,struct bpf_insn_access_aux * info)3309 static inline bool bpf_sock_is_valid_access(int off, int size,
3310 enum bpf_access_type type,
3311 struct bpf_insn_access_aux *info)
3312 {
3313 return false;
3314 }
bpf_sock_convert_ctx_access(enum bpf_access_type type,const struct bpf_insn * si,struct bpf_insn * insn_buf,struct bpf_prog * prog,u32 * target_size)3315 static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
3316 const struct bpf_insn *si,
3317 struct bpf_insn *insn_buf,
3318 struct bpf_prog *prog,
3319 u32 *target_size)
3320 {
3321 return 0;
3322 }
bpf_dynptr_from_skb_rdonly(struct __sk_buff * skb,u64 flags,struct bpf_dynptr * ptr)3323 static inline int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags,
3324 struct bpf_dynptr *ptr)
3325 {
3326 return -EOPNOTSUPP;
3327 }
3328 #endif
3329
3330 #ifdef CONFIG_INET
3331 struct sk_reuseport_kern {
3332 struct sk_buff *skb;
3333 struct sock *sk;
3334 struct sock *selected_sk;
3335 struct sock *migrating_sk;
3336 void *data_end;
3337 u32 hash;
3338 u32 reuseport_id;
3339 bool bind_inany;
3340 };
3341 bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
3342 struct bpf_insn_access_aux *info);
3343
3344 u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
3345 const struct bpf_insn *si,
3346 struct bpf_insn *insn_buf,
3347 struct bpf_prog *prog,
3348 u32 *target_size);
3349
3350 bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
3351 struct bpf_insn_access_aux *info);
3352
3353 u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
3354 const struct bpf_insn *si,
3355 struct bpf_insn *insn_buf,
3356 struct bpf_prog *prog,
3357 u32 *target_size);
3358 #else
bpf_tcp_sock_is_valid_access(int off,int size,enum bpf_access_type type,struct bpf_insn_access_aux * info)3359 static inline bool bpf_tcp_sock_is_valid_access(int off, int size,
3360 enum bpf_access_type type,
3361 struct bpf_insn_access_aux *info)
3362 {
3363 return false;
3364 }
3365
bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,const struct bpf_insn * si,struct bpf_insn * insn_buf,struct bpf_prog * prog,u32 * target_size)3366 static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
3367 const struct bpf_insn *si,
3368 struct bpf_insn *insn_buf,
3369 struct bpf_prog *prog,
3370 u32 *target_size)
3371 {
3372 return 0;
3373 }
bpf_xdp_sock_is_valid_access(int off,int size,enum bpf_access_type type,struct bpf_insn_access_aux * info)3374 static inline bool bpf_xdp_sock_is_valid_access(int off, int size,
3375 enum bpf_access_type type,
3376 struct bpf_insn_access_aux *info)
3377 {
3378 return false;
3379 }
3380
bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,const struct bpf_insn * si,struct bpf_insn * insn_buf,struct bpf_prog * prog,u32 * target_size)3381 static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
3382 const struct bpf_insn *si,
3383 struct bpf_insn *insn_buf,
3384 struct bpf_prog *prog,
3385 u32 *target_size)
3386 {
3387 return 0;
3388 }
3389 #endif /* CONFIG_INET */
3390
3391 enum bpf_text_poke_type {
3392 BPF_MOD_CALL,
3393 BPF_MOD_JUMP,
3394 };
3395
3396 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
3397 void *addr1, void *addr2);
3398
3399 void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,
3400 struct bpf_prog *new, struct bpf_prog *old);
3401
3402 void *bpf_arch_text_copy(void *dst, void *src, size_t len);
3403 int bpf_arch_text_invalidate(void *dst, size_t len);
3404
3405 struct btf_id_set;
3406 bool btf_id_set_contains(const struct btf_id_set *set, u32 id);
3407
3408 #define MAX_BPRINTF_VARARGS 12
3409 #define MAX_BPRINTF_BUF 1024
3410
3411 struct bpf_bprintf_data {
3412 u32 *bin_args;
3413 char *buf;
3414 bool get_bin_args;
3415 bool get_buf;
3416 };
3417
3418 int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args,
3419 u32 num_args, struct bpf_bprintf_data *data);
3420 void bpf_bprintf_cleanup(struct bpf_bprintf_data *data);
3421
3422 #ifdef CONFIG_BPF_LSM
3423 void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype);
3424 void bpf_cgroup_atype_put(int cgroup_atype);
3425 #else
bpf_cgroup_atype_get(u32 attach_btf_id,int cgroup_atype)3426 static inline void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) {}
bpf_cgroup_atype_put(int cgroup_atype)3427 static inline void bpf_cgroup_atype_put(int cgroup_atype) {}
3428 #endif /* CONFIG_BPF_LSM */
3429
3430 struct key;
3431
3432 #ifdef CONFIG_KEYS
3433 struct bpf_key {
3434 struct key *key;
3435 bool has_ref;
3436 };
3437 #endif /* CONFIG_KEYS */
3438
type_is_alloc(u32 type)3439 static inline bool type_is_alloc(u32 type)
3440 {
3441 return type & MEM_ALLOC;
3442 }
3443
bpf_memcg_flags(gfp_t flags)3444 static inline gfp_t bpf_memcg_flags(gfp_t flags)
3445 {
3446 if (memcg_bpf_enabled())
3447 return flags | __GFP_ACCOUNT;
3448 return flags;
3449 }
3450
bpf_is_subprog(const struct bpf_prog * prog)3451 static inline bool bpf_is_subprog(const struct bpf_prog *prog)
3452 {
3453 return prog->aux->func_idx != 0;
3454 }
3455
3456 #endif /* _LINUX_BPF_H */
3457