xref: /linux/kernel/bpf/arena.c (revision b90d77e5)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3 #include <linux/bpf.h>
4 #include <linux/btf.h>
5 #include <linux/err.h>
6 #include <linux/btf_ids.h>
7 #include <linux/vmalloc.h>
8 #include <linux/pagemap.h>
9 
10 /*
11  * bpf_arena is a sparsely populated shared memory region between bpf program and
12  * user space process.
13  *
14  * For example on x86-64 the values could be:
15  * user_vm_start 7f7d26200000     // picked by mmap()
16  * kern_vm_start ffffc90001e69000 // picked by get_vm_area()
17  * For user space all pointers within the arena are normal 8-byte addresses.
18  * In this example 7f7d26200000 is the address of the first page (pgoff=0).
19  * The bpf program will access it as: kern_vm_start + lower_32bit_of_user_ptr
20  * (u32)7f7d26200000 -> 26200000
21  * hence
22  * ffffc90001e69000 + 26200000 == ffffc90028069000 is "pgoff=0" within 4Gb
23  * kernel memory region.
24  *
25  * BPF JITs generate the following code to access arena:
26  *   mov eax, eax  // eax has lower 32-bit of user pointer
27  *   mov word ptr [rax + r12 + off], bx
28  * where r12 == kern_vm_start and off is s16.
29  * Hence allocate 4Gb + GUARD_SZ/2 on each side.
30  *
31  * Initially kernel vm_area and user vma are not populated.
32  * User space can fault-in any address which will insert the page
33  * into kernel and user vma.
34  * bpf program can allocate a page via bpf_arena_alloc_pages() kfunc
35  * which will insert it into kernel vm_area.
36  * The later fault-in from user space will populate that page into user vma.
37  */
38 
39 /* number of bytes addressable by LDX/STX insn with 16-bit 'off' field */
40 #define GUARD_SZ (1ull << sizeof_field(struct bpf_insn, off) * 8)
41 #define KERN_VM_SZ (SZ_4G + GUARD_SZ)
42 
43 struct bpf_arena {
44 	struct bpf_map map;
45 	u64 user_vm_start;
46 	u64 user_vm_end;
47 	struct vm_struct *kern_vm;
48 	struct maple_tree mt;
49 	struct list_head vma_list;
50 	struct mutex lock;
51 };
52 
bpf_arena_get_kern_vm_start(struct bpf_arena * arena)53 u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena)
54 {
55 	return arena ? (u64) (long) arena->kern_vm->addr + GUARD_SZ / 2 : 0;
56 }
57 
bpf_arena_get_user_vm_start(struct bpf_arena * arena)58 u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena)
59 {
60 	return arena ? arena->user_vm_start : 0;
61 }
62 
arena_map_peek_elem(struct bpf_map * map,void * value)63 static long arena_map_peek_elem(struct bpf_map *map, void *value)
64 {
65 	return -EOPNOTSUPP;
66 }
67 
arena_map_push_elem(struct bpf_map * map,void * value,u64 flags)68 static long arena_map_push_elem(struct bpf_map *map, void *value, u64 flags)
69 {
70 	return -EOPNOTSUPP;
71 }
72 
arena_map_pop_elem(struct bpf_map * map,void * value)73 static long arena_map_pop_elem(struct bpf_map *map, void *value)
74 {
75 	return -EOPNOTSUPP;
76 }
77 
arena_map_delete_elem(struct bpf_map * map,void * value)78 static long arena_map_delete_elem(struct bpf_map *map, void *value)
79 {
80 	return -EOPNOTSUPP;
81 }
82 
arena_map_get_next_key(struct bpf_map * map,void * key,void * next_key)83 static int arena_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
84 {
85 	return -EOPNOTSUPP;
86 }
87 
compute_pgoff(struct bpf_arena * arena,long uaddr)88 static long compute_pgoff(struct bpf_arena *arena, long uaddr)
89 {
90 	return (u32)(uaddr - (u32)arena->user_vm_start) >> PAGE_SHIFT;
91 }
92 
arena_map_alloc(union bpf_attr * attr)93 static struct bpf_map *arena_map_alloc(union bpf_attr *attr)
94 {
95 	struct vm_struct *kern_vm;
96 	int numa_node = bpf_map_attr_numa_node(attr);
97 	struct bpf_arena *arena;
98 	u64 vm_range;
99 	int err = -ENOMEM;
100 
101 	if (attr->key_size || attr->value_size || attr->max_entries == 0 ||
102 	    /* BPF_F_MMAPABLE must be set */
103 	    !(attr->map_flags & BPF_F_MMAPABLE) ||
104 	    /* No unsupported flags present */
105 	    (attr->map_flags & ~(BPF_F_SEGV_ON_FAULT | BPF_F_MMAPABLE | BPF_F_NO_USER_CONV)))
106 		return ERR_PTR(-EINVAL);
107 
108 	if (attr->map_extra & ~PAGE_MASK)
109 		/* If non-zero the map_extra is an expected user VMA start address */
110 		return ERR_PTR(-EINVAL);
111 
112 	vm_range = (u64)attr->max_entries * PAGE_SIZE;
113 	if (vm_range > SZ_4G)
114 		return ERR_PTR(-E2BIG);
115 
116 	if ((attr->map_extra >> 32) != ((attr->map_extra + vm_range - 1) >> 32))
117 		/* user vma must not cross 32-bit boundary */
118 		return ERR_PTR(-ERANGE);
119 
120 	kern_vm = get_vm_area(KERN_VM_SZ, VM_SPARSE | VM_USERMAP);
121 	if (!kern_vm)
122 		return ERR_PTR(-ENOMEM);
123 
124 	arena = bpf_map_area_alloc(sizeof(*arena), numa_node);
125 	if (!arena)
126 		goto err;
127 
128 	arena->kern_vm = kern_vm;
129 	arena->user_vm_start = attr->map_extra;
130 	if (arena->user_vm_start)
131 		arena->user_vm_end = arena->user_vm_start + vm_range;
132 
133 	INIT_LIST_HEAD(&arena->vma_list);
134 	bpf_map_init_from_attr(&arena->map, attr);
135 	mt_init_flags(&arena->mt, MT_FLAGS_ALLOC_RANGE);
136 	mutex_init(&arena->lock);
137 
138 	return &arena->map;
139 err:
140 	free_vm_area(kern_vm);
141 	return ERR_PTR(err);
142 }
143 
existing_page_cb(pte_t * ptep,unsigned long addr,void * data)144 static int existing_page_cb(pte_t *ptep, unsigned long addr, void *data)
145 {
146 	struct page *page;
147 	pte_t pte;
148 
149 	pte = ptep_get(ptep);
150 	if (!pte_present(pte)) /* sanity check */
151 		return 0;
152 	page = pte_page(pte);
153 	/*
154 	 * We do not update pte here:
155 	 * 1. Nobody should be accessing bpf_arena's range outside of a kernel bug
156 	 * 2. TLB flushing is batched or deferred. Even if we clear pte,
157 	 * the TLB entries can stick around and continue to permit access to
158 	 * the freed page. So it all relies on 1.
159 	 */
160 	__free_page(page);
161 	return 0;
162 }
163 
arena_map_free(struct bpf_map * map)164 static void arena_map_free(struct bpf_map *map)
165 {
166 	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);
167 
168 	/*
169 	 * Check that user vma-s are not around when bpf map is freed.
170 	 * mmap() holds vm_file which holds bpf_map refcnt.
171 	 * munmap() must have happened on vma followed by arena_vm_close()
172 	 * which would clear arena->vma_list.
173 	 */
174 	if (WARN_ON_ONCE(!list_empty(&arena->vma_list)))
175 		return;
176 
177 	/*
178 	 * free_vm_area() calls remove_vm_area() that calls free_unmap_vmap_area().
179 	 * It unmaps everything from vmalloc area and clears pgtables.
180 	 * Call apply_to_existing_page_range() first to find populated ptes and
181 	 * free those pages.
182 	 */
183 	apply_to_existing_page_range(&init_mm, bpf_arena_get_kern_vm_start(arena),
184 				     KERN_VM_SZ - GUARD_SZ, existing_page_cb, NULL);
185 	free_vm_area(arena->kern_vm);
186 	mtree_destroy(&arena->mt);
187 	bpf_map_area_free(arena);
188 }
189 
arena_map_lookup_elem(struct bpf_map * map,void * key)190 static void *arena_map_lookup_elem(struct bpf_map *map, void *key)
191 {
192 	return ERR_PTR(-EINVAL);
193 }
194 
arena_map_update_elem(struct bpf_map * map,void * key,void * value,u64 flags)195 static long arena_map_update_elem(struct bpf_map *map, void *key,
196 				  void *value, u64 flags)
197 {
198 	return -EOPNOTSUPP;
199 }
200 
arena_map_check_btf(const struct bpf_map * map,const struct btf * btf,const struct btf_type * key_type,const struct btf_type * value_type)201 static int arena_map_check_btf(const struct bpf_map *map, const struct btf *btf,
202 			       const struct btf_type *key_type, const struct btf_type *value_type)
203 {
204 	return 0;
205 }
206 
arena_map_mem_usage(const struct bpf_map * map)207 static u64 arena_map_mem_usage(const struct bpf_map *map)
208 {
209 	return 0;
210 }
211 
212 struct vma_list {
213 	struct vm_area_struct *vma;
214 	struct list_head head;
215 	atomic_t mmap_count;
216 };
217 
remember_vma(struct bpf_arena * arena,struct vm_area_struct * vma)218 static int remember_vma(struct bpf_arena *arena, struct vm_area_struct *vma)
219 {
220 	struct vma_list *vml;
221 
222 	vml = kmalloc(sizeof(*vml), GFP_KERNEL);
223 	if (!vml)
224 		return -ENOMEM;
225 	atomic_set(&vml->mmap_count, 1);
226 	vma->vm_private_data = vml;
227 	vml->vma = vma;
228 	list_add(&vml->head, &arena->vma_list);
229 	return 0;
230 }
231 
arena_vm_open(struct vm_area_struct * vma)232 static void arena_vm_open(struct vm_area_struct *vma)
233 {
234 	struct vma_list *vml = vma->vm_private_data;
235 
236 	atomic_inc(&vml->mmap_count);
237 }
238 
arena_vm_close(struct vm_area_struct * vma)239 static void arena_vm_close(struct vm_area_struct *vma)
240 {
241 	struct bpf_map *map = vma->vm_file->private_data;
242 	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);
243 	struct vma_list *vml = vma->vm_private_data;
244 
245 	if (!atomic_dec_and_test(&vml->mmap_count))
246 		return;
247 	guard(mutex)(&arena->lock);
248 	/* update link list under lock */
249 	list_del(&vml->head);
250 	vma->vm_private_data = NULL;
251 	kfree(vml);
252 }
253 
254 #define MT_ENTRY ((void *)&arena_map_ops) /* unused. has to be valid pointer */
255 
arena_vm_fault(struct vm_fault * vmf)256 static vm_fault_t arena_vm_fault(struct vm_fault *vmf)
257 {
258 	struct bpf_map *map = vmf->vma->vm_file->private_data;
259 	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);
260 	struct page *page;
261 	long kbase, kaddr;
262 	int ret;
263 
264 	kbase = bpf_arena_get_kern_vm_start(arena);
265 	kaddr = kbase + (u32)(vmf->address);
266 
267 	guard(mutex)(&arena->lock);
268 	page = vmalloc_to_page((void *)kaddr);
269 	if (page)
270 		/* already have a page vmap-ed */
271 		goto out;
272 
273 	if (arena->map.map_flags & BPF_F_SEGV_ON_FAULT)
274 		/* User space requested to segfault when page is not allocated by bpf prog */
275 		return VM_FAULT_SIGSEGV;
276 
277 	ret = mtree_insert(&arena->mt, vmf->pgoff, MT_ENTRY, GFP_KERNEL);
278 	if (ret)
279 		return VM_FAULT_SIGSEGV;
280 
281 	/* Account into memcg of the process that created bpf_arena */
282 	ret = bpf_map_alloc_pages(map, GFP_KERNEL | __GFP_ZERO, NUMA_NO_NODE, 1, &page);
283 	if (ret) {
284 		mtree_erase(&arena->mt, vmf->pgoff);
285 		return VM_FAULT_SIGSEGV;
286 	}
287 
288 	ret = vm_area_map_pages(arena->kern_vm, kaddr, kaddr + PAGE_SIZE, &page);
289 	if (ret) {
290 		mtree_erase(&arena->mt, vmf->pgoff);
291 		__free_page(page);
292 		return VM_FAULT_SIGSEGV;
293 	}
294 out:
295 	page_ref_add(page, 1);
296 	vmf->page = page;
297 	return 0;
298 }
299 
300 static const struct vm_operations_struct arena_vm_ops = {
301 	.open		= arena_vm_open,
302 	.close		= arena_vm_close,
303 	.fault          = arena_vm_fault,
304 };
305 
arena_get_unmapped_area(struct file * filp,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags)306 static unsigned long arena_get_unmapped_area(struct file *filp, unsigned long addr,
307 					     unsigned long len, unsigned long pgoff,
308 					     unsigned long flags)
309 {
310 	struct bpf_map *map = filp->private_data;
311 	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);
312 	long ret;
313 
314 	if (pgoff)
315 		return -EINVAL;
316 	if (len > SZ_4G)
317 		return -E2BIG;
318 
319 	/* if user_vm_start was specified at arena creation time */
320 	if (arena->user_vm_start) {
321 		if (len > arena->user_vm_end - arena->user_vm_start)
322 			return -E2BIG;
323 		if (len != arena->user_vm_end - arena->user_vm_start)
324 			return -EINVAL;
325 		if (addr != arena->user_vm_start)
326 			return -EINVAL;
327 	}
328 
329 	ret = mm_get_unmapped_area(current->mm, filp, addr, len * 2, 0, flags);
330 	if (IS_ERR_VALUE(ret))
331 		return ret;
332 	if ((ret >> 32) == ((ret + len - 1) >> 32))
333 		return ret;
334 	if (WARN_ON_ONCE(arena->user_vm_start))
335 		/* checks at map creation time should prevent this */
336 		return -EFAULT;
337 	return round_up(ret, SZ_4G);
338 }
339 
arena_map_mmap(struct bpf_map * map,struct vm_area_struct * vma)340 static int arena_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)
341 {
342 	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);
343 
344 	guard(mutex)(&arena->lock);
345 	if (arena->user_vm_start && arena->user_vm_start != vma->vm_start)
346 		/*
347 		 * If map_extra was not specified at arena creation time then
348 		 * 1st user process can do mmap(NULL, ...) to pick user_vm_start
349 		 * 2nd user process must pass the same addr to mmap(addr, MAP_FIXED..);
350 		 *   or
351 		 * specify addr in map_extra and
352 		 * use the same addr later with mmap(addr, MAP_FIXED..);
353 		 */
354 		return -EBUSY;
355 
356 	if (arena->user_vm_end && arena->user_vm_end != vma->vm_end)
357 		/* all user processes must have the same size of mmap-ed region */
358 		return -EBUSY;
359 
360 	/* Earlier checks should prevent this */
361 	if (WARN_ON_ONCE(vma->vm_end - vma->vm_start > SZ_4G || vma->vm_pgoff))
362 		return -EFAULT;
363 
364 	if (remember_vma(arena, vma))
365 		return -ENOMEM;
366 
367 	arena->user_vm_start = vma->vm_start;
368 	arena->user_vm_end = vma->vm_end;
369 	/*
370 	 * bpf_map_mmap() checks that it's being mmaped as VM_SHARED and
371 	 * clears VM_MAYEXEC. Set VM_DONTEXPAND as well to avoid
372 	 * potential change of user_vm_start.
373 	 */
374 	vm_flags_set(vma, VM_DONTEXPAND);
375 	vma->vm_ops = &arena_vm_ops;
376 	return 0;
377 }
378 
arena_map_direct_value_addr(const struct bpf_map * map,u64 * imm,u32 off)379 static int arena_map_direct_value_addr(const struct bpf_map *map, u64 *imm, u32 off)
380 {
381 	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);
382 
383 	if ((u64)off > arena->user_vm_end - arena->user_vm_start)
384 		return -ERANGE;
385 	*imm = (unsigned long)arena->user_vm_start;
386 	return 0;
387 }
388 
389 BTF_ID_LIST_SINGLE(bpf_arena_map_btf_ids, struct, bpf_arena)
390 const struct bpf_map_ops arena_map_ops = {
391 	.map_meta_equal = bpf_map_meta_equal,
392 	.map_alloc = arena_map_alloc,
393 	.map_free = arena_map_free,
394 	.map_direct_value_addr = arena_map_direct_value_addr,
395 	.map_mmap = arena_map_mmap,
396 	.map_get_unmapped_area = arena_get_unmapped_area,
397 	.map_get_next_key = arena_map_get_next_key,
398 	.map_push_elem = arena_map_push_elem,
399 	.map_peek_elem = arena_map_peek_elem,
400 	.map_pop_elem = arena_map_pop_elem,
401 	.map_lookup_elem = arena_map_lookup_elem,
402 	.map_update_elem = arena_map_update_elem,
403 	.map_delete_elem = arena_map_delete_elem,
404 	.map_check_btf = arena_map_check_btf,
405 	.map_mem_usage = arena_map_mem_usage,
406 	.map_btf_id = &bpf_arena_map_btf_ids[0],
407 };
408 
clear_lo32(u64 val)409 static u64 clear_lo32(u64 val)
410 {
411 	return val & ~(u64)~0U;
412 }
413 
414 /*
415  * Allocate pages and vmap them into kernel vmalloc area.
416  * Later the pages will be mmaped into user space vma.
417  */
arena_alloc_pages(struct bpf_arena * arena,long uaddr,long page_cnt,int node_id)418 static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt, int node_id)
419 {
420 	/* user_vm_end/start are fixed before bpf prog runs */
421 	long page_cnt_max = (arena->user_vm_end - arena->user_vm_start) >> PAGE_SHIFT;
422 	u64 kern_vm_start = bpf_arena_get_kern_vm_start(arena);
423 	struct page **pages;
424 	long pgoff = 0;
425 	u32 uaddr32;
426 	int ret, i;
427 
428 	if (page_cnt > page_cnt_max)
429 		return 0;
430 
431 	if (uaddr) {
432 		if (uaddr & ~PAGE_MASK)
433 			return 0;
434 		pgoff = compute_pgoff(arena, uaddr);
435 		if (pgoff > page_cnt_max - page_cnt)
436 			/* requested address will be outside of user VMA */
437 			return 0;
438 	}
439 
440 	/* zeroing is needed, since alloc_pages_bulk_array() only fills in non-zero entries */
441 	pages = kvcalloc(page_cnt, sizeof(struct page *), GFP_KERNEL);
442 	if (!pages)
443 		return 0;
444 
445 	guard(mutex)(&arena->lock);
446 
447 	if (uaddr)
448 		ret = mtree_insert_range(&arena->mt, pgoff, pgoff + page_cnt - 1,
449 					 MT_ENTRY, GFP_KERNEL);
450 	else
451 		ret = mtree_alloc_range(&arena->mt, &pgoff, MT_ENTRY,
452 					page_cnt, 0, page_cnt_max - 1, GFP_KERNEL);
453 	if (ret)
454 		goto out_free_pages;
455 
456 	ret = bpf_map_alloc_pages(&arena->map, GFP_KERNEL | __GFP_ZERO,
457 				  node_id, page_cnt, pages);
458 	if (ret)
459 		goto out;
460 
461 	uaddr32 = (u32)(arena->user_vm_start + pgoff * PAGE_SIZE);
462 	/* Earlier checks made sure that uaddr32 + page_cnt * PAGE_SIZE - 1
463 	 * will not overflow 32-bit. Lower 32-bit need to represent
464 	 * contiguous user address range.
465 	 * Map these pages at kern_vm_start base.
466 	 * kern_vm_start + uaddr32 + page_cnt * PAGE_SIZE - 1 can overflow
467 	 * lower 32-bit and it's ok.
468 	 */
469 	ret = vm_area_map_pages(arena->kern_vm, kern_vm_start + uaddr32,
470 				kern_vm_start + uaddr32 + page_cnt * PAGE_SIZE, pages);
471 	if (ret) {
472 		for (i = 0; i < page_cnt; i++)
473 			__free_page(pages[i]);
474 		goto out;
475 	}
476 	kvfree(pages);
477 	return clear_lo32(arena->user_vm_start) + uaddr32;
478 out:
479 	mtree_erase(&arena->mt, pgoff);
480 out_free_pages:
481 	kvfree(pages);
482 	return 0;
483 }
484 
485 /*
486  * If page is present in vmalloc area, unmap it from vmalloc area,
487  * unmap it from all user space vma-s,
488  * and free it.
489  */
zap_pages(struct bpf_arena * arena,long uaddr,long page_cnt)490 static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt)
491 {
492 	struct vma_list *vml;
493 
494 	list_for_each_entry(vml, &arena->vma_list, head)
495 		zap_page_range_single(vml->vma, uaddr,
496 				      PAGE_SIZE * page_cnt, NULL);
497 }
498 
arena_free_pages(struct bpf_arena * arena,long uaddr,long page_cnt)499 static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt)
500 {
501 	u64 full_uaddr, uaddr_end;
502 	long kaddr, pgoff, i;
503 	struct page *page;
504 
505 	/* only aligned lower 32-bit are relevant */
506 	uaddr = (u32)uaddr;
507 	uaddr &= PAGE_MASK;
508 	full_uaddr = clear_lo32(arena->user_vm_start) + uaddr;
509 	uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT));
510 	if (full_uaddr >= uaddr_end)
511 		return;
512 
513 	page_cnt = (uaddr_end - full_uaddr) >> PAGE_SHIFT;
514 
515 	guard(mutex)(&arena->lock);
516 
517 	pgoff = compute_pgoff(arena, uaddr);
518 	/* clear range */
519 	mtree_store_range(&arena->mt, pgoff, pgoff + page_cnt - 1, NULL, GFP_KERNEL);
520 
521 	if (page_cnt > 1)
522 		/* bulk zap if multiple pages being freed */
523 		zap_pages(arena, full_uaddr, page_cnt);
524 
525 	kaddr = bpf_arena_get_kern_vm_start(arena) + uaddr;
526 	for (i = 0; i < page_cnt; i++, kaddr += PAGE_SIZE, full_uaddr += PAGE_SIZE) {
527 		page = vmalloc_to_page((void *)kaddr);
528 		if (!page)
529 			continue;
530 		if (page_cnt == 1 && page_mapped(page)) /* mapped by some user process */
531 			/* Optimization for the common case of page_cnt==1:
532 			 * If page wasn't mapped into some user vma there
533 			 * is no need to call zap_pages which is slow. When
534 			 * page_cnt is big it's faster to do the batched zap.
535 			 */
536 			zap_pages(arena, full_uaddr, 1);
537 		vm_area_unmap_pages(arena->kern_vm, kaddr, kaddr + PAGE_SIZE);
538 		__free_page(page);
539 	}
540 }
541 
542 __bpf_kfunc_start_defs();
543 
bpf_arena_alloc_pages(void * p__map,void * addr__ign,u32 page_cnt,int node_id,u64 flags)544 __bpf_kfunc void *bpf_arena_alloc_pages(void *p__map, void *addr__ign, u32 page_cnt,
545 					int node_id, u64 flags)
546 {
547 	struct bpf_map *map = p__map;
548 	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);
549 
550 	if (map->map_type != BPF_MAP_TYPE_ARENA || flags || !page_cnt)
551 		return NULL;
552 
553 	return (void *)arena_alloc_pages(arena, (long)addr__ign, page_cnt, node_id);
554 }
555 
bpf_arena_free_pages(void * p__map,void * ptr__ign,u32 page_cnt)556 __bpf_kfunc void bpf_arena_free_pages(void *p__map, void *ptr__ign, u32 page_cnt)
557 {
558 	struct bpf_map *map = p__map;
559 	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);
560 
561 	if (map->map_type != BPF_MAP_TYPE_ARENA || !page_cnt || !ptr__ign)
562 		return;
563 	arena_free_pages(arena, (long)ptr__ign, page_cnt);
564 }
565 __bpf_kfunc_end_defs();
566 
567 BTF_KFUNCS_START(arena_kfuncs)
568 BTF_ID_FLAGS(func, bpf_arena_alloc_pages, KF_TRUSTED_ARGS | KF_SLEEPABLE)
569 BTF_ID_FLAGS(func, bpf_arena_free_pages, KF_TRUSTED_ARGS | KF_SLEEPABLE)
570 BTF_KFUNCS_END(arena_kfuncs)
571 
572 static const struct btf_kfunc_id_set common_kfunc_set = {
573 	.owner = THIS_MODULE,
574 	.set   = &arena_kfuncs,
575 };
576 
kfunc_init(void)577 static int __init kfunc_init(void)
578 {
579 	return register_btf_kfunc_id_set(BPF_PROG_TYPE_UNSPEC, &common_kfunc_set);
580 }
581 late_initcall(kfunc_init);
582