1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2019 Facebook */
3
4 #include <linux/bpf.h>
5 #include <linux/bpf_verifier.h>
6 #include <linux/btf.h>
7 #include <linux/filter.h>
8 #include <linux/slab.h>
9 #include <linux/numa.h>
10 #include <linux/seq_file.h>
11 #include <linux/refcount.h>
12 #include <linux/mutex.h>
13 #include <linux/btf_ids.h>
14 #include <linux/rcupdate_wait.h>
15
16 struct bpf_struct_ops_value {
17 struct bpf_struct_ops_common_value common;
18 char data[] ____cacheline_aligned_in_smp;
19 };
20
21 #define MAX_TRAMP_IMAGE_PAGES 8
22
23 struct bpf_struct_ops_map {
24 struct bpf_map map;
25 struct rcu_head rcu;
26 const struct bpf_struct_ops_desc *st_ops_desc;
27 /* protect map_update */
28 struct mutex lock;
29 /* link has all the bpf_links that is populated
30 * to the func ptr of the kernel's struct
31 * (in kvalue.data).
32 */
33 struct bpf_link **links;
34 u32 links_cnt;
35 u32 image_pages_cnt;
36 /* image_pages is an array of pages that has all the trampolines
37 * that stores the func args before calling the bpf_prog.
38 */
39 void *image_pages[MAX_TRAMP_IMAGE_PAGES];
40 /* The owner moduler's btf. */
41 struct btf *btf;
42 /* uvalue->data stores the kernel struct
43 * (e.g. tcp_congestion_ops) that is more useful
44 * to userspace than the kvalue. For example,
45 * the bpf_prog's id is stored instead of the kernel
46 * address of a func ptr.
47 */
48 struct bpf_struct_ops_value *uvalue;
49 /* kvalue.data stores the actual kernel's struct
50 * (e.g. tcp_congestion_ops) that will be
51 * registered to the kernel subsystem.
52 */
53 struct bpf_struct_ops_value kvalue;
54 };
55
56 struct bpf_struct_ops_link {
57 struct bpf_link link;
58 struct bpf_map __rcu *map;
59 };
60
61 static DEFINE_MUTEX(update_mutex);
62
63 #define VALUE_PREFIX "bpf_struct_ops_"
64 #define VALUE_PREFIX_LEN (sizeof(VALUE_PREFIX) - 1)
65
66 const struct bpf_verifier_ops bpf_struct_ops_verifier_ops = {
67 };
68
69 const struct bpf_prog_ops bpf_struct_ops_prog_ops = {
70 #ifdef CONFIG_NET
71 .test_run = bpf_struct_ops_test_run,
72 #endif
73 };
74
75 BTF_ID_LIST(st_ops_ids)
76 BTF_ID(struct, module)
77 BTF_ID(struct, bpf_struct_ops_common_value)
78
79 enum {
80 IDX_MODULE_ID,
81 IDX_ST_OPS_COMMON_VALUE_ID,
82 };
83
84 extern struct btf *btf_vmlinux;
85
is_valid_value_type(struct btf * btf,s32 value_id,const struct btf_type * type,const char * value_name)86 static bool is_valid_value_type(struct btf *btf, s32 value_id,
87 const struct btf_type *type,
88 const char *value_name)
89 {
90 const struct btf_type *common_value_type;
91 const struct btf_member *member;
92 const struct btf_type *vt, *mt;
93
94 vt = btf_type_by_id(btf, value_id);
95 if (btf_vlen(vt) != 2) {
96 pr_warn("The number of %s's members should be 2, but we get %d\n",
97 value_name, btf_vlen(vt));
98 return false;
99 }
100 member = btf_type_member(vt);
101 mt = btf_type_by_id(btf, member->type);
102 common_value_type = btf_type_by_id(btf_vmlinux,
103 st_ops_ids[IDX_ST_OPS_COMMON_VALUE_ID]);
104 if (mt != common_value_type) {
105 pr_warn("The first member of %s should be bpf_struct_ops_common_value\n",
106 value_name);
107 return false;
108 }
109 member++;
110 mt = btf_type_by_id(btf, member->type);
111 if (mt != type) {
112 pr_warn("The second member of %s should be %s\n",
113 value_name, btf_name_by_offset(btf, type->name_off));
114 return false;
115 }
116
117 return true;
118 }
119
bpf_struct_ops_image_alloc(void)120 static void *bpf_struct_ops_image_alloc(void)
121 {
122 void *image;
123 int err;
124
125 err = bpf_jit_charge_modmem(PAGE_SIZE);
126 if (err)
127 return ERR_PTR(err);
128 image = arch_alloc_bpf_trampoline(PAGE_SIZE);
129 if (!image) {
130 bpf_jit_uncharge_modmem(PAGE_SIZE);
131 return ERR_PTR(-ENOMEM);
132 }
133
134 return image;
135 }
136
bpf_struct_ops_image_free(void * image)137 void bpf_struct_ops_image_free(void *image)
138 {
139 if (image) {
140 arch_free_bpf_trampoline(image, PAGE_SIZE);
141 bpf_jit_uncharge_modmem(PAGE_SIZE);
142 }
143 }
144
145 #define MAYBE_NULL_SUFFIX "__nullable"
146 #define MAX_STUB_NAME 128
147
148 /* Return the type info of a stub function, if it exists.
149 *
150 * The name of a stub function is made up of the name of the struct_ops and
151 * the name of the function pointer member, separated by "__". For example,
152 * if the struct_ops type is named "foo_ops" and the function pointer
153 * member is named "bar", the stub function name would be "foo_ops__bar".
154 */
155 static const struct btf_type *
find_stub_func_proto(const struct btf * btf,const char * st_op_name,const char * member_name)156 find_stub_func_proto(const struct btf *btf, const char *st_op_name,
157 const char *member_name)
158 {
159 char stub_func_name[MAX_STUB_NAME];
160 const struct btf_type *func_type;
161 s32 btf_id;
162 int cp;
163
164 cp = snprintf(stub_func_name, MAX_STUB_NAME, "%s__%s",
165 st_op_name, member_name);
166 if (cp >= MAX_STUB_NAME) {
167 pr_warn("Stub function name too long\n");
168 return NULL;
169 }
170 btf_id = btf_find_by_name_kind(btf, stub_func_name, BTF_KIND_FUNC);
171 if (btf_id < 0)
172 return NULL;
173 func_type = btf_type_by_id(btf, btf_id);
174 if (!func_type)
175 return NULL;
176
177 return btf_type_by_id(btf, func_type->type); /* FUNC_PROTO */
178 }
179
180 /* Prepare argument info for every nullable argument of a member of a
181 * struct_ops type.
182 *
183 * Initialize a struct bpf_struct_ops_arg_info according to type info of
184 * the arguments of a stub function. (Check kCFI for more information about
185 * stub functions.)
186 *
187 * Each member in the struct_ops type has a struct bpf_struct_ops_arg_info
188 * to provide an array of struct bpf_ctx_arg_aux, which in turn provides
189 * the information that used by the verifier to check the arguments of the
190 * BPF struct_ops program assigned to the member. Here, we only care about
191 * the arguments that are marked as __nullable.
192 *
193 * The array of struct bpf_ctx_arg_aux is eventually assigned to
194 * prog->aux->ctx_arg_info of BPF struct_ops programs and passed to the
195 * verifier. (See check_struct_ops_btf_id())
196 *
197 * arg_info->info will be the list of struct bpf_ctx_arg_aux if success. If
198 * fails, it will be kept untouched.
199 */
prepare_arg_info(struct btf * btf,const char * st_ops_name,const char * member_name,const struct btf_type * func_proto,struct bpf_struct_ops_arg_info * arg_info)200 static int prepare_arg_info(struct btf *btf,
201 const char *st_ops_name,
202 const char *member_name,
203 const struct btf_type *func_proto,
204 struct bpf_struct_ops_arg_info *arg_info)
205 {
206 const struct btf_type *stub_func_proto, *pointed_type;
207 const struct btf_param *stub_args, *args;
208 struct bpf_ctx_arg_aux *info, *info_buf;
209 u32 nargs, arg_no, info_cnt = 0;
210 u32 arg_btf_id;
211 int offset;
212
213 stub_func_proto = find_stub_func_proto(btf, st_ops_name, member_name);
214 if (!stub_func_proto)
215 return 0;
216
217 /* Check if the number of arguments of the stub function is the same
218 * as the number of arguments of the function pointer.
219 */
220 nargs = btf_type_vlen(func_proto);
221 if (nargs != btf_type_vlen(stub_func_proto)) {
222 pr_warn("the number of arguments of the stub function %s__%s does not match the number of arguments of the member %s of struct %s\n",
223 st_ops_name, member_name, member_name, st_ops_name);
224 return -EINVAL;
225 }
226
227 if (!nargs)
228 return 0;
229
230 args = btf_params(func_proto);
231 stub_args = btf_params(stub_func_proto);
232
233 info_buf = kcalloc(nargs, sizeof(*info_buf), GFP_KERNEL);
234 if (!info_buf)
235 return -ENOMEM;
236
237 /* Prepare info for every nullable argument */
238 info = info_buf;
239 for (arg_no = 0; arg_no < nargs; arg_no++) {
240 /* Skip arguments that is not suffixed with
241 * "__nullable".
242 */
243 if (!btf_param_match_suffix(btf, &stub_args[arg_no],
244 MAYBE_NULL_SUFFIX))
245 continue;
246
247 /* Should be a pointer to struct */
248 pointed_type = btf_type_resolve_ptr(btf,
249 args[arg_no].type,
250 &arg_btf_id);
251 if (!pointed_type ||
252 !btf_type_is_struct(pointed_type)) {
253 pr_warn("stub function %s__%s has %s tagging to an unsupported type\n",
254 st_ops_name, member_name, MAYBE_NULL_SUFFIX);
255 goto err_out;
256 }
257
258 offset = btf_ctx_arg_offset(btf, func_proto, arg_no);
259 if (offset < 0) {
260 pr_warn("stub function %s__%s has an invalid trampoline ctx offset for arg#%u\n",
261 st_ops_name, member_name, arg_no);
262 goto err_out;
263 }
264
265 if (args[arg_no].type != stub_args[arg_no].type) {
266 pr_warn("arg#%u type in stub function %s__%s does not match with its original func_proto\n",
267 arg_no, st_ops_name, member_name);
268 goto err_out;
269 }
270
271 /* Fill the information of the new argument */
272 info->reg_type =
273 PTR_TRUSTED | PTR_TO_BTF_ID | PTR_MAYBE_NULL;
274 info->btf_id = arg_btf_id;
275 info->btf = btf;
276 info->offset = offset;
277
278 info++;
279 info_cnt++;
280 }
281
282 if (info_cnt) {
283 arg_info->info = info_buf;
284 arg_info->cnt = info_cnt;
285 } else {
286 kfree(info_buf);
287 }
288
289 return 0;
290
291 err_out:
292 kfree(info_buf);
293
294 return -EINVAL;
295 }
296
297 /* Clean up the arg_info in a struct bpf_struct_ops_desc. */
bpf_struct_ops_desc_release(struct bpf_struct_ops_desc * st_ops_desc)298 void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc)
299 {
300 struct bpf_struct_ops_arg_info *arg_info;
301 int i;
302
303 arg_info = st_ops_desc->arg_info;
304 for (i = 0; i < btf_type_vlen(st_ops_desc->type); i++)
305 kfree(arg_info[i].info);
306
307 kfree(arg_info);
308 }
309
bpf_struct_ops_desc_init(struct bpf_struct_ops_desc * st_ops_desc,struct btf * btf,struct bpf_verifier_log * log)310 int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,
311 struct btf *btf,
312 struct bpf_verifier_log *log)
313 {
314 struct bpf_struct_ops *st_ops = st_ops_desc->st_ops;
315 struct bpf_struct_ops_arg_info *arg_info;
316 const struct btf_member *member;
317 const struct btf_type *t;
318 s32 type_id, value_id;
319 char value_name[128];
320 const char *mname;
321 int i, err;
322
323 if (strlen(st_ops->name) + VALUE_PREFIX_LEN >=
324 sizeof(value_name)) {
325 pr_warn("struct_ops name %s is too long\n",
326 st_ops->name);
327 return -EINVAL;
328 }
329 sprintf(value_name, "%s%s", VALUE_PREFIX, st_ops->name);
330
331 if (!st_ops->cfi_stubs) {
332 pr_warn("struct_ops for %s has no cfi_stubs\n", st_ops->name);
333 return -EINVAL;
334 }
335
336 type_id = btf_find_by_name_kind(btf, st_ops->name,
337 BTF_KIND_STRUCT);
338 if (type_id < 0) {
339 pr_warn("Cannot find struct %s in %s\n",
340 st_ops->name, btf_get_name(btf));
341 return -EINVAL;
342 }
343 t = btf_type_by_id(btf, type_id);
344 if (btf_type_vlen(t) > BPF_STRUCT_OPS_MAX_NR_MEMBERS) {
345 pr_warn("Cannot support #%u members in struct %s\n",
346 btf_type_vlen(t), st_ops->name);
347 return -EINVAL;
348 }
349
350 value_id = btf_find_by_name_kind(btf, value_name,
351 BTF_KIND_STRUCT);
352 if (value_id < 0) {
353 pr_warn("Cannot find struct %s in %s\n",
354 value_name, btf_get_name(btf));
355 return -EINVAL;
356 }
357 if (!is_valid_value_type(btf, value_id, t, value_name))
358 return -EINVAL;
359
360 arg_info = kcalloc(btf_type_vlen(t), sizeof(*arg_info),
361 GFP_KERNEL);
362 if (!arg_info)
363 return -ENOMEM;
364
365 st_ops_desc->arg_info = arg_info;
366 st_ops_desc->type = t;
367 st_ops_desc->type_id = type_id;
368 st_ops_desc->value_id = value_id;
369 st_ops_desc->value_type = btf_type_by_id(btf, value_id);
370
371 for_each_member(i, t, member) {
372 const struct btf_type *func_proto;
373
374 mname = btf_name_by_offset(btf, member->name_off);
375 if (!*mname) {
376 pr_warn("anon member in struct %s is not supported\n",
377 st_ops->name);
378 err = -EOPNOTSUPP;
379 goto errout;
380 }
381
382 if (__btf_member_bitfield_size(t, member)) {
383 pr_warn("bit field member %s in struct %s is not supported\n",
384 mname, st_ops->name);
385 err = -EOPNOTSUPP;
386 goto errout;
387 }
388
389 func_proto = btf_type_resolve_func_ptr(btf,
390 member->type,
391 NULL);
392 if (!func_proto)
393 continue;
394
395 if (btf_distill_func_proto(log, btf,
396 func_proto, mname,
397 &st_ops->func_models[i])) {
398 pr_warn("Error in parsing func ptr %s in struct %s\n",
399 mname, st_ops->name);
400 err = -EINVAL;
401 goto errout;
402 }
403
404 err = prepare_arg_info(btf, st_ops->name, mname,
405 func_proto,
406 arg_info + i);
407 if (err)
408 goto errout;
409 }
410
411 if (st_ops->init(btf)) {
412 pr_warn("Error in init bpf_struct_ops %s\n",
413 st_ops->name);
414 err = -EINVAL;
415 goto errout;
416 }
417
418 return 0;
419
420 errout:
421 bpf_struct_ops_desc_release(st_ops_desc);
422
423 return err;
424 }
425
bpf_struct_ops_map_get_next_key(struct bpf_map * map,void * key,void * next_key)426 static int bpf_struct_ops_map_get_next_key(struct bpf_map *map, void *key,
427 void *next_key)
428 {
429 if (key && *(u32 *)key == 0)
430 return -ENOENT;
431
432 *(u32 *)next_key = 0;
433 return 0;
434 }
435
bpf_struct_ops_map_sys_lookup_elem(struct bpf_map * map,void * key,void * value)436 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
437 void *value)
438 {
439 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
440 struct bpf_struct_ops_value *uvalue, *kvalue;
441 enum bpf_struct_ops_state state;
442 s64 refcnt;
443
444 if (unlikely(*(u32 *)key != 0))
445 return -ENOENT;
446
447 kvalue = &st_map->kvalue;
448 /* Pair with smp_store_release() during map_update */
449 state = smp_load_acquire(&kvalue->common.state);
450 if (state == BPF_STRUCT_OPS_STATE_INIT) {
451 memset(value, 0, map->value_size);
452 return 0;
453 }
454
455 /* No lock is needed. state and refcnt do not need
456 * to be updated together under atomic context.
457 */
458 uvalue = value;
459 memcpy(uvalue, st_map->uvalue, map->value_size);
460 uvalue->common.state = state;
461
462 /* This value offers the user space a general estimate of how
463 * many sockets are still utilizing this struct_ops for TCP
464 * congestion control. The number might not be exact, but it
465 * should sufficiently meet our present goals.
466 */
467 refcnt = atomic64_read(&map->refcnt) - atomic64_read(&map->usercnt);
468 refcount_set(&uvalue->common.refcnt, max_t(s64, refcnt, 0));
469
470 return 0;
471 }
472
bpf_struct_ops_map_lookup_elem(struct bpf_map * map,void * key)473 static void *bpf_struct_ops_map_lookup_elem(struct bpf_map *map, void *key)
474 {
475 return ERR_PTR(-EINVAL);
476 }
477
bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map * st_map)478 static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map)
479 {
480 u32 i;
481
482 for (i = 0; i < st_map->links_cnt; i++) {
483 if (st_map->links[i]) {
484 bpf_link_put(st_map->links[i]);
485 st_map->links[i] = NULL;
486 }
487 }
488 }
489
bpf_struct_ops_map_free_image(struct bpf_struct_ops_map * st_map)490 static void bpf_struct_ops_map_free_image(struct bpf_struct_ops_map *st_map)
491 {
492 int i;
493
494 for (i = 0; i < st_map->image_pages_cnt; i++)
495 bpf_struct_ops_image_free(st_map->image_pages[i]);
496 st_map->image_pages_cnt = 0;
497 }
498
check_zero_holes(const struct btf * btf,const struct btf_type * t,void * data)499 static int check_zero_holes(const struct btf *btf, const struct btf_type *t, void *data)
500 {
501 const struct btf_member *member;
502 u32 i, moff, msize, prev_mend = 0;
503 const struct btf_type *mtype;
504
505 for_each_member(i, t, member) {
506 moff = __btf_member_bit_offset(t, member) / 8;
507 if (moff > prev_mend &&
508 memchr_inv(data + prev_mend, 0, moff - prev_mend))
509 return -EINVAL;
510
511 mtype = btf_type_by_id(btf, member->type);
512 mtype = btf_resolve_size(btf, mtype, &msize);
513 if (IS_ERR(mtype))
514 return PTR_ERR(mtype);
515 prev_mend = moff + msize;
516 }
517
518 if (t->size > prev_mend &&
519 memchr_inv(data + prev_mend, 0, t->size - prev_mend))
520 return -EINVAL;
521
522 return 0;
523 }
524
bpf_struct_ops_link_release(struct bpf_link * link)525 static void bpf_struct_ops_link_release(struct bpf_link *link)
526 {
527 }
528
bpf_struct_ops_link_dealloc(struct bpf_link * link)529 static void bpf_struct_ops_link_dealloc(struct bpf_link *link)
530 {
531 struct bpf_tramp_link *tlink = container_of(link, struct bpf_tramp_link, link);
532
533 kfree(tlink);
534 }
535
536 const struct bpf_link_ops bpf_struct_ops_link_lops = {
537 .release = bpf_struct_ops_link_release,
538 .dealloc = bpf_struct_ops_link_dealloc,
539 };
540
bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links * tlinks,struct bpf_tramp_link * link,const struct btf_func_model * model,void * stub_func,void ** _image,u32 * _image_off,bool allow_alloc)541 int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
542 struct bpf_tramp_link *link,
543 const struct btf_func_model *model,
544 void *stub_func,
545 void **_image, u32 *_image_off,
546 bool allow_alloc)
547 {
548 u32 image_off = *_image_off, flags = BPF_TRAMP_F_INDIRECT;
549 void *image = *_image;
550 int size;
551
552 tlinks[BPF_TRAMP_FENTRY].links[0] = link;
553 tlinks[BPF_TRAMP_FENTRY].nr_links = 1;
554
555 if (model->ret_size > 0)
556 flags |= BPF_TRAMP_F_RET_FENTRY_RET;
557
558 size = arch_bpf_trampoline_size(model, flags, tlinks, NULL);
559 if (size <= 0)
560 return size ? : -EFAULT;
561
562 /* Allocate image buffer if necessary */
563 if (!image || size > PAGE_SIZE - image_off) {
564 if (!allow_alloc)
565 return -E2BIG;
566
567 image = bpf_struct_ops_image_alloc();
568 if (IS_ERR(image))
569 return PTR_ERR(image);
570 image_off = 0;
571 }
572
573 size = arch_prepare_bpf_trampoline(NULL, image + image_off,
574 image + PAGE_SIZE,
575 model, flags, tlinks, stub_func);
576 if (size <= 0) {
577 if (image != *_image)
578 bpf_struct_ops_image_free(image);
579 return size ? : -EFAULT;
580 }
581
582 *_image = image;
583 *_image_off = image_off + size;
584 return 0;
585 }
586
bpf_struct_ops_map_update_elem(struct bpf_map * map,void * key,void * value,u64 flags)587 static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
588 void *value, u64 flags)
589 {
590 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
591 const struct bpf_struct_ops_desc *st_ops_desc = st_map->st_ops_desc;
592 const struct bpf_struct_ops *st_ops = st_ops_desc->st_ops;
593 struct bpf_struct_ops_value *uvalue, *kvalue;
594 const struct btf_type *module_type;
595 const struct btf_member *member;
596 const struct btf_type *t = st_ops_desc->type;
597 struct bpf_tramp_links *tlinks;
598 void *udata, *kdata;
599 int prog_fd, err;
600 u32 i, trampoline_start, image_off = 0;
601 void *cur_image = NULL, *image = NULL;
602
603 if (flags)
604 return -EINVAL;
605
606 if (*(u32 *)key != 0)
607 return -E2BIG;
608
609 err = check_zero_holes(st_map->btf, st_ops_desc->value_type, value);
610 if (err)
611 return err;
612
613 uvalue = value;
614 err = check_zero_holes(st_map->btf, t, uvalue->data);
615 if (err)
616 return err;
617
618 if (uvalue->common.state || refcount_read(&uvalue->common.refcnt))
619 return -EINVAL;
620
621 tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL);
622 if (!tlinks)
623 return -ENOMEM;
624
625 uvalue = (struct bpf_struct_ops_value *)st_map->uvalue;
626 kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue;
627
628 mutex_lock(&st_map->lock);
629
630 if (kvalue->common.state != BPF_STRUCT_OPS_STATE_INIT) {
631 err = -EBUSY;
632 goto unlock;
633 }
634
635 memcpy(uvalue, value, map->value_size);
636
637 udata = &uvalue->data;
638 kdata = &kvalue->data;
639
640 module_type = btf_type_by_id(btf_vmlinux, st_ops_ids[IDX_MODULE_ID]);
641 for_each_member(i, t, member) {
642 const struct btf_type *mtype, *ptype;
643 struct bpf_prog *prog;
644 struct bpf_tramp_link *link;
645 u32 moff;
646
647 moff = __btf_member_bit_offset(t, member) / 8;
648 ptype = btf_type_resolve_ptr(st_map->btf, member->type, NULL);
649 if (ptype == module_type) {
650 if (*(void **)(udata + moff))
651 goto reset_unlock;
652 *(void **)(kdata + moff) = BPF_MODULE_OWNER;
653 continue;
654 }
655
656 err = st_ops->init_member(t, member, kdata, udata);
657 if (err < 0)
658 goto reset_unlock;
659
660 /* The ->init_member() has handled this member */
661 if (err > 0)
662 continue;
663
664 /* If st_ops->init_member does not handle it,
665 * we will only handle func ptrs and zero-ed members
666 * here. Reject everything else.
667 */
668
669 /* All non func ptr member must be 0 */
670 if (!ptype || !btf_type_is_func_proto(ptype)) {
671 u32 msize;
672
673 mtype = btf_type_by_id(st_map->btf, member->type);
674 mtype = btf_resolve_size(st_map->btf, mtype, &msize);
675 if (IS_ERR(mtype)) {
676 err = PTR_ERR(mtype);
677 goto reset_unlock;
678 }
679
680 if (memchr_inv(udata + moff, 0, msize)) {
681 err = -EINVAL;
682 goto reset_unlock;
683 }
684
685 continue;
686 }
687
688 prog_fd = (int)(*(unsigned long *)(udata + moff));
689 /* Similar check as the attr->attach_prog_fd */
690 if (!prog_fd)
691 continue;
692
693 prog = bpf_prog_get(prog_fd);
694 if (IS_ERR(prog)) {
695 err = PTR_ERR(prog);
696 goto reset_unlock;
697 }
698
699 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
700 prog->aux->attach_btf_id != st_ops_desc->type_id ||
701 prog->expected_attach_type != i) {
702 bpf_prog_put(prog);
703 err = -EINVAL;
704 goto reset_unlock;
705 }
706
707 link = kzalloc(sizeof(*link), GFP_USER);
708 if (!link) {
709 bpf_prog_put(prog);
710 err = -ENOMEM;
711 goto reset_unlock;
712 }
713 bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS,
714 &bpf_struct_ops_link_lops, prog);
715 st_map->links[i] = &link->link;
716
717 trampoline_start = image_off;
718 err = bpf_struct_ops_prepare_trampoline(tlinks, link,
719 &st_ops->func_models[i],
720 *(void **)(st_ops->cfi_stubs + moff),
721 &image, &image_off,
722 st_map->image_pages_cnt < MAX_TRAMP_IMAGE_PAGES);
723 if (err)
724 goto reset_unlock;
725
726 if (cur_image != image) {
727 st_map->image_pages[st_map->image_pages_cnt++] = image;
728 cur_image = image;
729 trampoline_start = 0;
730 }
731
732 *(void **)(kdata + moff) = image + trampoline_start + cfi_get_offset();
733
734 /* put prog_id to udata */
735 *(unsigned long *)(udata + moff) = prog->aux->id;
736 }
737
738 if (st_ops->validate) {
739 err = st_ops->validate(kdata);
740 if (err)
741 goto reset_unlock;
742 }
743 for (i = 0; i < st_map->image_pages_cnt; i++) {
744 err = arch_protect_bpf_trampoline(st_map->image_pages[i],
745 PAGE_SIZE);
746 if (err)
747 goto reset_unlock;
748 }
749
750 if (st_map->map.map_flags & BPF_F_LINK) {
751 err = 0;
752 /* Let bpf_link handle registration & unregistration.
753 *
754 * Pair with smp_load_acquire() during lookup_elem().
755 */
756 smp_store_release(&kvalue->common.state, BPF_STRUCT_OPS_STATE_READY);
757 goto unlock;
758 }
759
760 err = st_ops->reg(kdata);
761 if (likely(!err)) {
762 /* This refcnt increment on the map here after
763 * 'st_ops->reg()' is secure since the state of the
764 * map must be set to INIT at this moment, and thus
765 * bpf_struct_ops_map_delete_elem() can't unregister
766 * or transition it to TOBEFREE concurrently.
767 */
768 bpf_map_inc(map);
769 /* Pair with smp_load_acquire() during lookup_elem().
770 * It ensures the above udata updates (e.g. prog->aux->id)
771 * can be seen once BPF_STRUCT_OPS_STATE_INUSE is set.
772 */
773 smp_store_release(&kvalue->common.state, BPF_STRUCT_OPS_STATE_INUSE);
774 goto unlock;
775 }
776
777 /* Error during st_ops->reg(). Can happen if this struct_ops needs to be
778 * verified as a whole, after all init_member() calls. Can also happen if
779 * there was a race in registering the struct_ops (under the same name) to
780 * a sub-system through different struct_ops's maps.
781 */
782
783 reset_unlock:
784 bpf_struct_ops_map_free_image(st_map);
785 bpf_struct_ops_map_put_progs(st_map);
786 memset(uvalue, 0, map->value_size);
787 memset(kvalue, 0, map->value_size);
788 unlock:
789 kfree(tlinks);
790 mutex_unlock(&st_map->lock);
791 return err;
792 }
793
bpf_struct_ops_map_delete_elem(struct bpf_map * map,void * key)794 static long bpf_struct_ops_map_delete_elem(struct bpf_map *map, void *key)
795 {
796 enum bpf_struct_ops_state prev_state;
797 struct bpf_struct_ops_map *st_map;
798
799 st_map = (struct bpf_struct_ops_map *)map;
800 if (st_map->map.map_flags & BPF_F_LINK)
801 return -EOPNOTSUPP;
802
803 prev_state = cmpxchg(&st_map->kvalue.common.state,
804 BPF_STRUCT_OPS_STATE_INUSE,
805 BPF_STRUCT_OPS_STATE_TOBEFREE);
806 switch (prev_state) {
807 case BPF_STRUCT_OPS_STATE_INUSE:
808 st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data);
809 bpf_map_put(map);
810 return 0;
811 case BPF_STRUCT_OPS_STATE_TOBEFREE:
812 return -EINPROGRESS;
813 case BPF_STRUCT_OPS_STATE_INIT:
814 return -ENOENT;
815 default:
816 WARN_ON_ONCE(1);
817 /* Should never happen. Treat it as not found. */
818 return -ENOENT;
819 }
820 }
821
bpf_struct_ops_map_seq_show_elem(struct bpf_map * map,void * key,struct seq_file * m)822 static void bpf_struct_ops_map_seq_show_elem(struct bpf_map *map, void *key,
823 struct seq_file *m)
824 {
825 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
826 void *value;
827 int err;
828
829 value = kmalloc(map->value_size, GFP_USER | __GFP_NOWARN);
830 if (!value)
831 return;
832
833 err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
834 if (!err) {
835 btf_type_seq_show(st_map->btf,
836 map->btf_vmlinux_value_type_id,
837 value, m);
838 seq_puts(m, "\n");
839 }
840
841 kfree(value);
842 }
843
__bpf_struct_ops_map_free(struct bpf_map * map)844 static void __bpf_struct_ops_map_free(struct bpf_map *map)
845 {
846 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
847
848 if (st_map->links)
849 bpf_struct_ops_map_put_progs(st_map);
850 bpf_map_area_free(st_map->links);
851 bpf_struct_ops_map_free_image(st_map);
852 bpf_map_area_free(st_map->uvalue);
853 bpf_map_area_free(st_map);
854 }
855
bpf_struct_ops_map_free(struct bpf_map * map)856 static void bpf_struct_ops_map_free(struct bpf_map *map)
857 {
858 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
859
860 /* st_ops->owner was acquired during map_alloc to implicitly holds
861 * the btf's refcnt. The acquire was only done when btf_is_module()
862 * st_map->btf cannot be NULL here.
863 */
864 if (btf_is_module(st_map->btf))
865 module_put(st_map->st_ops_desc->st_ops->owner);
866
867 /* The struct_ops's function may switch to another struct_ops.
868 *
869 * For example, bpf_tcp_cc_x->init() may switch to
870 * another tcp_cc_y by calling
871 * setsockopt(TCP_CONGESTION, "tcp_cc_y").
872 * During the switch, bpf_struct_ops_put(tcp_cc_x) is called
873 * and its refcount may reach 0 which then free its
874 * trampoline image while tcp_cc_x is still running.
875 *
876 * A vanilla rcu gp is to wait for all bpf-tcp-cc prog
877 * to finish. bpf-tcp-cc prog is non sleepable.
878 * A rcu_tasks gp is to wait for the last few insn
879 * in the tramopline image to finish before releasing
880 * the trampoline image.
881 */
882 synchronize_rcu_mult(call_rcu, call_rcu_tasks);
883
884 __bpf_struct_ops_map_free(map);
885 }
886
bpf_struct_ops_map_alloc_check(union bpf_attr * attr)887 static int bpf_struct_ops_map_alloc_check(union bpf_attr *attr)
888 {
889 if (attr->key_size != sizeof(unsigned int) || attr->max_entries != 1 ||
890 (attr->map_flags & ~(BPF_F_LINK | BPF_F_VTYPE_BTF_OBJ_FD)) ||
891 !attr->btf_vmlinux_value_type_id)
892 return -EINVAL;
893 return 0;
894 }
895
bpf_struct_ops_map_alloc(union bpf_attr * attr)896 static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr)
897 {
898 const struct bpf_struct_ops_desc *st_ops_desc;
899 size_t st_map_size;
900 struct bpf_struct_ops_map *st_map;
901 const struct btf_type *t, *vt;
902 struct module *mod = NULL;
903 struct bpf_map *map;
904 struct btf *btf;
905 int ret;
906
907 if (attr->map_flags & BPF_F_VTYPE_BTF_OBJ_FD) {
908 /* The map holds btf for its whole life time. */
909 btf = btf_get_by_fd(attr->value_type_btf_obj_fd);
910 if (IS_ERR(btf))
911 return ERR_CAST(btf);
912 if (!btf_is_module(btf)) {
913 btf_put(btf);
914 return ERR_PTR(-EINVAL);
915 }
916
917 mod = btf_try_get_module(btf);
918 /* mod holds a refcnt to btf. We don't need an extra refcnt
919 * here.
920 */
921 btf_put(btf);
922 if (!mod)
923 return ERR_PTR(-EINVAL);
924 } else {
925 btf = bpf_get_btf_vmlinux();
926 if (IS_ERR(btf))
927 return ERR_CAST(btf);
928 if (!btf)
929 return ERR_PTR(-ENOTSUPP);
930 }
931
932 st_ops_desc = bpf_struct_ops_find_value(btf, attr->btf_vmlinux_value_type_id);
933 if (!st_ops_desc) {
934 ret = -ENOTSUPP;
935 goto errout;
936 }
937
938 vt = st_ops_desc->value_type;
939 if (attr->value_size != vt->size) {
940 ret = -EINVAL;
941 goto errout;
942 }
943
944 t = st_ops_desc->type;
945
946 st_map_size = sizeof(*st_map) +
947 /* kvalue stores the
948 * struct bpf_struct_ops_tcp_congestions_ops
949 */
950 (vt->size - sizeof(struct bpf_struct_ops_value));
951
952 st_map = bpf_map_area_alloc(st_map_size, NUMA_NO_NODE);
953 if (!st_map) {
954 ret = -ENOMEM;
955 goto errout;
956 }
957
958 st_map->st_ops_desc = st_ops_desc;
959 map = &st_map->map;
960
961 st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE);
962 st_map->links_cnt = btf_type_vlen(t);
963 st_map->links =
964 bpf_map_area_alloc(st_map->links_cnt * sizeof(struct bpf_links *),
965 NUMA_NO_NODE);
966 if (!st_map->uvalue || !st_map->links) {
967 ret = -ENOMEM;
968 goto errout_free;
969 }
970 st_map->btf = btf;
971
972 mutex_init(&st_map->lock);
973 bpf_map_init_from_attr(map, attr);
974
975 return map;
976
977 errout_free:
978 __bpf_struct_ops_map_free(map);
979 errout:
980 module_put(mod);
981
982 return ERR_PTR(ret);
983 }
984
bpf_struct_ops_map_mem_usage(const struct bpf_map * map)985 static u64 bpf_struct_ops_map_mem_usage(const struct bpf_map *map)
986 {
987 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
988 const struct bpf_struct_ops_desc *st_ops_desc = st_map->st_ops_desc;
989 const struct btf_type *vt = st_ops_desc->value_type;
990 u64 usage;
991
992 usage = sizeof(*st_map) +
993 vt->size - sizeof(struct bpf_struct_ops_value);
994 usage += vt->size;
995 usage += btf_type_vlen(vt) * sizeof(struct bpf_links *);
996 usage += PAGE_SIZE;
997 return usage;
998 }
999
1000 BTF_ID_LIST_SINGLE(bpf_struct_ops_map_btf_ids, struct, bpf_struct_ops_map)
1001 const struct bpf_map_ops bpf_struct_ops_map_ops = {
1002 .map_alloc_check = bpf_struct_ops_map_alloc_check,
1003 .map_alloc = bpf_struct_ops_map_alloc,
1004 .map_free = bpf_struct_ops_map_free,
1005 .map_get_next_key = bpf_struct_ops_map_get_next_key,
1006 .map_lookup_elem = bpf_struct_ops_map_lookup_elem,
1007 .map_delete_elem = bpf_struct_ops_map_delete_elem,
1008 .map_update_elem = bpf_struct_ops_map_update_elem,
1009 .map_seq_show_elem = bpf_struct_ops_map_seq_show_elem,
1010 .map_mem_usage = bpf_struct_ops_map_mem_usage,
1011 .map_btf_id = &bpf_struct_ops_map_btf_ids[0],
1012 };
1013
1014 /* "const void *" because some subsystem is
1015 * passing a const (e.g. const struct tcp_congestion_ops *)
1016 */
bpf_struct_ops_get(const void * kdata)1017 bool bpf_struct_ops_get(const void *kdata)
1018 {
1019 struct bpf_struct_ops_value *kvalue;
1020 struct bpf_struct_ops_map *st_map;
1021 struct bpf_map *map;
1022
1023 kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
1024 st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue);
1025
1026 map = __bpf_map_inc_not_zero(&st_map->map, false);
1027 return !IS_ERR(map);
1028 }
1029
bpf_struct_ops_put(const void * kdata)1030 void bpf_struct_ops_put(const void *kdata)
1031 {
1032 struct bpf_struct_ops_value *kvalue;
1033 struct bpf_struct_ops_map *st_map;
1034
1035 kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
1036 st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue);
1037
1038 bpf_map_put(&st_map->map);
1039 }
1040
bpf_struct_ops_valid_to_reg(struct bpf_map * map)1041 static bool bpf_struct_ops_valid_to_reg(struct bpf_map *map)
1042 {
1043 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
1044
1045 return map->map_type == BPF_MAP_TYPE_STRUCT_OPS &&
1046 map->map_flags & BPF_F_LINK &&
1047 /* Pair with smp_store_release() during map_update */
1048 smp_load_acquire(&st_map->kvalue.common.state) == BPF_STRUCT_OPS_STATE_READY;
1049 }
1050
bpf_struct_ops_map_link_dealloc(struct bpf_link * link)1051 static void bpf_struct_ops_map_link_dealloc(struct bpf_link *link)
1052 {
1053 struct bpf_struct_ops_link *st_link;
1054 struct bpf_struct_ops_map *st_map;
1055
1056 st_link = container_of(link, struct bpf_struct_ops_link, link);
1057 st_map = (struct bpf_struct_ops_map *)
1058 rcu_dereference_protected(st_link->map, true);
1059 if (st_map) {
1060 /* st_link->map can be NULL if
1061 * bpf_struct_ops_link_create() fails to register.
1062 */
1063 st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data);
1064 bpf_map_put(&st_map->map);
1065 }
1066 kfree(st_link);
1067 }
1068
bpf_struct_ops_map_link_show_fdinfo(const struct bpf_link * link,struct seq_file * seq)1069 static void bpf_struct_ops_map_link_show_fdinfo(const struct bpf_link *link,
1070 struct seq_file *seq)
1071 {
1072 struct bpf_struct_ops_link *st_link;
1073 struct bpf_map *map;
1074
1075 st_link = container_of(link, struct bpf_struct_ops_link, link);
1076 rcu_read_lock();
1077 map = rcu_dereference(st_link->map);
1078 seq_printf(seq, "map_id:\t%d\n", map->id);
1079 rcu_read_unlock();
1080 }
1081
bpf_struct_ops_map_link_fill_link_info(const struct bpf_link * link,struct bpf_link_info * info)1082 static int bpf_struct_ops_map_link_fill_link_info(const struct bpf_link *link,
1083 struct bpf_link_info *info)
1084 {
1085 struct bpf_struct_ops_link *st_link;
1086 struct bpf_map *map;
1087
1088 st_link = container_of(link, struct bpf_struct_ops_link, link);
1089 rcu_read_lock();
1090 map = rcu_dereference(st_link->map);
1091 info->struct_ops.map_id = map->id;
1092 rcu_read_unlock();
1093 return 0;
1094 }
1095
bpf_struct_ops_map_link_update(struct bpf_link * link,struct bpf_map * new_map,struct bpf_map * expected_old_map)1096 static int bpf_struct_ops_map_link_update(struct bpf_link *link, struct bpf_map *new_map,
1097 struct bpf_map *expected_old_map)
1098 {
1099 struct bpf_struct_ops_map *st_map, *old_st_map;
1100 struct bpf_map *old_map;
1101 struct bpf_struct_ops_link *st_link;
1102 int err;
1103
1104 st_link = container_of(link, struct bpf_struct_ops_link, link);
1105 st_map = container_of(new_map, struct bpf_struct_ops_map, map);
1106
1107 if (!bpf_struct_ops_valid_to_reg(new_map))
1108 return -EINVAL;
1109
1110 if (!st_map->st_ops_desc->st_ops->update)
1111 return -EOPNOTSUPP;
1112
1113 mutex_lock(&update_mutex);
1114
1115 old_map = rcu_dereference_protected(st_link->map, lockdep_is_held(&update_mutex));
1116 if (expected_old_map && old_map != expected_old_map) {
1117 err = -EPERM;
1118 goto err_out;
1119 }
1120
1121 old_st_map = container_of(old_map, struct bpf_struct_ops_map, map);
1122 /* The new and old struct_ops must be the same type. */
1123 if (st_map->st_ops_desc != old_st_map->st_ops_desc) {
1124 err = -EINVAL;
1125 goto err_out;
1126 }
1127
1128 err = st_map->st_ops_desc->st_ops->update(st_map->kvalue.data, old_st_map->kvalue.data);
1129 if (err)
1130 goto err_out;
1131
1132 bpf_map_inc(new_map);
1133 rcu_assign_pointer(st_link->map, new_map);
1134 bpf_map_put(old_map);
1135
1136 err_out:
1137 mutex_unlock(&update_mutex);
1138
1139 return err;
1140 }
1141
1142 static const struct bpf_link_ops bpf_struct_ops_map_lops = {
1143 .dealloc = bpf_struct_ops_map_link_dealloc,
1144 .show_fdinfo = bpf_struct_ops_map_link_show_fdinfo,
1145 .fill_link_info = bpf_struct_ops_map_link_fill_link_info,
1146 .update_map = bpf_struct_ops_map_link_update,
1147 };
1148
bpf_struct_ops_link_create(union bpf_attr * attr)1149 int bpf_struct_ops_link_create(union bpf_attr *attr)
1150 {
1151 struct bpf_struct_ops_link *link = NULL;
1152 struct bpf_link_primer link_primer;
1153 struct bpf_struct_ops_map *st_map;
1154 struct bpf_map *map;
1155 int err;
1156
1157 map = bpf_map_get(attr->link_create.map_fd);
1158 if (IS_ERR(map))
1159 return PTR_ERR(map);
1160
1161 st_map = (struct bpf_struct_ops_map *)map;
1162
1163 if (!bpf_struct_ops_valid_to_reg(map)) {
1164 err = -EINVAL;
1165 goto err_out;
1166 }
1167
1168 link = kzalloc(sizeof(*link), GFP_USER);
1169 if (!link) {
1170 err = -ENOMEM;
1171 goto err_out;
1172 }
1173 bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, &bpf_struct_ops_map_lops, NULL);
1174
1175 err = bpf_link_prime(&link->link, &link_primer);
1176 if (err)
1177 goto err_out;
1178
1179 err = st_map->st_ops_desc->st_ops->reg(st_map->kvalue.data);
1180 if (err) {
1181 bpf_link_cleanup(&link_primer);
1182 link = NULL;
1183 goto err_out;
1184 }
1185 RCU_INIT_POINTER(link->map, map);
1186
1187 return bpf_link_settle(&link_primer);
1188
1189 err_out:
1190 bpf_map_put(map);
1191 kfree(link);
1192 return err;
1193 }
1194
bpf_map_struct_ops_info_fill(struct bpf_map_info * info,struct bpf_map * map)1195 void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map)
1196 {
1197 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
1198
1199 info->btf_vmlinux_id = btf_obj_id(st_map->btf);
1200 }
1201