1 // SPDX-License-Identifier: LGPL-2.1
2
3 /*
4 * common eBPF ELF operations.
5 *
6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
8 * Copyright (C) 2015 Huawei Inc.
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public
12 * License as published by the Free Software Foundation;
13 * version 2.1 of the License (not later!)
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU Lesser General Public License for more details.
19 *
20 * You should have received a copy of the GNU Lesser General Public
21 * License along with this program; if not, see <http://www.gnu.org/licenses>
22 */
23
24 #include <stdlib.h>
25 #include <memory.h>
26 #include <unistd.h>
27 #include <asm/unistd.h>
28 #include <linux/bpf.h>
29 #include "bpf.h"
30 #include <errno.h>
31
32 /*
33 * When building perf, unistd.h is overridden. __NR_bpf is
34 * required to be defined explicitly.
35 */
36 #ifndef __NR_bpf
37 # if defined(__i386__)
38 # define __NR_bpf 357
39 # elif defined(__x86_64__)
40 # define __NR_bpf 321
41 # elif defined(__aarch64__)
42 # define __NR_bpf 280
43 # elif defined(__sparc__)
44 # define __NR_bpf 349
45 # elif defined(__s390__)
46 # define __NR_bpf 351
47 # else
48 # error __NR_bpf not defined. libbpf does not support your arch.
49 # endif
50 #endif
51
52 #ifndef min
53 #define min(x, y) ((x) < (y) ? (x) : (y))
54 #endif
55
ptr_to_u64(const void * ptr)56 static inline uint64_t ptr_to_u64(const void *ptr)
57 {
58 return (uint64_t) (unsigned long) ptr;
59 }
60
sys_bpf(enum bpf_cmd cmd,union bpf_attr * attr,unsigned int size)61 static inline int sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr,
62 unsigned int size)
63 {
64 return syscall(__NR_bpf, cmd, attr, size);
65 }
66
bpf_create_map_xattr(const struct bpf_create_map_attr * create_attr)67 int bpf_create_map_xattr(const struct bpf_create_map_attr *create_attr)
68 {
69 uint32_t name_len = create_attr->name ? strlen(create_attr->name) : 0;
70 union bpf_attr attr = {};
71
72 attr.map_type = create_attr->map_type;
73 attr.key_size = create_attr->key_size;
74 attr.value_size = create_attr->value_size;
75 attr.max_entries = create_attr->max_entries;
76 attr.map_flags = create_attr->map_flags;
77 memcpy(attr.map_name, create_attr->name,
78 min(name_len, BPF_OBJ_NAME_LEN - 1));
79 attr.numa_node = create_attr->numa_node;
80 attr.btf_fd = create_attr->btf_fd;
81 attr.btf_key_type_id = create_attr->btf_key_type_id;
82 attr.btf_value_type_id = create_attr->btf_value_type_id;
83 attr.map_ifindex = create_attr->map_ifindex;
84
85 return sys_bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
86 }
87
bpf_create_map_node(enum bpf_map_type map_type,const char * name,int key_size,int value_size,int max_entries,uint32_t map_flags,int node)88 int bpf_create_map_node(enum bpf_map_type map_type, const char *name,
89 int key_size, int value_size, int max_entries,
90 uint32_t map_flags, int node)
91 {
92 struct bpf_create_map_attr map_attr = {};
93
94 map_attr.name = name;
95 map_attr.map_type = map_type;
96 map_attr.map_flags = map_flags;
97 map_attr.key_size = key_size;
98 map_attr.value_size = value_size;
99 map_attr.max_entries = max_entries;
100 if (node >= 0) {
101 map_attr.numa_node = node;
102 map_attr.map_flags |= BPF_F_NUMA_NODE;
103 }
104
105 return bpf_create_map_xattr(&map_attr);
106 }
107
bpf_create_map(enum bpf_map_type map_type,int key_size,int value_size,int max_entries,uint32_t map_flags)108 int bpf_create_map(enum bpf_map_type map_type, int key_size,
109 int value_size, int max_entries, uint32_t map_flags)
110 {
111 struct bpf_create_map_attr map_attr = {};
112
113 map_attr.map_type = map_type;
114 map_attr.map_flags = map_flags;
115 map_attr.key_size = key_size;
116 map_attr.value_size = value_size;
117 map_attr.max_entries = max_entries;
118
119 return bpf_create_map_xattr(&map_attr);
120 }
121
bpf_create_map_name(enum bpf_map_type map_type,const char * name,int key_size,int value_size,int max_entries,uint32_t map_flags)122 int bpf_create_map_name(enum bpf_map_type map_type, const char *name,
123 int key_size, int value_size, int max_entries,
124 uint32_t map_flags)
125 {
126 struct bpf_create_map_attr map_attr = {};
127
128 map_attr.name = name;
129 map_attr.map_type = map_type;
130 map_attr.map_flags = map_flags;
131 map_attr.key_size = key_size;
132 map_attr.value_size = value_size;
133 map_attr.max_entries = max_entries;
134
135 return bpf_create_map_xattr(&map_attr);
136 }
137
bpf_create_map_in_map_node(enum bpf_map_type map_type,const char * name,int key_size,int inner_map_fd,int max_entries,uint32_t map_flags,int node)138 int bpf_create_map_in_map_node(enum bpf_map_type map_type, const char *name,
139 int key_size, int inner_map_fd, int max_entries,
140 uint32_t map_flags, int node)
141 {
142 uint32_t name_len = name ? strlen(name) : 0;
143 union bpf_attr attr = {};
144
145 attr.map_type = map_type;
146 attr.key_size = key_size;
147 attr.value_size = 4;
148 attr.inner_map_fd = inner_map_fd;
149 attr.max_entries = max_entries;
150 attr.map_flags = map_flags;
151 memcpy(attr.map_name, name, min(name_len, BPF_OBJ_NAME_LEN - 1));
152
153 if (node >= 0) {
154 attr.map_flags |= BPF_F_NUMA_NODE;
155 attr.numa_node = node;
156 }
157
158 return sys_bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
159 }
160
bpf_create_map_in_map(enum bpf_map_type map_type,const char * name,int key_size,int inner_map_fd,int max_entries,uint32_t map_flags)161 int bpf_create_map_in_map(enum bpf_map_type map_type, const char *name,
162 int key_size, int inner_map_fd, int max_entries,
163 uint32_t map_flags)
164 {
165 return bpf_create_map_in_map_node(map_type, name, key_size,
166 inner_map_fd, max_entries, map_flags,
167 -1);
168 }
169
bpf_load_program_xattr(const struct bpf_load_program_attr * load_attr,char * log_buf,size_t log_buf_sz)170 int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
171 char *log_buf, size_t log_buf_sz)
172 {
173 union bpf_attr attr;
174 uint32_t name_len;
175 int fd;
176
177 if (!load_attr)
178 return -EINVAL;
179
180 name_len = load_attr->name ? strlen(load_attr->name) : 0;
181
182 memset(&attr, 0, sizeof(attr));
183 attr.prog_type = load_attr->prog_type;
184 attr.expected_attach_type = load_attr->expected_attach_type;
185 attr.insn_cnt = (uint32_t)load_attr->insns_cnt;
186 attr.insns = ptr_to_u64(load_attr->insns);
187 attr.license = ptr_to_u64(load_attr->license);
188 attr.log_buf = ptr_to_u64(NULL);
189 attr.log_size = 0;
190 attr.log_level = 0;
191 attr.kern_version = load_attr->kern_version;
192 attr.prog_ifindex = load_attr->prog_ifindex;
193 memcpy(attr.prog_name, load_attr->name,
194 min(name_len, BPF_OBJ_NAME_LEN - 1));
195
196 fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
197 if (fd >= 0 || !log_buf || !log_buf_sz)
198 return fd;
199
200 /* Try again with log */
201 attr.log_buf = ptr_to_u64(log_buf);
202 attr.log_size = log_buf_sz;
203 attr.log_level = 1;
204 log_buf[0] = 0;
205 return sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
206 }
207
bpf_load_program(enum bpf_prog_type type,const struct bpf_insn * insns,size_t insns_cnt,const char * name,const char * license,uint32_t kern_version,char * log_buf,size_t log_buf_sz)208 int bpf_load_program(enum bpf_prog_type type, const struct bpf_insn *insns,
209 size_t insns_cnt, const char *name, const char *license,
210 uint32_t kern_version, char *log_buf,
211 size_t log_buf_sz)
212 {
213 struct bpf_load_program_attr load_attr = {};
214
215 load_attr.prog_type = type;
216 load_attr.expected_attach_type = 0;
217 load_attr.name = name;
218 load_attr.insns = insns;
219 load_attr.insns_cnt = insns_cnt;
220 load_attr.license = license;
221 load_attr.kern_version = kern_version;
222
223 return bpf_load_program_xattr(&load_attr, log_buf, log_buf_sz);
224 }
225
bpf_verify_program(enum bpf_prog_type type,const struct bpf_insn * insns,size_t insns_cnt,int strict_alignment,const char * license,uint32_t kern_version,char * log_buf,size_t log_buf_sz,int log_level)226 int bpf_verify_program(enum bpf_prog_type type, const struct bpf_insn *insns,
227 size_t insns_cnt, int strict_alignment,
228 const char *license, uint32_t kern_version,
229 char *log_buf, size_t log_buf_sz, int log_level)
230 {
231 union bpf_attr attr = {};
232
233 attr.prog_type = type;
234 attr.insn_cnt = (uint32_t)insns_cnt;
235 attr.insns = ptr_to_u64(insns);
236 attr.license = ptr_to_u64(license);
237 attr.log_buf = ptr_to_u64(log_buf);
238 attr.log_size = log_buf_sz;
239 attr.log_level = log_level;
240 log_buf[0] = 0;
241 attr.kern_version = kern_version;
242 attr.prog_flags = strict_alignment ? BPF_F_STRICT_ALIGNMENT : 0;
243
244 return sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
245 }
246
bpf_map_update_elem(int fd,const void * key,const void * value,uint64_t flags)247 int bpf_map_update_elem(int fd, const void *key, const void *value,
248 uint64_t flags)
249 {
250 union bpf_attr attr = {};
251
252 attr.map_fd = fd;
253 attr.key = ptr_to_u64(key);
254 attr.value = ptr_to_u64(value);
255 attr.flags = flags;
256
257 return sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr));
258 }
259
bpf_map_lookup_elem(int fd,const void * key,void * value)260 int bpf_map_lookup_elem(int fd, const void *key, void *value)
261 {
262 union bpf_attr attr = {};
263
264 attr.map_fd = fd;
265 attr.key = ptr_to_u64(key);
266 attr.value = ptr_to_u64(value);
267
268 return sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, sizeof(attr));
269 }
270
bpf_map_delete_elem(int fd,const void * key)271 int bpf_map_delete_elem(int fd, const void *key)
272 {
273 union bpf_attr attr = {};
274
275 attr.map_fd = fd;
276 attr.key = ptr_to_u64(key);
277
278 return sys_bpf(BPF_MAP_DELETE_ELEM, &attr, sizeof(attr));
279 }
280
bpf_map_get_next_key(int fd,const void * key,void * next_key)281 int bpf_map_get_next_key(int fd, const void *key, void *next_key)
282 {
283 union bpf_attr attr = {};
284
285 attr.map_fd = fd;
286 attr.key = ptr_to_u64(key);
287 attr.next_key = ptr_to_u64(next_key);
288
289 return sys_bpf(BPF_MAP_GET_NEXT_KEY, &attr, sizeof(attr));
290 }
291
bpf_obj_pin(int fd,const char * pathname)292 int bpf_obj_pin(int fd, const char *pathname)
293 {
294 union bpf_attr attr = {};
295
296 attr.pathname = ptr_to_u64((void *)pathname);
297 attr.bpf_fd = fd;
298
299 return sys_bpf(BPF_OBJ_PIN, &attr, sizeof(attr));
300 }
301
bpf_obj_get(const char * pathname)302 int bpf_obj_get(const char *pathname)
303 {
304 union bpf_attr attr = {};
305
306 attr.pathname = ptr_to_u64((void *)pathname);
307
308 return sys_bpf(BPF_OBJ_GET, &attr, sizeof(attr));
309 }
310
bpf_prog_attach(int prog_fd,int target_fd,enum bpf_attach_type type,unsigned int flags)311 int bpf_prog_attach(int prog_fd, int target_fd, enum bpf_attach_type type,
312 unsigned int flags)
313 {
314 union bpf_attr attr = {};
315
316 attr.target_fd = target_fd;
317 attr.attach_bpf_fd = prog_fd;
318 attr.attach_type = type;
319 attr.attach_flags = flags;
320
321 return sys_bpf(BPF_PROG_ATTACH, &attr, sizeof(attr));
322 }
323
bpf_prog_detach(int target_fd,enum bpf_attach_type type)324 int bpf_prog_detach(int target_fd, enum bpf_attach_type type)
325 {
326 union bpf_attr attr = {};
327
328 attr.target_fd = target_fd;
329 attr.attach_type = type;
330
331 return sys_bpf(BPF_PROG_DETACH, &attr, sizeof(attr));
332 }
333
bpf_prog_detach2(int prog_fd,int target_fd,enum bpf_attach_type type)334 int bpf_prog_detach2(int prog_fd, int target_fd, enum bpf_attach_type type)
335 {
336 union bpf_attr attr = {};
337
338 attr.target_fd = target_fd;
339 attr.attach_bpf_fd = prog_fd;
340 attr.attach_type = type;
341
342 return sys_bpf(BPF_PROG_DETACH, &attr, sizeof(attr));
343 }
344
bpf_prog_query(int target_fd,enum bpf_attach_type type,uint32_t query_flags,uint32_t * attach_flags,uint32_t * prog_ids,uint32_t * prog_cnt)345 int bpf_prog_query(int target_fd, enum bpf_attach_type type, uint32_t query_flags,
346 uint32_t *attach_flags, uint32_t *prog_ids, uint32_t *prog_cnt)
347 {
348 union bpf_attr attr = {};
349 int ret;
350
351 attr.query.target_fd = target_fd;
352 attr.query.attach_type = type;
353 attr.query.query_flags = query_flags;
354 attr.query.prog_cnt = *prog_cnt;
355 attr.query.prog_ids = ptr_to_u64(prog_ids);
356
357 ret = sys_bpf(BPF_PROG_QUERY, &attr, sizeof(attr));
358 if (attach_flags)
359 *attach_flags = attr.query.attach_flags;
360 *prog_cnt = attr.query.prog_cnt;
361 return ret;
362 }
363
bpf_prog_test_run(int prog_fd,int repeat,void * data,uint32_t size,void * data_out,uint32_t * size_out,uint32_t * retval,uint32_t * duration)364 int bpf_prog_test_run(int prog_fd, int repeat, void *data, uint32_t size,
365 void *data_out, uint32_t *size_out, uint32_t *retval,
366 uint32_t *duration)
367 {
368 union bpf_attr attr = {};
369 int ret;
370
371 attr.test.prog_fd = prog_fd;
372 attr.test.data_in = ptr_to_u64(data);
373 attr.test.data_out = ptr_to_u64(data_out);
374 attr.test.data_size_in = size;
375 attr.test.repeat = repeat;
376
377 ret = sys_bpf(BPF_PROG_TEST_RUN, &attr, sizeof(attr));
378 if (size_out)
379 *size_out = attr.test.data_size_out;
380 if (retval)
381 *retval = attr.test.retval;
382 if (duration)
383 *duration = attr.test.duration;
384 return ret;
385 }
386
bpf_prog_get_next_id(uint32_t start_id,uint32_t * next_id)387 int bpf_prog_get_next_id(uint32_t start_id, uint32_t *next_id)
388 {
389 union bpf_attr attr = {};
390 int err;
391
392 attr.start_id = start_id;
393
394 err = sys_bpf(BPF_PROG_GET_NEXT_ID, &attr, sizeof(attr));
395 if (!err)
396 *next_id = attr.next_id;
397
398 return err;
399 }
400
bpf_map_get_next_id(uint32_t start_id,uint32_t * next_id)401 int bpf_map_get_next_id(uint32_t start_id, uint32_t *next_id)
402 {
403 union bpf_attr attr = {};
404 int err;
405
406 attr.start_id = start_id;
407
408 err = sys_bpf(BPF_MAP_GET_NEXT_ID, &attr, sizeof(attr));
409 if (!err)
410 *next_id = attr.next_id;
411
412 return err;
413 }
414
bpf_prog_get_fd_by_id(uint32_t id)415 int bpf_prog_get_fd_by_id(uint32_t id)
416 {
417 union bpf_attr attr = {};
418
419 attr.prog_id = id;
420
421 return sys_bpf(BPF_PROG_GET_FD_BY_ID, &attr, sizeof(attr));
422 }
423
bpf_map_get_fd_by_id(uint32_t id)424 int bpf_map_get_fd_by_id(uint32_t id)
425 {
426 union bpf_attr attr = {};
427
428 attr.map_id = id;
429
430 return sys_bpf(BPF_MAP_GET_FD_BY_ID, &attr, sizeof(attr));
431 }
432
bpf_btf_get_fd_by_id(uint32_t id)433 int bpf_btf_get_fd_by_id(uint32_t id)
434 {
435 union bpf_attr attr = {};
436
437 attr.btf_id = id;
438
439 return sys_bpf(BPF_BTF_GET_FD_BY_ID, &attr, sizeof(attr));
440 }
441
bpf_obj_get_info_by_fd(int prog_fd,void * info,uint32_t * info_len)442 int bpf_obj_get_info_by_fd(int prog_fd, void *info, uint32_t *info_len)
443 {
444 union bpf_attr attr = {};
445 int err;
446
447 attr.info.bpf_fd = prog_fd;
448 attr.info.info_len = *info_len;
449 attr.info.info = ptr_to_u64(info);
450
451 err = sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, sizeof(attr));
452 if (!err)
453 *info_len = attr.info.info_len;
454
455 return err;
456 }
457
bpf_raw_tracepoint_open(const char * name,int prog_fd)458 int bpf_raw_tracepoint_open(const char *name, int prog_fd)
459 {
460 union bpf_attr attr = {};
461
462 attr.raw_tracepoint.name = ptr_to_u64(name);
463 attr.raw_tracepoint.prog_fd = prog_fd;
464
465 return sys_bpf(BPF_RAW_TRACEPOINT_OPEN, &attr, sizeof(attr));
466 }
467
bpf_load_btf(void * btf,uint32_t btf_size,char * log_buf,uint32_t log_buf_size,bool do_log)468 int bpf_load_btf(void *btf, uint32_t btf_size, char *log_buf, uint32_t log_buf_size,
469 bool do_log)
470 {
471 union bpf_attr attr = {};
472 int fd;
473
474 attr.btf = ptr_to_u64(btf);
475 attr.btf_size = btf_size;
476
477 retry:
478 if (do_log && log_buf && log_buf_size) {
479 attr.btf_log_level = 1;
480 attr.btf_log_size = log_buf_size;
481 attr.btf_log_buf = ptr_to_u64(log_buf);
482 }
483
484 fd = sys_bpf(BPF_BTF_LOAD, &attr, sizeof(attr));
485 if (fd == -1 && !do_log && log_buf && log_buf_size) {
486 do_log = true;
487 goto retry;
488 }
489
490 return fd;
491 }
492