1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __BPF_HELPERS_H 3 #define __BPF_HELPERS_H 4 5 /* helper macro to place programs, maps, license in 6 * different sections in elf_bpf file. Section names 7 * are interpreted by elf_bpf loader 8 */ 9 #define SEC(NAME) __attribute__((section(NAME), used)) 10 11 /* helper functions called from eBPF programs written in C */ 12 static void *(*bpf_map_lookup_elem)(void *map, void *key) = 13 (void *) BPF_FUNC_map_lookup_elem; 14 static int (*bpf_map_update_elem)(void *map, void *key, void *value, 15 unsigned long long flags) = 16 (void *) BPF_FUNC_map_update_elem; 17 static int (*bpf_map_delete_elem)(void *map, void *key) = 18 (void *) BPF_FUNC_map_delete_elem; 19 static int (*bpf_map_push_elem)(void *map, void *value, 20 unsigned long long flags) = 21 (void *) BPF_FUNC_map_push_elem; 22 static int (*bpf_map_pop_elem)(void *map, void *value) = 23 (void *) BPF_FUNC_map_pop_elem; 24 static int (*bpf_map_peek_elem)(void *map, void *value) = 25 (void *) BPF_FUNC_map_peek_elem; 26 static int (*bpf_probe_read)(void *dst, int size, void *unsafe_ptr) = 27 (void *) BPF_FUNC_probe_read; 28 static unsigned long long (*bpf_ktime_get_ns)(void) = 29 (void *) BPF_FUNC_ktime_get_ns; 30 static int (*bpf_trace_printk)(const char *fmt, int fmt_size, ...) = 31 (void *) BPF_FUNC_trace_printk; 32 static void (*bpf_tail_call)(void *ctx, void *map, int index) = 33 (void *) BPF_FUNC_tail_call; 34 static unsigned long long (*bpf_get_smp_processor_id)(void) = 35 (void *) BPF_FUNC_get_smp_processor_id; 36 static unsigned long long (*bpf_get_current_pid_tgid)(void) = 37 (void *) BPF_FUNC_get_current_pid_tgid; 38 static unsigned long long (*bpf_get_current_uid_gid)(void) = 39 (void *) BPF_FUNC_get_current_uid_gid; 40 static int (*bpf_get_current_comm)(void *buf, int buf_size) = 41 (void *) BPF_FUNC_get_current_comm; 42 static unsigned long long (*bpf_perf_event_read)(void *map, 43 unsigned long long flags) = 44 (void *) BPF_FUNC_perf_event_read; 45 static int (*bpf_clone_redirect)(void *ctx, int ifindex, int flags) = 46 (void *) BPF_FUNC_clone_redirect; 47 static int (*bpf_redirect)(int ifindex, int flags) = 48 (void *) BPF_FUNC_redirect; 49 static int (*bpf_redirect_map)(void *map, int key, int flags) = 50 (void *) BPF_FUNC_redirect_map; 51 static int (*bpf_perf_event_output)(void *ctx, void *map, 52 unsigned long long flags, void *data, 53 int size) = 54 (void *) BPF_FUNC_perf_event_output; 55 static int (*bpf_get_stackid)(void *ctx, void *map, int flags) = 56 (void *) BPF_FUNC_get_stackid; 57 static int (*bpf_probe_write_user)(void *dst, void *src, int size) = 58 (void *) BPF_FUNC_probe_write_user; 59 static int (*bpf_current_task_under_cgroup)(void *map, int index) = 60 (void *) BPF_FUNC_current_task_under_cgroup; 61 static int (*bpf_skb_get_tunnel_key)(void *ctx, void *key, int size, int flags) = 62 (void *) BPF_FUNC_skb_get_tunnel_key; 63 static int (*bpf_skb_set_tunnel_key)(void *ctx, void *key, int size, int flags) = 64 (void *) BPF_FUNC_skb_set_tunnel_key; 65 static int (*bpf_skb_get_tunnel_opt)(void *ctx, void *md, int size) = 66 (void *) BPF_FUNC_skb_get_tunnel_opt; 67 static int (*bpf_skb_set_tunnel_opt)(void *ctx, void *md, int size) = 68 (void *) BPF_FUNC_skb_set_tunnel_opt; 69 static unsigned long long (*bpf_get_prandom_u32)(void) = 70 (void *) BPF_FUNC_get_prandom_u32; 71 static int (*bpf_xdp_adjust_head)(void *ctx, int offset) = 72 (void *) BPF_FUNC_xdp_adjust_head; 73 static int (*bpf_xdp_adjust_meta)(void *ctx, int offset) = 74 (void *) BPF_FUNC_xdp_adjust_meta; 75 static int (*bpf_get_socket_cookie)(void *ctx) = 76 (void *) BPF_FUNC_get_socket_cookie; 77 static int (*bpf_setsockopt)(void *ctx, int level, int optname, void *optval, 78 int optlen) = 79 (void *) BPF_FUNC_setsockopt; 80 static int (*bpf_getsockopt)(void *ctx, int level, int optname, void *optval, 81 int optlen) = 82 (void *) BPF_FUNC_getsockopt; 83 static int (*bpf_sock_ops_cb_flags_set)(void *ctx, int flags) = 84 (void *) BPF_FUNC_sock_ops_cb_flags_set; 85 static int (*bpf_sk_redirect_map)(void *ctx, void *map, int key, int flags) = 86 (void *) BPF_FUNC_sk_redirect_map; 87 static int (*bpf_sk_redirect_hash)(void *ctx, void *map, void *key, int flags) = 88 (void *) BPF_FUNC_sk_redirect_hash; 89 static int (*bpf_sock_map_update)(void *map, void *key, void *value, 90 unsigned long long flags) = 91 (void *) BPF_FUNC_sock_map_update; 92 static int (*bpf_sock_hash_update)(void *map, void *key, void *value, 93 unsigned long long flags) = 94 (void *) BPF_FUNC_sock_hash_update; 95 static int (*bpf_perf_event_read_value)(void *map, unsigned long long flags, 96 void *buf, unsigned int buf_size) = 97 (void *) BPF_FUNC_perf_event_read_value; 98 static int (*bpf_perf_prog_read_value)(void *ctx, void *buf, 99 unsigned int buf_size) = 100 (void *) BPF_FUNC_perf_prog_read_value; 101 static int (*bpf_override_return)(void *ctx, unsigned long rc) = 102 (void *) BPF_FUNC_override_return; 103 static int (*bpf_msg_redirect_map)(void *ctx, void *map, int key, int flags) = 104 (void *) BPF_FUNC_msg_redirect_map; 105 static int (*bpf_msg_redirect_hash)(void *ctx, 106 void *map, void *key, int flags) = 107 (void *) BPF_FUNC_msg_redirect_hash; 108 static int (*bpf_msg_apply_bytes)(void *ctx, int len) = 109 (void *) BPF_FUNC_msg_apply_bytes; 110 static int (*bpf_msg_cork_bytes)(void *ctx, int len) = 111 (void *) BPF_FUNC_msg_cork_bytes; 112 static int (*bpf_msg_pull_data)(void *ctx, int start, int end, int flags) = 113 (void *) BPF_FUNC_msg_pull_data; 114 static int (*bpf_msg_push_data)(void *ctx, int start, int end, int flags) = 115 (void *) BPF_FUNC_msg_push_data; 116 static int (*bpf_msg_pop_data)(void *ctx, int start, int cut, int flags) = 117 (void *) BPF_FUNC_msg_pop_data; 118 static int (*bpf_bind)(void *ctx, void *addr, int addr_len) = 119 (void *) BPF_FUNC_bind; 120 static int (*bpf_xdp_adjust_tail)(void *ctx, int offset) = 121 (void *) BPF_FUNC_xdp_adjust_tail; 122 static int (*bpf_skb_get_xfrm_state)(void *ctx, int index, void *state, 123 int size, int flags) = 124 (void *) BPF_FUNC_skb_get_xfrm_state; 125 static int (*bpf_sk_select_reuseport)(void *ctx, void *map, void *key, uint32_t flags) = 126 (void *) BPF_FUNC_sk_select_reuseport; 127 static int (*bpf_get_stack)(void *ctx, void *buf, int size, int flags) = 128 (void *) BPF_FUNC_get_stack; 129 static int (*bpf_fib_lookup)(void *ctx, struct bpf_fib_lookup *params, 130 int plen, uint32_t flags) = 131 (void *) BPF_FUNC_fib_lookup; 132 static int (*bpf_lwt_push_encap)(void *ctx, unsigned int type, void *hdr, 133 unsigned int len) = 134 (void *) BPF_FUNC_lwt_push_encap; 135 static int (*bpf_lwt_seg6_store_bytes)(void *ctx, unsigned int offset, 136 void *from, unsigned int len) = 137 (void *) BPF_FUNC_lwt_seg6_store_bytes; 138 static int (*bpf_lwt_seg6_action)(void *ctx, unsigned int action, void *param, 139 unsigned int param_len) = 140 (void *) BPF_FUNC_lwt_seg6_action; 141 static int (*bpf_lwt_seg6_adjust_srh)(void *ctx, unsigned int offset, 142 unsigned int len) = 143 (void *) BPF_FUNC_lwt_seg6_adjust_srh; 144 static int (*bpf_rc_repeat)(void *ctx) = 145 (void *) BPF_FUNC_rc_repeat; 146 static int (*bpf_rc_keydown)(void *ctx, unsigned int protocol, 147 unsigned long long scancode, unsigned int toggle) = 148 (void *) BPF_FUNC_rc_keydown; 149 static unsigned long long (*bpf_get_current_cgroup_id)(void) = 150 (void *) BPF_FUNC_get_current_cgroup_id; 151 static void *(*bpf_get_local_storage)(void *map, unsigned long long flags) = 152 (void *) BPF_FUNC_get_local_storage; 153 static unsigned long long (*bpf_skb_cgroup_id)(void *ctx) = 154 (void *) BPF_FUNC_skb_cgroup_id; 155 static unsigned long long (*bpf_skb_ancestor_cgroup_id)(void *ctx, int level) = 156 (void *) BPF_FUNC_skb_ancestor_cgroup_id; 157 static struct bpf_sock *(*bpf_sk_lookup_tcp)(void *ctx, 158 struct bpf_sock_tuple *tuple, 159 int size, unsigned long long netns_id, 160 unsigned long long flags) = 161 (void *) BPF_FUNC_sk_lookup_tcp; 162 static struct bpf_sock *(*bpf_sk_lookup_udp)(void *ctx, 163 struct bpf_sock_tuple *tuple, 164 int size, unsigned long long netns_id, 165 unsigned long long flags) = 166 (void *) BPF_FUNC_sk_lookup_udp; 167 static int (*bpf_sk_release)(struct bpf_sock *sk) = 168 (void *) BPF_FUNC_sk_release; 169 static int (*bpf_skb_vlan_push)(void *ctx, uint16_t vlan_proto, uint16_t vlan_tci) = 170 (void *) BPF_FUNC_skb_vlan_push; 171 static int (*bpf_skb_vlan_pop)(void *ctx) = 172 (void *) BPF_FUNC_skb_vlan_pop; 173 static int (*bpf_rc_pointer_rel)(void *ctx, int rel_x, int rel_y) = 174 (void *) BPF_FUNC_rc_pointer_rel; 175 176 /* llvm builtin functions that eBPF C program may use to 177 * emit BPF_LD_ABS and BPF_LD_IND instructions 178 */ 179 struct sk_buff; 180 unsigned long long load_byte(void *skb, 181 unsigned long long off) asm("llvm.bpf.load.byte"); 182 unsigned long long load_half(void *skb, 183 unsigned long long off) asm("llvm.bpf.load.half"); 184 unsigned long long load_word(void *skb, 185 unsigned long long off) asm("llvm.bpf.load.word"); 186 187 /* a helper structure used by eBPF C program 188 * to describe map attributes to elf_bpf loader 189 */ 190 struct bpf_map_def { 191 unsigned int type; 192 unsigned int key_size; 193 unsigned int value_size; 194 unsigned int max_entries; 195 unsigned int map_flags; 196 unsigned int inner_map_idx; 197 unsigned int numa_node; 198 }; 199 200 #define BPF_ANNOTATE_KV_PAIR(name, type_key, type_val) \ 201 struct ____btf_map_##name { \ 202 type_key key; \ 203 type_val value; \ 204 }; \ 205 struct ____btf_map_##name \ 206 __attribute__ ((section(".maps." #name), used)) \ 207 ____btf_map_##name = { } 208 209 static int (*bpf_skb_load_bytes)(void *ctx, int off, void *to, int len) = 210 (void *) BPF_FUNC_skb_load_bytes; 211 static int (*bpf_skb_load_bytes_relative)(void *ctx, int off, void *to, int len, uint32_t start_header) = 212 (void *) BPF_FUNC_skb_load_bytes_relative; 213 static int (*bpf_skb_store_bytes)(void *ctx, int off, void *from, int len, int flags) = 214 (void *) BPF_FUNC_skb_store_bytes; 215 static int (*bpf_l3_csum_replace)(void *ctx, int off, int from, int to, int flags) = 216 (void *) BPF_FUNC_l3_csum_replace; 217 static int (*bpf_l4_csum_replace)(void *ctx, int off, int from, int to, int flags) = 218 (void *) BPF_FUNC_l4_csum_replace; 219 static int (*bpf_csum_diff)(void *from, int from_size, void *to, int to_size, int seed) = 220 (void *) BPF_FUNC_csum_diff; 221 static int (*bpf_skb_under_cgroup)(void *ctx, void *map, int index) = 222 (void *) BPF_FUNC_skb_under_cgroup; 223 static int (*bpf_skb_change_head)(void *, int len, int flags) = 224 (void *) BPF_FUNC_skb_change_head; 225 static int (*bpf_skb_pull_data)(void *, int len) = 226 (void *) BPF_FUNC_skb_pull_data; 227 228 /* Scan the ARCH passed in from ARCH env variable (see Makefile) */ 229 #if defined(__TARGET_ARCH_x86) 230 #define bpf_target_x86 231 #define bpf_target_defined 232 #elif defined(__TARGET_ARCH_s930x) 233 #define bpf_target_s930x 234 #define bpf_target_defined 235 #elif defined(__TARGET_ARCH_arm64) 236 #define bpf_target_arm64 237 #define bpf_target_defined 238 #elif defined(__TARGET_ARCH_mips) 239 #define bpf_target_mips 240 #define bpf_target_defined 241 #elif defined(__TARGET_ARCH_powerpc) 242 #define bpf_target_powerpc 243 #define bpf_target_defined 244 #elif defined(__TARGET_ARCH_sparc) 245 #define bpf_target_sparc 246 #define bpf_target_defined 247 #else 248 #undef bpf_target_defined 249 #endif 250 251 /* Fall back to what the compiler says */ 252 #ifndef bpf_target_defined 253 #if defined(__x86_64__) 254 #define bpf_target_x86 255 #elif defined(__s390x__) 256 #define bpf_target_s930x 257 #elif defined(__aarch64__) 258 #define bpf_target_arm64 259 #elif defined(__mips__) 260 #define bpf_target_mips 261 #elif defined(__powerpc__) 262 #define bpf_target_powerpc 263 #elif defined(__sparc__) 264 #define bpf_target_sparc 265 #endif 266 #endif 267 268 #if defined(bpf_target_x86) 269 270 #define PT_REGS_PARM1(x) ((x)->di) 271 #define PT_REGS_PARM2(x) ((x)->si) 272 #define PT_REGS_PARM3(x) ((x)->dx) 273 #define PT_REGS_PARM4(x) ((x)->cx) 274 #define PT_REGS_PARM5(x) ((x)->r8) 275 #define PT_REGS_RET(x) ((x)->sp) 276 #define PT_REGS_FP(x) ((x)->bp) 277 #define PT_REGS_RC(x) ((x)->ax) 278 #define PT_REGS_SP(x) ((x)->sp) 279 #define PT_REGS_IP(x) ((x)->ip) 280 281 #elif defined(bpf_target_s390x) 282 283 #define PT_REGS_PARM1(x) ((x)->gprs[2]) 284 #define PT_REGS_PARM2(x) ((x)->gprs[3]) 285 #define PT_REGS_PARM3(x) ((x)->gprs[4]) 286 #define PT_REGS_PARM4(x) ((x)->gprs[5]) 287 #define PT_REGS_PARM5(x) ((x)->gprs[6]) 288 #define PT_REGS_RET(x) ((x)->gprs[14]) 289 #define PT_REGS_FP(x) ((x)->gprs[11]) /* Works only with CONFIG_FRAME_POINTER */ 290 #define PT_REGS_RC(x) ((x)->gprs[2]) 291 #define PT_REGS_SP(x) ((x)->gprs[15]) 292 #define PT_REGS_IP(x) ((x)->psw.addr) 293 294 #elif defined(bpf_target_arm64) 295 296 #define PT_REGS_PARM1(x) ((x)->regs[0]) 297 #define PT_REGS_PARM2(x) ((x)->regs[1]) 298 #define PT_REGS_PARM3(x) ((x)->regs[2]) 299 #define PT_REGS_PARM4(x) ((x)->regs[3]) 300 #define PT_REGS_PARM5(x) ((x)->regs[4]) 301 #define PT_REGS_RET(x) ((x)->regs[30]) 302 #define PT_REGS_FP(x) ((x)->regs[29]) /* Works only with CONFIG_FRAME_POINTER */ 303 #define PT_REGS_RC(x) ((x)->regs[0]) 304 #define PT_REGS_SP(x) ((x)->sp) 305 #define PT_REGS_IP(x) ((x)->pc) 306 307 #elif defined(bpf_target_mips) 308 309 #define PT_REGS_PARM1(x) ((x)->regs[4]) 310 #define PT_REGS_PARM2(x) ((x)->regs[5]) 311 #define PT_REGS_PARM3(x) ((x)->regs[6]) 312 #define PT_REGS_PARM4(x) ((x)->regs[7]) 313 #define PT_REGS_PARM5(x) ((x)->regs[8]) 314 #define PT_REGS_RET(x) ((x)->regs[31]) 315 #define PT_REGS_FP(x) ((x)->regs[30]) /* Works only with CONFIG_FRAME_POINTER */ 316 #define PT_REGS_RC(x) ((x)->regs[1]) 317 #define PT_REGS_SP(x) ((x)->regs[29]) 318 #define PT_REGS_IP(x) ((x)->cp0_epc) 319 320 #elif defined(bpf_target_powerpc) 321 322 #define PT_REGS_PARM1(x) ((x)->gpr[3]) 323 #define PT_REGS_PARM2(x) ((x)->gpr[4]) 324 #define PT_REGS_PARM3(x) ((x)->gpr[5]) 325 #define PT_REGS_PARM4(x) ((x)->gpr[6]) 326 #define PT_REGS_PARM5(x) ((x)->gpr[7]) 327 #define PT_REGS_RC(x) ((x)->gpr[3]) 328 #define PT_REGS_SP(x) ((x)->sp) 329 #define PT_REGS_IP(x) ((x)->nip) 330 331 #elif defined(bpf_target_sparc) 332 333 #define PT_REGS_PARM1(x) ((x)->u_regs[UREG_I0]) 334 #define PT_REGS_PARM2(x) ((x)->u_regs[UREG_I1]) 335 #define PT_REGS_PARM3(x) ((x)->u_regs[UREG_I2]) 336 #define PT_REGS_PARM4(x) ((x)->u_regs[UREG_I3]) 337 #define PT_REGS_PARM5(x) ((x)->u_regs[UREG_I4]) 338 #define PT_REGS_RET(x) ((x)->u_regs[UREG_I7]) 339 #define PT_REGS_RC(x) ((x)->u_regs[UREG_I0]) 340 #define PT_REGS_SP(x) ((x)->u_regs[UREG_FP]) 341 342 /* Should this also be a bpf_target check for the sparc case? */ 343 #if defined(__arch64__) 344 #define PT_REGS_IP(x) ((x)->tpc) 345 #else 346 #define PT_REGS_IP(x) ((x)->pc) 347 #endif 348 349 #endif 350 351 #ifdef bpf_target_powerpc 352 #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = (ctx)->link; }) 353 #define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP 354 #elif bpf_target_sparc 355 #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = PT_REGS_RET(ctx); }) 356 #define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP 357 #else 358 #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ \ 359 bpf_probe_read(&(ip), sizeof(ip), (void *)PT_REGS_RET(ctx)); }) 360 #define BPF_KRETPROBE_READ_RET_IP(ip, ctx) ({ \ 361 bpf_probe_read(&(ip), sizeof(ip), \ 362 (void *)(PT_REGS_FP(ctx) + sizeof(ip))); }) 363 #endif 364 365 #endif 366