1fec56f58SAlexei Starovoitov // SPDX-License-Identifier: GPL-2.0-only 2fec56f58SAlexei Starovoitov /* Copyright (c) 2019 Facebook */ 3fec56f58SAlexei Starovoitov #include <linux/hash.h> 4fec56f58SAlexei Starovoitov #include <linux/bpf.h> 5fec56f58SAlexei Starovoitov #include <linux/filter.h> 6b91e014fSAlexei Starovoitov #include <linux/ftrace.h> 7e9b4e606SJiri Olsa #include <linux/rbtree_latch.h> 8a108f7dcSJiri Olsa #include <linux/perf_event.h> 99e4e01dfSKP Singh #include <linux/btf.h> 101e6c62a8SAlexei Starovoitov #include <linux/rcupdate_trace.h> 111e6c62a8SAlexei Starovoitov #include <linux/rcupdate_wait.h> 12861de02eSJiri Olsa #include <linux/module.h> 13856c02dbSSong Liu #include <linux/static_call.h> 1469fd337aSStanislav Fomichev #include <linux/bpf_verifier.h> 1569fd337aSStanislav Fomichev #include <linux/bpf_lsm.h> 16fec56f58SAlexei Starovoitov 17be8704ffSAlexei Starovoitov /* dummy _ops. The verifier will operate on target program's ops. */ 18be8704ffSAlexei Starovoitov const struct bpf_verifier_ops bpf_extension_verifier_ops = { 19be8704ffSAlexei Starovoitov }; 20be8704ffSAlexei Starovoitov const struct bpf_prog_ops bpf_extension_prog_ops = { 21be8704ffSAlexei Starovoitov }; 22be8704ffSAlexei Starovoitov 23fec56f58SAlexei Starovoitov /* btf_vmlinux has ~22k attachable functions. 1k htab is enough. */ 24fec56f58SAlexei Starovoitov #define TRAMPOLINE_HASH_BITS 10 25fec56f58SAlexei Starovoitov #define TRAMPOLINE_TABLE_SIZE (1 << TRAMPOLINE_HASH_BITS) 26fec56f58SAlexei Starovoitov 27fec56f58SAlexei Starovoitov static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE]; 28fec56f58SAlexei Starovoitov 297ac88ebaSJiri Olsa /* serializes access to trampoline_table */ 30fec56f58SAlexei Starovoitov static DEFINE_MUTEX(trampoline_mutex); 31fec56f58SAlexei Starovoitov 32f92c1e18SJiri Olsa bool bpf_prog_has_trampoline(const struct bpf_prog *prog) 33f92c1e18SJiri Olsa { 34f92c1e18SJiri Olsa enum bpf_attach_type eatype = prog->expected_attach_type; 352fcc8241SKui-Feng Lee enum bpf_prog_type ptype = prog->type; 36f92c1e18SJiri Olsa 372fcc8241SKui-Feng Lee return (ptype == BPF_PROG_TYPE_TRACING && 382fcc8241SKui-Feng Lee (eatype == BPF_TRACE_FENTRY || eatype == BPF_TRACE_FEXIT || 392fcc8241SKui-Feng Lee eatype == BPF_MODIFY_RETURN)) || 402fcc8241SKui-Feng Lee (ptype == BPF_PROG_TYPE_LSM && eatype == BPF_LSM_MAC); 41f92c1e18SJiri Olsa } 42f92c1e18SJiri Olsa 437ac88ebaSJiri Olsa void *bpf_jit_alloc_exec_page(void) 4498e8627eSBjörn Töpel { 4598e8627eSBjörn Töpel void *image; 4698e8627eSBjörn Töpel 4798e8627eSBjörn Töpel image = bpf_jit_alloc_exec(PAGE_SIZE); 4898e8627eSBjörn Töpel if (!image) 4998e8627eSBjörn Töpel return NULL; 5098e8627eSBjörn Töpel 5198e8627eSBjörn Töpel set_vm_flush_reset_perms(image); 5298e8627eSBjörn Töpel /* Keep image as writeable. The alternative is to keep flipping ro/rw 5398e8627eSBjörn Töpel * every time new program is attached or detached. 5498e8627eSBjörn Töpel */ 5598e8627eSBjörn Töpel set_memory_x((long)image, 1); 5698e8627eSBjörn Töpel return image; 5798e8627eSBjörn Töpel } 5898e8627eSBjörn Töpel 59a108f7dcSJiri Olsa void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym) 60a108f7dcSJiri Olsa { 61a108f7dcSJiri Olsa ksym->start = (unsigned long) data; 627ac88ebaSJiri Olsa ksym->end = ksym->start + PAGE_SIZE; 63a108f7dcSJiri Olsa bpf_ksym_add(ksym); 64a108f7dcSJiri Olsa perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start, 657ac88ebaSJiri Olsa PAGE_SIZE, false, ksym->name); 66a108f7dcSJiri Olsa } 67a108f7dcSJiri Olsa 68a108f7dcSJiri Olsa void bpf_image_ksym_del(struct bpf_ksym *ksym) 69a108f7dcSJiri Olsa { 70a108f7dcSJiri Olsa bpf_ksym_del(ksym); 71a108f7dcSJiri Olsa perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start, 727ac88ebaSJiri Olsa PAGE_SIZE, true, ksym->name); 73a108f7dcSJiri Olsa } 74a108f7dcSJiri Olsa 75f7b12b6fSToke Høiland-Jørgensen static struct bpf_trampoline *bpf_trampoline_lookup(u64 key) 76fec56f58SAlexei Starovoitov { 77fec56f58SAlexei Starovoitov struct bpf_trampoline *tr; 78fec56f58SAlexei Starovoitov struct hlist_head *head; 79fec56f58SAlexei Starovoitov int i; 80fec56f58SAlexei Starovoitov 81fec56f58SAlexei Starovoitov mutex_lock(&trampoline_mutex); 82fec56f58SAlexei Starovoitov head = &trampoline_table[hash_64(key, TRAMPOLINE_HASH_BITS)]; 83fec56f58SAlexei Starovoitov hlist_for_each_entry(tr, head, hlist) { 84fec56f58SAlexei Starovoitov if (tr->key == key) { 85fec56f58SAlexei Starovoitov refcount_inc(&tr->refcnt); 86fec56f58SAlexei Starovoitov goto out; 87fec56f58SAlexei Starovoitov } 88fec56f58SAlexei Starovoitov } 89fec56f58SAlexei Starovoitov tr = kzalloc(sizeof(*tr), GFP_KERNEL); 90fec56f58SAlexei Starovoitov if (!tr) 91fec56f58SAlexei Starovoitov goto out; 92fec56f58SAlexei Starovoitov 93fec56f58SAlexei Starovoitov tr->key = key; 94fec56f58SAlexei Starovoitov INIT_HLIST_NODE(&tr->hlist); 95fec56f58SAlexei Starovoitov hlist_add_head(&tr->hlist, head); 96fec56f58SAlexei Starovoitov refcount_set(&tr->refcnt, 1); 97fec56f58SAlexei Starovoitov mutex_init(&tr->mutex); 98fec56f58SAlexei Starovoitov for (i = 0; i < BPF_TRAMP_MAX; i++) 99fec56f58SAlexei Starovoitov INIT_HLIST_HEAD(&tr->progs_hlist[i]); 100fec56f58SAlexei Starovoitov out: 101fec56f58SAlexei Starovoitov mutex_unlock(&trampoline_mutex); 102fec56f58SAlexei Starovoitov return tr; 103fec56f58SAlexei Starovoitov } 104fec56f58SAlexei Starovoitov 105861de02eSJiri Olsa static int bpf_trampoline_module_get(struct bpf_trampoline *tr) 106861de02eSJiri Olsa { 107861de02eSJiri Olsa struct module *mod; 108861de02eSJiri Olsa int err = 0; 109861de02eSJiri Olsa 110861de02eSJiri Olsa preempt_disable(); 111861de02eSJiri Olsa mod = __module_text_address((unsigned long) tr->func.addr); 112861de02eSJiri Olsa if (mod && !try_module_get(mod)) 113861de02eSJiri Olsa err = -ENOENT; 114861de02eSJiri Olsa preempt_enable(); 115861de02eSJiri Olsa tr->mod = mod; 116861de02eSJiri Olsa return err; 117861de02eSJiri Olsa } 118861de02eSJiri Olsa 119861de02eSJiri Olsa static void bpf_trampoline_module_put(struct bpf_trampoline *tr) 120861de02eSJiri Olsa { 121861de02eSJiri Olsa module_put(tr->mod); 122861de02eSJiri Olsa tr->mod = NULL; 123861de02eSJiri Olsa } 124861de02eSJiri Olsa 125b91e014fSAlexei Starovoitov static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr) 126b91e014fSAlexei Starovoitov { 127b91e014fSAlexei Starovoitov void *ip = tr->func.addr; 128b91e014fSAlexei Starovoitov int ret; 129b91e014fSAlexei Starovoitov 130b91e014fSAlexei Starovoitov if (tr->func.ftrace_managed) 131b91e014fSAlexei Starovoitov ret = unregister_ftrace_direct((long)ip, (long)old_addr); 132b91e014fSAlexei Starovoitov else 133b91e014fSAlexei Starovoitov ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, NULL); 134861de02eSJiri Olsa 135861de02eSJiri Olsa if (!ret) 136861de02eSJiri Olsa bpf_trampoline_module_put(tr); 137b91e014fSAlexei Starovoitov return ret; 138b91e014fSAlexei Starovoitov } 139b91e014fSAlexei Starovoitov 140b91e014fSAlexei Starovoitov static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_addr) 141b91e014fSAlexei Starovoitov { 142b91e014fSAlexei Starovoitov void *ip = tr->func.addr; 143b91e014fSAlexei Starovoitov int ret; 144b91e014fSAlexei Starovoitov 145b91e014fSAlexei Starovoitov if (tr->func.ftrace_managed) 146b91e014fSAlexei Starovoitov ret = modify_ftrace_direct((long)ip, (long)old_addr, (long)new_addr); 147b91e014fSAlexei Starovoitov else 148b91e014fSAlexei Starovoitov ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, new_addr); 149b91e014fSAlexei Starovoitov return ret; 150b91e014fSAlexei Starovoitov } 151b91e014fSAlexei Starovoitov 152b91e014fSAlexei Starovoitov /* first time registering */ 153b91e014fSAlexei Starovoitov static int register_fentry(struct bpf_trampoline *tr, void *new_addr) 154b91e014fSAlexei Starovoitov { 155b91e014fSAlexei Starovoitov void *ip = tr->func.addr; 156aebfd125SPeter Zijlstra unsigned long faddr; 157b91e014fSAlexei Starovoitov int ret; 158b91e014fSAlexei Starovoitov 159aebfd125SPeter Zijlstra faddr = ftrace_location((unsigned long)ip); 160aebfd125SPeter Zijlstra if (faddr) 161aebfd125SPeter Zijlstra tr->func.ftrace_managed = true; 162b91e014fSAlexei Starovoitov 163861de02eSJiri Olsa if (bpf_trampoline_module_get(tr)) 164861de02eSJiri Olsa return -ENOENT; 165861de02eSJiri Olsa 166b91e014fSAlexei Starovoitov if (tr->func.ftrace_managed) 167b91e014fSAlexei Starovoitov ret = register_ftrace_direct((long)ip, (long)new_addr); 168b91e014fSAlexei Starovoitov else 169b91e014fSAlexei Starovoitov ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, NULL, new_addr); 170861de02eSJiri Olsa 171861de02eSJiri Olsa if (ret) 172861de02eSJiri Olsa bpf_trampoline_module_put(tr); 173b91e014fSAlexei Starovoitov return ret; 174b91e014fSAlexei Starovoitov } 175b91e014fSAlexei Starovoitov 176f7e0beafSKui-Feng Lee static struct bpf_tramp_links * 1771e37392cSJiri Olsa bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_arg) 17888fd9e53SKP Singh { 179f7e0beafSKui-Feng Lee struct bpf_tramp_link *link; 180f7e0beafSKui-Feng Lee struct bpf_tramp_links *tlinks; 181f7e0beafSKui-Feng Lee struct bpf_tramp_link **links; 18288fd9e53SKP Singh int kind; 18388fd9e53SKP Singh 18488fd9e53SKP Singh *total = 0; 185f7e0beafSKui-Feng Lee tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL); 186f7e0beafSKui-Feng Lee if (!tlinks) 18788fd9e53SKP Singh return ERR_PTR(-ENOMEM); 18888fd9e53SKP Singh 18988fd9e53SKP Singh for (kind = 0; kind < BPF_TRAMP_MAX; kind++) { 190f7e0beafSKui-Feng Lee tlinks[kind].nr_links = tr->progs_cnt[kind]; 19188fd9e53SKP Singh *total += tr->progs_cnt[kind]; 192f7e0beafSKui-Feng Lee links = tlinks[kind].links; 19388fd9e53SKP Singh 194f7e0beafSKui-Feng Lee hlist_for_each_entry(link, &tr->progs_hlist[kind], tramp_hlist) { 195f7e0beafSKui-Feng Lee *ip_arg |= link->link.prog->call_get_func_ip; 196f7e0beafSKui-Feng Lee *links++ = link; 19788fd9e53SKP Singh } 1981e37392cSJiri Olsa } 199f7e0beafSKui-Feng Lee return tlinks; 20088fd9e53SKP Singh } 201fec56f58SAlexei Starovoitov 202e21aa341SAlexei Starovoitov static void __bpf_tramp_image_put_deferred(struct work_struct *work) 203e21aa341SAlexei Starovoitov { 204e21aa341SAlexei Starovoitov struct bpf_tramp_image *im; 205e21aa341SAlexei Starovoitov 206e21aa341SAlexei Starovoitov im = container_of(work, struct bpf_tramp_image, work); 207e21aa341SAlexei Starovoitov bpf_image_ksym_del(&im->ksym); 208e21aa341SAlexei Starovoitov bpf_jit_free_exec(im->image); 2093486beddSSong Liu bpf_jit_uncharge_modmem(PAGE_SIZE); 210e21aa341SAlexei Starovoitov percpu_ref_exit(&im->pcref); 211e21aa341SAlexei Starovoitov kfree_rcu(im, rcu); 212e21aa341SAlexei Starovoitov } 213e21aa341SAlexei Starovoitov 214e21aa341SAlexei Starovoitov /* callback, fexit step 3 or fentry step 2 */ 215e21aa341SAlexei Starovoitov static void __bpf_tramp_image_put_rcu(struct rcu_head *rcu) 216e21aa341SAlexei Starovoitov { 217e21aa341SAlexei Starovoitov struct bpf_tramp_image *im; 218e21aa341SAlexei Starovoitov 219e21aa341SAlexei Starovoitov im = container_of(rcu, struct bpf_tramp_image, rcu); 220e21aa341SAlexei Starovoitov INIT_WORK(&im->work, __bpf_tramp_image_put_deferred); 221e21aa341SAlexei Starovoitov schedule_work(&im->work); 222e21aa341SAlexei Starovoitov } 223e21aa341SAlexei Starovoitov 224e21aa341SAlexei Starovoitov /* callback, fexit step 2. Called after percpu_ref_kill confirms. */ 225e21aa341SAlexei Starovoitov static void __bpf_tramp_image_release(struct percpu_ref *pcref) 226e21aa341SAlexei Starovoitov { 227e21aa341SAlexei Starovoitov struct bpf_tramp_image *im; 228e21aa341SAlexei Starovoitov 229e21aa341SAlexei Starovoitov im = container_of(pcref, struct bpf_tramp_image, pcref); 230e21aa341SAlexei Starovoitov call_rcu_tasks(&im->rcu, __bpf_tramp_image_put_rcu); 231e21aa341SAlexei Starovoitov } 232e21aa341SAlexei Starovoitov 233e21aa341SAlexei Starovoitov /* callback, fexit or fentry step 1 */ 234e21aa341SAlexei Starovoitov static void __bpf_tramp_image_put_rcu_tasks(struct rcu_head *rcu) 235e21aa341SAlexei Starovoitov { 236e21aa341SAlexei Starovoitov struct bpf_tramp_image *im; 237e21aa341SAlexei Starovoitov 238e21aa341SAlexei Starovoitov im = container_of(rcu, struct bpf_tramp_image, rcu); 239e21aa341SAlexei Starovoitov if (im->ip_after_call) 240e21aa341SAlexei Starovoitov /* the case of fmod_ret/fexit trampoline and CONFIG_PREEMPTION=y */ 241e21aa341SAlexei Starovoitov percpu_ref_kill(&im->pcref); 242e21aa341SAlexei Starovoitov else 243e21aa341SAlexei Starovoitov /* the case of fentry trampoline */ 244e21aa341SAlexei Starovoitov call_rcu_tasks(&im->rcu, __bpf_tramp_image_put_rcu); 245e21aa341SAlexei Starovoitov } 246e21aa341SAlexei Starovoitov 247e21aa341SAlexei Starovoitov static void bpf_tramp_image_put(struct bpf_tramp_image *im) 248e21aa341SAlexei Starovoitov { 249e21aa341SAlexei Starovoitov /* The trampoline image that calls original function is using: 250e21aa341SAlexei Starovoitov * rcu_read_lock_trace to protect sleepable bpf progs 251e21aa341SAlexei Starovoitov * rcu_read_lock to protect normal bpf progs 252e21aa341SAlexei Starovoitov * percpu_ref to protect trampoline itself 253e21aa341SAlexei Starovoitov * rcu tasks to protect trampoline asm not covered by percpu_ref 254e21aa341SAlexei Starovoitov * (which are few asm insns before __bpf_tramp_enter and 255e21aa341SAlexei Starovoitov * after __bpf_tramp_exit) 256e21aa341SAlexei Starovoitov * 257e21aa341SAlexei Starovoitov * The trampoline is unreachable before bpf_tramp_image_put(). 258e21aa341SAlexei Starovoitov * 259e21aa341SAlexei Starovoitov * First, patch the trampoline to avoid calling into fexit progs. 260e21aa341SAlexei Starovoitov * The progs will be freed even if the original function is still 261e21aa341SAlexei Starovoitov * executing or sleeping. 262e21aa341SAlexei Starovoitov * In case of CONFIG_PREEMPT=y use call_rcu_tasks() to wait on 263e21aa341SAlexei Starovoitov * first few asm instructions to execute and call into 264e21aa341SAlexei Starovoitov * __bpf_tramp_enter->percpu_ref_get. 265e21aa341SAlexei Starovoitov * Then use percpu_ref_kill to wait for the trampoline and the original 266e21aa341SAlexei Starovoitov * function to finish. 267e21aa341SAlexei Starovoitov * Then use call_rcu_tasks() to make sure few asm insns in 268e21aa341SAlexei Starovoitov * the trampoline epilogue are done as well. 269e21aa341SAlexei Starovoitov * 270e21aa341SAlexei Starovoitov * In !PREEMPT case the task that got interrupted in the first asm 271e21aa341SAlexei Starovoitov * insns won't go through an RCU quiescent state which the 272e21aa341SAlexei Starovoitov * percpu_ref_kill will be waiting for. Hence the first 273e21aa341SAlexei Starovoitov * call_rcu_tasks() is not necessary. 274e21aa341SAlexei Starovoitov */ 275e21aa341SAlexei Starovoitov if (im->ip_after_call) { 276e21aa341SAlexei Starovoitov int err = bpf_arch_text_poke(im->ip_after_call, BPF_MOD_JUMP, 277e21aa341SAlexei Starovoitov NULL, im->ip_epilogue); 278e21aa341SAlexei Starovoitov WARN_ON(err); 279e21aa341SAlexei Starovoitov if (IS_ENABLED(CONFIG_PREEMPTION)) 280e21aa341SAlexei Starovoitov call_rcu_tasks(&im->rcu, __bpf_tramp_image_put_rcu_tasks); 281e21aa341SAlexei Starovoitov else 282e21aa341SAlexei Starovoitov percpu_ref_kill(&im->pcref); 283e21aa341SAlexei Starovoitov return; 284e21aa341SAlexei Starovoitov } 285e21aa341SAlexei Starovoitov 286e21aa341SAlexei Starovoitov /* The trampoline without fexit and fmod_ret progs doesn't call original 287e21aa341SAlexei Starovoitov * function and doesn't use percpu_ref. 288e21aa341SAlexei Starovoitov * Use call_rcu_tasks_trace() to wait for sleepable progs to finish. 289e21aa341SAlexei Starovoitov * Then use call_rcu_tasks() to wait for the rest of trampoline asm 290e21aa341SAlexei Starovoitov * and normal progs. 291e21aa341SAlexei Starovoitov */ 292e21aa341SAlexei Starovoitov call_rcu_tasks_trace(&im->rcu, __bpf_tramp_image_put_rcu_tasks); 293e21aa341SAlexei Starovoitov } 294e21aa341SAlexei Starovoitov 295e21aa341SAlexei Starovoitov static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, u32 idx) 296e21aa341SAlexei Starovoitov { 297e21aa341SAlexei Starovoitov struct bpf_tramp_image *im; 298e21aa341SAlexei Starovoitov struct bpf_ksym *ksym; 299e21aa341SAlexei Starovoitov void *image; 300e21aa341SAlexei Starovoitov int err = -ENOMEM; 301e21aa341SAlexei Starovoitov 302e21aa341SAlexei Starovoitov im = kzalloc(sizeof(*im), GFP_KERNEL); 303e21aa341SAlexei Starovoitov if (!im) 304e21aa341SAlexei Starovoitov goto out; 305e21aa341SAlexei Starovoitov 3063486beddSSong Liu err = bpf_jit_charge_modmem(PAGE_SIZE); 307e21aa341SAlexei Starovoitov if (err) 308e21aa341SAlexei Starovoitov goto out_free_im; 309e21aa341SAlexei Starovoitov 310e21aa341SAlexei Starovoitov err = -ENOMEM; 311e21aa341SAlexei Starovoitov im->image = image = bpf_jit_alloc_exec_page(); 312e21aa341SAlexei Starovoitov if (!image) 313e21aa341SAlexei Starovoitov goto out_uncharge; 314e21aa341SAlexei Starovoitov 315e21aa341SAlexei Starovoitov err = percpu_ref_init(&im->pcref, __bpf_tramp_image_release, 0, GFP_KERNEL); 316e21aa341SAlexei Starovoitov if (err) 317e21aa341SAlexei Starovoitov goto out_free_image; 318e21aa341SAlexei Starovoitov 319e21aa341SAlexei Starovoitov ksym = &im->ksym; 320e21aa341SAlexei Starovoitov INIT_LIST_HEAD_RCU(&ksym->lnode); 321e21aa341SAlexei Starovoitov snprintf(ksym->name, KSYM_NAME_LEN, "bpf_trampoline_%llu_%u", key, idx); 322e21aa341SAlexei Starovoitov bpf_image_ksym_add(image, ksym); 323e21aa341SAlexei Starovoitov return im; 324e21aa341SAlexei Starovoitov 325e21aa341SAlexei Starovoitov out_free_image: 326e21aa341SAlexei Starovoitov bpf_jit_free_exec(im->image); 327e21aa341SAlexei Starovoitov out_uncharge: 3283486beddSSong Liu bpf_jit_uncharge_modmem(PAGE_SIZE); 329e21aa341SAlexei Starovoitov out_free_im: 330e21aa341SAlexei Starovoitov kfree(im); 331e21aa341SAlexei Starovoitov out: 332e21aa341SAlexei Starovoitov return ERR_PTR(err); 333e21aa341SAlexei Starovoitov } 334e21aa341SAlexei Starovoitov 335fec56f58SAlexei Starovoitov static int bpf_trampoline_update(struct bpf_trampoline *tr) 336fec56f58SAlexei Starovoitov { 337e21aa341SAlexei Starovoitov struct bpf_tramp_image *im; 338f7e0beafSKui-Feng Lee struct bpf_tramp_links *tlinks; 339fec56f58SAlexei Starovoitov u32 flags = BPF_TRAMP_F_RESTORE_REGS; 3401e37392cSJiri Olsa bool ip_arg = false; 34188fd9e53SKP Singh int err, total; 342fec56f58SAlexei Starovoitov 343f7e0beafSKui-Feng Lee tlinks = bpf_trampoline_get_progs(tr, &total, &ip_arg); 344f7e0beafSKui-Feng Lee if (IS_ERR(tlinks)) 345f7e0beafSKui-Feng Lee return PTR_ERR(tlinks); 34688fd9e53SKP Singh 34788fd9e53SKP Singh if (total == 0) { 348e21aa341SAlexei Starovoitov err = unregister_fentry(tr, tr->cur_image->image); 349e21aa341SAlexei Starovoitov bpf_tramp_image_put(tr->cur_image); 350e21aa341SAlexei Starovoitov tr->cur_image = NULL; 351fec56f58SAlexei Starovoitov tr->selector = 0; 352fec56f58SAlexei Starovoitov goto out; 353fec56f58SAlexei Starovoitov } 354fec56f58SAlexei Starovoitov 355e21aa341SAlexei Starovoitov im = bpf_tramp_image_alloc(tr->key, tr->selector); 356e21aa341SAlexei Starovoitov if (IS_ERR(im)) { 357e21aa341SAlexei Starovoitov err = PTR_ERR(im); 358e21aa341SAlexei Starovoitov goto out; 359e21aa341SAlexei Starovoitov } 360e21aa341SAlexei Starovoitov 361f7e0beafSKui-Feng Lee if (tlinks[BPF_TRAMP_FEXIT].nr_links || 362f7e0beafSKui-Feng Lee tlinks[BPF_TRAMP_MODIFY_RETURN].nr_links) 363535a57a7SXu Kuohai /* NOTE: BPF_TRAMP_F_RESTORE_REGS and BPF_TRAMP_F_SKIP_FRAME 364535a57a7SXu Kuohai * should not be set together. 365535a57a7SXu Kuohai */ 366fec56f58SAlexei Starovoitov flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME; 367fec56f58SAlexei Starovoitov 3681e37392cSJiri Olsa if (ip_arg) 3691e37392cSJiri Olsa flags |= BPF_TRAMP_F_IP_ARG; 3701e37392cSJiri Olsa 371e21aa341SAlexei Starovoitov err = arch_prepare_bpf_trampoline(im, im->image, im->image + PAGE_SIZE, 372f7e0beafSKui-Feng Lee &tr->func.model, flags, tlinks, 373fec56f58SAlexei Starovoitov tr->func.addr); 37485d33df3SMartin KaFai Lau if (err < 0) 375fec56f58SAlexei Starovoitov goto out; 376fec56f58SAlexei Starovoitov 377e21aa341SAlexei Starovoitov WARN_ON(tr->cur_image && tr->selector == 0); 378e21aa341SAlexei Starovoitov WARN_ON(!tr->cur_image && tr->selector); 379e21aa341SAlexei Starovoitov if (tr->cur_image) 380fec56f58SAlexei Starovoitov /* progs already running at this address */ 381e21aa341SAlexei Starovoitov err = modify_fentry(tr, tr->cur_image->image, im->image); 382fec56f58SAlexei Starovoitov else 383fec56f58SAlexei Starovoitov /* first time registering */ 384e21aa341SAlexei Starovoitov err = register_fentry(tr, im->image); 385fec56f58SAlexei Starovoitov if (err) 386fec56f58SAlexei Starovoitov goto out; 387e21aa341SAlexei Starovoitov if (tr->cur_image) 388e21aa341SAlexei Starovoitov bpf_tramp_image_put(tr->cur_image); 389e21aa341SAlexei Starovoitov tr->cur_image = im; 390fec56f58SAlexei Starovoitov tr->selector++; 391fec56f58SAlexei Starovoitov out: 392f7e0beafSKui-Feng Lee kfree(tlinks); 393fec56f58SAlexei Starovoitov return err; 394fec56f58SAlexei Starovoitov } 395fec56f58SAlexei Starovoitov 3969e4e01dfSKP Singh static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) 397fec56f58SAlexei Starovoitov { 3989e4e01dfSKP Singh switch (prog->expected_attach_type) { 399fec56f58SAlexei Starovoitov case BPF_TRACE_FENTRY: 400fec56f58SAlexei Starovoitov return BPF_TRAMP_FENTRY; 401ae240823SKP Singh case BPF_MODIFY_RETURN: 402ae240823SKP Singh return BPF_TRAMP_MODIFY_RETURN; 403be8704ffSAlexei Starovoitov case BPF_TRACE_FEXIT: 404fec56f58SAlexei Starovoitov return BPF_TRAMP_FEXIT; 4059e4e01dfSKP Singh case BPF_LSM_MAC: 4069e4e01dfSKP Singh if (!prog->aux->attach_func_proto->type) 4079e4e01dfSKP Singh /* The function returns void, we cannot modify its 4089e4e01dfSKP Singh * return value. 4099e4e01dfSKP Singh */ 4109e4e01dfSKP Singh return BPF_TRAMP_FEXIT; 4119e4e01dfSKP Singh else 4129e4e01dfSKP Singh return BPF_TRAMP_MODIFY_RETURN; 413be8704ffSAlexei Starovoitov default: 414be8704ffSAlexei Starovoitov return BPF_TRAMP_REPLACE; 415fec56f58SAlexei Starovoitov } 416fec56f58SAlexei Starovoitov } 417fec56f58SAlexei Starovoitov 418af3f4134SStanislav Fomichev static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr) 419fec56f58SAlexei Starovoitov { 420fec56f58SAlexei Starovoitov enum bpf_tramp_prog_type kind; 421f7e0beafSKui-Feng Lee struct bpf_tramp_link *link_exiting; 422fec56f58SAlexei Starovoitov int err = 0; 423a2aa95b7SYuntao Wang int cnt = 0, i; 424fec56f58SAlexei Starovoitov 425f7e0beafSKui-Feng Lee kind = bpf_attach_type_to_tramp(link->link.prog); 426af3f4134SStanislav Fomichev if (tr->extension_prog) 427be8704ffSAlexei Starovoitov /* cannot attach fentry/fexit if extension prog is attached. 428be8704ffSAlexei Starovoitov * cannot overwrite extension prog either. 429be8704ffSAlexei Starovoitov */ 430af3f4134SStanislav Fomichev return -EBUSY; 431a2aa95b7SYuntao Wang 432a2aa95b7SYuntao Wang for (i = 0; i < BPF_TRAMP_MAX; i++) 433a2aa95b7SYuntao Wang cnt += tr->progs_cnt[i]; 434a2aa95b7SYuntao Wang 435be8704ffSAlexei Starovoitov if (kind == BPF_TRAMP_REPLACE) { 436be8704ffSAlexei Starovoitov /* Cannot attach extension if fentry/fexit are in use. */ 437af3f4134SStanislav Fomichev if (cnt) 438af3f4134SStanislav Fomichev return -EBUSY; 439f7e0beafSKui-Feng Lee tr->extension_prog = link->link.prog; 440af3f4134SStanislav Fomichev return bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL, 441f7e0beafSKui-Feng Lee link->link.prog->bpf_func); 442be8704ffSAlexei Starovoitov } 443af3f4134SStanislav Fomichev if (cnt >= BPF_MAX_TRAMP_LINKS) 444af3f4134SStanislav Fomichev return -E2BIG; 445af3f4134SStanislav Fomichev if (!hlist_unhashed(&link->tramp_hlist)) 446fec56f58SAlexei Starovoitov /* prog already linked */ 447af3f4134SStanislav Fomichev return -EBUSY; 448f7e0beafSKui-Feng Lee hlist_for_each_entry(link_exiting, &tr->progs_hlist[kind], tramp_hlist) { 449f7e0beafSKui-Feng Lee if (link_exiting->link.prog != link->link.prog) 450f7e0beafSKui-Feng Lee continue; 451f7e0beafSKui-Feng Lee /* prog already linked */ 452af3f4134SStanislav Fomichev return -EBUSY; 453f7e0beafSKui-Feng Lee } 454f7e0beafSKui-Feng Lee 455f7e0beafSKui-Feng Lee hlist_add_head(&link->tramp_hlist, &tr->progs_hlist[kind]); 456fec56f58SAlexei Starovoitov tr->progs_cnt[kind]++; 4573aac1eadSToke Høiland-Jørgensen err = bpf_trampoline_update(tr); 458fec56f58SAlexei Starovoitov if (err) { 459f7e0beafSKui-Feng Lee hlist_del_init(&link->tramp_hlist); 460fec56f58SAlexei Starovoitov tr->progs_cnt[kind]--; 461fec56f58SAlexei Starovoitov } 462af3f4134SStanislav Fomichev return err; 463af3f4134SStanislav Fomichev } 464af3f4134SStanislav Fomichev 465af3f4134SStanislav Fomichev int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr) 466af3f4134SStanislav Fomichev { 467af3f4134SStanislav Fomichev int err; 468af3f4134SStanislav Fomichev 469af3f4134SStanislav Fomichev mutex_lock(&tr->mutex); 470af3f4134SStanislav Fomichev err = __bpf_trampoline_link_prog(link, tr); 471fec56f58SAlexei Starovoitov mutex_unlock(&tr->mutex); 472fec56f58SAlexei Starovoitov return err; 473fec56f58SAlexei Starovoitov } 474fec56f58SAlexei Starovoitov 475af3f4134SStanislav Fomichev static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr) 476fec56f58SAlexei Starovoitov { 477fec56f58SAlexei Starovoitov enum bpf_tramp_prog_type kind; 478fec56f58SAlexei Starovoitov int err; 479fec56f58SAlexei Starovoitov 480f7e0beafSKui-Feng Lee kind = bpf_attach_type_to_tramp(link->link.prog); 481be8704ffSAlexei Starovoitov if (kind == BPF_TRAMP_REPLACE) { 482be8704ffSAlexei Starovoitov WARN_ON_ONCE(!tr->extension_prog); 483be8704ffSAlexei Starovoitov err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, 484be8704ffSAlexei Starovoitov tr->extension_prog->bpf_func, NULL); 485be8704ffSAlexei Starovoitov tr->extension_prog = NULL; 486af3f4134SStanislav Fomichev return err; 487be8704ffSAlexei Starovoitov } 488f7e0beafSKui-Feng Lee hlist_del_init(&link->tramp_hlist); 489fec56f58SAlexei Starovoitov tr->progs_cnt[kind]--; 490af3f4134SStanislav Fomichev return bpf_trampoline_update(tr); 491af3f4134SStanislav Fomichev } 492af3f4134SStanislav Fomichev 493af3f4134SStanislav Fomichev /* bpf_trampoline_unlink_prog() should never fail. */ 494af3f4134SStanislav Fomichev int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr) 495af3f4134SStanislav Fomichev { 496af3f4134SStanislav Fomichev int err; 497af3f4134SStanislav Fomichev 498af3f4134SStanislav Fomichev mutex_lock(&tr->mutex); 499af3f4134SStanislav Fomichev err = __bpf_trampoline_unlink_prog(link, tr); 500fec56f58SAlexei Starovoitov mutex_unlock(&tr->mutex); 501fec56f58SAlexei Starovoitov return err; 502fec56f58SAlexei Starovoitov } 503fec56f58SAlexei Starovoitov 504*3908fcddSStanislav Fomichev #if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM) 50569fd337aSStanislav Fomichev static void bpf_shim_tramp_link_release(struct bpf_link *link) 50669fd337aSStanislav Fomichev { 50769fd337aSStanislav Fomichev struct bpf_shim_tramp_link *shim_link = 50869fd337aSStanislav Fomichev container_of(link, struct bpf_shim_tramp_link, link.link); 50969fd337aSStanislav Fomichev 51069fd337aSStanislav Fomichev /* paired with 'shim_link->trampoline = tr' in bpf_trampoline_link_cgroup_shim */ 51169fd337aSStanislav Fomichev if (!shim_link->trampoline) 51269fd337aSStanislav Fomichev return; 51369fd337aSStanislav Fomichev 51469fd337aSStanislav Fomichev WARN_ON_ONCE(bpf_trampoline_unlink_prog(&shim_link->link, shim_link->trampoline)); 51569fd337aSStanislav Fomichev bpf_trampoline_put(shim_link->trampoline); 51669fd337aSStanislav Fomichev } 51769fd337aSStanislav Fomichev 51869fd337aSStanislav Fomichev static void bpf_shim_tramp_link_dealloc(struct bpf_link *link) 51969fd337aSStanislav Fomichev { 52069fd337aSStanislav Fomichev struct bpf_shim_tramp_link *shim_link = 52169fd337aSStanislav Fomichev container_of(link, struct bpf_shim_tramp_link, link.link); 52269fd337aSStanislav Fomichev 52369fd337aSStanislav Fomichev kfree(shim_link); 52469fd337aSStanislav Fomichev } 52569fd337aSStanislav Fomichev 52669fd337aSStanislav Fomichev static const struct bpf_link_ops bpf_shim_tramp_link_lops = { 52769fd337aSStanislav Fomichev .release = bpf_shim_tramp_link_release, 52869fd337aSStanislav Fomichev .dealloc = bpf_shim_tramp_link_dealloc, 52969fd337aSStanislav Fomichev }; 53069fd337aSStanislav Fomichev 53169fd337aSStanislav Fomichev static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog, 53269fd337aSStanislav Fomichev bpf_func_t bpf_func, 53369fd337aSStanislav Fomichev int cgroup_atype) 53469fd337aSStanislav Fomichev { 53569fd337aSStanislav Fomichev struct bpf_shim_tramp_link *shim_link = NULL; 53669fd337aSStanislav Fomichev struct bpf_prog *p; 53769fd337aSStanislav Fomichev 53869fd337aSStanislav Fomichev shim_link = kzalloc(sizeof(*shim_link), GFP_USER); 53969fd337aSStanislav Fomichev if (!shim_link) 54069fd337aSStanislav Fomichev return NULL; 54169fd337aSStanislav Fomichev 54269fd337aSStanislav Fomichev p = bpf_prog_alloc(1, 0); 54369fd337aSStanislav Fomichev if (!p) { 54469fd337aSStanislav Fomichev kfree(shim_link); 54569fd337aSStanislav Fomichev return NULL; 54669fd337aSStanislav Fomichev } 54769fd337aSStanislav Fomichev 54869fd337aSStanislav Fomichev p->jited = false; 54969fd337aSStanislav Fomichev p->bpf_func = bpf_func; 55069fd337aSStanislav Fomichev 55169fd337aSStanislav Fomichev p->aux->cgroup_atype = cgroup_atype; 55269fd337aSStanislav Fomichev p->aux->attach_func_proto = prog->aux->attach_func_proto; 55369fd337aSStanislav Fomichev p->aux->attach_btf_id = prog->aux->attach_btf_id; 55469fd337aSStanislav Fomichev p->aux->attach_btf = prog->aux->attach_btf; 55569fd337aSStanislav Fomichev btf_get(p->aux->attach_btf); 55669fd337aSStanislav Fomichev p->type = BPF_PROG_TYPE_LSM; 55769fd337aSStanislav Fomichev p->expected_attach_type = BPF_LSM_MAC; 55869fd337aSStanislav Fomichev bpf_prog_inc(p); 55969fd337aSStanislav Fomichev bpf_link_init(&shim_link->link.link, BPF_LINK_TYPE_UNSPEC, 56069fd337aSStanislav Fomichev &bpf_shim_tramp_link_lops, p); 561c0e19f2cSStanislav Fomichev bpf_cgroup_atype_get(p->aux->attach_btf_id, cgroup_atype); 56269fd337aSStanislav Fomichev 56369fd337aSStanislav Fomichev return shim_link; 56469fd337aSStanislav Fomichev } 56569fd337aSStanislav Fomichev 56669fd337aSStanislav Fomichev static struct bpf_shim_tramp_link *cgroup_shim_find(struct bpf_trampoline *tr, 56769fd337aSStanislav Fomichev bpf_func_t bpf_func) 56869fd337aSStanislav Fomichev { 56969fd337aSStanislav Fomichev struct bpf_tramp_link *link; 57069fd337aSStanislav Fomichev int kind; 57169fd337aSStanislav Fomichev 57269fd337aSStanislav Fomichev for (kind = 0; kind < BPF_TRAMP_MAX; kind++) { 57369fd337aSStanislav Fomichev hlist_for_each_entry(link, &tr->progs_hlist[kind], tramp_hlist) { 57469fd337aSStanislav Fomichev struct bpf_prog *p = link->link.prog; 57569fd337aSStanislav Fomichev 57669fd337aSStanislav Fomichev if (p->bpf_func == bpf_func) 57769fd337aSStanislav Fomichev return container_of(link, struct bpf_shim_tramp_link, link); 57869fd337aSStanislav Fomichev } 57969fd337aSStanislav Fomichev } 58069fd337aSStanislav Fomichev 58169fd337aSStanislav Fomichev return NULL; 58269fd337aSStanislav Fomichev } 58369fd337aSStanislav Fomichev 58469fd337aSStanislav Fomichev int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, 58569fd337aSStanislav Fomichev int cgroup_atype) 58669fd337aSStanislav Fomichev { 58769fd337aSStanislav Fomichev struct bpf_shim_tramp_link *shim_link = NULL; 58869fd337aSStanislav Fomichev struct bpf_attach_target_info tgt_info = {}; 58969fd337aSStanislav Fomichev struct bpf_trampoline *tr; 59069fd337aSStanislav Fomichev bpf_func_t bpf_func; 59169fd337aSStanislav Fomichev u64 key; 59269fd337aSStanislav Fomichev int err; 59369fd337aSStanislav Fomichev 59469fd337aSStanislav Fomichev err = bpf_check_attach_target(NULL, prog, NULL, 59569fd337aSStanislav Fomichev prog->aux->attach_btf_id, 59669fd337aSStanislav Fomichev &tgt_info); 59769fd337aSStanislav Fomichev if (err) 59869fd337aSStanislav Fomichev return err; 59969fd337aSStanislav Fomichev 60069fd337aSStanislav Fomichev key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, 60169fd337aSStanislav Fomichev prog->aux->attach_btf_id); 60269fd337aSStanislav Fomichev 60369fd337aSStanislav Fomichev bpf_lsm_find_cgroup_shim(prog, &bpf_func); 60469fd337aSStanislav Fomichev tr = bpf_trampoline_get(key, &tgt_info); 60569fd337aSStanislav Fomichev if (!tr) 60669fd337aSStanislav Fomichev return -ENOMEM; 60769fd337aSStanislav Fomichev 60869fd337aSStanislav Fomichev mutex_lock(&tr->mutex); 60969fd337aSStanislav Fomichev 61069fd337aSStanislav Fomichev shim_link = cgroup_shim_find(tr, bpf_func); 61169fd337aSStanislav Fomichev if (shim_link) { 61269fd337aSStanislav Fomichev /* Reusing existing shim attached by the other program. */ 61369fd337aSStanislav Fomichev bpf_link_inc(&shim_link->link.link); 61469fd337aSStanislav Fomichev 61569fd337aSStanislav Fomichev mutex_unlock(&tr->mutex); 61669fd337aSStanislav Fomichev bpf_trampoline_put(tr); /* bpf_trampoline_get above */ 61769fd337aSStanislav Fomichev return 0; 61869fd337aSStanislav Fomichev } 61969fd337aSStanislav Fomichev 62069fd337aSStanislav Fomichev /* Allocate and install new shim. */ 62169fd337aSStanislav Fomichev 62269fd337aSStanislav Fomichev shim_link = cgroup_shim_alloc(prog, bpf_func, cgroup_atype); 62369fd337aSStanislav Fomichev if (!shim_link) { 62469fd337aSStanislav Fomichev err = -ENOMEM; 62569fd337aSStanislav Fomichev goto err; 62669fd337aSStanislav Fomichev } 62769fd337aSStanislav Fomichev 62869fd337aSStanislav Fomichev err = __bpf_trampoline_link_prog(&shim_link->link, tr); 62969fd337aSStanislav Fomichev if (err) 63069fd337aSStanislav Fomichev goto err; 63169fd337aSStanislav Fomichev 63269fd337aSStanislav Fomichev shim_link->trampoline = tr; 63369fd337aSStanislav Fomichev /* note, we're still holding tr refcnt from above */ 63469fd337aSStanislav Fomichev 63569fd337aSStanislav Fomichev mutex_unlock(&tr->mutex); 63669fd337aSStanislav Fomichev 63769fd337aSStanislav Fomichev return 0; 63869fd337aSStanislav Fomichev err: 63969fd337aSStanislav Fomichev mutex_unlock(&tr->mutex); 64069fd337aSStanislav Fomichev 64169fd337aSStanislav Fomichev if (shim_link) 64269fd337aSStanislav Fomichev bpf_link_put(&shim_link->link.link); 64369fd337aSStanislav Fomichev 64469fd337aSStanislav Fomichev /* have to release tr while _not_ holding its mutex */ 64569fd337aSStanislav Fomichev bpf_trampoline_put(tr); /* bpf_trampoline_get above */ 64669fd337aSStanislav Fomichev 64769fd337aSStanislav Fomichev return err; 64869fd337aSStanislav Fomichev } 64969fd337aSStanislav Fomichev 65069fd337aSStanislav Fomichev void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog) 65169fd337aSStanislav Fomichev { 65269fd337aSStanislav Fomichev struct bpf_shim_tramp_link *shim_link = NULL; 65369fd337aSStanislav Fomichev struct bpf_trampoline *tr; 65469fd337aSStanislav Fomichev bpf_func_t bpf_func; 65569fd337aSStanislav Fomichev u64 key; 65669fd337aSStanislav Fomichev 65769fd337aSStanislav Fomichev key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, 65869fd337aSStanislav Fomichev prog->aux->attach_btf_id); 65969fd337aSStanislav Fomichev 66069fd337aSStanislav Fomichev bpf_lsm_find_cgroup_shim(prog, &bpf_func); 66169fd337aSStanislav Fomichev tr = bpf_trampoline_lookup(key); 66269fd337aSStanislav Fomichev if (WARN_ON_ONCE(!tr)) 66369fd337aSStanislav Fomichev return; 66469fd337aSStanislav Fomichev 66569fd337aSStanislav Fomichev mutex_lock(&tr->mutex); 66669fd337aSStanislav Fomichev shim_link = cgroup_shim_find(tr, bpf_func); 66769fd337aSStanislav Fomichev mutex_unlock(&tr->mutex); 66869fd337aSStanislav Fomichev 66969fd337aSStanislav Fomichev if (shim_link) 67069fd337aSStanislav Fomichev bpf_link_put(&shim_link->link.link); 67169fd337aSStanislav Fomichev 67269fd337aSStanislav Fomichev bpf_trampoline_put(tr); /* bpf_trampoline_lookup above */ 67369fd337aSStanislav Fomichev } 67469fd337aSStanislav Fomichev #endif 67569fd337aSStanislav Fomichev 676f7b12b6fSToke Høiland-Jørgensen struct bpf_trampoline *bpf_trampoline_get(u64 key, 677f7b12b6fSToke Høiland-Jørgensen struct bpf_attach_target_info *tgt_info) 678f7b12b6fSToke Høiland-Jørgensen { 679f7b12b6fSToke Høiland-Jørgensen struct bpf_trampoline *tr; 680f7b12b6fSToke Høiland-Jørgensen 681f7b12b6fSToke Høiland-Jørgensen tr = bpf_trampoline_lookup(key); 682f7b12b6fSToke Høiland-Jørgensen if (!tr) 683f7b12b6fSToke Høiland-Jørgensen return NULL; 684f7b12b6fSToke Høiland-Jørgensen 685f7b12b6fSToke Høiland-Jørgensen mutex_lock(&tr->mutex); 686f7b12b6fSToke Høiland-Jørgensen if (tr->func.addr) 687f7b12b6fSToke Høiland-Jørgensen goto out; 688f7b12b6fSToke Høiland-Jørgensen 689f7b12b6fSToke Høiland-Jørgensen memcpy(&tr->func.model, &tgt_info->fmodel, sizeof(tgt_info->fmodel)); 690f7b12b6fSToke Høiland-Jørgensen tr->func.addr = (void *)tgt_info->tgt_addr; 691f7b12b6fSToke Høiland-Jørgensen out: 692f7b12b6fSToke Høiland-Jørgensen mutex_unlock(&tr->mutex); 693f7b12b6fSToke Høiland-Jørgensen return tr; 694f7b12b6fSToke Høiland-Jørgensen } 695f7b12b6fSToke Høiland-Jørgensen 696fec56f58SAlexei Starovoitov void bpf_trampoline_put(struct bpf_trampoline *tr) 697fec56f58SAlexei Starovoitov { 698a2aa95b7SYuntao Wang int i; 699a2aa95b7SYuntao Wang 700fec56f58SAlexei Starovoitov if (!tr) 701fec56f58SAlexei Starovoitov return; 702fec56f58SAlexei Starovoitov mutex_lock(&trampoline_mutex); 703fec56f58SAlexei Starovoitov if (!refcount_dec_and_test(&tr->refcnt)) 704fec56f58SAlexei Starovoitov goto out; 705fec56f58SAlexei Starovoitov WARN_ON_ONCE(mutex_is_locked(&tr->mutex)); 706a2aa95b7SYuntao Wang 707a2aa95b7SYuntao Wang for (i = 0; i < BPF_TRAMP_MAX; i++) 708a2aa95b7SYuntao Wang if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[i]))) 709fec56f58SAlexei Starovoitov goto out; 710a2aa95b7SYuntao Wang 711e21aa341SAlexei Starovoitov /* This code will be executed even when the last bpf_tramp_image 712e21aa341SAlexei Starovoitov * is alive. All progs are detached from the trampoline and the 713e21aa341SAlexei Starovoitov * trampoline image is patched with jmp into epilogue to skip 714e21aa341SAlexei Starovoitov * fexit progs. The fentry-only trampoline will be freed via 715e21aa341SAlexei Starovoitov * multiple rcu callbacks. 7161e6c62a8SAlexei Starovoitov */ 717fec56f58SAlexei Starovoitov hlist_del(&tr->hlist); 718fec56f58SAlexei Starovoitov kfree(tr); 719fec56f58SAlexei Starovoitov out: 720fec56f58SAlexei Starovoitov mutex_unlock(&trampoline_mutex); 721fec56f58SAlexei Starovoitov } 722fec56f58SAlexei Starovoitov 723ca06f55bSAlexei Starovoitov #define NO_START_TIME 1 724856c02dbSSong Liu static __always_inline u64 notrace bpf_prog_start_time(void) 725f2dd3b39SAlexei Starovoitov { 726f2dd3b39SAlexei Starovoitov u64 start = NO_START_TIME; 727f2dd3b39SAlexei Starovoitov 728ca06f55bSAlexei Starovoitov if (static_branch_unlikely(&bpf_stats_enabled_key)) { 729f2dd3b39SAlexei Starovoitov start = sched_clock(); 730ca06f55bSAlexei Starovoitov if (unlikely(!start)) 731ca06f55bSAlexei Starovoitov start = NO_START_TIME; 732ca06f55bSAlexei Starovoitov } 733f2dd3b39SAlexei Starovoitov return start; 734f2dd3b39SAlexei Starovoitov } 735f2dd3b39SAlexei Starovoitov 7369ed9e9baSAlexei Starovoitov static void notrace inc_misses_counter(struct bpf_prog *prog) 7379ed9e9baSAlexei Starovoitov { 7389ed9e9baSAlexei Starovoitov struct bpf_prog_stats *stats; 7390e3135d3SHe Fengqing unsigned int flags; 7409ed9e9baSAlexei Starovoitov 7419ed9e9baSAlexei Starovoitov stats = this_cpu_ptr(prog->stats); 7420e3135d3SHe Fengqing flags = u64_stats_update_begin_irqsave(&stats->syncp); 74361a0abaeSEric Dumazet u64_stats_inc(&stats->misses); 7440e3135d3SHe Fengqing u64_stats_update_end_irqrestore(&stats->syncp, flags); 7459ed9e9baSAlexei Starovoitov } 7469ed9e9baSAlexei Starovoitov 747fb7dd8bcSAndrii Nakryiko /* The logic is similar to bpf_prog_run(), but with an explicit 74802ad0596SDavid Miller * rcu_read_lock() and migrate_disable() which are required 74902ad0596SDavid Miller * for the trampoline. The macro is split into 750f2dd3b39SAlexei Starovoitov * call __bpf_prog_enter 751fec56f58SAlexei Starovoitov * call prog->bpf_func 752fec56f58SAlexei Starovoitov * call __bpf_prog_exit 753ca06f55bSAlexei Starovoitov * 754ca06f55bSAlexei Starovoitov * __bpf_prog_enter returns: 755ca06f55bSAlexei Starovoitov * 0 - skip execution of the bpf prog 756ca06f55bSAlexei Starovoitov * 1 - execute bpf prog 7578fb33b60SZhen Lei * [2..MAX_U64] - execute bpf prog and record execution time. 758ca06f55bSAlexei Starovoitov * This is start time. 759fec56f58SAlexei Starovoitov */ 760e384c7b7SKui-Feng Lee u64 notrace __bpf_prog_enter(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx) 761dcce11d5SJules Irenge __acquires(RCU) 762fec56f58SAlexei Starovoitov { 763fec56f58SAlexei Starovoitov rcu_read_lock(); 76402ad0596SDavid Miller migrate_disable(); 765e384c7b7SKui-Feng Lee 766e384c7b7SKui-Feng Lee run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx); 767e384c7b7SKui-Feng Lee 7689ed9e9baSAlexei Starovoitov if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) { 7699ed9e9baSAlexei Starovoitov inc_misses_counter(prog); 770ca06f55bSAlexei Starovoitov return 0; 7719ed9e9baSAlexei Starovoitov } 772f2dd3b39SAlexei Starovoitov return bpf_prog_start_time(); 773fec56f58SAlexei Starovoitov } 774fec56f58SAlexei Starovoitov 775f2dd3b39SAlexei Starovoitov static void notrace update_prog_stats(struct bpf_prog *prog, 776f2dd3b39SAlexei Starovoitov u64 start) 777fec56f58SAlexei Starovoitov { 778fec56f58SAlexei Starovoitov struct bpf_prog_stats *stats; 779fec56f58SAlexei Starovoitov 780fec56f58SAlexei Starovoitov if (static_branch_unlikely(&bpf_stats_enabled_key) && 781f2dd3b39SAlexei Starovoitov /* static_key could be enabled in __bpf_prog_enter* 782f2dd3b39SAlexei Starovoitov * and disabled in __bpf_prog_exit*. 783fec56f58SAlexei Starovoitov * And vice versa. 784f2dd3b39SAlexei Starovoitov * Hence check that 'start' is valid. 785fec56f58SAlexei Starovoitov */ 786f2dd3b39SAlexei Starovoitov start > NO_START_TIME) { 787d979617aSEric Dumazet unsigned long flags; 788d979617aSEric Dumazet 789700d4796SAlexei Starovoitov stats = this_cpu_ptr(prog->stats); 790d979617aSEric Dumazet flags = u64_stats_update_begin_irqsave(&stats->syncp); 79161a0abaeSEric Dumazet u64_stats_inc(&stats->cnt); 79261a0abaeSEric Dumazet u64_stats_add(&stats->nsecs, sched_clock() - start); 793d979617aSEric Dumazet u64_stats_update_end_irqrestore(&stats->syncp, flags); 794fec56f58SAlexei Starovoitov } 795f2dd3b39SAlexei Starovoitov } 796f2dd3b39SAlexei Starovoitov 797e384c7b7SKui-Feng Lee void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx) 798f2dd3b39SAlexei Starovoitov __releases(RCU) 799f2dd3b39SAlexei Starovoitov { 800e384c7b7SKui-Feng Lee bpf_reset_run_ctx(run_ctx->saved_run_ctx); 801e384c7b7SKui-Feng Lee 802f2dd3b39SAlexei Starovoitov update_prog_stats(prog, start); 803ca06f55bSAlexei Starovoitov __this_cpu_dec(*(prog->active)); 80402ad0596SDavid Miller migrate_enable(); 805fec56f58SAlexei Starovoitov rcu_read_unlock(); 806fec56f58SAlexei Starovoitov } 807fec56f58SAlexei Starovoitov 80869fd337aSStanislav Fomichev u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog, 80969fd337aSStanislav Fomichev struct bpf_tramp_run_ctx *run_ctx) 81069fd337aSStanislav Fomichev __acquires(RCU) 81169fd337aSStanislav Fomichev { 81269fd337aSStanislav Fomichev /* Runtime stats are exported via actual BPF_LSM_CGROUP 81369fd337aSStanislav Fomichev * programs, not the shims. 81469fd337aSStanislav Fomichev */ 81569fd337aSStanislav Fomichev rcu_read_lock(); 81669fd337aSStanislav Fomichev migrate_disable(); 81769fd337aSStanislav Fomichev 81869fd337aSStanislav Fomichev run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx); 81969fd337aSStanislav Fomichev 82069fd337aSStanislav Fomichev return NO_START_TIME; 82169fd337aSStanislav Fomichev } 82269fd337aSStanislav Fomichev 82369fd337aSStanislav Fomichev void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start, 82469fd337aSStanislav Fomichev struct bpf_tramp_run_ctx *run_ctx) 82569fd337aSStanislav Fomichev __releases(RCU) 82669fd337aSStanislav Fomichev { 82769fd337aSStanislav Fomichev bpf_reset_run_ctx(run_ctx->saved_run_ctx); 82869fd337aSStanislav Fomichev 82969fd337aSStanislav Fomichev migrate_enable(); 83069fd337aSStanislav Fomichev rcu_read_unlock(); 83169fd337aSStanislav Fomichev } 83269fd337aSStanislav Fomichev 833e384c7b7SKui-Feng Lee u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx) 8341e6c62a8SAlexei Starovoitov { 8351e6c62a8SAlexei Starovoitov rcu_read_lock_trace(); 836031d6e02SAlexei Starovoitov migrate_disable(); 837f56407faSAlexei Starovoitov might_fault(); 838e384c7b7SKui-Feng Lee 8399ed9e9baSAlexei Starovoitov if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) { 8409ed9e9baSAlexei Starovoitov inc_misses_counter(prog); 841ca06f55bSAlexei Starovoitov return 0; 8429ed9e9baSAlexei Starovoitov } 843e384c7b7SKui-Feng Lee 844e384c7b7SKui-Feng Lee run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx); 845e384c7b7SKui-Feng Lee 846f2dd3b39SAlexei Starovoitov return bpf_prog_start_time(); 8471e6c62a8SAlexei Starovoitov } 8481e6c62a8SAlexei Starovoitov 849e384c7b7SKui-Feng Lee void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start, 850e384c7b7SKui-Feng Lee struct bpf_tramp_run_ctx *run_ctx) 8511e6c62a8SAlexei Starovoitov { 852e384c7b7SKui-Feng Lee bpf_reset_run_ctx(run_ctx->saved_run_ctx); 853e384c7b7SKui-Feng Lee 854f2dd3b39SAlexei Starovoitov update_prog_stats(prog, start); 855ca06f55bSAlexei Starovoitov __this_cpu_dec(*(prog->active)); 856031d6e02SAlexei Starovoitov migrate_enable(); 8571e6c62a8SAlexei Starovoitov rcu_read_unlock_trace(); 8581e6c62a8SAlexei Starovoitov } 8591e6c62a8SAlexei Starovoitov 860e21aa341SAlexei Starovoitov void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr) 861e21aa341SAlexei Starovoitov { 862e21aa341SAlexei Starovoitov percpu_ref_get(&tr->pcref); 863e21aa341SAlexei Starovoitov } 864e21aa341SAlexei Starovoitov 865e21aa341SAlexei Starovoitov void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr) 866e21aa341SAlexei Starovoitov { 867e21aa341SAlexei Starovoitov percpu_ref_put(&tr->pcref); 868e21aa341SAlexei Starovoitov } 869e21aa341SAlexei Starovoitov 870fec56f58SAlexei Starovoitov int __weak 871e21aa341SAlexei Starovoitov arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end, 87285d33df3SMartin KaFai Lau const struct btf_func_model *m, u32 flags, 873f7e0beafSKui-Feng Lee struct bpf_tramp_links *tlinks, 874fec56f58SAlexei Starovoitov void *orig_call) 875fec56f58SAlexei Starovoitov { 876fec56f58SAlexei Starovoitov return -ENOTSUPP; 877fec56f58SAlexei Starovoitov } 878fec56f58SAlexei Starovoitov 879fec56f58SAlexei Starovoitov static int __init init_trampolines(void) 880fec56f58SAlexei Starovoitov { 881fec56f58SAlexei Starovoitov int i; 882fec56f58SAlexei Starovoitov 883fec56f58SAlexei Starovoitov for (i = 0; i < TRAMPOLINE_TABLE_SIZE; i++) 884fec56f58SAlexei Starovoitov INIT_HLIST_HEAD(&trampoline_table[i]); 885fec56f58SAlexei Starovoitov return 0; 886fec56f58SAlexei Starovoitov } 887fec56f58SAlexei Starovoitov late_initcall(init_trampolines); 888