1fec56f58SAlexei Starovoitov // SPDX-License-Identifier: GPL-2.0-only
2fec56f58SAlexei Starovoitov /* Copyright (c) 2019 Facebook */
3fec56f58SAlexei Starovoitov #include <linux/hash.h>
4fec56f58SAlexei Starovoitov #include <linux/bpf.h>
5fec56f58SAlexei Starovoitov #include <linux/filter.h>
6b91e014fSAlexei Starovoitov #include <linux/ftrace.h>
7e9b4e606SJiri Olsa #include <linux/rbtree_latch.h>
8a108f7dcSJiri Olsa #include <linux/perf_event.h>
99e4e01dfSKP Singh #include <linux/btf.h>
101e6c62a8SAlexei Starovoitov #include <linux/rcupdate_trace.h>
111e6c62a8SAlexei Starovoitov #include <linux/rcupdate_wait.h>
12856c02dbSSong Liu #include <linux/static_call.h>
1369fd337aSStanislav Fomichev #include <linux/bpf_verifier.h>
1469fd337aSStanislav Fomichev #include <linux/bpf_lsm.h>
1500963a2eSSong Liu #include <linux/delay.h>
16fec56f58SAlexei Starovoitov
17be8704ffSAlexei Starovoitov /* dummy _ops. The verifier will operate on target program's ops. */
18be8704ffSAlexei Starovoitov const struct bpf_verifier_ops bpf_extension_verifier_ops = {
19be8704ffSAlexei Starovoitov };
20be8704ffSAlexei Starovoitov const struct bpf_prog_ops bpf_extension_prog_ops = {
21be8704ffSAlexei Starovoitov };
22be8704ffSAlexei Starovoitov
23fec56f58SAlexei Starovoitov /* btf_vmlinux has ~22k attachable functions. 1k htab is enough. */
24fec56f58SAlexei Starovoitov #define TRAMPOLINE_HASH_BITS 10
25fec56f58SAlexei Starovoitov #define TRAMPOLINE_TABLE_SIZE (1 << TRAMPOLINE_HASH_BITS)
26fec56f58SAlexei Starovoitov
27fec56f58SAlexei Starovoitov static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE];
28fec56f58SAlexei Starovoitov
297ac88ebaSJiri Olsa /* serializes access to trampoline_table */
30fec56f58SAlexei Starovoitov static DEFINE_MUTEX(trampoline_mutex);
31fec56f58SAlexei Starovoitov
3200963a2eSSong Liu #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
3300963a2eSSong Liu static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex);
3400963a2eSSong Liu
bpf_tramp_ftrace_ops_func(struct ftrace_ops * ops,enum ftrace_ops_cmd cmd)3500963a2eSSong Liu static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, enum ftrace_ops_cmd cmd)
3600963a2eSSong Liu {
3700963a2eSSong Liu struct bpf_trampoline *tr = ops->private;
3800963a2eSSong Liu int ret = 0;
3900963a2eSSong Liu
4000963a2eSSong Liu if (cmd == FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF) {
4100963a2eSSong Liu /* This is called inside register_ftrace_direct_multi(), so
4200963a2eSSong Liu * tr->mutex is already locked.
4300963a2eSSong Liu */
4400963a2eSSong Liu lockdep_assert_held_once(&tr->mutex);
4500963a2eSSong Liu
4600963a2eSSong Liu /* Instead of updating the trampoline here, we propagate
47da8bdfbdSFlorent Revest * -EAGAIN to register_ftrace_direct(). Then we can
48da8bdfbdSFlorent Revest * retry register_ftrace_direct() after updating the
4900963a2eSSong Liu * trampoline.
5000963a2eSSong Liu */
5100963a2eSSong Liu if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) &&
5200963a2eSSong Liu !(tr->flags & BPF_TRAMP_F_ORIG_STACK)) {
5300963a2eSSong Liu if (WARN_ON_ONCE(tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY))
5400963a2eSSong Liu return -EBUSY;
5500963a2eSSong Liu
5600963a2eSSong Liu tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY;
5700963a2eSSong Liu return -EAGAIN;
5800963a2eSSong Liu }
5900963a2eSSong Liu
6000963a2eSSong Liu return 0;
6100963a2eSSong Liu }
6200963a2eSSong Liu
6300963a2eSSong Liu /* The normal locking order is
6400963a2eSSong Liu * tr->mutex => direct_mutex (ftrace.c) => ftrace_lock (ftrace.c)
6500963a2eSSong Liu *
6600963a2eSSong Liu * The following two commands are called from
6700963a2eSSong Liu *
6800963a2eSSong Liu * prepare_direct_functions_for_ipmodify
6900963a2eSSong Liu * cleanup_direct_functions_after_ipmodify
7000963a2eSSong Liu *
7100963a2eSSong Liu * In both cases, direct_mutex is already locked. Use
7200963a2eSSong Liu * mutex_trylock(&tr->mutex) to avoid deadlock in race condition
7300963a2eSSong Liu * (something else is making changes to this same trampoline).
7400963a2eSSong Liu */
7500963a2eSSong Liu if (!mutex_trylock(&tr->mutex)) {
7600963a2eSSong Liu /* sleep 1 ms to make sure whatever holding tr->mutex makes
7700963a2eSSong Liu * some progress.
7800963a2eSSong Liu */
7900963a2eSSong Liu msleep(1);
8000963a2eSSong Liu return -EAGAIN;
8100963a2eSSong Liu }
8200963a2eSSong Liu
8300963a2eSSong Liu switch (cmd) {
8400963a2eSSong Liu case FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER:
8500963a2eSSong Liu tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY;
8600963a2eSSong Liu
8700963a2eSSong Liu if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) &&
8800963a2eSSong Liu !(tr->flags & BPF_TRAMP_F_ORIG_STACK))
8900963a2eSSong Liu ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */);
9000963a2eSSong Liu break;
9100963a2eSSong Liu case FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER:
9200963a2eSSong Liu tr->flags &= ~BPF_TRAMP_F_SHARE_IPMODIFY;
9300963a2eSSong Liu
9400963a2eSSong Liu if (tr->flags & BPF_TRAMP_F_ORIG_STACK)
9500963a2eSSong Liu ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */);
9600963a2eSSong Liu break;
9700963a2eSSong Liu default:
9800963a2eSSong Liu ret = -EINVAL;
9900963a2eSSong Liu break;
10014250fa4SYang Li }
10100963a2eSSong Liu
10200963a2eSSong Liu mutex_unlock(&tr->mutex);
10300963a2eSSong Liu return ret;
10400963a2eSSong Liu }
10500963a2eSSong Liu #endif
10600963a2eSSong Liu
bpf_prog_has_trampoline(const struct bpf_prog * prog)107f92c1e18SJiri Olsa bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
108f92c1e18SJiri Olsa {
109f92c1e18SJiri Olsa enum bpf_attach_type eatype = prog->expected_attach_type;
1102fcc8241SKui-Feng Lee enum bpf_prog_type ptype = prog->type;
111f92c1e18SJiri Olsa
1122fcc8241SKui-Feng Lee return (ptype == BPF_PROG_TYPE_TRACING &&
1132fcc8241SKui-Feng Lee (eatype == BPF_TRACE_FENTRY || eatype == BPF_TRACE_FEXIT ||
1142fcc8241SKui-Feng Lee eatype == BPF_MODIFY_RETURN)) ||
1152fcc8241SKui-Feng Lee (ptype == BPF_PROG_TYPE_LSM && eatype == BPF_LSM_MAC);
116f92c1e18SJiri Olsa }
117f92c1e18SJiri Olsa
bpf_image_ksym_add(void * data,unsigned int size,struct bpf_ksym * ksym)11826ef208cSSong Liu void bpf_image_ksym_add(void *data, unsigned int size, struct bpf_ksym *ksym)
119a108f7dcSJiri Olsa {
120a108f7dcSJiri Olsa ksym->start = (unsigned long) data;
12126ef208cSSong Liu ksym->end = ksym->start + size;
122a108f7dcSJiri Olsa bpf_ksym_add(ksym);
123a108f7dcSJiri Olsa perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start,
1247ac88ebaSJiri Olsa PAGE_SIZE, false, ksym->name);
125a108f7dcSJiri Olsa }
126a108f7dcSJiri Olsa
bpf_image_ksym_del(struct bpf_ksym * ksym)127a108f7dcSJiri Olsa void bpf_image_ksym_del(struct bpf_ksym *ksym)
128a108f7dcSJiri Olsa {
129a108f7dcSJiri Olsa bpf_ksym_del(ksym);
130a108f7dcSJiri Olsa perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start,
1317ac88ebaSJiri Olsa PAGE_SIZE, true, ksym->name);
132a108f7dcSJiri Olsa }
133a108f7dcSJiri Olsa
bpf_trampoline_lookup(u64 key)134f7b12b6fSToke Høiland-Jørgensen static struct bpf_trampoline *bpf_trampoline_lookup(u64 key)
135fec56f58SAlexei Starovoitov {
136fec56f58SAlexei Starovoitov struct bpf_trampoline *tr;
137fec56f58SAlexei Starovoitov struct hlist_head *head;
138fec56f58SAlexei Starovoitov int i;
139fec56f58SAlexei Starovoitov
140fec56f58SAlexei Starovoitov mutex_lock(&trampoline_mutex);
141fec56f58SAlexei Starovoitov head = &trampoline_table[hash_64(key, TRAMPOLINE_HASH_BITS)];
142fec56f58SAlexei Starovoitov hlist_for_each_entry(tr, head, hlist) {
143fec56f58SAlexei Starovoitov if (tr->key == key) {
144fec56f58SAlexei Starovoitov refcount_inc(&tr->refcnt);
145fec56f58SAlexei Starovoitov goto out;
146fec56f58SAlexei Starovoitov }
147fec56f58SAlexei Starovoitov }
148fec56f58SAlexei Starovoitov tr = kzalloc(sizeof(*tr), GFP_KERNEL);
149fec56f58SAlexei Starovoitov if (!tr)
150fec56f58SAlexei Starovoitov goto out;
15100963a2eSSong Liu #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
15200963a2eSSong Liu tr->fops = kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL);
15300963a2eSSong Liu if (!tr->fops) {
15400963a2eSSong Liu kfree(tr);
15500963a2eSSong Liu tr = NULL;
15600963a2eSSong Liu goto out;
15700963a2eSSong Liu }
15800963a2eSSong Liu tr->fops->private = tr;
15900963a2eSSong Liu tr->fops->ops_func = bpf_tramp_ftrace_ops_func;
16000963a2eSSong Liu #endif
161fec56f58SAlexei Starovoitov
162fec56f58SAlexei Starovoitov tr->key = key;
163fec56f58SAlexei Starovoitov INIT_HLIST_NODE(&tr->hlist);
164fec56f58SAlexei Starovoitov hlist_add_head(&tr->hlist, head);
165fec56f58SAlexei Starovoitov refcount_set(&tr->refcnt, 1);
166fec56f58SAlexei Starovoitov mutex_init(&tr->mutex);
167fec56f58SAlexei Starovoitov for (i = 0; i < BPF_TRAMP_MAX; i++)
168fec56f58SAlexei Starovoitov INIT_HLIST_HEAD(&tr->progs_hlist[i]);
169fec56f58SAlexei Starovoitov out:
170fec56f58SAlexei Starovoitov mutex_unlock(&trampoline_mutex);
171fec56f58SAlexei Starovoitov return tr;
172fec56f58SAlexei Starovoitov }
173fec56f58SAlexei Starovoitov
unregister_fentry(struct bpf_trampoline * tr,void * old_addr)174b91e014fSAlexei Starovoitov static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr)
175b91e014fSAlexei Starovoitov {
176b91e014fSAlexei Starovoitov void *ip = tr->func.addr;
177b91e014fSAlexei Starovoitov int ret;
178b91e014fSAlexei Starovoitov
179b91e014fSAlexei Starovoitov if (tr->func.ftrace_managed)
180da8bdfbdSFlorent Revest ret = unregister_ftrace_direct(tr->fops, (long)old_addr, false);
181b91e014fSAlexei Starovoitov else
182b91e014fSAlexei Starovoitov ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, NULL);
183861de02eSJiri Olsa
184b91e014fSAlexei Starovoitov return ret;
185b91e014fSAlexei Starovoitov }
186b91e014fSAlexei Starovoitov
modify_fentry(struct bpf_trampoline * tr,void * old_addr,void * new_addr,bool lock_direct_mutex)18700963a2eSSong Liu static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_addr,
18800963a2eSSong Liu bool lock_direct_mutex)
189b91e014fSAlexei Starovoitov {
190b91e014fSAlexei Starovoitov void *ip = tr->func.addr;
191b91e014fSAlexei Starovoitov int ret;
192b91e014fSAlexei Starovoitov
19300963a2eSSong Liu if (tr->func.ftrace_managed) {
19400963a2eSSong Liu if (lock_direct_mutex)
195da8bdfbdSFlorent Revest ret = modify_ftrace_direct(tr->fops, (long)new_addr);
196b91e014fSAlexei Starovoitov else
197da8bdfbdSFlorent Revest ret = modify_ftrace_direct_nolock(tr->fops, (long)new_addr);
19800963a2eSSong Liu } else {
199b91e014fSAlexei Starovoitov ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, new_addr);
20000963a2eSSong Liu }
201b91e014fSAlexei Starovoitov return ret;
202b91e014fSAlexei Starovoitov }
203b91e014fSAlexei Starovoitov
204b91e014fSAlexei Starovoitov /* first time registering */
register_fentry(struct bpf_trampoline * tr,void * new_addr)205b91e014fSAlexei Starovoitov static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
206b91e014fSAlexei Starovoitov {
207b91e014fSAlexei Starovoitov void *ip = tr->func.addr;
208aebfd125SPeter Zijlstra unsigned long faddr;
209b91e014fSAlexei Starovoitov int ret;
210b91e014fSAlexei Starovoitov
211aebfd125SPeter Zijlstra faddr = ftrace_location((unsigned long)ip);
2123b317abcSXu Kuohai if (faddr) {
2133b317abcSXu Kuohai if (!tr->fops)
2143b317abcSXu Kuohai return -ENOTSUPP;
215aebfd125SPeter Zijlstra tr->func.ftrace_managed = true;
2163b317abcSXu Kuohai }
217b91e014fSAlexei Starovoitov
21800963a2eSSong Liu if (tr->func.ftrace_managed) {
219dc81f8d1SSong Liu ftrace_set_filter_ip(tr->fops, (unsigned long)ip, 0, 1);
220da8bdfbdSFlorent Revest ret = register_ftrace_direct(tr->fops, (long)new_addr);
22100963a2eSSong Liu } else {
222b91e014fSAlexei Starovoitov ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, NULL, new_addr);
22300963a2eSSong Liu }
224861de02eSJiri Olsa
225b91e014fSAlexei Starovoitov return ret;
226b91e014fSAlexei Starovoitov }
227b91e014fSAlexei Starovoitov
228f7e0beafSKui-Feng Lee static struct bpf_tramp_links *
bpf_trampoline_get_progs(const struct bpf_trampoline * tr,int * total,bool * ip_arg)2291e37392cSJiri Olsa bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_arg)
23088fd9e53SKP Singh {
231f7e0beafSKui-Feng Lee struct bpf_tramp_link *link;
232f7e0beafSKui-Feng Lee struct bpf_tramp_links *tlinks;
233f7e0beafSKui-Feng Lee struct bpf_tramp_link **links;
23488fd9e53SKP Singh int kind;
23588fd9e53SKP Singh
23688fd9e53SKP Singh *total = 0;
237f7e0beafSKui-Feng Lee tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL);
238f7e0beafSKui-Feng Lee if (!tlinks)
23988fd9e53SKP Singh return ERR_PTR(-ENOMEM);
24088fd9e53SKP Singh
24188fd9e53SKP Singh for (kind = 0; kind < BPF_TRAMP_MAX; kind++) {
242f7e0beafSKui-Feng Lee tlinks[kind].nr_links = tr->progs_cnt[kind];
24388fd9e53SKP Singh *total += tr->progs_cnt[kind];
244f7e0beafSKui-Feng Lee links = tlinks[kind].links;
24588fd9e53SKP Singh
246f7e0beafSKui-Feng Lee hlist_for_each_entry(link, &tr->progs_hlist[kind], tramp_hlist) {
247f7e0beafSKui-Feng Lee *ip_arg |= link->link.prog->call_get_func_ip;
248f7e0beafSKui-Feng Lee *links++ = link;
24988fd9e53SKP Singh }
2501e37392cSJiri Olsa }
251f7e0beafSKui-Feng Lee return tlinks;
25288fd9e53SKP Singh }
253fec56f58SAlexei Starovoitov
bpf_tramp_image_free(struct bpf_tramp_image * im)254108598c3SYafang Shao static void bpf_tramp_image_free(struct bpf_tramp_image *im)
255e21aa341SAlexei Starovoitov {
256e21aa341SAlexei Starovoitov bpf_image_ksym_del(&im->ksym);
25726ef208cSSong Liu arch_free_bpf_trampoline(im->image, im->size);
25826ef208cSSong Liu bpf_jit_uncharge_modmem(im->size);
259e21aa341SAlexei Starovoitov percpu_ref_exit(&im->pcref);
260e21aa341SAlexei Starovoitov kfree_rcu(im, rcu);
261e21aa341SAlexei Starovoitov }
262e21aa341SAlexei Starovoitov
__bpf_tramp_image_put_deferred(struct work_struct * work)263108598c3SYafang Shao static void __bpf_tramp_image_put_deferred(struct work_struct *work)
264108598c3SYafang Shao {
265108598c3SYafang Shao struct bpf_tramp_image *im;
266108598c3SYafang Shao
267108598c3SYafang Shao im = container_of(work, struct bpf_tramp_image, work);
268108598c3SYafang Shao bpf_tramp_image_free(im);
269108598c3SYafang Shao }
270108598c3SYafang Shao
271e21aa341SAlexei Starovoitov /* callback, fexit step 3 or fentry step 2 */
__bpf_tramp_image_put_rcu(struct rcu_head * rcu)272e21aa341SAlexei Starovoitov static void __bpf_tramp_image_put_rcu(struct rcu_head *rcu)
273e21aa341SAlexei Starovoitov {
274e21aa341SAlexei Starovoitov struct bpf_tramp_image *im;
275e21aa341SAlexei Starovoitov
276e21aa341SAlexei Starovoitov im = container_of(rcu, struct bpf_tramp_image, rcu);
277e21aa341SAlexei Starovoitov INIT_WORK(&im->work, __bpf_tramp_image_put_deferred);
278e21aa341SAlexei Starovoitov schedule_work(&im->work);
279e21aa341SAlexei Starovoitov }
280e21aa341SAlexei Starovoitov
281e21aa341SAlexei Starovoitov /* callback, fexit step 2. Called after percpu_ref_kill confirms. */
__bpf_tramp_image_release(struct percpu_ref * pcref)282e21aa341SAlexei Starovoitov static void __bpf_tramp_image_release(struct percpu_ref *pcref)
283e21aa341SAlexei Starovoitov {
284e21aa341SAlexei Starovoitov struct bpf_tramp_image *im;
285e21aa341SAlexei Starovoitov
286e21aa341SAlexei Starovoitov im = container_of(pcref, struct bpf_tramp_image, pcref);
287e21aa341SAlexei Starovoitov call_rcu_tasks(&im->rcu, __bpf_tramp_image_put_rcu);
288e21aa341SAlexei Starovoitov }
289e21aa341SAlexei Starovoitov
290e21aa341SAlexei Starovoitov /* callback, fexit or fentry step 1 */
__bpf_tramp_image_put_rcu_tasks(struct rcu_head * rcu)291e21aa341SAlexei Starovoitov static void __bpf_tramp_image_put_rcu_tasks(struct rcu_head *rcu)
292e21aa341SAlexei Starovoitov {
293e21aa341SAlexei Starovoitov struct bpf_tramp_image *im;
294e21aa341SAlexei Starovoitov
295e21aa341SAlexei Starovoitov im = container_of(rcu, struct bpf_tramp_image, rcu);
296e21aa341SAlexei Starovoitov if (im->ip_after_call)
297e21aa341SAlexei Starovoitov /* the case of fmod_ret/fexit trampoline and CONFIG_PREEMPTION=y */
298e21aa341SAlexei Starovoitov percpu_ref_kill(&im->pcref);
299e21aa341SAlexei Starovoitov else
300e21aa341SAlexei Starovoitov /* the case of fentry trampoline */
301e21aa341SAlexei Starovoitov call_rcu_tasks(&im->rcu, __bpf_tramp_image_put_rcu);
302e21aa341SAlexei Starovoitov }
303e21aa341SAlexei Starovoitov
bpf_tramp_image_put(struct bpf_tramp_image * im)304e21aa341SAlexei Starovoitov static void bpf_tramp_image_put(struct bpf_tramp_image *im)
305e21aa341SAlexei Starovoitov {
306e21aa341SAlexei Starovoitov /* The trampoline image that calls original function is using:
307e21aa341SAlexei Starovoitov * rcu_read_lock_trace to protect sleepable bpf progs
308e21aa341SAlexei Starovoitov * rcu_read_lock to protect normal bpf progs
309e21aa341SAlexei Starovoitov * percpu_ref to protect trampoline itself
310e21aa341SAlexei Starovoitov * rcu tasks to protect trampoline asm not covered by percpu_ref
311e21aa341SAlexei Starovoitov * (which are few asm insns before __bpf_tramp_enter and
312e21aa341SAlexei Starovoitov * after __bpf_tramp_exit)
313e21aa341SAlexei Starovoitov *
314e21aa341SAlexei Starovoitov * The trampoline is unreachable before bpf_tramp_image_put().
315e21aa341SAlexei Starovoitov *
316e21aa341SAlexei Starovoitov * First, patch the trampoline to avoid calling into fexit progs.
317e21aa341SAlexei Starovoitov * The progs will be freed even if the original function is still
318e21aa341SAlexei Starovoitov * executing or sleeping.
319e21aa341SAlexei Starovoitov * In case of CONFIG_PREEMPT=y use call_rcu_tasks() to wait on
320e21aa341SAlexei Starovoitov * first few asm instructions to execute and call into
321e21aa341SAlexei Starovoitov * __bpf_tramp_enter->percpu_ref_get.
322e21aa341SAlexei Starovoitov * Then use percpu_ref_kill to wait for the trampoline and the original
323e21aa341SAlexei Starovoitov * function to finish.
324e21aa341SAlexei Starovoitov * Then use call_rcu_tasks() to make sure few asm insns in
325e21aa341SAlexei Starovoitov * the trampoline epilogue are done as well.
326e21aa341SAlexei Starovoitov *
327e21aa341SAlexei Starovoitov * In !PREEMPT case the task that got interrupted in the first asm
328e21aa341SAlexei Starovoitov * insns won't go through an RCU quiescent state which the
329e21aa341SAlexei Starovoitov * percpu_ref_kill will be waiting for. Hence the first
330e21aa341SAlexei Starovoitov * call_rcu_tasks() is not necessary.
331e21aa341SAlexei Starovoitov */
332e21aa341SAlexei Starovoitov if (im->ip_after_call) {
333e21aa341SAlexei Starovoitov int err = bpf_arch_text_poke(im->ip_after_call, BPF_MOD_JUMP,
334e21aa341SAlexei Starovoitov NULL, im->ip_epilogue);
335e21aa341SAlexei Starovoitov WARN_ON(err);
336*1e52af7fSPaul E. McKenney if (IS_ENABLED(CONFIG_TASKS_RCU))
337e21aa341SAlexei Starovoitov call_rcu_tasks(&im->rcu, __bpf_tramp_image_put_rcu_tasks);
338e21aa341SAlexei Starovoitov else
339e21aa341SAlexei Starovoitov percpu_ref_kill(&im->pcref);
340e21aa341SAlexei Starovoitov return;
341e21aa341SAlexei Starovoitov }
342e21aa341SAlexei Starovoitov
343e21aa341SAlexei Starovoitov /* The trampoline without fexit and fmod_ret progs doesn't call original
344e21aa341SAlexei Starovoitov * function and doesn't use percpu_ref.
345e21aa341SAlexei Starovoitov * Use call_rcu_tasks_trace() to wait for sleepable progs to finish.
346e21aa341SAlexei Starovoitov * Then use call_rcu_tasks() to wait for the rest of trampoline asm
347e21aa341SAlexei Starovoitov * and normal progs.
348e21aa341SAlexei Starovoitov */
349e21aa341SAlexei Starovoitov call_rcu_tasks_trace(&im->rcu, __bpf_tramp_image_put_rcu_tasks);
350e21aa341SAlexei Starovoitov }
351e21aa341SAlexei Starovoitov
bpf_tramp_image_alloc(u64 key,int size)35226ef208cSSong Liu static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size)
353e21aa341SAlexei Starovoitov {
354e21aa341SAlexei Starovoitov struct bpf_tramp_image *im;
355e21aa341SAlexei Starovoitov struct bpf_ksym *ksym;
356e21aa341SAlexei Starovoitov void *image;
357e21aa341SAlexei Starovoitov int err = -ENOMEM;
358e21aa341SAlexei Starovoitov
359e21aa341SAlexei Starovoitov im = kzalloc(sizeof(*im), GFP_KERNEL);
360e21aa341SAlexei Starovoitov if (!im)
361e21aa341SAlexei Starovoitov goto out;
362e21aa341SAlexei Starovoitov
36326ef208cSSong Liu err = bpf_jit_charge_modmem(size);
364e21aa341SAlexei Starovoitov if (err)
365e21aa341SAlexei Starovoitov goto out_free_im;
36626ef208cSSong Liu im->size = size;
367e21aa341SAlexei Starovoitov
368e21aa341SAlexei Starovoitov err = -ENOMEM;
36926ef208cSSong Liu im->image = image = arch_alloc_bpf_trampoline(size);
370e21aa341SAlexei Starovoitov if (!image)
371e21aa341SAlexei Starovoitov goto out_uncharge;
372e21aa341SAlexei Starovoitov
373e21aa341SAlexei Starovoitov err = percpu_ref_init(&im->pcref, __bpf_tramp_image_release, 0, GFP_KERNEL);
374e21aa341SAlexei Starovoitov if (err)
375e21aa341SAlexei Starovoitov goto out_free_image;
376e21aa341SAlexei Starovoitov
377e21aa341SAlexei Starovoitov ksym = &im->ksym;
378e21aa341SAlexei Starovoitov INIT_LIST_HEAD_RCU(&ksym->lnode);
37947e79cbeSYafang Shao snprintf(ksym->name, KSYM_NAME_LEN, "bpf_trampoline_%llu", key);
38026ef208cSSong Liu bpf_image_ksym_add(image, size, ksym);
381e21aa341SAlexei Starovoitov return im;
382e21aa341SAlexei Starovoitov
383e21aa341SAlexei Starovoitov out_free_image:
38426ef208cSSong Liu arch_free_bpf_trampoline(im->image, im->size);
385e21aa341SAlexei Starovoitov out_uncharge:
38626ef208cSSong Liu bpf_jit_uncharge_modmem(size);
387e21aa341SAlexei Starovoitov out_free_im:
388e21aa341SAlexei Starovoitov kfree(im);
389e21aa341SAlexei Starovoitov out:
390e21aa341SAlexei Starovoitov return ERR_PTR(err);
391e21aa341SAlexei Starovoitov }
392e21aa341SAlexei Starovoitov
bpf_trampoline_update(struct bpf_trampoline * tr,bool lock_direct_mutex)39300963a2eSSong Liu static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex)
394fec56f58SAlexei Starovoitov {
395e21aa341SAlexei Starovoitov struct bpf_tramp_image *im;
396f7e0beafSKui-Feng Lee struct bpf_tramp_links *tlinks;
39700963a2eSSong Liu u32 orig_flags = tr->flags;
3981e37392cSJiri Olsa bool ip_arg = false;
39926ef208cSSong Liu int err, total, size;
400fec56f58SAlexei Starovoitov
401f7e0beafSKui-Feng Lee tlinks = bpf_trampoline_get_progs(tr, &total, &ip_arg);
402f7e0beafSKui-Feng Lee if (IS_ERR(tlinks))
403f7e0beafSKui-Feng Lee return PTR_ERR(tlinks);
40488fd9e53SKP Singh
40588fd9e53SKP Singh if (total == 0) {
406e21aa341SAlexei Starovoitov err = unregister_fentry(tr, tr->cur_image->image);
407e21aa341SAlexei Starovoitov bpf_tramp_image_put(tr->cur_image);
408e21aa341SAlexei Starovoitov tr->cur_image = NULL;
409fec56f58SAlexei Starovoitov goto out;
410fec56f58SAlexei Starovoitov }
411fec56f58SAlexei Starovoitov
4122b5dcb31SLeon Hwang /* clear all bits except SHARE_IPMODIFY and TAIL_CALL_CTX */
4132b5dcb31SLeon Hwang tr->flags &= (BPF_TRAMP_F_SHARE_IPMODIFY | BPF_TRAMP_F_TAIL_CALL_CTX);
41400963a2eSSong Liu
415f7e0beafSKui-Feng Lee if (tlinks[BPF_TRAMP_FEXIT].nr_links ||
41600963a2eSSong Liu tlinks[BPF_TRAMP_MODIFY_RETURN].nr_links) {
417535a57a7SXu Kuohai /* NOTE: BPF_TRAMP_F_RESTORE_REGS and BPF_TRAMP_F_SKIP_FRAME
418535a57a7SXu Kuohai * should not be set together.
419535a57a7SXu Kuohai */
42000963a2eSSong Liu tr->flags |= BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME;
42100963a2eSSong Liu } else {
42200963a2eSSong Liu tr->flags |= BPF_TRAMP_F_RESTORE_REGS;
42300963a2eSSong Liu }
424fec56f58SAlexei Starovoitov
4251e37392cSJiri Olsa if (ip_arg)
42600963a2eSSong Liu tr->flags |= BPF_TRAMP_F_IP_ARG;
42700963a2eSSong Liu
42800963a2eSSong Liu #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
42900963a2eSSong Liu again:
43000963a2eSSong Liu if ((tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY) &&
43100963a2eSSong Liu (tr->flags & BPF_TRAMP_F_CALL_ORIG))
43200963a2eSSong Liu tr->flags |= BPF_TRAMP_F_ORIG_STACK;
43300963a2eSSong Liu #endif
4341e37392cSJiri Olsa
43526ef208cSSong Liu size = arch_bpf_trampoline_size(&tr->func.model, tr->flags,
43626ef208cSSong Liu tlinks, tr->func.addr);
43726ef208cSSong Liu if (size < 0) {
43826ef208cSSong Liu err = size;
43926ef208cSSong Liu goto out;
44026ef208cSSong Liu }
44126ef208cSSong Liu
44226ef208cSSong Liu if (size > PAGE_SIZE) {
44326ef208cSSong Liu err = -E2BIG;
44426ef208cSSong Liu goto out;
44526ef208cSSong Liu }
44626ef208cSSong Liu
44726ef208cSSong Liu im = bpf_tramp_image_alloc(tr->key, size);
44826ef208cSSong Liu if (IS_ERR(im)) {
44926ef208cSSong Liu err = PTR_ERR(im);
45026ef208cSSong Liu goto out;
45126ef208cSSong Liu }
45226ef208cSSong Liu
45326ef208cSSong Liu err = arch_prepare_bpf_trampoline(im, im->image, im->image + size,
45400963a2eSSong Liu &tr->func.model, tr->flags, tlinks,
455fec56f58SAlexei Starovoitov tr->func.addr);
45685d33df3SMartin KaFai Lau if (err < 0)
457108598c3SYafang Shao goto out_free;
458fec56f58SAlexei Starovoitov
45926ef208cSSong Liu err = arch_protect_bpf_trampoline(im->image, im->size);
4605b0d1c7bSSong Liu if (err)
46147e79cbeSYafang Shao goto out_free;
462e21aa341SAlexei Starovoitov
463fec56f58SAlexei Starovoitov WARN_ON(tr->cur_image && total == 0);
46400963a2eSSong Liu if (tr->cur_image)
465fec56f58SAlexei Starovoitov /* progs already running at this address */
466fec56f58SAlexei Starovoitov err = modify_fentry(tr, tr->cur_image->image, im->image, lock_direct_mutex);
467e21aa341SAlexei Starovoitov else
46800963a2eSSong Liu /* first time registering */
46900963a2eSSong Liu err = register_fentry(tr, im->image);
47000963a2eSSong Liu
47100963a2eSSong Liu #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
47200963a2eSSong Liu if (err == -EAGAIN) {
47300963a2eSSong Liu /* -EAGAIN from bpf_tramp_ftrace_ops_func. Now
47400963a2eSSong Liu * BPF_TRAMP_F_SHARE_IPMODIFY is set, we can generate the
47500963a2eSSong Liu * trampoline again, and retry register.
47600963a2eSSong Liu */
47700963a2eSSong Liu /* reset fops->func and fops->trampoline for re-register */
4789ed1d9aeSChuang Wang tr->fops->func = NULL;
47926ef208cSSong Liu tr->fops->trampoline = 0;
48026ef208cSSong Liu
48100963a2eSSong Liu /* free im memory and reallocate later */
48200963a2eSSong Liu bpf_tramp_image_free(im);
48300963a2eSSong Liu goto again;
484fec56f58SAlexei Starovoitov }
485108598c3SYafang Shao #endif
48600963a2eSSong Liu if (err)
487e21aa341SAlexei Starovoitov goto out_free;
488e21aa341SAlexei Starovoitov
489e21aa341SAlexei Starovoitov if (tr->cur_image)
490fec56f58SAlexei Starovoitov bpf_tramp_image_put(tr->cur_image);
49100963a2eSSong Liu tr->cur_image = im;
49200963a2eSSong Liu out:
49300963a2eSSong Liu /* If any error happens, restore previous flags */
494f7e0beafSKui-Feng Lee if (err)
495fec56f58SAlexei Starovoitov tr->flags = orig_flags;
496108598c3SYafang Shao kfree(tlinks);
497108598c3SYafang Shao return err;
498108598c3SYafang Shao
499108598c3SYafang Shao out_free:
500fec56f58SAlexei Starovoitov bpf_tramp_image_free(im);
501fec56f58SAlexei Starovoitov goto out;
5029e4e01dfSKP Singh }
503fec56f58SAlexei Starovoitov
bpf_attach_type_to_tramp(struct bpf_prog * prog)5049e4e01dfSKP Singh static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)
505fec56f58SAlexei Starovoitov {
506fec56f58SAlexei Starovoitov switch (prog->expected_attach_type) {
507ae240823SKP Singh case BPF_TRACE_FENTRY:
508ae240823SKP Singh return BPF_TRAMP_FENTRY;
509be8704ffSAlexei Starovoitov case BPF_MODIFY_RETURN:
510fec56f58SAlexei Starovoitov return BPF_TRAMP_MODIFY_RETURN;
5119e4e01dfSKP Singh case BPF_TRACE_FEXIT:
5129e4e01dfSKP Singh return BPF_TRAMP_FEXIT;
5139e4e01dfSKP Singh case BPF_LSM_MAC:
5149e4e01dfSKP Singh if (!prog->aux->attach_func_proto->type)
5159e4e01dfSKP Singh /* The function returns void, we cannot modify its
5169e4e01dfSKP Singh * return value.
5179e4e01dfSKP Singh */
5189e4e01dfSKP Singh return BPF_TRAMP_FEXIT;
519be8704ffSAlexei Starovoitov else
520be8704ffSAlexei Starovoitov return BPF_TRAMP_MODIFY_RETURN;
521fec56f58SAlexei Starovoitov default:
522fec56f58SAlexei Starovoitov return BPF_TRAMP_REPLACE;
523fec56f58SAlexei Starovoitov }
524af3f4134SStanislav Fomichev }
525fec56f58SAlexei Starovoitov
__bpf_trampoline_link_prog(struct bpf_tramp_link * link,struct bpf_trampoline * tr)526fec56f58SAlexei Starovoitov static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)
527f7e0beafSKui-Feng Lee {
528fec56f58SAlexei Starovoitov enum bpf_tramp_prog_type kind;
529a2aa95b7SYuntao Wang struct bpf_tramp_link *link_exiting;
530fec56f58SAlexei Starovoitov int err = 0;
531f7e0beafSKui-Feng Lee int cnt = 0, i;
532af3f4134SStanislav Fomichev
533be8704ffSAlexei Starovoitov kind = bpf_attach_type_to_tramp(link->link.prog);
534be8704ffSAlexei Starovoitov if (tr->extension_prog)
535be8704ffSAlexei Starovoitov /* cannot attach fentry/fexit if extension prog is attached.
536af3f4134SStanislav Fomichev * cannot overwrite extension prog either.
537a2aa95b7SYuntao Wang */
538a2aa95b7SYuntao Wang return -EBUSY;
539a2aa95b7SYuntao Wang
540a2aa95b7SYuntao Wang for (i = 0; i < BPF_TRAMP_MAX; i++)
541be8704ffSAlexei Starovoitov cnt += tr->progs_cnt[i];
542be8704ffSAlexei Starovoitov
543af3f4134SStanislav Fomichev if (kind == BPF_TRAMP_REPLACE) {
544af3f4134SStanislav Fomichev /* Cannot attach extension if fentry/fexit are in use. */
545f7e0beafSKui-Feng Lee if (cnt)
546af3f4134SStanislav Fomichev return -EBUSY;
547f7e0beafSKui-Feng Lee tr->extension_prog = link->link.prog;
548be8704ffSAlexei Starovoitov return bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL,
549af3f4134SStanislav Fomichev link->link.prog->bpf_func);
550af3f4134SStanislav Fomichev }
551af3f4134SStanislav Fomichev if (cnt >= BPF_MAX_TRAMP_LINKS)
552fec56f58SAlexei Starovoitov return -E2BIG;
553af3f4134SStanislav Fomichev if (!hlist_unhashed(&link->tramp_hlist))
554f7e0beafSKui-Feng Lee /* prog already linked */
555f7e0beafSKui-Feng Lee return -EBUSY;
556f7e0beafSKui-Feng Lee hlist_for_each_entry(link_exiting, &tr->progs_hlist[kind], tramp_hlist) {
557f7e0beafSKui-Feng Lee if (link_exiting->link.prog != link->link.prog)
558af3f4134SStanislav Fomichev continue;
559f7e0beafSKui-Feng Lee /* prog already linked */
560f7e0beafSKui-Feng Lee return -EBUSY;
561f7e0beafSKui-Feng Lee }
562fec56f58SAlexei Starovoitov
56300963a2eSSong Liu hlist_add_head(&link->tramp_hlist, &tr->progs_hlist[kind]);
564fec56f58SAlexei Starovoitov tr->progs_cnt[kind]++;
565f7e0beafSKui-Feng Lee err = bpf_trampoline_update(tr, true /* lock_direct_mutex */);
566fec56f58SAlexei Starovoitov if (err) {
567fec56f58SAlexei Starovoitov hlist_del_init(&link->tramp_hlist);
568af3f4134SStanislav Fomichev tr->progs_cnt[kind]--;
569af3f4134SStanislav Fomichev }
570af3f4134SStanislav Fomichev return err;
571af3f4134SStanislav Fomichev }
572af3f4134SStanislav Fomichev
bpf_trampoline_link_prog(struct bpf_tramp_link * link,struct bpf_trampoline * tr)573af3f4134SStanislav Fomichev int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)
574af3f4134SStanislav Fomichev {
575af3f4134SStanislav Fomichev int err;
576af3f4134SStanislav Fomichev
577fec56f58SAlexei Starovoitov mutex_lock(&tr->mutex);
578fec56f58SAlexei Starovoitov err = __bpf_trampoline_link_prog(link, tr);
579fec56f58SAlexei Starovoitov mutex_unlock(&tr->mutex);
580fec56f58SAlexei Starovoitov return err;
581af3f4134SStanislav Fomichev }
582fec56f58SAlexei Starovoitov
__bpf_trampoline_unlink_prog(struct bpf_tramp_link * link,struct bpf_trampoline * tr)583fec56f58SAlexei Starovoitov static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)
584fec56f58SAlexei Starovoitov {
585fec56f58SAlexei Starovoitov enum bpf_tramp_prog_type kind;
586f7e0beafSKui-Feng Lee int err;
587be8704ffSAlexei Starovoitov
588be8704ffSAlexei Starovoitov kind = bpf_attach_type_to_tramp(link->link.prog);
589be8704ffSAlexei Starovoitov if (kind == BPF_TRAMP_REPLACE) {
590be8704ffSAlexei Starovoitov WARN_ON_ONCE(!tr->extension_prog);
591be8704ffSAlexei Starovoitov err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP,
592af3f4134SStanislav Fomichev tr->extension_prog->bpf_func, NULL);
593be8704ffSAlexei Starovoitov tr->extension_prog = NULL;
594f7e0beafSKui-Feng Lee return err;
595fec56f58SAlexei Starovoitov }
59600963a2eSSong Liu hlist_del_init(&link->tramp_hlist);
597af3f4134SStanislav Fomichev tr->progs_cnt[kind]--;
598af3f4134SStanislav Fomichev return bpf_trampoline_update(tr, true /* lock_direct_mutex */);
599af3f4134SStanislav Fomichev }
600af3f4134SStanislav Fomichev
601af3f4134SStanislav Fomichev /* bpf_trampoline_unlink_prog() should never fail. */
bpf_trampoline_unlink_prog(struct bpf_tramp_link * link,struct bpf_trampoline * tr)602af3f4134SStanislav Fomichev int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)
603af3f4134SStanislav Fomichev {
604af3f4134SStanislav Fomichev int err;
605af3f4134SStanislav Fomichev
606fec56f58SAlexei Starovoitov mutex_lock(&tr->mutex);
607fec56f58SAlexei Starovoitov err = __bpf_trampoline_unlink_prog(link, tr);
608fec56f58SAlexei Starovoitov mutex_unlock(&tr->mutex);
609fec56f58SAlexei Starovoitov return err;
6103908fcddSStanislav Fomichev }
61169fd337aSStanislav Fomichev
61269fd337aSStanislav Fomichev #if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)
bpf_shim_tramp_link_release(struct bpf_link * link)61369fd337aSStanislav Fomichev static void bpf_shim_tramp_link_release(struct bpf_link *link)
61469fd337aSStanislav Fomichev {
61569fd337aSStanislav Fomichev struct bpf_shim_tramp_link *shim_link =
61669fd337aSStanislav Fomichev container_of(link, struct bpf_shim_tramp_link, link.link);
61769fd337aSStanislav Fomichev
61869fd337aSStanislav Fomichev /* paired with 'shim_link->trampoline = tr' in bpf_trampoline_link_cgroup_shim */
61969fd337aSStanislav Fomichev if (!shim_link->trampoline)
62069fd337aSStanislav Fomichev return;
62169fd337aSStanislav Fomichev
62269fd337aSStanislav Fomichev WARN_ON_ONCE(bpf_trampoline_unlink_prog(&shim_link->link, shim_link->trampoline));
62369fd337aSStanislav Fomichev bpf_trampoline_put(shim_link->trampoline);
62469fd337aSStanislav Fomichev }
62569fd337aSStanislav Fomichev
bpf_shim_tramp_link_dealloc(struct bpf_link * link)62669fd337aSStanislav Fomichev static void bpf_shim_tramp_link_dealloc(struct bpf_link *link)
62769fd337aSStanislav Fomichev {
62869fd337aSStanislav Fomichev struct bpf_shim_tramp_link *shim_link =
62969fd337aSStanislav Fomichev container_of(link, struct bpf_shim_tramp_link, link.link);
63069fd337aSStanislav Fomichev
63169fd337aSStanislav Fomichev kfree(shim_link);
63269fd337aSStanislav Fomichev }
63369fd337aSStanislav Fomichev
63469fd337aSStanislav Fomichev static const struct bpf_link_ops bpf_shim_tramp_link_lops = {
63569fd337aSStanislav Fomichev .release = bpf_shim_tramp_link_release,
63669fd337aSStanislav Fomichev .dealloc = bpf_shim_tramp_link_dealloc,
63769fd337aSStanislav Fomichev };
63869fd337aSStanislav Fomichev
cgroup_shim_alloc(const struct bpf_prog * prog,bpf_func_t bpf_func,int cgroup_atype)63969fd337aSStanislav Fomichev static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog,
64069fd337aSStanislav Fomichev bpf_func_t bpf_func,
64169fd337aSStanislav Fomichev int cgroup_atype)
64269fd337aSStanislav Fomichev {
64369fd337aSStanislav Fomichev struct bpf_shim_tramp_link *shim_link = NULL;
64469fd337aSStanislav Fomichev struct bpf_prog *p;
64569fd337aSStanislav Fomichev
64669fd337aSStanislav Fomichev shim_link = kzalloc(sizeof(*shim_link), GFP_USER);
64769fd337aSStanislav Fomichev if (!shim_link)
64869fd337aSStanislav Fomichev return NULL;
64969fd337aSStanislav Fomichev
65069fd337aSStanislav Fomichev p = bpf_prog_alloc(1, 0);
65169fd337aSStanislav Fomichev if (!p) {
65269fd337aSStanislav Fomichev kfree(shim_link);
65369fd337aSStanislav Fomichev return NULL;
65469fd337aSStanislav Fomichev }
65569fd337aSStanislav Fomichev
65669fd337aSStanislav Fomichev p->jited = false;
65769fd337aSStanislav Fomichev p->bpf_func = bpf_func;
65869fd337aSStanislav Fomichev
65969fd337aSStanislav Fomichev p->aux->cgroup_atype = cgroup_atype;
66069fd337aSStanislav Fomichev p->aux->attach_func_proto = prog->aux->attach_func_proto;
66169fd337aSStanislav Fomichev p->aux->attach_btf_id = prog->aux->attach_btf_id;
66269fd337aSStanislav Fomichev p->aux->attach_btf = prog->aux->attach_btf;
66369fd337aSStanislav Fomichev btf_get(p->aux->attach_btf);
66469fd337aSStanislav Fomichev p->type = BPF_PROG_TYPE_LSM;
66569fd337aSStanislav Fomichev p->expected_attach_type = BPF_LSM_MAC;
66669fd337aSStanislav Fomichev bpf_prog_inc(p);
667c0e19f2cSStanislav Fomichev bpf_link_init(&shim_link->link.link, BPF_LINK_TYPE_UNSPEC,
66869fd337aSStanislav Fomichev &bpf_shim_tramp_link_lops, p);
66969fd337aSStanislav Fomichev bpf_cgroup_atype_get(p->aux->attach_btf_id, cgroup_atype);
67069fd337aSStanislav Fomichev
67169fd337aSStanislav Fomichev return shim_link;
67269fd337aSStanislav Fomichev }
67369fd337aSStanislav Fomichev
cgroup_shim_find(struct bpf_trampoline * tr,bpf_func_t bpf_func)67469fd337aSStanislav Fomichev static struct bpf_shim_tramp_link *cgroup_shim_find(struct bpf_trampoline *tr,
67569fd337aSStanislav Fomichev bpf_func_t bpf_func)
67669fd337aSStanislav Fomichev {
67769fd337aSStanislav Fomichev struct bpf_tramp_link *link;
67869fd337aSStanislav Fomichev int kind;
67969fd337aSStanislav Fomichev
68069fd337aSStanislav Fomichev for (kind = 0; kind < BPF_TRAMP_MAX; kind++) {
68169fd337aSStanislav Fomichev hlist_for_each_entry(link, &tr->progs_hlist[kind], tramp_hlist) {
68269fd337aSStanislav Fomichev struct bpf_prog *p = link->link.prog;
68369fd337aSStanislav Fomichev
68469fd337aSStanislav Fomichev if (p->bpf_func == bpf_func)
68569fd337aSStanislav Fomichev return container_of(link, struct bpf_shim_tramp_link, link);
68669fd337aSStanislav Fomichev }
68769fd337aSStanislav Fomichev }
68869fd337aSStanislav Fomichev
68969fd337aSStanislav Fomichev return NULL;
69069fd337aSStanislav Fomichev }
69169fd337aSStanislav Fomichev
bpf_trampoline_link_cgroup_shim(struct bpf_prog * prog,int cgroup_atype)69269fd337aSStanislav Fomichev int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
69369fd337aSStanislav Fomichev int cgroup_atype)
69469fd337aSStanislav Fomichev {
69569fd337aSStanislav Fomichev struct bpf_shim_tramp_link *shim_link = NULL;
69669fd337aSStanislav Fomichev struct bpf_attach_target_info tgt_info = {};
69769fd337aSStanislav Fomichev struct bpf_trampoline *tr;
69869fd337aSStanislav Fomichev bpf_func_t bpf_func;
69969fd337aSStanislav Fomichev u64 key;
70069fd337aSStanislav Fomichev int err;
70169fd337aSStanislav Fomichev
70269fd337aSStanislav Fomichev err = bpf_check_attach_target(NULL, prog, NULL,
70369fd337aSStanislav Fomichev prog->aux->attach_btf_id,
70469fd337aSStanislav Fomichev &tgt_info);
70569fd337aSStanislav Fomichev if (err)
70669fd337aSStanislav Fomichev return err;
70769fd337aSStanislav Fomichev
70869fd337aSStanislav Fomichev key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,
70969fd337aSStanislav Fomichev prog->aux->attach_btf_id);
71069fd337aSStanislav Fomichev
71169fd337aSStanislav Fomichev bpf_lsm_find_cgroup_shim(prog, &bpf_func);
71269fd337aSStanislav Fomichev tr = bpf_trampoline_get(key, &tgt_info);
71369fd337aSStanislav Fomichev if (!tr)
71469fd337aSStanislav Fomichev return -ENOMEM;
71569fd337aSStanislav Fomichev
71669fd337aSStanislav Fomichev mutex_lock(&tr->mutex);
71769fd337aSStanislav Fomichev
71869fd337aSStanislav Fomichev shim_link = cgroup_shim_find(tr, bpf_func);
71969fd337aSStanislav Fomichev if (shim_link) {
72069fd337aSStanislav Fomichev /* Reusing existing shim attached by the other program. */
72169fd337aSStanislav Fomichev bpf_link_inc(&shim_link->link.link);
72269fd337aSStanislav Fomichev
72369fd337aSStanislav Fomichev mutex_unlock(&tr->mutex);
72469fd337aSStanislav Fomichev bpf_trampoline_put(tr); /* bpf_trampoline_get above */
72569fd337aSStanislav Fomichev return 0;
72669fd337aSStanislav Fomichev }
72769fd337aSStanislav Fomichev
72869fd337aSStanislav Fomichev /* Allocate and install new shim. */
72969fd337aSStanislav Fomichev
73069fd337aSStanislav Fomichev shim_link = cgroup_shim_alloc(prog, bpf_func, cgroup_atype);
73169fd337aSStanislav Fomichev if (!shim_link) {
73269fd337aSStanislav Fomichev err = -ENOMEM;
73369fd337aSStanislav Fomichev goto err;
73469fd337aSStanislav Fomichev }
73569fd337aSStanislav Fomichev
73669fd337aSStanislav Fomichev err = __bpf_trampoline_link_prog(&shim_link->link, tr);
73769fd337aSStanislav Fomichev if (err)
73869fd337aSStanislav Fomichev goto err;
73969fd337aSStanislav Fomichev
74069fd337aSStanislav Fomichev shim_link->trampoline = tr;
74169fd337aSStanislav Fomichev /* note, we're still holding tr refcnt from above */
74269fd337aSStanislav Fomichev
74369fd337aSStanislav Fomichev mutex_unlock(&tr->mutex);
74469fd337aSStanislav Fomichev
74569fd337aSStanislav Fomichev return 0;
74669fd337aSStanislav Fomichev err:
74769fd337aSStanislav Fomichev mutex_unlock(&tr->mutex);
74869fd337aSStanislav Fomichev
74969fd337aSStanislav Fomichev if (shim_link)
75069fd337aSStanislav Fomichev bpf_link_put(&shim_link->link.link);
75169fd337aSStanislav Fomichev
75269fd337aSStanislav Fomichev /* have to release tr while _not_ holding its mutex */
75369fd337aSStanislav Fomichev bpf_trampoline_put(tr); /* bpf_trampoline_get above */
75469fd337aSStanislav Fomichev
75569fd337aSStanislav Fomichev return err;
75669fd337aSStanislav Fomichev }
75769fd337aSStanislav Fomichev
bpf_trampoline_unlink_cgroup_shim(struct bpf_prog * prog)75869fd337aSStanislav Fomichev void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)
75969fd337aSStanislav Fomichev {
76069fd337aSStanislav Fomichev struct bpf_shim_tramp_link *shim_link = NULL;
76169fd337aSStanislav Fomichev struct bpf_trampoline *tr;
76269fd337aSStanislav Fomichev bpf_func_t bpf_func;
76369fd337aSStanislav Fomichev u64 key;
76469fd337aSStanislav Fomichev
76569fd337aSStanislav Fomichev key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,
76669fd337aSStanislav Fomichev prog->aux->attach_btf_id);
76769fd337aSStanislav Fomichev
76869fd337aSStanislav Fomichev bpf_lsm_find_cgroup_shim(prog, &bpf_func);
76969fd337aSStanislav Fomichev tr = bpf_trampoline_lookup(key);
77069fd337aSStanislav Fomichev if (WARN_ON_ONCE(!tr))
77169fd337aSStanislav Fomichev return;
77269fd337aSStanislav Fomichev
77369fd337aSStanislav Fomichev mutex_lock(&tr->mutex);
77469fd337aSStanislav Fomichev shim_link = cgroup_shim_find(tr, bpf_func);
77569fd337aSStanislav Fomichev mutex_unlock(&tr->mutex);
77669fd337aSStanislav Fomichev
77769fd337aSStanislav Fomichev if (shim_link)
77869fd337aSStanislav Fomichev bpf_link_put(&shim_link->link.link);
77969fd337aSStanislav Fomichev
78069fd337aSStanislav Fomichev bpf_trampoline_put(tr); /* bpf_trampoline_lookup above */
78169fd337aSStanislav Fomichev }
782f7b12b6fSToke Høiland-Jørgensen #endif
783f7b12b6fSToke Høiland-Jørgensen
bpf_trampoline_get(u64 key,struct bpf_attach_target_info * tgt_info)784f7b12b6fSToke Høiland-Jørgensen struct bpf_trampoline *bpf_trampoline_get(u64 key,
785f7b12b6fSToke Høiland-Jørgensen struct bpf_attach_target_info *tgt_info)
786f7b12b6fSToke Høiland-Jørgensen {
787f7b12b6fSToke Høiland-Jørgensen struct bpf_trampoline *tr;
788f7b12b6fSToke Høiland-Jørgensen
789f7b12b6fSToke Høiland-Jørgensen tr = bpf_trampoline_lookup(key);
790f7b12b6fSToke Høiland-Jørgensen if (!tr)
791f7b12b6fSToke Høiland-Jørgensen return NULL;
792f7b12b6fSToke Høiland-Jørgensen
793f7b12b6fSToke Høiland-Jørgensen mutex_lock(&tr->mutex);
794f7b12b6fSToke Høiland-Jørgensen if (tr->func.addr)
795f7b12b6fSToke Høiland-Jørgensen goto out;
796f7b12b6fSToke Høiland-Jørgensen
797f7b12b6fSToke Høiland-Jørgensen memcpy(&tr->func.model, &tgt_info->fmodel, sizeof(tgt_info->fmodel));
798f7b12b6fSToke Høiland-Jørgensen tr->func.addr = (void *)tgt_info->tgt_addr;
799f7b12b6fSToke Høiland-Jørgensen out:
800f7b12b6fSToke Høiland-Jørgensen mutex_unlock(&tr->mutex);
801f7b12b6fSToke Høiland-Jørgensen return tr;
802fec56f58SAlexei Starovoitov }
803fec56f58SAlexei Starovoitov
bpf_trampoline_put(struct bpf_trampoline * tr)804a2aa95b7SYuntao Wang void bpf_trampoline_put(struct bpf_trampoline *tr)
805a2aa95b7SYuntao Wang {
806fec56f58SAlexei Starovoitov int i;
807fec56f58SAlexei Starovoitov
808fec56f58SAlexei Starovoitov if (!tr)
809fec56f58SAlexei Starovoitov return;
810fec56f58SAlexei Starovoitov mutex_lock(&trampoline_mutex);
811fec56f58SAlexei Starovoitov if (!refcount_dec_and_test(&tr->refcnt))
812a2aa95b7SYuntao Wang goto out;
813a2aa95b7SYuntao Wang WARN_ON_ONCE(mutex_is_locked(&tr->mutex));
814a2aa95b7SYuntao Wang
815fec56f58SAlexei Starovoitov for (i = 0; i < BPF_TRAMP_MAX; i++)
816a2aa95b7SYuntao Wang if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[i])))
817e21aa341SAlexei Starovoitov goto out;
818e21aa341SAlexei Starovoitov
819e21aa341SAlexei Starovoitov /* This code will be executed even when the last bpf_tramp_image
820e21aa341SAlexei Starovoitov * is alive. All progs are detached from the trampoline and the
821e21aa341SAlexei Starovoitov * trampoline image is patched with jmp into epilogue to skip
8221e6c62a8SAlexei Starovoitov * fexit progs. The fentry-only trampoline will be freed via
823fec56f58SAlexei Starovoitov * multiple rcu callbacks.
82462d468e5SJiri Olsa */
82562d468e5SJiri Olsa hlist_del(&tr->hlist);
82600963a2eSSong Liu if (tr->fops) {
82762d468e5SJiri Olsa ftrace_free_filter(tr->fops);
828fec56f58SAlexei Starovoitov kfree(tr->fops);
829fec56f58SAlexei Starovoitov }
830fec56f58SAlexei Starovoitov kfree(tr);
831fec56f58SAlexei Starovoitov out:
832fec56f58SAlexei Starovoitov mutex_unlock(&trampoline_mutex);
833ca06f55bSAlexei Starovoitov }
834856c02dbSSong Liu
835f2dd3b39SAlexei Starovoitov #define NO_START_TIME 1
bpf_prog_start_time(void)836f2dd3b39SAlexei Starovoitov static __always_inline u64 notrace bpf_prog_start_time(void)
837f2dd3b39SAlexei Starovoitov {
838ca06f55bSAlexei Starovoitov u64 start = NO_START_TIME;
839f2dd3b39SAlexei Starovoitov
840ca06f55bSAlexei Starovoitov if (static_branch_unlikely(&bpf_stats_enabled_key)) {
841ca06f55bSAlexei Starovoitov start = sched_clock();
842ca06f55bSAlexei Starovoitov if (unlikely(!start))
843f2dd3b39SAlexei Starovoitov start = NO_START_TIME;
844f2dd3b39SAlexei Starovoitov }
845f2dd3b39SAlexei Starovoitov return start;
846fb7dd8bcSAndrii Nakryiko }
84702ad0596SDavid Miller
84802ad0596SDavid Miller /* The logic is similar to bpf_prog_run(), but with an explicit
849f2dd3b39SAlexei Starovoitov * rcu_read_lock() and migrate_disable() which are required
850fec56f58SAlexei Starovoitov * for the trampoline. The macro is split into
851fec56f58SAlexei Starovoitov * call __bpf_prog_enter
852ca06f55bSAlexei Starovoitov * call prog->bpf_func
853ca06f55bSAlexei Starovoitov * call __bpf_prog_exit
854ca06f55bSAlexei Starovoitov *
855ca06f55bSAlexei Starovoitov * __bpf_prog_enter returns:
8568fb33b60SZhen Lei * 0 - skip execution of the bpf prog
857ca06f55bSAlexei Starovoitov * 1 - execute bpf prog
858fec56f58SAlexei Starovoitov * [2..MAX_U64] - execute bpf prog and record execution time.
859271de525SMartin KaFai Lau * This is start time.
860dcce11d5SJules Irenge */
__bpf_prog_enter_recur(struct bpf_prog * prog,struct bpf_tramp_run_ctx * run_ctx)861fec56f58SAlexei Starovoitov static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)
862fec56f58SAlexei Starovoitov __acquires(RCU)
86302ad0596SDavid Miller {
864e384c7b7SKui-Feng Lee rcu_read_lock();
865e384c7b7SKui-Feng Lee migrate_disable();
866e384c7b7SKui-Feng Lee
867c89e843aSHou Tao run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
86805b24ff9SJiri Olsa
869ca06f55bSAlexei Starovoitov if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
8709ed9e9baSAlexei Starovoitov bpf_prog_inc_misses_counter(prog);
871f2dd3b39SAlexei Starovoitov return 0;
872fec56f58SAlexei Starovoitov }
873fec56f58SAlexei Starovoitov return bpf_prog_start_time();
874f2dd3b39SAlexei Starovoitov }
875f2dd3b39SAlexei Starovoitov
update_prog_stats(struct bpf_prog * prog,u64 start)876fec56f58SAlexei Starovoitov static void notrace update_prog_stats(struct bpf_prog *prog,
877fec56f58SAlexei Starovoitov u64 start)
878fec56f58SAlexei Starovoitov {
879fec56f58SAlexei Starovoitov struct bpf_prog_stats *stats;
880f2dd3b39SAlexei Starovoitov
881f2dd3b39SAlexei Starovoitov if (static_branch_unlikely(&bpf_stats_enabled_key) &&
882fec56f58SAlexei Starovoitov /* static_key could be enabled in __bpf_prog_enter*
883f2dd3b39SAlexei Starovoitov * and disabled in __bpf_prog_exit*.
884fec56f58SAlexei Starovoitov * And vice versa.
885f2dd3b39SAlexei Starovoitov * Hence check that 'start' is valid.
886d979617aSEric Dumazet */
887d979617aSEric Dumazet start > NO_START_TIME) {
888700d4796SAlexei Starovoitov u64 duration = sched_clock() - start;
889d979617aSEric Dumazet unsigned long flags;
89061a0abaeSEric Dumazet
89161a0abaeSEric Dumazet stats = this_cpu_ptr(prog->stats);
892d979617aSEric Dumazet flags = u64_stats_update_begin_irqsave(&stats->syncp);
893fec56f58SAlexei Starovoitov u64_stats_inc(&stats->cnt);
894f2dd3b39SAlexei Starovoitov u64_stats_add(&stats->nsecs, duration);
895f2dd3b39SAlexei Starovoitov u64_stats_update_end_irqrestore(&stats->syncp, flags);
896271de525SMartin KaFai Lau }
897271de525SMartin KaFai Lau }
898f2dd3b39SAlexei Starovoitov
__bpf_prog_exit_recur(struct bpf_prog * prog,u64 start,struct bpf_tramp_run_ctx * run_ctx)899f2dd3b39SAlexei Starovoitov static void notrace __bpf_prog_exit_recur(struct bpf_prog *prog, u64 start,
900e384c7b7SKui-Feng Lee struct bpf_tramp_run_ctx *run_ctx)
901e384c7b7SKui-Feng Lee __releases(RCU)
902f2dd3b39SAlexei Starovoitov {
903c89e843aSHou Tao bpf_reset_run_ctx(run_ctx->saved_run_ctx);
90402ad0596SDavid Miller
905fec56f58SAlexei Starovoitov update_prog_stats(prog, start);
906fec56f58SAlexei Starovoitov this_cpu_dec(*(prog->active));
907fec56f58SAlexei Starovoitov migrate_enable();
908271de525SMartin KaFai Lau rcu_read_unlock();
90969fd337aSStanislav Fomichev }
91069fd337aSStanislav Fomichev
__bpf_prog_enter_lsm_cgroup(struct bpf_prog * prog,struct bpf_tramp_run_ctx * run_ctx)91169fd337aSStanislav Fomichev static u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,
91269fd337aSStanislav Fomichev struct bpf_tramp_run_ctx *run_ctx)
91369fd337aSStanislav Fomichev __acquires(RCU)
91469fd337aSStanislav Fomichev {
91569fd337aSStanislav Fomichev /* Runtime stats are exported via actual BPF_LSM_CGROUP
91669fd337aSStanislav Fomichev * programs, not the shims.
91769fd337aSStanislav Fomichev */
91869fd337aSStanislav Fomichev rcu_read_lock();
91969fd337aSStanislav Fomichev migrate_disable();
92069fd337aSStanislav Fomichev
92169fd337aSStanislav Fomichev run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
92269fd337aSStanislav Fomichev
923271de525SMartin KaFai Lau return NO_START_TIME;
92469fd337aSStanislav Fomichev }
92569fd337aSStanislav Fomichev
__bpf_prog_exit_lsm_cgroup(struct bpf_prog * prog,u64 start,struct bpf_tramp_run_ctx * run_ctx)92669fd337aSStanislav Fomichev static void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,
92769fd337aSStanislav Fomichev struct bpf_tramp_run_ctx *run_ctx)
92869fd337aSStanislav Fomichev __releases(RCU)
92969fd337aSStanislav Fomichev {
93069fd337aSStanislav Fomichev bpf_reset_run_ctx(run_ctx->saved_run_ctx);
93169fd337aSStanislav Fomichev
93269fd337aSStanislav Fomichev migrate_enable();
933271de525SMartin KaFai Lau rcu_read_unlock();
934271de525SMartin KaFai Lau }
9351e6c62a8SAlexei Starovoitov
__bpf_prog_enter_sleepable_recur(struct bpf_prog * prog,struct bpf_tramp_run_ctx * run_ctx)9361e6c62a8SAlexei Starovoitov u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
937031d6e02SAlexei Starovoitov struct bpf_tramp_run_ctx *run_ctx)
938f56407faSAlexei Starovoitov {
939e384c7b7SKui-Feng Lee rcu_read_lock_trace();
9406764e767SSebastian Andrzej Siewior migrate_disable();
9416764e767SSebastian Andrzej Siewior might_fault();
942c89e843aSHou Tao
94305b24ff9SJiri Olsa run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
944ca06f55bSAlexei Starovoitov
9459ed9e9baSAlexei Starovoitov if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
946f2dd3b39SAlexei Starovoitov bpf_prog_inc_misses_counter(prog);
9471e6c62a8SAlexei Starovoitov return 0;
9481e6c62a8SAlexei Starovoitov }
949271de525SMartin KaFai Lau return bpf_prog_start_time();
950e384c7b7SKui-Feng Lee }
9511e6c62a8SAlexei Starovoitov
__bpf_prog_exit_sleepable_recur(struct bpf_prog * prog,u64 start,struct bpf_tramp_run_ctx * run_ctx)952e384c7b7SKui-Feng Lee void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,
953e384c7b7SKui-Feng Lee struct bpf_tramp_run_ctx *run_ctx)
954f2dd3b39SAlexei Starovoitov {
955c89e843aSHou Tao bpf_reset_run_ctx(run_ctx->saved_run_ctx);
956031d6e02SAlexei Starovoitov
9571e6c62a8SAlexei Starovoitov update_prog_stats(prog, start);
9581e6c62a8SAlexei Starovoitov this_cpu_dec(*(prog->active));
9591e6c62a8SAlexei Starovoitov migrate_enable();
960271de525SMartin KaFai Lau rcu_read_unlock_trace();
961271de525SMartin KaFai Lau }
962271de525SMartin KaFai Lau
__bpf_prog_enter_sleepable(struct bpf_prog * prog,struct bpf_tramp_run_ctx * run_ctx)963271de525SMartin KaFai Lau static u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog,
964271de525SMartin KaFai Lau struct bpf_tramp_run_ctx *run_ctx)
965271de525SMartin KaFai Lau {
966271de525SMartin KaFai Lau rcu_read_lock_trace();
967271de525SMartin KaFai Lau migrate_disable();
968271de525SMartin KaFai Lau might_fault();
969271de525SMartin KaFai Lau
970271de525SMartin KaFai Lau run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
971271de525SMartin KaFai Lau
972271de525SMartin KaFai Lau return bpf_prog_start_time();
973271de525SMartin KaFai Lau }
974271de525SMartin KaFai Lau
__bpf_prog_exit_sleepable(struct bpf_prog * prog,u64 start,struct bpf_tramp_run_ctx * run_ctx)975271de525SMartin KaFai Lau static void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,
976271de525SMartin KaFai Lau struct bpf_tramp_run_ctx *run_ctx)
977271de525SMartin KaFai Lau {
978271de525SMartin KaFai Lau bpf_reset_run_ctx(run_ctx->saved_run_ctx);
979271de525SMartin KaFai Lau
980271de525SMartin KaFai Lau update_prog_stats(prog, start);
981271de525SMartin KaFai Lau migrate_enable();
982271de525SMartin KaFai Lau rcu_read_unlock_trace();
98364696c40SMartin KaFai Lau }
98464696c40SMartin KaFai Lau
__bpf_prog_enter(struct bpf_prog * prog,struct bpf_tramp_run_ctx * run_ctx)98564696c40SMartin KaFai Lau static u64 notrace __bpf_prog_enter(struct bpf_prog *prog,
98664696c40SMartin KaFai Lau struct bpf_tramp_run_ctx *run_ctx)
98764696c40SMartin KaFai Lau __acquires(RCU)
98864696c40SMartin KaFai Lau {
98964696c40SMartin KaFai Lau rcu_read_lock();
99064696c40SMartin KaFai Lau migrate_disable();
99164696c40SMartin KaFai Lau
99264696c40SMartin KaFai Lau run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
99364696c40SMartin KaFai Lau
994271de525SMartin KaFai Lau return bpf_prog_start_time();
99564696c40SMartin KaFai Lau }
99664696c40SMartin KaFai Lau
__bpf_prog_exit(struct bpf_prog * prog,u64 start,struct bpf_tramp_run_ctx * run_ctx)99764696c40SMartin KaFai Lau static void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start,
99864696c40SMartin KaFai Lau struct bpf_tramp_run_ctx *run_ctx)
99964696c40SMartin KaFai Lau __releases(RCU)
100064696c40SMartin KaFai Lau {
100164696c40SMartin KaFai Lau bpf_reset_run_ctx(run_ctx->saved_run_ctx);
100264696c40SMartin KaFai Lau
100364696c40SMartin KaFai Lau update_prog_stats(prog, start);
100464696c40SMartin KaFai Lau migrate_enable();
1005e21aa341SAlexei Starovoitov rcu_read_unlock();
1006e21aa341SAlexei Starovoitov }
1007e21aa341SAlexei Starovoitov
__bpf_tramp_enter(struct bpf_tramp_image * tr)1008e21aa341SAlexei Starovoitov void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr)
1009e21aa341SAlexei Starovoitov {
1010e21aa341SAlexei Starovoitov percpu_ref_get(&tr->pcref);
1011e21aa341SAlexei Starovoitov }
1012e21aa341SAlexei Starovoitov
__bpf_tramp_exit(struct bpf_tramp_image * tr)1013e21aa341SAlexei Starovoitov void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr)
1014e21aa341SAlexei Starovoitov {
1015271de525SMartin KaFai Lau percpu_ref_put(&tr->pcref);
1016271de525SMartin KaFai Lau }
101766c84731SAndrii Nakryiko
bpf_trampoline_enter(const struct bpf_prog * prog)1018271de525SMartin KaFai Lau bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog)
1019271de525SMartin KaFai Lau {
1020271de525SMartin KaFai Lau bool sleepable = prog->sleepable;
1021271de525SMartin KaFai Lau
1022271de525SMartin KaFai Lau if (bpf_prog_check_recur(prog))
1023271de525SMartin KaFai Lau return sleepable ? __bpf_prog_enter_sleepable_recur :
1024271de525SMartin KaFai Lau __bpf_prog_enter_recur;
1025271de525SMartin KaFai Lau
1026271de525SMartin KaFai Lau if (resolve_prog_type(prog) == BPF_PROG_TYPE_LSM &&
1027271de525SMartin KaFai Lau prog->expected_attach_type == BPF_LSM_CGROUP)
1028271de525SMartin KaFai Lau return __bpf_prog_enter_lsm_cgroup;
1029271de525SMartin KaFai Lau
1030271de525SMartin KaFai Lau return sleepable ? __bpf_prog_enter_sleepable : __bpf_prog_enter;
1031271de525SMartin KaFai Lau }
103266c84731SAndrii Nakryiko
bpf_trampoline_exit(const struct bpf_prog * prog)1033271de525SMartin KaFai Lau bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog)
1034271de525SMartin KaFai Lau {
1035271de525SMartin KaFai Lau bool sleepable = prog->sleepable;
1036271de525SMartin KaFai Lau
1037271de525SMartin KaFai Lau if (bpf_prog_check_recur(prog))
1038271de525SMartin KaFai Lau return sleepable ? __bpf_prog_exit_sleepable_recur :
1039271de525SMartin KaFai Lau __bpf_prog_exit_recur;
1040271de525SMartin KaFai Lau
1041271de525SMartin KaFai Lau if (resolve_prog_type(prog) == BPF_PROG_TYPE_LSM &&
1042271de525SMartin KaFai Lau prog->expected_attach_type == BPF_LSM_CGROUP)
1043271de525SMartin KaFai Lau return __bpf_prog_exit_lsm_cgroup;
1044271de525SMartin KaFai Lau
1045fec56f58SAlexei Starovoitov return sleepable ? __bpf_prog_exit_sleepable : __bpf_prog_exit;
10467a3d9a15SSong Liu }
104785d33df3SMartin KaFai Lau
1048f7e0beafSKui-Feng Lee int __weak
arch_prepare_bpf_trampoline(struct bpf_tramp_image * im,void * image,void * image_end,const struct btf_func_model * m,u32 flags,struct bpf_tramp_links * tlinks,void * func_addr)10497a3d9a15SSong Liu arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
1050fec56f58SAlexei Starovoitov const struct btf_func_model *m, u32 flags,
1051fec56f58SAlexei Starovoitov struct bpf_tramp_links *tlinks,
1052fec56f58SAlexei Starovoitov void *func_addr)
1053fec56f58SAlexei Starovoitov {
105482583daaSSong Liu return -ENOTSUPP;
105582583daaSSong Liu }
105682583daaSSong Liu
arch_alloc_bpf_trampoline(unsigned int size)105782583daaSSong Liu void * __weak arch_alloc_bpf_trampoline(unsigned int size)
105882583daaSSong Liu {
105982583daaSSong Liu void *image;
106082583daaSSong Liu
106182583daaSSong Liu if (WARN_ON_ONCE(size > PAGE_SIZE))
106282583daaSSong Liu return NULL;
106382583daaSSong Liu image = bpf_jit_alloc_exec(PAGE_SIZE);
106482583daaSSong Liu if (image)
106582583daaSSong Liu set_vm_flush_reset_perms(image);
106682583daaSSong Liu return image;
106782583daaSSong Liu }
106882583daaSSong Liu
arch_free_bpf_trampoline(void * image,unsigned int size)106982583daaSSong Liu void __weak arch_free_bpf_trampoline(void *image, unsigned int size)
107082583daaSSong Liu {
107182583daaSSong Liu WARN_ON_ONCE(size > PAGE_SIZE);
107282583daaSSong Liu /* bpf_jit_free_exec doesn't need "size", but
107382583daaSSong Liu * bpf_prog_pack_free() needs it.
107482583daaSSong Liu */
107582583daaSSong Liu bpf_jit_free_exec(image);
107682583daaSSong Liu }
107782583daaSSong Liu
arch_protect_bpf_trampoline(void * image,unsigned int size)107882583daaSSong Liu int __weak arch_protect_bpf_trampoline(void *image, unsigned int size)
107982583daaSSong Liu {
108082583daaSSong Liu WARN_ON_ONCE(size > PAGE_SIZE);
108182583daaSSong Liu return set_memory_rox((long)image, 1);
108282583daaSSong Liu }
108382583daaSSong Liu
arch_bpf_trampoline_size(const struct btf_func_model * m,u32 flags,struct bpf_tramp_links * tlinks,void * func_addr)108482583daaSSong Liu int __weak arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
108582583daaSSong Liu struct bpf_tramp_links *tlinks, void *func_addr)
108682583daaSSong Liu {
108782583daaSSong Liu return -ENOTSUPP;
108896d1b7c0SSong Liu }
108996d1b7c0SSong Liu
init_trampolines(void)109096d1b7c0SSong Liu static int __init init_trampolines(void)
109196d1b7c0SSong Liu {
109296d1b7c0SSong Liu int i;
109396d1b7c0SSong Liu
1094fec56f58SAlexei Starovoitov for (i = 0; i < TRAMPOLINE_TABLE_SIZE; i++)
1095fec56f58SAlexei Starovoitov INIT_HLIST_HEAD(&trampoline_table[i]);
1096fec56f58SAlexei Starovoitov return 0;
1097fec56f58SAlexei Starovoitov }
1098fec56f58SAlexei Starovoitov late_initcall(init_trampolines);
1099fec56f58SAlexei Starovoitov