| /linux/include/net/ |
| H A D | xdp.h | 120 xdp->rxq = rxq; in xdp_init_buff()
121 xdp->flags = 0; in xdp_init_buff()
131 xdp->data = data; in xdp_prepare_buff()
143 ((xdp)->data_hard_start + (xdp)->frame_sz - \
154 unsigned int len = xdp->data_end - xdp->data; in xdp_get_buff_len()
269 headroom = xdp->data - xdp->data_hard_start; in xdp_update_frame_from_buff()
270 metasize = xdp->data - xdp->data_meta; in xdp_update_frame_from_buff()
276 if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) { in xdp_update_frame_from_buff()
282 xdp_frame->len = xdp->data_end - xdp->data; in xdp_update_frame_from_buff()
361 xdp->data_meta = xdp->data + 1; in xdp_set_data_meta_invalid()
[all …]
|
| H A D | xdp_sock_drv.h | 87 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_xdp_get_dma()
94 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_xdp_get_frame_dma()
122 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_free()
141 struct xdp_buff_xsk *frag = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_add_frag()
156 ret = &frag->xdp; in xsk_buff_get_frag()
176 return &frag->xdp; in xsk_buff_get_tail()
181 xdp->data = xdp->data_hard_start + XDP_PACKET_HEADROOM; in xsk_buff_set_size()
182 xdp->data_meta = xdp->data; in xsk_buff_set_size()
183 xdp->data_end = xdp->data + size; in xsk_buff_set_size()
184 xdp->flags = 0; in xsk_buff_set_size()
[all …]
|
| /linux/Documentation/bpf/ |
| H A D | redirect.rst | 25 :doc: xdp redirect
49 @cnt[tracepoint:xdp:mem_connect]: 18
50 @cnt[tracepoint:xdp:mem_disconnect]: 18
51 @cnt[tracepoint:xdp:xdp_exception]: 19605
53 @cnt[tracepoint:xdp:xdp_redirect]: 22292200
56 The various xdp tracepoints can be found in ``source/include/trace/events/xdp.h``
73 perf record -a -e xdp:xdp_redirect_err \
74 -e xdp:xdp_redirect_map_err \
75 -e xdp:xdp_exception \
76 -e xdp:xdp_devmap_xmit
[all …]
|
| /linux/tools/testing/selftests/bpf/progs/ |
| H A D | xdp_features.c | 68 void *data = (void *)(long)xdp->data; in xdp_process_echo_packet()
139 if (xdp_process_echo_packet(xdp, tx)) in xdp_update_stats()
158 xdp_update_stats(xdp, true, false); in xdp_tester_check_tx()
166 xdp_update_stats(xdp, false, false); in xdp_tester_check_rx()
174 int xdp_do_pass(struct xdp_md *xdp) in xdp_do_pass() argument
176 xdp_update_stats(xdp, true, true); in xdp_do_pass()
182 int xdp_do_drop(struct xdp_md *xdp) in xdp_do_drop() argument
191 int xdp_do_aborted(struct xdp_md *xdp) in xdp_do_aborted() argument
200 int xdp_do_tx(struct xdp_md *xdp) in xdp_do_tx() argument
202 void *data = (void *)(long)xdp->data; in xdp_do_tx()
[all …]
|
| H A D | xsk_xdp_progs.c | 19 SEC("xdp.frags") int xsk_def_prog(struct xdp_md *xdp) in xsk_def_prog() argument
24 SEC("xdp.frags") int xsk_xdp_drop(struct xdp_md *xdp) in xsk_xdp_drop() argument
33 SEC("xdp.frags") int xsk_xdp_populate_metadata(struct xdp_md *xdp) in xsk_xdp_populate_metadata() argument
40 err = bpf_xdp_adjust_meta(xdp, -(int)sizeof(struct xdp_info)); in xsk_xdp_populate_metadata()
44 data = (void *)(long)xdp->data; in xsk_xdp_populate_metadata()
45 data_meta = (void *)(long)xdp->data_meta; in xsk_xdp_populate_metadata()
56 SEC("xdp") int xsk_xdp_shared_umem(struct xdp_md *xdp) in xsk_xdp_shared_umem() argument
58 void *data = (void *)(long)xdp->data; in xsk_xdp_shared_umem()
59 void *data_end = (void *)(long)xdp->data_end; in xsk_xdp_shared_umem()
|
| H A D | test_xdp_loop.c | 77 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
78 void *data = (void *)(long)xdp->data; in handle_ipv4()
111 data = (void *)(long)xdp->data; in handle_ipv4()
112 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
151 void *data = (void *)(long)xdp->data; in handle_ipv6()
181 data = (void *)(long)xdp->data; in handle_ipv6()
182 data_end = (void *)(long)xdp->data_end; in handle_ipv6()
209 int _xdp_tx_iptunnel(struct xdp_md *xdp) in _xdp_tx_iptunnel() argument
212 void *data = (void *)(long)xdp->data; in _xdp_tx_iptunnel()
222 return handle_ipv4(xdp); in _xdp_tx_iptunnel()
[all …]
|
| H A D | test_xdp.c | 81 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
82 void *data = (void *)(long)xdp->data; in handle_ipv4()
115 data = (void *)(long)xdp->data; in handle_ipv4()
116 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
155 void *data = (void *)(long)xdp->data; in handle_ipv6()
185 data = (void *)(long)xdp->data; in handle_ipv6()
186 data_end = (void *)(long)xdp->data_end; in handle_ipv6()
213 int _xdp_tx_iptunnel(struct xdp_md *xdp) in _xdp_tx_iptunnel() argument
216 void *data = (void *)(long)xdp->data; in _xdp_tx_iptunnel()
226 return handle_ipv4(xdp); in _xdp_tx_iptunnel()
[all …]
|
| H A D | test_xdp_do_redirect.c | 29 int xdp_redirect(struct xdp_md *xdp) in xdp_redirect() argument
31 __u32 *metadata = (void *)(long)xdp->data_meta; in xdp_redirect()
32 void *data_end = (void *)(long)xdp->data_end; in xdp_redirect()
33 void *data = (void *)(long)xdp->data; in xdp_redirect()
41 if (xdp->ingress_ifindex != (__u32)ifindex_in) in xdp_redirect()
55 if (bpf_xdp_adjust_meta(xdp, sizeof(__u64))) in xdp_redirect()
86 int xdp_count_pkts(struct xdp_md *xdp) in xdp_count_pkts() argument
88 void *data = (void *)(long)xdp->data; in xdp_count_pkts()
89 void *data_end = (void *)(long)xdp->data_end; in xdp_count_pkts()
|
| H A D | test_xdp_bpf2bpf.c | 45 int BPF_PROG(trace_on_entry, struct xdp_buff *xdp) in BPF_PROG() argument
49 meta.ifindex = xdp->rxq->dev->ifindex; in BPF_PROG()
50 meta.pkt_len = bpf_xdp_get_buff_len((struct xdp_md *)xdp); in BPF_PROG()
51 bpf_xdp_output(xdp, &perf_buf_map, in BPF_PROG()
56 test_result_fentry = xdp->rxq->dev->ifindex; in BPF_PROG()
62 int BPF_PROG(trace_on_exit, struct xdp_buff *xdp, int ret) in BPF_PROG() argument
|
| H A D | test_xdp_adjust_tail_shrink.c | 13 int _xdp_adjust_tail_shrink(struct xdp_md *xdp) in _xdp_adjust_tail_shrink() argument
15 __u8 *data_end = (void *)(long)xdp->data_end; in _xdp_adjust_tail_shrink()
16 __u8 *data = (void *)(long)xdp->data; in _xdp_adjust_tail_shrink()
19 switch (bpf_xdp_get_buff_len(xdp)) { in _xdp_adjust_tail_shrink()
47 if (bpf_xdp_adjust_tail(xdp, 0 - offset)) in _xdp_adjust_tail_shrink()
|
| H A D | test_parse_tcp_hdr_opt.c | 36 static int parse_hdr_opt(const struct xdp_md *xdp, struct hdr_opt_state *state) in parse_hdr_opt() argument
38 const void *data = (void *)(long)xdp->data; in parse_hdr_opt()
39 const void *data_end = (void *)(long)xdp->data_end; in parse_hdr_opt()
82 int xdp_ingress_v6(struct xdp_md *xdp) in xdp_ingress_v6() argument
84 const void *data = (void *)(long)xdp->data; in xdp_ingress_v6()
85 const void *data_end = (void *)(long)xdp->data_end; in xdp_ingress_v6()
106 err = parse_hdr_opt(xdp, &opt_state); in xdp_ingress_v6()
|
| /linux/drivers/net/ethernet/broadcom/bnxt/ |
| H A D | bnxt_xdp.c | 28 struct xdp_buff *xdp) in bnxt_xmit_bd() argument
38 if (xdp && xdp_buff_has_frags(xdp)) { in bnxt_xmit_bd()
47 if (xdp) in bnxt_xmit_bd()
184 struct xdp_buff *xdp) in bnxt_xdp_buff_init() argument
209 if (!xdp || !xdp_buff_has_frags(xdp)) in bnxt_xdp_buff_frags_free()
247 orig_data = xdp->data; in bnxt_rx_xdp()
258 *len = xdp->data_end - xdp->data; in bnxt_rx_xdp()
260 offset = xdp->data - xdp->data_hard_start; in bnxt_rx_xdp()
363 mapping = dma_map_single(&pdev->dev, xdp->data, xdp->len, in bnxt_xdp_xmit()
369 __bnxt_xmit_xdp_redirect(bp, txr, mapping, xdp->len, xdp); in bnxt_xdp_xmit()
[all …]
|
| H A D | bnxt_xdp.h | 18 struct xdp_buff *xdp);
21 struct xdp_buff *xdp, struct page *page, u8 **data_ptr,
23 int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp);
31 struct xdp_buff *xdp);
33 struct xdp_buff *xdp);
36 struct xdp_buff *xdp,
|
| /linux/drivers/net/ethernet/intel/ixgbe/ |
| H A D | ixgbe_xsk.c | 169 if (!bi->xdp) { in ixgbe_alloc_rx_buffers_zc()
216 unsigned int totalsize = xdp->data_end - xdp->data_meta; in ixgbe_construct_skb_zc()
217 unsigned int metasize = xdp->data - xdp->data_meta; in ixgbe_construct_skb_zc()
289 bi->xdp = NULL; in ixgbe_clean_rx_irq_zc()
299 bi->xdp = NULL; in ixgbe_clean_rx_irq_zc()
305 bi->xdp->data_end = bi->xdp->data + size; in ixgbe_clean_rx_irq_zc()
320 bi->xdp = NULL; in ixgbe_clean_rx_irq_zc()
336 xsk_buff_free(bi->xdp); in ixgbe_clean_rx_irq_zc()
337 bi->xdp = NULL; in ixgbe_clean_rx_irq_zc()
383 if (!bi->xdp) in ixgbe_xsk_clean_rx_ring()
[all …]
|
| /linux/samples/bpf/ |
| H A D | xdp_adjust_tail_kern.c | 74 if (bpf_xdp_adjust_head(xdp, 0 - headroom)) in send_icmp4_too_big()
76 void *data = (void *)(long)xdp->data; in send_icmp4_too_big()
77 void *data_end = (void *)(long)xdp->data_end; in send_icmp4_too_big()
122 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
123 void *data = (void *)(long)xdp->data; in handle_ipv4()
129 if (bpf_xdp_adjust_tail(xdp, 0 - offset)) in handle_ipv4()
131 return send_icmp4_too_big(xdp); in handle_ipv4()
137 int _xdp_icmp(struct xdp_md *xdp) in _xdp_icmp() argument
139 void *data_end = (void *)(long)xdp->data_end; in _xdp_icmp()
140 void *data = (void *)(long)xdp->data; in _xdp_icmp()
[all …]
|
| H A D | xdp_tx_iptunnel_kern.c | 79 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
80 void *data = (void *)(long)xdp->data; in handle_ipv4()
115 data = (void *)(long)xdp->data; in handle_ipv4()
116 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
155 void *data = (void *)(long)xdp->data; in handle_ipv6()
187 data = (void *)(long)xdp->data; in handle_ipv6()
188 data_end = (void *)(long)xdp->data_end; in handle_ipv6()
216 int _xdp_tx_iptunnel(struct xdp_md *xdp) in _xdp_tx_iptunnel() argument
219 void *data = (void *)(long)xdp->data; in _xdp_tx_iptunnel()
229 return handle_ipv4(xdp); in _xdp_tx_iptunnel()
[all …]
|
| /linux/drivers/net/ethernet/microchip/lan966x/ |
| H A D | lan966x_xdp.c | 9 static int lan966x_xdp_setup(struct net_device *dev, struct netdev_bpf *xdp) in lan966x_xdp_setup() argument
18 NL_SET_ERR_MSG_MOD(xdp->extack, in lan966x_xdp_setup()
24 old_prog = xchg(&port->xdp_prog, xdp->prog); in lan966x_xdp_setup()
43 int lan966x_xdp(struct net_device *dev, struct netdev_bpf *xdp) in lan966x_xdp() argument
45 switch (xdp->command) { in lan966x_xdp()
47 return lan966x_xdp_setup(dev, xdp); in lan966x_xdp()
79 struct xdp_buff xdp; in lan966x_xdp_run() local
82 xdp_init_buff(&xdp, PAGE_SIZE << lan966x->rx.page_order, in lan966x_xdp_run()
84 xdp_prepare_buff(&xdp, page_address(page), in lan966x_xdp_run()
87 act = bpf_prog_run_xdp(xdp_prog, &xdp); in lan966x_xdp_run()
[all …]
|
| /linux/drivers/net/ethernet/intel/ice/ |
| H A D | ice_xsk.c | 456 xdp++; in ice_fill_rx_descs()
481 struct xdp_buff **xdp; in __ice_alloc_rx_bufs_zc() local
551 unsigned int totalsize = xdp->data_end - xdp->data_meta; in ice_construct_skb_zc()
552 unsigned int metasize = xdp->data - xdp->data_meta; in ice_construct_skb_zc()
598 xsk_buff_free(xdp); in ice_construct_skb_zc()
682 u32 size = xdp->data_end - xdp->data; in ice_xmit_xdp_tx_zc()
707 head = xdp; in ice_xmit_xdp_tx_zc()
715 tx_buf->xdp = xdp; in ice_xmit_xdp_tx_zc()
868 struct xdp_buff *xdp; in ice_clean_rx_irq_zc() local
897 first = xdp; in ice_clean_rx_irq_zc()
[all …]
|
| /linux/drivers/net/vmxnet3/ |
| H A D | vmxnet3_xdp.c | 260 act = bpf_prog_run_xdp(prog, xdp); in vmxnet3_run_xdp()
261 page = virt_to_page(xdp->data_hard_start); in vmxnet3_run_xdp()
276 xdpf = xdp_convert_buff_to_frame(xdp); in vmxnet3_run_xdp()
304 const struct xdp_buff *xdp) in vmxnet3_build_skb() argument
316 skb_reserve(skb, xdp->data - xdp->data_hard_start); in vmxnet3_build_skb()
317 skb_put(skb, xdp->data_end - xdp->data); in vmxnet3_build_skb()
331 struct xdp_buff xdp; in vmxnet3_process_xdp_small() local
344 xdp_buff_clear_frags_flag(&xdp); in vmxnet3_process_xdp_small()
347 memcpy(xdp.data, data, len); in vmxnet3_process_xdp_small()
377 struct xdp_buff xdp; in vmxnet3_process_xdp() local
[all …]
|
| /linux/drivers/net/ethernet/netronome/nfp/ |
| H A D | nfp_net_xsk.c | 18 struct xdp_buff *xdp) in nfp_net_xsk_rx_bufs_stash() argument
27 rx_ring->xsk_rxbufs[idx].xdp = xdp; in nfp_net_xsk_rx_bufs_stash()
29 xsk_buff_xdp_get_frame_dma(xdp) + headroom; in nfp_net_xsk_rx_bufs_stash()
35 rxbuf->xdp = NULL; in nfp_net_xsk_rx_unstash()
40 if (rxbuf->xdp) in nfp_net_xsk_rx_free()
41 xsk_buff_free(rxbuf->xdp); in nfp_net_xsk_rx_free()
62 struct xdp_buff *xdp; in nfp_net_xsk_rx_ring_fill_freelist() local
67 xdp = xsk_buff_alloc(pool); in nfp_net_xsk_rx_ring_fill_freelist()
68 if (!xdp) in nfp_net_xsk_rx_ring_fill_freelist()
71 nfp_net_xsk_rx_bufs_stash(rx_ring, wr_idx, xdp); in nfp_net_xsk_rx_ring_fill_freelist()
|
| /linux/drivers/net/ethernet/netronome/nfp/nfd3/ |
| H A D | xsk.c | 35 txbuf->xdp = xrxbuf->xdp; in nfp_nfd3_xsk_tx_xdp()
105 xrxbuf->xdp->data - xrxbuf->xdp->data_meta); in nfp_nfd3_xsk_rx_skb()
184 xrxbuf->xdp->data += meta_len; in nfp_nfd3_xsk_rx()
185 xrxbuf->xdp->data_end = xrxbuf->xdp->data + pkt_len; in nfp_nfd3_xsk_rx()
188 net_prefetch(xrxbuf->xdp->data); in nfp_nfd3_xsk_rx()
192 xrxbuf->xdp->data - in nfp_nfd3_xsk_rx()
194 xrxbuf->xdp->data, in nfp_nfd3_xsk_rx()
221 pkt_len = xrxbuf->xdp->data_end - xrxbuf->xdp->data; in nfp_nfd3_xsk_rx()
222 pkt_off = xrxbuf->xdp->data - xrxbuf->xdp->data_hard_start; in nfp_nfd3_xsk_rx()
269 xsk_buff_free(txbuf->xdp); in nfp_nfd3_xsk_tx_free()
[all …]
|
| /linux/include/trace/events/ |
| H A D | xdp.h | 3 #define TRACE_SYSTEM xdp
32 const struct bpf_prog *xdp, u32 act),
34 TP_ARGS(dev, xdp, act),
43 __entry->prog_id = xdp->aux->id;
93 const struct bpf_prog *xdp,
124 __entry->prog_id = xdp->aux->id;
143 const struct bpf_prog *xdp,
152 const struct bpf_prog *xdp,
159 #define _trace_xdp_redirect(dev, xdp, to) \ argument
174 const struct bpf_prog *xdp,
[all …]
|
| /linux/drivers/net/hyperv/ |
| H A D | netvsc_bpf.c | 25 struct xdp_buff *xdp) in netvsc_run_xdp() argument
35 xdp->data_hard_start = NULL; in netvsc_run_xdp()
59 memcpy(xdp->data, data, len); in netvsc_run_xdp()
61 act = bpf_prog_run_xdp(prog, xdp); in netvsc_run_xdp()
73 if (!xdp_do_redirect(ndev, xdp, prog)) { in netvsc_run_xdp()
107 xdp->data_hard_start = NULL; in netvsc_run_xdp()
167 struct netdev_bpf xdp; in netvsc_vf_setxdp() local
178 memset(&xdp, 0, sizeof(xdp)); in netvsc_vf_setxdp()
183 xdp.command = XDP_SETUP_PROG; in netvsc_vf_setxdp()
184 xdp.prog = prog; in netvsc_vf_setxdp()
[all …]
|
| /linux/net/bpf/ |
| H A D | test_run.c | 171 .init_arg = xdp, in xdp_test_run_setup()
175 if (!xdp->frames) in xdp_test_run_setup()
179 if (!xdp->skbs) in xdp_test_run_setup()
193 xdp->pp = pp; in xdp_test_run_setup()
220 kfree(xdp->skbs); in xdp_test_run_teardown()
360 ret = xdp_recv_frames(frames, nframes, xdp->skbs, xdp->dev); in xdp_test_run_batch()
430 if (xdp) in bpf_test_run()
1180 xdp->data = xdp->data_meta + xdp_md->data; in xdp_convert_md_to_buff()
1193 xdp_md->data = xdp->data - xdp->data_meta; in xdp_convert_buff_to_md()
1194 xdp_md->data_end = xdp->data_end - xdp->data_meta; in xdp_convert_buff_to_md()
[all …]
|
| /linux/tools/testing/selftests/bpf/ |
| H A D | test_tcp_check_syncookie_user.c | 21 static int get_map_fd_by_prog_id(int prog_id, bool *xdp) in get_map_fd_by_prog_id() argument
48 *xdp = info.type == BPF_PROG_TYPE_XDP; in get_map_fd_by_prog_id()
59 static int run_test(int server_fd, int results_fd, bool xdp) in run_test() argument
110 if (xdp && value_gen == 0) { in run_test()
164 bool xdp; in main() local
174 results = get_map_fd_by_prog_id(atoi(argv[1]), &xdp); in main()
194 if (run_test(server, results, xdp)) in main()
197 if (run_test(server_v6, results, xdp)) in main()
200 if (run_test(server_dual, results, xdp)) in main()
|