1dac09149SBjörn Töpel /* SPDX-License-Identifier: GPL-2.0 */
2dac09149SBjörn Töpel /* AF_XDP internal functions
3c0c77d8fSBjörn Töpel * Copyright(c) 2018 Intel Corporation.
4c0c77d8fSBjörn Töpel */
5c0c77d8fSBjörn Töpel
6c0c77d8fSBjörn Töpel #ifndef _LINUX_XDP_SOCK_H
7c0c77d8fSBjörn Töpel #define _LINUX_XDP_SOCK_H
8c0c77d8fSBjörn Töpel
9b6459415SJakub Kicinski #include <linux/bpf.h>
10e61e62b9SBjörn Töpel #include <linux/workqueue.h>
11e61e62b9SBjörn Töpel #include <linux/if_xdp.h>
12c0c77d8fSBjörn Töpel #include <linux/mutex.h>
13ac98d8aaSMagnus Karlsson #include <linux/spinlock.h>
14e61e62b9SBjörn Töpel #include <linux/mm.h>
15c0c77d8fSBjörn Töpel #include <net/sock.h>
16c0c77d8fSBjörn Töpel
17d609f3d2STirthendu Sarkar #define XDP_UMEM_SG_FLAG (1 << 1)
18d609f3d2STirthendu Sarkar
19b9b6b68eSBjörn Töpel struct net_device;
20b9b6b68eSBjörn Töpel struct xsk_queue;
21a71506a4SMagnus Karlsson struct xdp_buff;
22e61e62b9SBjörn Töpel
23e61e62b9SBjörn Töpel struct xdp_umem {
247f7ffa4eSMagnus Karlsson void *addrs;
2593ee30f3SMagnus Karlsson u64 size;
26e61e62b9SBjörn Töpel u32 headroom;
272b43470aSBjörn Töpel u32 chunk_size;
281c1efc2aSMagnus Karlsson u32 chunks;
298ef4e27eSMagnus Karlsson u32 npgs;
30e61e62b9SBjörn Töpel struct user_struct *user;
31e61e62b9SBjörn Töpel refcount_t users;
3277cd0d7bSMagnus Karlsson u8 flags;
33341ac980SStanislav Fomichev u8 tx_metadata_len;
34173d3adbSBjörn Töpel bool zc;
358ef4e27eSMagnus Karlsson struct page **pgs;
368ef4e27eSMagnus Karlsson int id;
37921b6869SMagnus Karlsson struct list_head xsk_dma_list;
38537cf4e3SMagnus Karlsson struct work_struct work;
39e61e62b9SBjörn Töpel };
40c0c77d8fSBjörn Töpel
41d817991cSBjörn Töpel struct xsk_map {
42d817991cSBjörn Töpel struct bpf_map map;
43d817991cSBjörn Töpel spinlock_t lock; /* Synchronize map updates */
44b4fd0d67SYafang Shao atomic_t count;
45782347b6SToke Høiland-Jørgensen struct xdp_sock __rcu *xsk_map[];
46d817991cSBjörn Töpel };
47d817991cSBjörn Töpel
48c0c77d8fSBjörn Töpel struct xdp_sock {
49c0c77d8fSBjörn Töpel /* struct sock must be the first member of struct xdp_sock */
50c0c77d8fSBjörn Töpel struct sock sk;
518ef4e27eSMagnus Karlsson struct xsk_queue *rx ____cacheline_aligned_in_smp;
52b9b6b68eSBjörn Töpel struct net_device *dev;
53c0c77d8fSBjörn Töpel struct xdp_umem *umem;
54fbfc504aSBjörn Töpel struct list_head flush_node;
55c4655761SMagnus Karlsson struct xsk_buff_pool *pool;
56965a9909SMagnus Karlsson u16 queue_id;
57ac98d8aaSMagnus Karlsson bool zc;
5881470b5cSTirthendu Sarkar bool sg;
59455302d1SIlya Maximets enum {
60455302d1SIlya Maximets XSK_READY = 0,
61455302d1SIlya Maximets XSK_BOUND,
62455302d1SIlya Maximets XSK_UNBOUND,
63455302d1SIlya Maximets } state;
648ef4e27eSMagnus Karlsson
65fada7fdcSJonathan Lemon struct xsk_queue *tx ____cacheline_aligned_in_smp;
66a5aa8e52SMagnus Karlsson struct list_head tx_list;
6799b29a49SAlbert Huang /* record the number of tx descriptors sent by this xsk and
6899b29a49SAlbert Huang * when it exceeds MAX_PER_SOCKET_BUDGET, an opportunity needs
6999b29a49SAlbert Huang * to be given to other xsks for sending tx descriptors, thereby
7099b29a49SAlbert Huang * preventing other XSKs from being starved.
7199b29a49SAlbert Huang */
7299b29a49SAlbert Huang u32 tx_budget_spent;
7399b29a49SAlbert Huang
74bf0bdd13SIlya Maximets /* Protects generic receive. */
75bf0bdd13SIlya Maximets spinlock_t rx_lock;
768aa5a335SCiara Loftus
778aa5a335SCiara Loftus /* Statistics */
78c497176cSBjörn Töpel u64 rx_dropped;
798aa5a335SCiara Loftus u64 rx_queue_full;
808aa5a335SCiara Loftus
81b7f72a30STirthendu Sarkar /* When __xsk_generic_xmit() must return before it sees the EOP descriptor for the current
82b7f72a30STirthendu Sarkar * packet, the partially built skb is saved here so that packet building can resume in next
83b7f72a30STirthendu Sarkar * call of __xsk_generic_xmit().
84b7f72a30STirthendu Sarkar */
85b7f72a30STirthendu Sarkar struct sk_buff *skb;
86b7f72a30STirthendu Sarkar
870402acd6SBjörn Töpel struct list_head map_list;
880402acd6SBjörn Töpel /* Protects map_list */
890402acd6SBjörn Töpel spinlock_t map_list_lock;
908ef4e27eSMagnus Karlsson /* Protects multiple processes in the control path */
918ef4e27eSMagnus Karlsson struct mutex mutex;
927361f9c3SMagnus Karlsson struct xsk_queue *fq_tmp; /* Only as tmp storage before bind */
937361f9c3SMagnus Karlsson struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */
94c0c77d8fSBjörn Töpel };
95c0c77d8fSBjörn Töpel
9648eb03ddSStanislav Fomichev /*
9748eb03ddSStanislav Fomichev * AF_XDP TX metadata hooks for network devices.
9848eb03ddSStanislav Fomichev * The following hooks can be defined; unless noted otherwise, they are
9948eb03ddSStanislav Fomichev * optional and can be filled with a null pointer.
10048eb03ddSStanislav Fomichev *
10148eb03ddSStanislav Fomichev * void (*tmo_request_timestamp)(void *priv)
10248eb03ddSStanislav Fomichev * Called when AF_XDP frame requested egress timestamp.
10348eb03ddSStanislav Fomichev *
10448eb03ddSStanislav Fomichev * u64 (*tmo_fill_timestamp)(void *priv)
10548eb03ddSStanislav Fomichev * Called when AF_XDP frame, that had requested egress timestamp,
10648eb03ddSStanislav Fomichev * received a completion. The hook needs to return the actual HW timestamp.
10748eb03ddSStanislav Fomichev *
10848eb03ddSStanislav Fomichev * void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv)
10948eb03ddSStanislav Fomichev * Called when AF_XDP frame requested HW checksum offload. csum_start
11048eb03ddSStanislav Fomichev * indicates position where checksumming should start.
11148eb03ddSStanislav Fomichev * csum_offset indicates position where checksum should be stored.
11248eb03ddSStanislav Fomichev *
11348eb03ddSStanislav Fomichev */
11448eb03ddSStanislav Fomichev struct xsk_tx_metadata_ops {
11548eb03ddSStanislav Fomichev void (*tmo_request_timestamp)(void *priv);
11648eb03ddSStanislav Fomichev u64 (*tmo_fill_timestamp)(void *priv);
11748eb03ddSStanislav Fomichev void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv);
11848eb03ddSStanislav Fomichev };
11948eb03ddSStanislav Fomichev
120c497176cSBjörn Töpel #ifdef CONFIG_XDP_SOCKETS
12190254034SBjörn Töpel
122a71506a4SMagnus Karlsson int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
123e312b9e7SBjörn Töpel int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);
124e312b9e7SBjörn Töpel void __xsk_map_flush(void);
125d817991cSBjörn Töpel
12648eb03ddSStanislav Fomichev /**
12748eb03ddSStanislav Fomichev * xsk_tx_metadata_to_compl - Save enough relevant metadata information
12848eb03ddSStanislav Fomichev * to perform tx completion in the future.
12948eb03ddSStanislav Fomichev * @meta: pointer to AF_XDP metadata area
13048eb03ddSStanislav Fomichev * @compl: pointer to output struct xsk_tx_metadata_to_compl
13148eb03ddSStanislav Fomichev *
13248eb03ddSStanislav Fomichev * This function should be called by the networking device when
13348eb03ddSStanislav Fomichev * it prepares AF_XDP egress packet. The value of @compl should be stored
13448eb03ddSStanislav Fomichev * and passed to xsk_tx_metadata_complete upon TX completion.
13548eb03ddSStanislav Fomichev */
xsk_tx_metadata_to_compl(struct xsk_tx_metadata * meta,struct xsk_tx_metadata_compl * compl)13648eb03ddSStanislav Fomichev static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
13748eb03ddSStanislav Fomichev struct xsk_tx_metadata_compl *compl)
13848eb03ddSStanislav Fomichev {
13948eb03ddSStanislav Fomichev if (!meta)
14048eb03ddSStanislav Fomichev return;
14148eb03ddSStanislav Fomichev
14248eb03ddSStanislav Fomichev if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
14348eb03ddSStanislav Fomichev compl->tx_timestamp = &meta->completion.tx_timestamp;
14448eb03ddSStanislav Fomichev else
14548eb03ddSStanislav Fomichev compl->tx_timestamp = NULL;
14648eb03ddSStanislav Fomichev }
14748eb03ddSStanislav Fomichev
14848eb03ddSStanislav Fomichev /**
14948eb03ddSStanislav Fomichev * xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission
15048eb03ddSStanislav Fomichev * and call appropriate xsk_tx_metadata_ops operation.
15148eb03ddSStanislav Fomichev * @meta: pointer to AF_XDP metadata area
15248eb03ddSStanislav Fomichev * @ops: pointer to struct xsk_tx_metadata_ops
15348eb03ddSStanislav Fomichev * @priv: pointer to driver-private aread
15448eb03ddSStanislav Fomichev *
15548eb03ddSStanislav Fomichev * This function should be called by the networking device when
15648eb03ddSStanislav Fomichev * it prepares AF_XDP egress packet.
15748eb03ddSStanislav Fomichev */
xsk_tx_metadata_request(const struct xsk_tx_metadata * meta,const struct xsk_tx_metadata_ops * ops,void * priv)15848eb03ddSStanislav Fomichev static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta,
15948eb03ddSStanislav Fomichev const struct xsk_tx_metadata_ops *ops,
16048eb03ddSStanislav Fomichev void *priv)
16148eb03ddSStanislav Fomichev {
16248eb03ddSStanislav Fomichev if (!meta)
16348eb03ddSStanislav Fomichev return;
16448eb03ddSStanislav Fomichev
16548eb03ddSStanislav Fomichev if (ops->tmo_request_timestamp)
16648eb03ddSStanislav Fomichev if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
16748eb03ddSStanislav Fomichev ops->tmo_request_timestamp(priv);
16848eb03ddSStanislav Fomichev
16948eb03ddSStanislav Fomichev if (ops->tmo_request_checksum)
17048eb03ddSStanislav Fomichev if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM)
17148eb03ddSStanislav Fomichev ops->tmo_request_checksum(meta->request.csum_start,
17248eb03ddSStanislav Fomichev meta->request.csum_offset, priv);
17348eb03ddSStanislav Fomichev }
17448eb03ddSStanislav Fomichev
17548eb03ddSStanislav Fomichev /**
17648eb03ddSStanislav Fomichev * xsk_tx_metadata_complete - Evaluate AF_XDP TX metadata at completion
17748eb03ddSStanislav Fomichev * and call appropriate xsk_tx_metadata_ops operation.
17848eb03ddSStanislav Fomichev * @compl: pointer to completion metadata produced from xsk_tx_metadata_to_compl
17948eb03ddSStanislav Fomichev * @ops: pointer to struct xsk_tx_metadata_ops
18048eb03ddSStanislav Fomichev * @priv: pointer to driver-private aread
18148eb03ddSStanislav Fomichev *
18248eb03ddSStanislav Fomichev * This function should be called by the networking device upon
18348eb03ddSStanislav Fomichev * AF_XDP egress completion.
18448eb03ddSStanislav Fomichev */
xsk_tx_metadata_complete(struct xsk_tx_metadata_compl * compl,const struct xsk_tx_metadata_ops * ops,void * priv)18548eb03ddSStanislav Fomichev static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
18648eb03ddSStanislav Fomichev const struct xsk_tx_metadata_ops *ops,
18748eb03ddSStanislav Fomichev void *priv)
18848eb03ddSStanislav Fomichev {
18948eb03ddSStanislav Fomichev if (!compl)
19048eb03ddSStanislav Fomichev return;
191*f6e92236SStanislav Fomichev if (!compl->tx_timestamp)
192*f6e92236SStanislav Fomichev return;
19348eb03ddSStanislav Fomichev
19448eb03ddSStanislav Fomichev *compl->tx_timestamp = ops->tmo_fill_timestamp(priv);
19548eb03ddSStanislav Fomichev }
19648eb03ddSStanislav Fomichev
197c497176cSBjörn Töpel #else
198a71506a4SMagnus Karlsson
xsk_generic_rcv(struct xdp_sock * xs,struct xdp_buff * xdp)199c497176cSBjörn Töpel static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
200c497176cSBjörn Töpel {
201c497176cSBjörn Töpel return -ENOTSUPP;
202c497176cSBjörn Töpel }
203c497176cSBjörn Töpel
__xsk_map_redirect(struct xdp_sock * xs,struct xdp_buff * xdp)204a71506a4SMagnus Karlsson static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
205d57d7642SMaxim Mikityanskiy {
206a71506a4SMagnus Karlsson return -EOPNOTSUPP;
207d57d7642SMaxim Mikityanskiy }
208d57d7642SMaxim Mikityanskiy
__xsk_map_flush(void)209a71506a4SMagnus Karlsson static inline void __xsk_map_flush(void)
21090254034SBjörn Töpel {
21190254034SBjörn Töpel }
21290254034SBjörn Töpel
xsk_tx_metadata_to_compl(struct xsk_tx_metadata * meta,struct xsk_tx_metadata_compl * compl)21348eb03ddSStanislav Fomichev static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
21448eb03ddSStanislav Fomichev struct xsk_tx_metadata_compl *compl)
21548eb03ddSStanislav Fomichev {
21648eb03ddSStanislav Fomichev }
21748eb03ddSStanislav Fomichev
xsk_tx_metadata_request(struct xsk_tx_metadata * meta,const struct xsk_tx_metadata_ops * ops,void * priv)21848eb03ddSStanislav Fomichev static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta,
21948eb03ddSStanislav Fomichev const struct xsk_tx_metadata_ops *ops,
22048eb03ddSStanislav Fomichev void *priv)
22148eb03ddSStanislav Fomichev {
22248eb03ddSStanislav Fomichev }
22348eb03ddSStanislav Fomichev
xsk_tx_metadata_complete(struct xsk_tx_metadata_compl * compl,const struct xsk_tx_metadata_ops * ops,void * priv)22448eb03ddSStanislav Fomichev static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
22548eb03ddSStanislav Fomichev const struct xsk_tx_metadata_ops *ops,
22648eb03ddSStanislav Fomichev void *priv)
22748eb03ddSStanislav Fomichev {
22848eb03ddSStanislav Fomichev }
22948eb03ddSStanislav Fomichev
230c497176cSBjörn Töpel #endif /* CONFIG_XDP_SOCKETS */
231c497176cSBjörn Töpel
2329a675ba5SSebastian Andrzej Siewior #if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET)
2339a675ba5SSebastian Andrzej Siewior bool xsk_map_check_flush(void);
2349a675ba5SSebastian Andrzej Siewior #else
xsk_map_check_flush(void)2359a675ba5SSebastian Andrzej Siewior static inline bool xsk_map_check_flush(void)
2369a675ba5SSebastian Andrzej Siewior {
2379a675ba5SSebastian Andrzej Siewior return false;
2389a675ba5SSebastian Andrzej Siewior }
2399a675ba5SSebastian Andrzej Siewior #endif
2409a675ba5SSebastian Andrzej Siewior
241c0c77d8fSBjörn Töpel #endif /* _LINUX_XDP_SOCK_H */
242