1dac09149SBjörn Töpel /* SPDX-License-Identifier: GPL-2.0 */ 2dac09149SBjörn Töpel /* AF_XDP internal functions 3c0c77d8fSBjörn Töpel * Copyright(c) 2018 Intel Corporation. 4c0c77d8fSBjörn Töpel */ 5c0c77d8fSBjörn Töpel 6c0c77d8fSBjörn Töpel #ifndef _LINUX_XDP_SOCK_H 7c0c77d8fSBjörn Töpel #define _LINUX_XDP_SOCK_H 8c0c77d8fSBjörn Töpel 9b6459415SJakub Kicinski #include <linux/bpf.h> 10e61e62b9SBjörn Töpel #include <linux/workqueue.h> 11e61e62b9SBjörn Töpel #include <linux/if_xdp.h> 12c0c77d8fSBjörn Töpel #include <linux/mutex.h> 13ac98d8aaSMagnus Karlsson #include <linux/spinlock.h> 14e61e62b9SBjörn Töpel #include <linux/mm.h> 15c0c77d8fSBjörn Töpel #include <net/sock.h> 16c0c77d8fSBjörn Töpel 17b9b6b68eSBjörn Töpel struct net_device; 18b9b6b68eSBjörn Töpel struct xsk_queue; 19a71506a4SMagnus Karlsson struct xdp_buff; 20e61e62b9SBjörn Töpel 21e61e62b9SBjörn Töpel struct xdp_umem { 227f7ffa4eSMagnus Karlsson void *addrs; 2393ee30f3SMagnus Karlsson u64 size; 24e61e62b9SBjörn Töpel u32 headroom; 252b43470aSBjörn Töpel u32 chunk_size; 261c1efc2aSMagnus Karlsson u32 chunks; 278ef4e27eSMagnus Karlsson u32 npgs; 28e61e62b9SBjörn Töpel struct user_struct *user; 29e61e62b9SBjörn Töpel refcount_t users; 3077cd0d7bSMagnus Karlsson u8 flags; 31173d3adbSBjörn Töpel bool zc; 328ef4e27eSMagnus Karlsson struct page **pgs; 338ef4e27eSMagnus Karlsson int id; 34921b6869SMagnus Karlsson struct list_head xsk_dma_list; 35537cf4e3SMagnus Karlsson struct work_struct work; 36e61e62b9SBjörn Töpel }; 37c0c77d8fSBjörn Töpel 38d817991cSBjörn Töpel struct xsk_map { 39d817991cSBjörn Töpel struct bpf_map map; 40d817991cSBjörn Töpel spinlock_t lock; /* Synchronize map updates */ 41b4fd0d67SYafang Shao atomic_t count; 42782347b6SToke Høiland-Jørgensen struct xdp_sock __rcu *xsk_map[]; 43d817991cSBjörn Töpel }; 44d817991cSBjörn Töpel 45c0c77d8fSBjörn Töpel struct xdp_sock { 46c0c77d8fSBjörn Töpel /* struct sock must be the first member of struct xdp_sock */ 47c0c77d8fSBjörn Töpel struct sock sk; 488ef4e27eSMagnus Karlsson struct xsk_queue *rx ____cacheline_aligned_in_smp; 49b9b6b68eSBjörn Töpel struct net_device *dev; 50c0c77d8fSBjörn Töpel struct xdp_umem *umem; 51fbfc504aSBjörn Töpel struct list_head flush_node; 52c4655761SMagnus Karlsson struct xsk_buff_pool *pool; 53965a9909SMagnus Karlsson u16 queue_id; 54ac98d8aaSMagnus Karlsson bool zc; 55*81470b5cSTirthendu Sarkar bool sg; 56455302d1SIlya Maximets enum { 57455302d1SIlya Maximets XSK_READY = 0, 58455302d1SIlya Maximets XSK_BOUND, 59455302d1SIlya Maximets XSK_UNBOUND, 60455302d1SIlya Maximets } state; 618ef4e27eSMagnus Karlsson 62fada7fdcSJonathan Lemon struct xsk_queue *tx ____cacheline_aligned_in_smp; 63a5aa8e52SMagnus Karlsson struct list_head tx_list; 64bf0bdd13SIlya Maximets /* Protects generic receive. */ 65bf0bdd13SIlya Maximets spinlock_t rx_lock; 668aa5a335SCiara Loftus 678aa5a335SCiara Loftus /* Statistics */ 68c497176cSBjörn Töpel u64 rx_dropped; 698aa5a335SCiara Loftus u64 rx_queue_full; 708aa5a335SCiara Loftus 710402acd6SBjörn Töpel struct list_head map_list; 720402acd6SBjörn Töpel /* Protects map_list */ 730402acd6SBjörn Töpel spinlock_t map_list_lock; 748ef4e27eSMagnus Karlsson /* Protects multiple processes in the control path */ 758ef4e27eSMagnus Karlsson struct mutex mutex; 767361f9c3SMagnus Karlsson struct xsk_queue *fq_tmp; /* Only as tmp storage before bind */ 777361f9c3SMagnus Karlsson struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */ 78c0c77d8fSBjörn Töpel }; 79c0c77d8fSBjörn Töpel 80c497176cSBjörn Töpel #ifdef CONFIG_XDP_SOCKETS 8190254034SBjörn Töpel 82a71506a4SMagnus Karlsson int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp); 83e312b9e7SBjörn Töpel int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp); 84e312b9e7SBjörn Töpel void __xsk_map_flush(void); 85d817991cSBjörn Töpel 86c497176cSBjörn Töpel #else 87a71506a4SMagnus Karlsson 88c497176cSBjörn Töpel static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) 89c497176cSBjörn Töpel { 90c497176cSBjörn Töpel return -ENOTSUPP; 91c497176cSBjörn Töpel } 92c497176cSBjörn Töpel 93a71506a4SMagnus Karlsson static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp) 94d57d7642SMaxim Mikityanskiy { 95a71506a4SMagnus Karlsson return -EOPNOTSUPP; 96d57d7642SMaxim Mikityanskiy } 97d57d7642SMaxim Mikityanskiy 98a71506a4SMagnus Karlsson static inline void __xsk_map_flush(void) 9990254034SBjörn Töpel { 10090254034SBjörn Töpel } 10190254034SBjörn Töpel 102c497176cSBjörn Töpel #endif /* CONFIG_XDP_SOCKETS */ 103c497176cSBjörn Töpel 104c0c77d8fSBjörn Töpel #endif /* _LINUX_XDP_SOCK_H */ 105