1dac09149SBjörn Töpel /* SPDX-License-Identifier: GPL-2.0 */ 2dac09149SBjörn Töpel /* AF_XDP internal functions 3c0c77d8fSBjörn Töpel * Copyright(c) 2018 Intel Corporation. 4c0c77d8fSBjörn Töpel */ 5c0c77d8fSBjörn Töpel 6c0c77d8fSBjörn Töpel #ifndef _LINUX_XDP_SOCK_H 7c0c77d8fSBjörn Töpel #define _LINUX_XDP_SOCK_H 8c0c77d8fSBjörn Töpel 9e61e62b9SBjörn Töpel #include <linux/workqueue.h> 10e61e62b9SBjörn Töpel #include <linux/if_xdp.h> 11c0c77d8fSBjörn Töpel #include <linux/mutex.h> 12ac98d8aaSMagnus Karlsson #include <linux/spinlock.h> 13e61e62b9SBjörn Töpel #include <linux/mm.h> 14c0c77d8fSBjörn Töpel #include <net/sock.h> 15c0c77d8fSBjörn Töpel 16b9b6b68eSBjörn Töpel struct net_device; 17b9b6b68eSBjörn Töpel struct xsk_queue; 18e61e62b9SBjörn Töpel 198aef7340SBjörn Töpel struct xdp_umem_page { 208aef7340SBjörn Töpel void *addr; 21173d3adbSBjörn Töpel dma_addr_t dma; 228aef7340SBjörn Töpel }; 238aef7340SBjörn Töpel 24e61e62b9SBjörn Töpel struct xdp_umem { 25e61e62b9SBjörn Töpel struct xsk_queue *fq; 26e61e62b9SBjörn Töpel struct xsk_queue *cq; 278aef7340SBjörn Töpel struct xdp_umem_page *pages; 28*93ee30f3SMagnus Karlsson u64 chunk_mask; 29*93ee30f3SMagnus Karlsson u64 size; 30e61e62b9SBjörn Töpel u32 headroom; 31e61e62b9SBjörn Töpel u32 chunk_size_nohr; 32e61e62b9SBjörn Töpel struct user_struct *user; 33e61e62b9SBjörn Töpel struct pid *pid; 34e61e62b9SBjörn Töpel unsigned long address; 35e61e62b9SBjörn Töpel refcount_t users; 36e61e62b9SBjörn Töpel struct work_struct work; 378aef7340SBjörn Töpel struct page **pgs; 38e61e62b9SBjörn Töpel u32 npgs; 39173d3adbSBjörn Töpel struct net_device *dev; 40173d3adbSBjörn Töpel u16 queue_id; 41173d3adbSBjörn Töpel bool zc; 42ac98d8aaSMagnus Karlsson spinlock_t xsk_list_lock; 43ac98d8aaSMagnus Karlsson struct list_head xsk_list; 44e61e62b9SBjörn Töpel }; 45c0c77d8fSBjörn Töpel 46c0c77d8fSBjörn Töpel struct xdp_sock { 47c0c77d8fSBjörn Töpel /* struct sock must be the first member of struct xdp_sock */ 48c0c77d8fSBjörn Töpel struct sock sk; 49b9b6b68eSBjörn Töpel struct xsk_queue *rx; 50b9b6b68eSBjörn Töpel struct net_device *dev; 51c0c77d8fSBjörn Töpel struct xdp_umem *umem; 52fbfc504aSBjörn Töpel struct list_head flush_node; 53965a9909SMagnus Karlsson u16 queue_id; 54f6145903SMagnus Karlsson struct xsk_queue *tx ____cacheline_aligned_in_smp; 55ac98d8aaSMagnus Karlsson struct list_head list; 56ac98d8aaSMagnus Karlsson bool zc; 57c0c77d8fSBjörn Töpel /* Protects multiple processes in the control path */ 58c0c77d8fSBjörn Töpel struct mutex mutex; 59a9744f7cSMagnus Karlsson /* Mutual exclusion of NAPI TX thread and sendmsg error paths 60a9744f7cSMagnus Karlsson * in the SKB destructor callback. 61a9744f7cSMagnus Karlsson */ 62a9744f7cSMagnus Karlsson spinlock_t tx_completion_lock; 63c497176cSBjörn Töpel u64 rx_dropped; 64c0c77d8fSBjörn Töpel }; 65c0c77d8fSBjörn Töpel 66c497176cSBjörn Töpel struct xdp_buff; 67c497176cSBjörn Töpel #ifdef CONFIG_XDP_SOCKETS 68c497176cSBjörn Töpel int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp); 69c497176cSBjörn Töpel int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp); 70c497176cSBjörn Töpel void xsk_flush(struct xdp_sock *xs); 71fbfc504aSBjörn Töpel bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs); 72ac98d8aaSMagnus Karlsson /* Used from netdev driver */ 73173d3adbSBjörn Töpel u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr); 74173d3adbSBjörn Töpel void xsk_umem_discard_addr(struct xdp_umem *umem); 75ac98d8aaSMagnus Karlsson void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries); 76ac98d8aaSMagnus Karlsson bool xsk_umem_consume_tx(struct xdp_umem *umem, dma_addr_t *dma, u32 *len); 77ac98d8aaSMagnus Karlsson void xsk_umem_consume_tx_done(struct xdp_umem *umem); 7890254034SBjörn Töpel 7990254034SBjörn Töpel static inline char *xdp_umem_get_data(struct xdp_umem *umem, u64 addr) 8090254034SBjörn Töpel { 8190254034SBjörn Töpel return umem->pages[addr >> PAGE_SHIFT].addr + (addr & (PAGE_SIZE - 1)); 8290254034SBjörn Töpel } 8390254034SBjörn Töpel 8490254034SBjörn Töpel static inline dma_addr_t xdp_umem_get_dma(struct xdp_umem *umem, u64 addr) 8590254034SBjörn Töpel { 8690254034SBjörn Töpel return umem->pages[addr >> PAGE_SHIFT].dma + (addr & (PAGE_SIZE - 1)); 8790254034SBjörn Töpel } 88c497176cSBjörn Töpel #else 89c497176cSBjörn Töpel static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) 90c497176cSBjörn Töpel { 91c497176cSBjörn Töpel return -ENOTSUPP; 92c497176cSBjörn Töpel } 93c497176cSBjörn Töpel 94c497176cSBjörn Töpel static inline int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) 95c497176cSBjörn Töpel { 96c497176cSBjörn Töpel return -ENOTSUPP; 97c497176cSBjörn Töpel } 98c497176cSBjörn Töpel 99c497176cSBjörn Töpel static inline void xsk_flush(struct xdp_sock *xs) 100c497176cSBjörn Töpel { 101c497176cSBjörn Töpel } 102fbfc504aSBjörn Töpel 103fbfc504aSBjörn Töpel static inline bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs) 104fbfc504aSBjörn Töpel { 105fbfc504aSBjörn Töpel return false; 106fbfc504aSBjörn Töpel } 10790254034SBjörn Töpel 10890254034SBjörn Töpel static inline u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr) 10990254034SBjörn Töpel { 11090254034SBjörn Töpel return NULL; 11190254034SBjörn Töpel } 11290254034SBjörn Töpel 11390254034SBjörn Töpel static inline void xsk_umem_discard_addr(struct xdp_umem *umem) 11490254034SBjörn Töpel { 11590254034SBjörn Töpel } 11690254034SBjörn Töpel 11790254034SBjörn Töpel static inline void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries) 11890254034SBjörn Töpel { 11990254034SBjörn Töpel } 12090254034SBjörn Töpel 12190254034SBjörn Töpel static inline bool xsk_umem_consume_tx(struct xdp_umem *umem, dma_addr_t *dma, 12290254034SBjörn Töpel u32 *len) 12390254034SBjörn Töpel { 12490254034SBjörn Töpel return false; 12590254034SBjörn Töpel } 12690254034SBjörn Töpel 12790254034SBjörn Töpel static inline void xsk_umem_consume_tx_done(struct xdp_umem *umem) 12890254034SBjörn Töpel { 12990254034SBjörn Töpel } 13090254034SBjörn Töpel 13190254034SBjörn Töpel static inline char *xdp_umem_get_data(struct xdp_umem *umem, u64 addr) 13290254034SBjörn Töpel { 13390254034SBjörn Töpel return NULL; 13490254034SBjörn Töpel } 13590254034SBjörn Töpel 13690254034SBjörn Töpel static inline dma_addr_t xdp_umem_get_dma(struct xdp_umem *umem, u64 addr) 13790254034SBjörn Töpel { 13890254034SBjörn Töpel return 0; 13990254034SBjörn Töpel } 140c497176cSBjörn Töpel #endif /* CONFIG_XDP_SOCKETS */ 141c497176cSBjörn Töpel 142c0c77d8fSBjörn Töpel #endif /* _LINUX_XDP_SOCK_H */ 143