1 /* SPDX-License-Identifier: GPL-2.0 */ 2 3 #ifndef _NET_PAGE_POOL_TYPES_H 4 #define _NET_PAGE_POOL_TYPES_H 5 6 #include <linux/dma-direction.h> 7 #include <linux/ptr_ring.h> 8 9 #define PP_FLAG_DMA_MAP BIT(0) /* Should page_pool do the DMA 10 * map/unmap 11 */ 12 #define PP_FLAG_DMA_SYNC_DEV BIT(1) /* If set all pages that the driver gets 13 * from page_pool will be 14 * DMA-synced-for-device according to 15 * the length provided by the device 16 * driver. 17 * Please note DMA-sync-for-CPU is still 18 * device driver responsibility 19 */ 20 #define PP_FLAG_PAGE_FRAG BIT(2) /* for page frag feature */ 21 #define PP_FLAG_ALL (PP_FLAG_DMA_MAP |\ 22 PP_FLAG_DMA_SYNC_DEV |\ 23 PP_FLAG_PAGE_FRAG) 24 25 /* 26 * Fast allocation side cache array/stack 27 * 28 * The cache size and refill watermark is related to the network 29 * use-case. The NAPI budget is 64 packets. After a NAPI poll the RX 30 * ring is usually refilled and the max consumed elements will be 64, 31 * thus a natural max size of objects needed in the cache. 32 * 33 * Keeping room for more objects, is due to XDP_DROP use-case. As 34 * XDP_DROP allows the opportunity to recycle objects directly into 35 * this array, as it shares the same softirq/NAPI protection. If 36 * cache is already full (or partly full) then the XDP_DROP recycles 37 * would have to take a slower code path. 38 */ 39 #define PP_ALLOC_CACHE_SIZE 128 40 #define PP_ALLOC_CACHE_REFILL 64 41 struct pp_alloc_cache { 42 u32 count; 43 struct page *cache[PP_ALLOC_CACHE_SIZE]; 44 }; 45 46 /** 47 * struct page_pool_params - page pool parameters 48 * @flags: PP_FLAG_DMA_MAP, PP_FLAG_DMA_SYNC_DEV, PP_FLAG_PAGE_FRAG 49 * @order: 2^order pages on allocation 50 * @pool_size: size of the ptr_ring 51 * @nid: NUMA node id to allocate from pages from 52 * @dev: device, for DMA pre-mapping purposes 53 * @napi: NAPI which is the sole consumer of pages, otherwise NULL 54 * @dma_dir: DMA mapping direction 55 * @max_len: max DMA sync memory size for PP_FLAG_DMA_SYNC_DEV 56 * @offset: DMA sync address offset for PP_FLAG_DMA_SYNC_DEV 57 */ 58 struct page_pool_params { 59 unsigned int flags; 60 unsigned int order; 61 unsigned int pool_size; 62 int nid; 63 struct device *dev; 64 struct napi_struct *napi; 65 enum dma_data_direction dma_dir; 66 unsigned int max_len; 67 unsigned int offset; 68 /* private: used by test code only */ 69 void (*init_callback)(struct page *page, void *arg); 70 void *init_arg; 71 }; 72 73 #ifdef CONFIG_PAGE_POOL_STATS 74 /** 75 * struct page_pool_alloc_stats - allocation statistics 76 * @fast: successful fast path allocations 77 * @slow: slow path order-0 allocations 78 * @slow_high_order: slow path high order allocations 79 * @empty: ptr ring is empty, so a slow path allocation was forced 80 * @refill: an allocation which triggered a refill of the cache 81 * @waive: pages obtained from the ptr ring that cannot be added to 82 * the cache due to a NUMA mismatch 83 */ 84 struct page_pool_alloc_stats { 85 u64 fast; 86 u64 slow; 87 u64 slow_high_order; 88 u64 empty; 89 u64 refill; 90 u64 waive; 91 }; 92 93 /** 94 * struct page_pool_recycle_stats - recycling (freeing) statistics 95 * @cached: recycling placed page in the page pool cache 96 * @cache_full: page pool cache was full 97 * @ring: page placed into the ptr ring 98 * @ring_full: page released from page pool because the ptr ring was full 99 * @released_refcnt: page released (and not recycled) because refcnt > 1 100 */ 101 struct page_pool_recycle_stats { 102 u64 cached; 103 u64 cache_full; 104 u64 ring; 105 u64 ring_full; 106 u64 released_refcnt; 107 }; 108 109 /** 110 * struct page_pool_stats - combined page pool use statistics 111 * @alloc_stats: see struct page_pool_alloc_stats 112 * @recycle_stats: see struct page_pool_recycle_stats 113 * 114 * Wrapper struct for combining page pool stats with different storage 115 * requirements. 116 */ 117 struct page_pool_stats { 118 struct page_pool_alloc_stats alloc_stats; 119 struct page_pool_recycle_stats recycle_stats; 120 }; 121 #endif 122 123 struct page_pool { 124 struct page_pool_params p; 125 126 long frag_users; 127 struct page *frag_page; 128 unsigned int frag_offset; 129 u32 pages_state_hold_cnt; 130 131 struct delayed_work release_dw; 132 void (*disconnect)(void *pool); 133 unsigned long defer_start; 134 unsigned long defer_warn; 135 136 #ifdef CONFIG_PAGE_POOL_STATS 137 /* these stats are incremented while in softirq context */ 138 struct page_pool_alloc_stats alloc_stats; 139 #endif 140 u32 xdp_mem_id; 141 142 /* 143 * Data structure for allocation side 144 * 145 * Drivers allocation side usually already perform some kind 146 * of resource protection. Piggyback on this protection, and 147 * require driver to protect allocation side. 148 * 149 * For NIC drivers this means, allocate a page_pool per 150 * RX-queue. As the RX-queue is already protected by 151 * Softirq/BH scheduling and napi_schedule. NAPI schedule 152 * guarantee that a single napi_struct will only be scheduled 153 * on a single CPU (see napi_schedule). 154 */ 155 struct pp_alloc_cache alloc ____cacheline_aligned_in_smp; 156 157 /* Data structure for storing recycled pages. 158 * 159 * Returning/freeing pages is more complicated synchronization 160 * wise, because free's can happen on remote CPUs, with no 161 * association with allocation resource. 162 * 163 * Use ptr_ring, as it separates consumer and producer 164 * efficiently, it a way that doesn't bounce cache-lines. 165 * 166 * TODO: Implement bulk return pages into this structure. 167 */ 168 struct ptr_ring ring; 169 170 #ifdef CONFIG_PAGE_POOL_STATS 171 /* recycle stats are per-cpu to avoid locking */ 172 struct page_pool_recycle_stats __percpu *recycle_stats; 173 #endif 174 atomic_t pages_state_release_cnt; 175 176 /* A page_pool is strictly tied to a single RX-queue being 177 * protected by NAPI, due to above pp_alloc_cache. This 178 * refcnt serves purpose is to simplify drivers error handling. 179 */ 180 refcount_t user_cnt; 181 182 u64 destroy_cnt; 183 }; 184 185 struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp); 186 struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset, 187 unsigned int size, gfp_t gfp); 188 struct page_pool *page_pool_create(const struct page_pool_params *params); 189 190 struct xdp_mem_info; 191 192 #ifdef CONFIG_PAGE_POOL 193 void page_pool_unlink_napi(struct page_pool *pool); 194 void page_pool_destroy(struct page_pool *pool); 195 void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *), 196 struct xdp_mem_info *mem); 197 void page_pool_put_page_bulk(struct page_pool *pool, void **data, 198 int count); 199 #else 200 static inline void page_pool_unlink_napi(struct page_pool *pool) 201 { 202 } 203 204 static inline void page_pool_destroy(struct page_pool *pool) 205 { 206 } 207 208 static inline void page_pool_use_xdp_mem(struct page_pool *pool, 209 void (*disconnect)(void *), 210 struct xdp_mem_info *mem) 211 { 212 } 213 214 static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data, 215 int count) 216 { 217 } 218 #endif 219 220 void page_pool_put_defragged_page(struct page_pool *pool, struct page *page, 221 unsigned int dma_sync_size, 222 bool allow_direct); 223 224 static inline bool is_page_pool_compiled_in(void) 225 { 226 #ifdef CONFIG_PAGE_POOL 227 return true; 228 #else 229 return false; 230 #endif 231 } 232 233 /* Caller must provide appropriate safe context, e.g. NAPI. */ 234 void page_pool_update_nid(struct page_pool *pool, int new_nid); 235 236 #endif /* _NET_PAGE_POOL_H */ 237