1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __LINUX_SWIOTLB_H 3 #define __LINUX_SWIOTLB_H 4 5 #include <linux/device.h> 6 #include <linux/dma-direction.h> 7 #include <linux/init.h> 8 #include <linux/types.h> 9 #include <linux/limits.h> 10 #include <linux/spinlock.h> 11 #include <linux/workqueue.h> 12 13 struct device; 14 struct page; 15 struct scatterlist; 16 17 #define SWIOTLB_VERBOSE (1 << 0) /* verbose initialization */ 18 #define SWIOTLB_FORCE (1 << 1) /* force bounce buffering */ 19 #define SWIOTLB_ANY (1 << 2) /* allow any memory for the buffer */ 20 21 /* 22 * Maximum allowable number of contiguous slabs to map, 23 * must be a power of 2. What is the appropriate value ? 24 * The complexity of {map,unmap}_single is linearly dependent on this value. 25 */ 26 #define IO_TLB_SEGSIZE 128 27 28 /* 29 * log of the size of each IO TLB slab. The number of slabs is command line 30 * controllable. 31 */ 32 #define IO_TLB_SHIFT 11 33 #define IO_TLB_SIZE (1 << IO_TLB_SHIFT) 34 35 /* default to 64MB */ 36 #define IO_TLB_DEFAULT_SIZE (64UL<<20) 37 38 unsigned long swiotlb_size_or_default(void); 39 void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags, 40 int (*remap)(void *tlb, unsigned long nslabs)); 41 int swiotlb_init_late(size_t size, gfp_t gfp_mask, 42 int (*remap)(void *tlb, unsigned long nslabs)); 43 extern void __init swiotlb_update_mem_attributes(void); 44 45 phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys, 46 size_t mapping_size, size_t alloc_size, 47 unsigned int alloc_aligned_mask, enum dma_data_direction dir, 48 unsigned long attrs); 49 50 extern void swiotlb_tbl_unmap_single(struct device *hwdev, 51 phys_addr_t tlb_addr, 52 size_t mapping_size, 53 enum dma_data_direction dir, 54 unsigned long attrs); 55 56 void swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr, 57 size_t size, enum dma_data_direction dir); 58 void swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr, 59 size_t size, enum dma_data_direction dir); 60 dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys, 61 size_t size, enum dma_data_direction dir, unsigned long attrs); 62 63 #ifdef CONFIG_SWIOTLB 64 65 /** 66 * struct io_tlb_pool - IO TLB memory pool descriptor 67 * @start: The start address of the swiotlb memory pool. Used to do a quick 68 * range check to see if the memory was in fact allocated by this 69 * API. 70 * @end: The end address of the swiotlb memory pool. Used to do a quick 71 * range check to see if the memory was in fact allocated by this 72 * API. 73 * @vaddr: The vaddr of the swiotlb memory pool. The swiotlb memory pool 74 * may be remapped in the memory encrypted case and store virtual 75 * address for bounce buffer operation. 76 * @nslabs: The number of IO TLB slots between @start and @end. For the 77 * default swiotlb, this can be adjusted with a boot parameter, 78 * see setup_io_tlb_npages(). 79 * @late_alloc: %true if allocated using the page allocator. 80 * @nareas: Number of areas in the pool. 81 * @area_nslabs: Number of slots in each area. 82 * @areas: Array of memory area descriptors. 83 * @slots: Array of slot descriptors. 84 * @node: Member of the IO TLB memory pool list. 85 * @rcu: RCU head for swiotlb_dyn_free(). 86 * @transient: %true if transient memory pool. 87 */ 88 struct io_tlb_pool { 89 phys_addr_t start; 90 phys_addr_t end; 91 void *vaddr; 92 unsigned long nslabs; 93 bool late_alloc; 94 unsigned int nareas; 95 unsigned int area_nslabs; 96 struct io_tlb_area *areas; 97 struct io_tlb_slot *slots; 98 #ifdef CONFIG_SWIOTLB_DYNAMIC 99 struct list_head node; 100 struct rcu_head rcu; 101 bool transient; 102 #endif 103 }; 104 105 /** 106 * struct io_tlb_mem - Software IO TLB allocator 107 * @defpool: Default (initial) IO TLB memory pool descriptor. 108 * @pool: IO TLB memory pool descriptor (if not dynamic). 109 * @nslabs: Total number of IO TLB slabs in all pools. 110 * @debugfs: The dentry to debugfs. 111 * @force_bounce: %true if swiotlb bouncing is forced 112 * @for_alloc: %true if the pool is used for memory allocation 113 * @can_grow: %true if more pools can be allocated dynamically. 114 * @phys_limit: Maximum allowed physical address. 115 * @lock: Lock to synchronize changes to the list. 116 * @pools: List of IO TLB memory pool descriptors (if dynamic). 117 * @dyn_alloc: Dynamic IO TLB pool allocation work. 118 * @total_used: The total number of slots in the pool that are currently used 119 * across all areas. Used only for calculating used_hiwater in 120 * debugfs. 121 * @used_hiwater: The high water mark for total_used. Used only for reporting 122 * in debugfs. 123 */ 124 struct io_tlb_mem { 125 struct io_tlb_pool defpool; 126 unsigned long nslabs; 127 struct dentry *debugfs; 128 bool force_bounce; 129 bool for_alloc; 130 #ifdef CONFIG_SWIOTLB_DYNAMIC 131 bool can_grow; 132 u64 phys_limit; 133 spinlock_t lock; 134 struct list_head pools; 135 struct work_struct dyn_alloc; 136 #endif 137 #ifdef CONFIG_DEBUG_FS 138 atomic_long_t total_used; 139 atomic_long_t used_hiwater; 140 #endif 141 }; 142 143 #ifdef CONFIG_SWIOTLB_DYNAMIC 144 145 struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr); 146 147 #else 148 149 static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev, 150 phys_addr_t paddr) 151 { 152 return &dev->dma_io_tlb_mem->defpool; 153 } 154 155 #endif 156 157 /** 158 * is_swiotlb_buffer() - check if a physical address belongs to a swiotlb 159 * @dev: Device which has mapped the buffer. 160 * @paddr: Physical address within the DMA buffer. 161 * 162 * Check if @paddr points into a bounce buffer. 163 * 164 * Return: 165 * * %true if @paddr points into a bounce buffer 166 * * %false otherwise 167 */ 168 static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr) 169 { 170 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; 171 172 if (!mem) 173 return false; 174 175 if (IS_ENABLED(CONFIG_SWIOTLB_DYNAMIC)) { 176 /* Pairs with smp_wmb() in swiotlb_find_slots() and 177 * swiotlb_dyn_alloc(), which modify the RCU lists. 178 */ 179 smp_rmb(); 180 return swiotlb_find_pool(dev, paddr); 181 } 182 return paddr >= mem->defpool.start && paddr < mem->defpool.end; 183 } 184 185 static inline bool is_swiotlb_force_bounce(struct device *dev) 186 { 187 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; 188 189 return mem && mem->force_bounce; 190 } 191 192 void swiotlb_init(bool addressing_limited, unsigned int flags); 193 void __init swiotlb_exit(void); 194 void swiotlb_dev_init(struct device *dev); 195 size_t swiotlb_max_mapping_size(struct device *dev); 196 bool is_swiotlb_allocated(void); 197 bool is_swiotlb_active(struct device *dev); 198 void __init swiotlb_adjust_size(unsigned long size); 199 phys_addr_t default_swiotlb_base(void); 200 phys_addr_t default_swiotlb_limit(void); 201 #else 202 static inline void swiotlb_init(bool addressing_limited, unsigned int flags) 203 { 204 } 205 206 static inline void swiotlb_dev_init(struct device *dev) 207 { 208 } 209 210 static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr) 211 { 212 return false; 213 } 214 static inline bool is_swiotlb_force_bounce(struct device *dev) 215 { 216 return false; 217 } 218 static inline void swiotlb_exit(void) 219 { 220 } 221 static inline size_t swiotlb_max_mapping_size(struct device *dev) 222 { 223 return SIZE_MAX; 224 } 225 226 static inline bool is_swiotlb_allocated(void) 227 { 228 return false; 229 } 230 231 static inline bool is_swiotlb_active(struct device *dev) 232 { 233 return false; 234 } 235 236 static inline void swiotlb_adjust_size(unsigned long size) 237 { 238 } 239 240 static inline phys_addr_t default_swiotlb_base(void) 241 { 242 return 0; 243 } 244 245 static inline phys_addr_t default_swiotlb_limit(void) 246 { 247 return 0; 248 } 249 #endif /* CONFIG_SWIOTLB */ 250 251 extern void swiotlb_print_info(void); 252 253 #ifdef CONFIG_DMA_RESTRICTED_POOL 254 struct page *swiotlb_alloc(struct device *dev, size_t size); 255 bool swiotlb_free(struct device *dev, struct page *page, size_t size); 256 257 static inline bool is_swiotlb_for_alloc(struct device *dev) 258 { 259 return dev->dma_io_tlb_mem->for_alloc; 260 } 261 #else 262 static inline struct page *swiotlb_alloc(struct device *dev, size_t size) 263 { 264 return NULL; 265 } 266 static inline bool swiotlb_free(struct device *dev, struct page *page, 267 size_t size) 268 { 269 return false; 270 } 271 static inline bool is_swiotlb_for_alloc(struct device *dev) 272 { 273 return false; 274 } 275 #endif /* CONFIG_DMA_RESTRICTED_POOL */ 276 277 #endif /* __LINUX_SWIOTLB_H */ 278