1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __LINUX_SWIOTLB_H 3 #define __LINUX_SWIOTLB_H 4 5 #include <linux/device.h> 6 #include <linux/dma-direction.h> 7 #include <linux/init.h> 8 #include <linux/types.h> 9 #include <linux/limits.h> 10 #include <linux/spinlock.h> 11 #include <linux/workqueue.h> 12 13 struct device; 14 struct page; 15 struct scatterlist; 16 17 #define SWIOTLB_VERBOSE (1 << 0) /* verbose initialization */ 18 #define SWIOTLB_FORCE (1 << 1) /* force bounce buffering */ 19 #define SWIOTLB_ANY (1 << 2) /* allow any memory for the buffer */ 20 21 /* 22 * Maximum allowable number of contiguous slabs to map, 23 * must be a power of 2. What is the appropriate value ? 24 * The complexity of {map,unmap}_single is linearly dependent on this value. 25 */ 26 #define IO_TLB_SEGSIZE 128 27 28 /* 29 * log of the size of each IO TLB slab. The number of slabs is command line 30 * controllable. 31 */ 32 #define IO_TLB_SHIFT 11 33 #define IO_TLB_SIZE (1 << IO_TLB_SHIFT) 34 35 /* default to 64MB */ 36 #define IO_TLB_DEFAULT_SIZE (64UL<<20) 37 38 unsigned long swiotlb_size_or_default(void); 39 void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags, 40 int (*remap)(void *tlb, unsigned long nslabs)); 41 int swiotlb_init_late(size_t size, gfp_t gfp_mask, 42 int (*remap)(void *tlb, unsigned long nslabs)); 43 extern void __init swiotlb_update_mem_attributes(void); 44 45 phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys, 46 size_t mapping_size, size_t alloc_size, 47 unsigned int alloc_aligned_mask, enum dma_data_direction dir, 48 unsigned long attrs); 49 50 extern void swiotlb_tbl_unmap_single(struct device *hwdev, 51 phys_addr_t tlb_addr, 52 size_t mapping_size, 53 enum dma_data_direction dir, 54 unsigned long attrs); 55 56 void swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr, 57 size_t size, enum dma_data_direction dir); 58 void swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr, 59 size_t size, enum dma_data_direction dir); 60 dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys, 61 size_t size, enum dma_data_direction dir, unsigned long attrs); 62 63 #ifdef CONFIG_SWIOTLB 64 65 /** 66 * struct io_tlb_pool - IO TLB memory pool descriptor 67 * @start: The start address of the swiotlb memory pool. Used to do a quick 68 * range check to see if the memory was in fact allocated by this 69 * API. 70 * @end: The end address of the swiotlb memory pool. Used to do a quick 71 * range check to see if the memory was in fact allocated by this 72 * API. 73 * @vaddr: The vaddr of the swiotlb memory pool. The swiotlb memory pool 74 * may be remapped in the memory encrypted case and store virtual 75 * address for bounce buffer operation. 76 * @nslabs: The number of IO TLB slots between @start and @end. For the 77 * default swiotlb, this can be adjusted with a boot parameter, 78 * see setup_io_tlb_npages(). 79 * @late_alloc: %true if allocated using the page allocator. 80 * @nareas: Number of areas in the pool. 81 * @area_nslabs: Number of slots in each area. 82 * @areas: Array of memory area descriptors. 83 * @slots: Array of slot descriptors. 84 * @node: Member of the IO TLB memory pool list. 85 * @rcu: RCU head for swiotlb_dyn_free(). 86 * @transient: %true if transient memory pool. 87 */ 88 struct io_tlb_pool { 89 phys_addr_t start; 90 phys_addr_t end; 91 void *vaddr; 92 unsigned long nslabs; 93 bool late_alloc; 94 unsigned int nareas; 95 unsigned int area_nslabs; 96 struct io_tlb_area *areas; 97 struct io_tlb_slot *slots; 98 #ifdef CONFIG_SWIOTLB_DYNAMIC 99 struct list_head node; 100 struct rcu_head rcu; 101 bool transient; 102 #endif 103 }; 104 105 /** 106 * struct io_tlb_mem - Software IO TLB allocator 107 * @defpool: Default (initial) IO TLB memory pool descriptor. 108 * @pool: IO TLB memory pool descriptor (if not dynamic). 109 * @nslabs: Total number of IO TLB slabs in all pools. 110 * @debugfs: The dentry to debugfs. 111 * @force_bounce: %true if swiotlb bouncing is forced 112 * @for_alloc: %true if the pool is used for memory allocation 113 * @can_grow: %true if more pools can be allocated dynamically. 114 * @phys_limit: Maximum allowed physical address. 115 * @lock: Lock to synchronize changes to the list. 116 * @pools: List of IO TLB memory pool descriptors (if dynamic). 117 * @dyn_alloc: Dynamic IO TLB pool allocation work. 118 * @total_used: The total number of slots in the pool that are currently used 119 * across all areas. Used only for calculating used_hiwater in 120 * debugfs. 121 * @used_hiwater: The high water mark for total_used. Used only for reporting 122 * in debugfs. 123 */ 124 struct io_tlb_mem { 125 struct io_tlb_pool defpool; 126 unsigned long nslabs; 127 struct dentry *debugfs; 128 bool force_bounce; 129 bool for_alloc; 130 #ifdef CONFIG_SWIOTLB_DYNAMIC 131 bool can_grow; 132 u64 phys_limit; 133 spinlock_t lock; 134 struct list_head pools; 135 struct work_struct dyn_alloc; 136 #endif 137 #ifdef CONFIG_DEBUG_FS 138 atomic_long_t total_used; 139 atomic_long_t used_hiwater; 140 #endif 141 }; 142 143 #ifdef CONFIG_SWIOTLB_DYNAMIC 144 145 struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr); 146 147 #else 148 149 static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev, 150 phys_addr_t paddr) 151 { 152 return &dev->dma_io_tlb_mem->defpool; 153 } 154 155 #endif 156 157 /** 158 * is_swiotlb_buffer() - check if a physical address belongs to a swiotlb 159 * @dev: Device which has mapped the buffer. 160 * @paddr: Physical address within the DMA buffer. 161 * 162 * Check if @paddr points into a bounce buffer. 163 * 164 * Return: 165 * * %true if @paddr points into a bounce buffer 166 * * %false otherwise 167 */ 168 static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr) 169 { 170 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; 171 172 if (!mem) 173 return false; 174 175 #ifdef CONFIG_SWIOTLB_DYNAMIC 176 /* 177 * All SWIOTLB buffer addresses must have been returned by 178 * swiotlb_tbl_map_single() and passed to a device driver. 179 * If a SWIOTLB address is checked on another CPU, then it was 180 * presumably loaded by the device driver from an unspecified private 181 * data structure. Make sure that this load is ordered before reading 182 * dev->dma_uses_io_tlb here and mem->pools in swiotlb_find_pool(). 183 * 184 * This barrier pairs with smp_mb() in swiotlb_find_slots(). 185 */ 186 smp_rmb(); 187 return READ_ONCE(dev->dma_uses_io_tlb) && 188 swiotlb_find_pool(dev, paddr); 189 #else 190 return paddr >= mem->defpool.start && paddr < mem->defpool.end; 191 #endif 192 } 193 194 static inline bool is_swiotlb_force_bounce(struct device *dev) 195 { 196 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; 197 198 return mem && mem->force_bounce; 199 } 200 201 void swiotlb_init(bool addressing_limited, unsigned int flags); 202 void __init swiotlb_exit(void); 203 void swiotlb_dev_init(struct device *dev); 204 size_t swiotlb_max_mapping_size(struct device *dev); 205 bool is_swiotlb_allocated(void); 206 bool is_swiotlb_active(struct device *dev); 207 void __init swiotlb_adjust_size(unsigned long size); 208 phys_addr_t default_swiotlb_base(void); 209 phys_addr_t default_swiotlb_limit(void); 210 #else 211 static inline void swiotlb_init(bool addressing_limited, unsigned int flags) 212 { 213 } 214 215 static inline void swiotlb_dev_init(struct device *dev) 216 { 217 } 218 219 static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr) 220 { 221 return false; 222 } 223 static inline bool is_swiotlb_force_bounce(struct device *dev) 224 { 225 return false; 226 } 227 static inline void swiotlb_exit(void) 228 { 229 } 230 static inline size_t swiotlb_max_mapping_size(struct device *dev) 231 { 232 return SIZE_MAX; 233 } 234 235 static inline bool is_swiotlb_allocated(void) 236 { 237 return false; 238 } 239 240 static inline bool is_swiotlb_active(struct device *dev) 241 { 242 return false; 243 } 244 245 static inline void swiotlb_adjust_size(unsigned long size) 246 { 247 } 248 249 static inline phys_addr_t default_swiotlb_base(void) 250 { 251 return 0; 252 } 253 254 static inline phys_addr_t default_swiotlb_limit(void) 255 { 256 return 0; 257 } 258 #endif /* CONFIG_SWIOTLB */ 259 260 extern void swiotlb_print_info(void); 261 262 #ifdef CONFIG_DMA_RESTRICTED_POOL 263 struct page *swiotlb_alloc(struct device *dev, size_t size); 264 bool swiotlb_free(struct device *dev, struct page *page, size_t size); 265 266 static inline bool is_swiotlb_for_alloc(struct device *dev) 267 { 268 return dev->dma_io_tlb_mem->for_alloc; 269 } 270 #else 271 static inline struct page *swiotlb_alloc(struct device *dev, size_t size) 272 { 273 return NULL; 274 } 275 static inline bool swiotlb_free(struct device *dev, struct page *page, 276 size_t size) 277 { 278 return false; 279 } 280 static inline bool is_swiotlb_for_alloc(struct device *dev) 281 { 282 return false; 283 } 284 #endif /* CONFIG_DMA_RESTRICTED_POOL */ 285 286 #endif /* __LINUX_SWIOTLB_H */ 287