1 /*
2 * Copyright 2020 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Christian König
23 */
24
25 #ifndef _TTM_RESOURCE_H_
26 #define _TTM_RESOURCE_H_
27
28 #include <linux/types.h>
29 #include <linux/list.h>
30 #include <linux/mutex.h>
31 #include <linux/iosys-map.h>
32 #include <linux/dma-fence.h>
33
34 #include <drm/drm_print.h>
35 #include <drm/ttm/ttm_caching.h>
36 #include <drm/ttm/ttm_kmap_iter.h>
37
38 #define TTM_MAX_BO_PRIORITY 4U
39 #define TTM_NUM_MEM_TYPES 8
40
41 struct ttm_device;
42 struct ttm_resource_manager;
43 struct ttm_resource;
44 struct ttm_place;
45 struct ttm_buffer_object;
46 struct ttm_placement;
47 struct iosys_map;
48 struct io_mapping;
49 struct sg_table;
50 struct scatterlist;
51
52 struct ttm_resource_manager_func {
53 /**
54 * struct ttm_resource_manager_func member alloc
55 *
56 * @man: Pointer to a memory type manager.
57 * @bo: Pointer to the buffer object we're allocating space for.
58 * @place: Placement details.
59 * @res: Resulting pointer to the ttm_resource.
60 *
61 * This function should allocate space in the memory type managed
62 * by @man. Placement details if applicable are given by @place. If
63 * successful, a filled in ttm_resource object should be returned in
64 * @res. @res::start should be set to a value identifying the beginning
65 * of the range allocated, and the function should return zero.
66 * If the manager can't fulfill the request -ENOSPC should be returned.
67 * If a system error occurred, preventing the request to be fulfilled,
68 * the function should return a negative error code.
69 *
70 * This function may not be called from within atomic context and needs
71 * to take care of its own locking to protect any data structures
72 * managing the space.
73 */
74 int (*alloc)(struct ttm_resource_manager *man,
75 struct ttm_buffer_object *bo,
76 const struct ttm_place *place,
77 struct ttm_resource **res);
78
79 /**
80 * struct ttm_resource_manager_func member free
81 *
82 * @man: Pointer to a memory type manager.
83 * @res: Pointer to a struct ttm_resource to be freed.
84 *
85 * This function frees memory type resources previously allocated.
86 * May not be called from within atomic context.
87 */
88 void (*free)(struct ttm_resource_manager *man,
89 struct ttm_resource *res);
90
91 /**
92 * struct ttm_resource_manager_func member intersects
93 *
94 * @man: Pointer to a memory type manager.
95 * @res: Pointer to a struct ttm_resource to be checked.
96 * @place: Placement to check against.
97 * @size: Size of the check.
98 *
99 * Test if @res intersects with @place + @size. Used to judge if
100 * evictions are valueable or not.
101 */
102 bool (*intersects)(struct ttm_resource_manager *man,
103 struct ttm_resource *res,
104 const struct ttm_place *place,
105 size_t size);
106
107 /**
108 * struct ttm_resource_manager_func member compatible
109 *
110 * @man: Pointer to a memory type manager.
111 * @res: Pointer to a struct ttm_resource to be checked.
112 * @place: Placement to check against.
113 * @size: Size of the check.
114 *
115 * Test if @res compatible with @place + @size. Used to check of
116 * the need to move the backing store or not.
117 */
118 bool (*compatible)(struct ttm_resource_manager *man,
119 struct ttm_resource *res,
120 const struct ttm_place *place,
121 size_t size);
122
123 /**
124 * struct ttm_resource_manager_func member debug
125 *
126 * @man: Pointer to a memory type manager.
127 * @printer: Prefix to be used in printout to identify the caller.
128 *
129 * This function is called to print out the state of the memory
130 * type manager to aid debugging of out-of-memory conditions.
131 * It may not be called from within atomic context.
132 */
133 void (*debug)(struct ttm_resource_manager *man,
134 struct drm_printer *printer);
135 };
136
137 /**
138 * struct ttm_resource_manager
139 *
140 * @use_type: The memory type is enabled.
141 * @use_tt: If a TT object should be used for the backing store.
142 * @size: Size of the managed region.
143 * @bdev: ttm device this manager belongs to
144 * @func: structure pointer implementing the range manager. See above
145 * @move_lock: lock for move fence
146 * @move: The fence of the last pipelined move operation.
147 * @lru: The lru list for this memory type.
148 *
149 * This structure is used to identify and manage memory types for a device.
150 */
151 struct ttm_resource_manager {
152 /*
153 * No protection. Constant from start.
154 */
155 bool use_type;
156 bool use_tt;
157 struct ttm_device *bdev;
158 uint64_t size;
159 const struct ttm_resource_manager_func *func;
160 spinlock_t move_lock;
161
162 /*
163 * Protected by @move_lock.
164 */
165 struct dma_fence *move;
166
167 /*
168 * Protected by the bdev->lru_lock.
169 */
170 struct list_head lru[TTM_MAX_BO_PRIORITY];
171
172 /**
173 * @usage: How much of the resources are used, protected by the
174 * bdev->lru_lock.
175 */
176 uint64_t usage;
177 };
178
179 /**
180 * struct ttm_bus_placement
181 *
182 * @addr: mapped virtual address
183 * @offset: physical addr
184 * @is_iomem: is this io memory ?
185 * @caching: See enum ttm_caching
186 *
187 * Structure indicating the bus placement of an object.
188 */
189 struct ttm_bus_placement {
190 void *addr;
191 phys_addr_t offset;
192 bool is_iomem;
193 enum ttm_caching caching;
194 bus_space_handle_t bsh;
195 };
196
197 /**
198 * struct ttm_resource
199 *
200 * @start: Start of the allocation.
201 * @size: Actual size of resource in bytes.
202 * @mem_type: Resource type of the allocation.
203 * @placement: Placement flags.
204 * @bus: Placement on io bus accessible to the CPU
205 * @bo: weak reference to the BO, protected by ttm_device::lru_lock
206 *
207 * Structure indicating the placement and space resources used by a
208 * buffer object.
209 */
210 struct ttm_resource {
211 unsigned long start;
212 size_t size;
213 uint32_t mem_type;
214 uint32_t placement;
215 struct ttm_bus_placement bus;
216 struct ttm_buffer_object *bo;
217
218 /**
219 * @lru: Least recently used list, see &ttm_resource_manager.lru
220 */
221 struct list_head lru;
222 };
223
224 /**
225 * struct ttm_resource_cursor
226 *
227 * @priority: the current priority
228 *
229 * Cursor to iterate over the resources in a manager.
230 */
231 struct ttm_resource_cursor {
232 unsigned int priority;
233 };
234
235 /**
236 * struct ttm_lru_bulk_move_pos
237 *
238 * @first: first res in the bulk move range
239 * @last: last res in the bulk move range
240 *
241 * Range of resources for a lru bulk move.
242 */
243 struct ttm_lru_bulk_move_pos {
244 struct ttm_resource *first;
245 struct ttm_resource *last;
246 };
247
248 /**
249 * struct ttm_lru_bulk_move
250 *
251 * @pos: first/last lru entry for resources in the each domain/priority
252 *
253 * Container for the current bulk move state. Should be used with
254 * ttm_lru_bulk_move_init() and ttm_bo_set_bulk_move().
255 */
256 struct ttm_lru_bulk_move {
257 struct ttm_lru_bulk_move_pos pos[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY];
258 };
259
260 /**
261 * struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping +
262 * struct sg_table backed struct ttm_resource.
263 * @base: Embedded struct ttm_kmap_iter providing the usage interface.
264 * @iomap: struct io_mapping representing the underlying linear io_memory.
265 * @st: sg_table into @iomap, representing the memory of the struct ttm_resource.
266 * @start: Offset that needs to be subtracted from @st to make
267 * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
268 * @cache: Scatterlist traversal cache for fast lookups.
269 * @cache.sg: Pointer to the currently cached scatterlist segment.
270 * @cache.i: First index of @sg. PAGE_SIZE granularity.
271 * @cache.end: Last index + 1 of @sg. PAGE_SIZE granularity.
272 * @cache.offs: First offset into @iomap of @sg. PAGE_SIZE granularity.
273 */
274 struct ttm_kmap_iter_iomap {
275 struct ttm_kmap_iter base;
276 struct io_mapping *iomap;
277 struct sg_table *st;
278 resource_size_t start;
279 struct {
280 struct scatterlist *sg;
281 pgoff_t i;
282 pgoff_t end;
283 pgoff_t offs;
284 } cache;
285 };
286
287 /**
288 * struct ttm_kmap_iter_linear_io - Iterator specialization for linear io
289 * @base: The base iterator
290 * @dmap: Points to the starting address of the region
291 * @needs_unmap: Whether we need to unmap on fini
292 */
293 struct ttm_kmap_iter_linear_io {
294 struct ttm_kmap_iter base;
295 struct iosys_map dmap;
296 bool needs_unmap;
297 };
298
299 /**
300 * ttm_resource_manager_set_used
301 *
302 * @man: A memory manager object.
303 * @used: usage state to set.
304 *
305 * Set the manager in use flag. If disabled the manager is no longer
306 * used for object placement.
307 */
308 static inline void
ttm_resource_manager_set_used(struct ttm_resource_manager * man,bool used)309 ttm_resource_manager_set_used(struct ttm_resource_manager *man, bool used)
310 {
311 int i;
312
313 for (i = 0; i < TTM_MAX_BO_PRIORITY; i++)
314 WARN_ON(!list_empty(&man->lru[i]));
315 man->use_type = used;
316 }
317
318 /**
319 * ttm_resource_manager_used
320 *
321 * @man: Manager to get used state for
322 *
323 * Get the in use flag for a manager.
324 * Returns:
325 * true is used, false if not.
326 */
ttm_resource_manager_used(struct ttm_resource_manager * man)327 static inline bool ttm_resource_manager_used(struct ttm_resource_manager *man)
328 {
329 return man->use_type;
330 }
331
332 /**
333 * ttm_resource_manager_cleanup
334 *
335 * @man: A memory manager object.
336 *
337 * Cleanup the move fences from the memory manager object.
338 */
339 static inline void
ttm_resource_manager_cleanup(struct ttm_resource_manager * man)340 ttm_resource_manager_cleanup(struct ttm_resource_manager *man)
341 {
342 dma_fence_put(man->move);
343 man->move = NULL;
344 }
345
346 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk);
347 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk);
348
349 void ttm_resource_add_bulk_move(struct ttm_resource *res,
350 struct ttm_buffer_object *bo);
351 void ttm_resource_del_bulk_move(struct ttm_resource *res,
352 struct ttm_buffer_object *bo);
353 void ttm_resource_move_to_lru_tail(struct ttm_resource *res);
354
355 void ttm_resource_init(struct ttm_buffer_object *bo,
356 const struct ttm_place *place,
357 struct ttm_resource *res);
358 void ttm_resource_fini(struct ttm_resource_manager *man,
359 struct ttm_resource *res);
360
361 int ttm_resource_alloc(struct ttm_buffer_object *bo,
362 const struct ttm_place *place,
363 struct ttm_resource **res);
364 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res);
365 bool ttm_resource_intersects(struct ttm_device *bdev,
366 struct ttm_resource *res,
367 const struct ttm_place *place,
368 size_t size);
369 bool ttm_resource_compatible(struct ttm_device *bdev,
370 struct ttm_resource *res,
371 const struct ttm_place *place,
372 size_t size);
373 bool ttm_resource_compat(struct ttm_resource *res,
374 struct ttm_placement *placement);
375 void ttm_resource_set_bo(struct ttm_resource *res,
376 struct ttm_buffer_object *bo);
377
378 void ttm_resource_manager_init(struct ttm_resource_manager *man,
379 struct ttm_device *bdev,
380 uint64_t size);
381
382 int ttm_resource_manager_evict_all(struct ttm_device *bdev,
383 struct ttm_resource_manager *man);
384
385 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man);
386 void ttm_resource_manager_debug(struct ttm_resource_manager *man,
387 struct drm_printer *p);
388
389 struct ttm_resource *
390 ttm_resource_manager_first(struct ttm_resource_manager *man,
391 struct ttm_resource_cursor *cursor);
392 struct ttm_resource *
393 ttm_resource_manager_next(struct ttm_resource_manager *man,
394 struct ttm_resource_cursor *cursor,
395 struct ttm_resource *res);
396
397 /**
398 * ttm_resource_manager_for_each_res - iterate over all resources
399 * @man: the resource manager
400 * @cursor: struct ttm_resource_cursor for the current position
401 * @res: the current resource
402 *
403 * Iterate over all the evictable resources in a resource manager.
404 */
405 #define ttm_resource_manager_for_each_res(man, cursor, res) \
406 for (res = ttm_resource_manager_first(man, cursor); res; \
407 res = ttm_resource_manager_next(man, cursor, res))
408
409 struct ttm_kmap_iter *
410 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
411 struct io_mapping *iomap,
412 struct sg_table *st,
413 resource_size_t start);
414
415 struct ttm_kmap_iter_linear_io;
416
417 struct ttm_kmap_iter *
418 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
419 struct ttm_device *bdev,
420 struct ttm_resource *mem);
421
422 void ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
423 struct ttm_device *bdev,
424 struct ttm_resource *mem);
425
426 #ifdef __linux__
427 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
428 struct dentry * parent,
429 const char *name);
430 #endif
431 #endif
432