1 /*
2 * Copyright (c) 2011 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 *
23 * Authors:
24 * Chris Wilson <chris@chris-wilson.co.uk>
25 *
26 */
27
28 #ifndef KGEM_H
29 #define KGEM_H
30
31 #include <stdint.h>
32 #include <stdbool.h>
33 #include <stdarg.h>
34
35 #include <i915_drm.h>
36
37 #include "compiler.h"
38 #include "debug.h"
39
40 struct kgem_bo {
41 struct kgem_request *rq;
42 #define RQ(rq) ((struct kgem_request *)((uintptr_t)(rq) & ~3))
43 #define RQ_RING(rq) ((uintptr_t)(rq) & 3)
44 #define RQ_IS_BLT(rq) (RQ_RING(rq) == KGEM_BLT)
45 #define RQ_IS_RENDER(rq) (RQ_RING(rq) == KGEM_RENDER)
46 #define MAKE_REQUEST(rq, ring) ((struct kgem_request *)((uintptr_t)(rq) | (ring)))
47
48 struct drm_i915_gem_exec_object2 *exec;
49
50 struct kgem_bo *proxy;
51
52 struct list list;
53 struct list request;
54 struct list vma;
55
56 void *map__cpu;
57 void *map__gtt;
58 void *map__wc;
59 #define MAP(ptr) ((void*)((uintptr_t)(ptr) & ~3))
60
61 struct kgem_bo_binding {
62 struct kgem_bo_binding *next;
63 uint32_t format;
64 uint16_t offset;
65 } binding;
66
67 uint64_t presumed_offset;
68 uint32_t unique_id;
69 uint32_t refcnt;
70 uint32_t handle;
71 uint32_t target_handle;
72 uint32_t delta;
73 uint32_t active_scanout;
74 union {
75 struct {
76 uint32_t count:27;
77 #define PAGE_SIZE 4096
78 uint32_t bucket:5;
79 #define NUM_CACHE_BUCKETS 16
80 #define MAX_CACHE_SIZE (1 << (NUM_CACHE_BUCKETS+12))
81 } pages;
82 uint32_t bytes;
83 } size;
84 uint32_t pitch : 18; /* max 128k */
85 uint32_t tiling : 2;
86 uint32_t reusable : 1;
87 uint32_t gpu_dirty : 1;
88 uint32_t gtt_dirty : 1;
89 uint32_t domain : 2;
90 uint32_t needs_flush : 1;
91 uint32_t snoop : 1;
92 uint32_t io : 1;
93 uint32_t flush : 1;
94 uint32_t scanout : 1;
95 uint32_t prime : 1;
96 uint32_t purged : 1;
97 };
98 #define DOMAIN_NONE 0
99 #define DOMAIN_CPU 1
100 #define DOMAIN_GTT 2
101 #define DOMAIN_GPU 3
102
103 struct kgem_request {
104 struct list list;
105 struct kgem_bo *bo;
106 struct list buffers;
107 unsigned ring;
108 };
109
110 enum {
111 MAP_GTT = 0,
112 MAP_CPU,
113 NUM_MAP_TYPES,
114 };
115
116 typedef void (*memcpy_box_func)(const void *src, void *dst, int bpp,
117 int32_t src_stride, int32_t dst_stride,
118 int16_t src_x, int16_t src_y,
119 int16_t dst_x, int16_t dst_y,
120 uint16_t width, uint16_t height);
121
122 struct kgem {
123 unsigned wedged;
124 int fd;
125 unsigned gen;
126
127 uint32_t unique_id;
128
129 uint16_t nbatch;
130 uint16_t surface;
131 uint16_t nexec;
132 uint16_t nreloc;
133 uint16_t nreloc__self;
134 uint16_t nfence;
135 uint16_t batch_size;
136
137 uint32_t *batch;
138
139 enum kgem_mode {
140 /* order matches I915_EXEC_RING ordering */
141 KGEM_NONE = 0,
142 KGEM_RENDER,
143 KGEM_BSD,
144 KGEM_BLT,
145 } mode, ring;
146
147 struct list flushing;
148 struct list large;
149 struct list large_inactive;
150 struct list active[NUM_CACHE_BUCKETS][3];
151 struct list inactive[NUM_CACHE_BUCKETS];
152 struct list pinned_batches[2];
153 struct list snoop;
154 struct list scanout;
155 struct list batch_buffers, active_buffers;
156
157 struct list requests[2];
158 struct kgem_request *fence[2];
159 struct kgem_request *next_request;
160 struct kgem_request static_request;
161
162 struct {
163 struct list inactive[NUM_CACHE_BUCKETS];
164 int16_t count;
165 } vma[NUM_MAP_TYPES];
166
167 uint32_t bcs_state;
168
169 uint32_t batch_flags;
170 uint32_t batch_flags_base;
171 #define I915_EXEC_SECURE (1<<9)
172 #define LOCAL_EXEC_OBJECT_WRITE (1<<2)
173
174 uint32_t flush:1;
175 uint32_t need_expire:1;
176 uint32_t need_purge:1;
177 uint32_t need_retire:1;
178 uint32_t need_throttle:1;
179 uint32_t needs_semaphore:1;
180 uint32_t needs_reservation:1;
181 uint32_t scanout_busy:1;
182 uint32_t busy:1;
183
184 uint32_t has_create2 :1;
185 uint32_t has_userptr :1;
186 uint32_t has_blt :1;
187 uint32_t has_relaxed_fencing :1;
188 uint32_t has_relaxed_delta :1;
189 uint32_t has_semaphores :1;
190 uint32_t has_secure_batches :1;
191 uint32_t has_pinned_batches :1;
192 uint32_t has_caching :1;
193 uint32_t has_coherent_mmap_gtt :1;
194 uint32_t has_full_ppgtt :1;
195 uint32_t has_llc :1;
196 uint32_t has_wt :1;
197 uint32_t has_no_reloc :1;
198 uint32_t has_handle_lut :1;
199 uint32_t has_wc_mmap :1;
200 uint32_t has_dirtyfb :1;
201
202 uint32_t can_fence :1;
203 uint32_t can_blt_cpu :1;
204 uint32_t can_blt_y :1;
205 uint32_t can_render_y :1;
206 uint32_t can_scanout_y :1;
207
208 uint32_t needs_dirtyfb :1;
209
210 uint16_t fence_max;
211 uint16_t half_cpu_cache_pages;
212 uint32_t aperture_total, aperture_high, aperture_low, aperture_mappable, aperture_fenceable;
213 uint32_t aperture, aperture_fenced, aperture_max_fence;
214 uint32_t max_upload_tile_size, max_copy_tile_size;
215 uint32_t max_gpu_size, max_cpu_size;
216 uint32_t large_object_size, max_object_size;
217 uint32_t buffer_size;
218
219 void (*context_switch)(struct kgem *kgem, int new_mode);
220 void (*retire)(struct kgem *kgem);
221 void (*expire)(struct kgem *kgem);
222
223 memcpy_box_func memcpy_to_tiled_x;
224 memcpy_box_func memcpy_from_tiled_x;
225 memcpy_box_func memcpy_between_tiled_x;
226
227 struct kgem_bo *batch_bo;
228
229 uint16_t reloc__self[256];
230 struct drm_i915_gem_exec_object2 exec[384] page_aligned;
231 struct drm_i915_gem_relocation_entry reloc[8192] page_aligned;
232
233 #ifdef DEBUG_MEMORY
234 struct {
235 int bo_allocs;
236 size_t bo_bytes;
237 } debug_memory;
238 #endif
239 };
240
241 #define KGEM_MAX_DEFERRED_VBO 16
242
243 #define KGEM_BATCH_RESERVED 8 /* LRI(SWCTRL) + END */
244 #define KGEM_RELOC_RESERVED (KGEM_MAX_DEFERRED_VBO)
245 #define KGEM_EXEC_RESERVED (1+KGEM_MAX_DEFERRED_VBO)
246
247 #ifndef ARRAY_SIZE
248 #define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0]))
249 #endif
250
251 #define KGEM_BATCH_SIZE(K) ((K)->batch_size-KGEM_BATCH_RESERVED)
252 #define KGEM_EXEC_SIZE(K) (int)(ARRAY_SIZE((K)->exec)-KGEM_EXEC_RESERVED)
253 #define KGEM_RELOC_SIZE(K) (int)(ARRAY_SIZE((K)->reloc)-KGEM_RELOC_RESERVED)
254
255 void kgem_init(struct kgem *kgem, int fd, struct pci_device *dev, unsigned gen);
256 void kgem_reset(struct kgem *kgem);
257
258 struct kgem_bo *kgem_create_map(struct kgem *kgem,
259 void *ptr, uint32_t size,
260 bool read_only);
261
262 struct kgem_bo *kgem_create_for_name(struct kgem *kgem, uint32_t name);
263 struct kgem_bo *kgem_create_for_prime(struct kgem *kgem, int name, uint32_t size);
264 int kgem_bo_export_to_prime(struct kgem *kgem, struct kgem_bo *bo);
265
266 struct kgem_bo *kgem_create_linear(struct kgem *kgem, int size, unsigned flags);
267 struct kgem_bo *kgem_create_proxy(struct kgem *kgem,
268 struct kgem_bo *target,
269 int offset, int length);
270
271 struct kgem_bo *kgem_upload_source_image(struct kgem *kgem,
272 const void *data,
273 const BoxRec *box,
274 int stride, int bpp);
275 void kgem_proxy_bo_attach(struct kgem_bo *bo, struct kgem_bo **ptr);
276
277 int kgem_choose_tiling(struct kgem *kgem,
278 int tiling, int width, int height, int bpp);
279 unsigned kgem_can_create_2d(struct kgem *kgem, int width, int height, int depth);
280 #define KGEM_CAN_CREATE_GPU 0x1
281 #define KGEM_CAN_CREATE_CPU 0x2
282 #define KGEM_CAN_CREATE_LARGE 0x4
283 #define KGEM_CAN_CREATE_GTT 0x8
284 #define KGEM_CAN_CREATE_TILED 0x10
285
286 bool kgem_check_surface_size(struct kgem *kgem,
287 uint32_t width,
288 uint32_t height,
289 uint32_t bpp,
290 uint32_t tiling,
291 uint32_t pitch,
292 uint32_t size);
293
294 struct kgem_bo *
295 kgem_replace_bo(struct kgem *kgem,
296 struct kgem_bo *src,
297 uint32_t width,
298 uint32_t height,
299 uint32_t pitch,
300 uint32_t bpp);
301 enum {
302 CREATE_EXACT = 0x1,
303 CREATE_INACTIVE = 0x2,
304 CREATE_CPU_MAP = 0x4,
305 CREATE_GTT_MAP = 0x8,
306 CREATE_SCANOUT = 0x10,
307 CREATE_PRIME = 0x20,
308 CREATE_TEMPORARY = 0x40,
309 CREATE_CACHED = 0x80,
310 CREATE_UNCACHED = 0x100,
311 CREATE_NO_RETIRE = 0x200,
312 CREATE_NO_THROTTLE = 0x400,
313 };
314 struct kgem_bo *kgem_create_2d(struct kgem *kgem,
315 int width,
316 int height,
317 int bpp,
318 int tiling,
319 uint32_t flags);
320 struct kgem_bo *kgem_create_cpu_2d(struct kgem *kgem,
321 int width,
322 int height,
323 int bpp,
324 uint32_t flags);
325
326 bool kgem_bo_convert_to_gpu(struct kgem *kgem,
327 struct kgem_bo *bo,
328 unsigned flags);
329
330 bool kgem_bo_is_fenced(struct kgem *kgem, struct kgem_bo *bo);
331 uint32_t kgem_bo_get_binding(struct kgem_bo *bo, uint32_t format);
332 void kgem_bo_set_binding(struct kgem_bo *bo, uint32_t format, uint16_t offset);
333
334 bool kgem_retire(struct kgem *kgem);
335 void kgem_retire__buffers(struct kgem *kgem);
336
kgem_bo_discard_cache(struct kgem_bo * bo,bool force)337 static inline bool kgem_bo_discard_cache(struct kgem_bo *bo, bool force)
338 {
339 if (bo == NULL || bo->proxy == NULL)
340 return false;
341
342 if (force)
343 return true;
344
345 if (bo->proxy->rq)
346 return false;
347
348 return bo->snoop;
349 }
350
351 bool __kgem_ring_is_idle(struct kgem *kgem, int ring);
kgem_ring_is_idle(struct kgem * kgem,int ring)352 static inline bool kgem_ring_is_idle(struct kgem *kgem, int ring)
353 {
354 ring = ring == KGEM_BLT;
355
356 if (kgem->needs_semaphore &&
357 !list_is_empty(&kgem->requests[!ring]) &&
358 !__kgem_ring_is_idle(kgem, !ring))
359 return false;
360
361 if (list_is_empty(&kgem->requests[ring]))
362 return true;
363
364 return __kgem_ring_is_idle(kgem, ring);
365 }
366
kgem_is_idle(struct kgem * kgem)367 static inline bool kgem_is_idle(struct kgem *kgem)
368 {
369 if (!kgem->need_retire)
370 return true;
371
372 return kgem_ring_is_idle(kgem, kgem->ring);
373 }
374
__kgem_ring_empty(struct kgem * kgem)375 static inline bool __kgem_ring_empty(struct kgem *kgem)
376 {
377 return list_is_empty(&kgem->requests[kgem->ring == KGEM_BLT]);
378 }
379
380 void _kgem_submit(struct kgem *kgem);
kgem_submit(struct kgem * kgem)381 static inline void kgem_submit(struct kgem *kgem)
382 {
383 if (kgem->nbatch)
384 _kgem_submit(kgem);
385 }
386
kgem_bo_submit(struct kgem * kgem,struct kgem_bo * bo)387 static inline void kgem_bo_submit(struct kgem *kgem, struct kgem_bo *bo)
388 {
389 if (bo->exec == NULL)
390 return;
391
392 assert(bo->refcnt);
393 _kgem_submit(kgem);
394 }
395
396 void kgem_scanout_flush(struct kgem *kgem, struct kgem_bo *bo);
397
kgem_bo_reference(struct kgem_bo * bo)398 static inline struct kgem_bo *kgem_bo_reference(struct kgem_bo *bo)
399 {
400 assert(bo->refcnt);
401 bo->refcnt++;
402 return bo;
403 }
404
405 void _kgem_bo_destroy(struct kgem *kgem, struct kgem_bo *bo);
kgem_bo_destroy(struct kgem * kgem,struct kgem_bo * bo)406 static inline void kgem_bo_destroy(struct kgem *kgem, struct kgem_bo *bo)
407 {
408 assert(bo->refcnt);
409 assert(bo->refcnt > bo->active_scanout);
410 if (--bo->refcnt == 0)
411 _kgem_bo_destroy(kgem, bo);
412 }
413
414 void kgem_clear_dirty(struct kgem *kgem);
415
kgem_set_mode(struct kgem * kgem,enum kgem_mode mode,struct kgem_bo * bo)416 static inline void kgem_set_mode(struct kgem *kgem,
417 enum kgem_mode mode,
418 struct kgem_bo *bo)
419 {
420 warn_unless(!kgem->wedged);
421
422 #if DEBUG_FLUSH_BATCH
423 kgem_submit(kgem);
424 #endif
425
426 if (kgem->nreloc && bo->rq == NULL && kgem_ring_is_idle(kgem, kgem->ring)) {
427 DBG(("%s: flushing before new bo\n", __FUNCTION__));
428 _kgem_submit(kgem);
429 }
430
431 if (kgem->mode == mode)
432 return;
433
434 kgem->context_switch(kgem, mode);
435 kgem->mode = mode;
436 }
437
_kgem_set_mode(struct kgem * kgem,enum kgem_mode mode)438 static inline void _kgem_set_mode(struct kgem *kgem, enum kgem_mode mode)
439 {
440 assert(kgem->mode == KGEM_NONE);
441 assert(kgem->nbatch == 0);
442 warn_unless(!kgem->wedged);
443 kgem->context_switch(kgem, mode);
444 kgem->mode = mode;
445 }
446
kgem_batch_space(struct kgem * kgem)447 static inline int kgem_batch_space(struct kgem *kgem)
448 {
449 int rem = kgem->surface - kgem->nbatch;
450 assert(rem > 0);
451 return rem - KGEM_BATCH_RESERVED;
452 }
453
kgem_check_batch(struct kgem * kgem,int num_dwords)454 static inline bool kgem_check_batch(struct kgem *kgem, int num_dwords)
455 {
456 assert(num_dwords > 0);
457 assert(kgem->nbatch < kgem->surface);
458 assert(kgem->surface <= kgem->batch_size);
459 return likely(kgem->nbatch + num_dwords + KGEM_BATCH_RESERVED <= kgem->surface);
460 }
461
kgem_check_reloc(struct kgem * kgem,int n)462 static inline bool kgem_check_reloc(struct kgem *kgem, int n)
463 {
464 assert(kgem->nreloc <= KGEM_RELOC_SIZE(kgem));
465 return likely(kgem->nreloc + n <= KGEM_RELOC_SIZE(kgem));
466 }
467
kgem_check_exec(struct kgem * kgem,int n)468 static inline bool kgem_check_exec(struct kgem *kgem, int n)
469 {
470 assert(kgem->nexec <= KGEM_EXEC_SIZE(kgem));
471 return likely(kgem->nexec + n <= KGEM_EXEC_SIZE(kgem));
472 }
473
kgem_check_reloc_and_exec(struct kgem * kgem,int n)474 static inline bool kgem_check_reloc_and_exec(struct kgem *kgem, int n)
475 {
476 return kgem_check_reloc(kgem, n) && kgem_check_exec(kgem, n);
477 }
478
kgem_check_batch_with_surfaces(struct kgem * kgem,int num_dwords,int num_surfaces)479 static inline bool kgem_check_batch_with_surfaces(struct kgem *kgem,
480 int num_dwords,
481 int num_surfaces)
482 {
483 return (int)(kgem->nbatch + num_dwords + KGEM_BATCH_RESERVED) <= (int)(kgem->surface - num_surfaces*8) &&
484 kgem_check_reloc(kgem, num_surfaces) &&
485 kgem_check_exec(kgem, num_surfaces);
486 }
487
kgem_get_batch(struct kgem * kgem)488 static inline uint32_t *kgem_get_batch(struct kgem *kgem)
489 {
490 if (kgem->nreloc) {
491 unsigned mode = kgem->mode;
492 _kgem_submit(kgem);
493 _kgem_set_mode(kgem, mode);
494 }
495
496 return kgem->batch + kgem->nbatch;
497 }
498
499 bool kgem_check_bo(struct kgem *kgem, ...) __attribute__((sentinel(0)));
500 bool kgem_check_bo_fenced(struct kgem *kgem, struct kgem_bo *bo);
501 bool kgem_check_many_bo_fenced(struct kgem *kgem, ...) __attribute__((sentinel(0)));
502
503 #define KGEM_RELOC_FENCED 0x8000
504 uint32_t kgem_add_reloc(struct kgem *kgem,
505 uint32_t pos,
506 struct kgem_bo *bo,
507 uint32_t read_write_domains,
508 uint32_t delta);
509 uint64_t kgem_add_reloc64(struct kgem *kgem,
510 uint32_t pos,
511 struct kgem_bo *bo,
512 uint32_t read_write_domains,
513 uint64_t delta);
514
515 void *kgem_bo_map(struct kgem *kgem, struct kgem_bo *bo);
516 void *kgem_bo_map__async(struct kgem *kgem, struct kgem_bo *bo);
517 void *kgem_bo_map__gtt(struct kgem *kgem, struct kgem_bo *bo);
518 void *kgem_bo_map__wc(struct kgem *kgem, struct kgem_bo *bo);
519 void kgem_bo_sync__gtt(struct kgem *kgem, struct kgem_bo *bo);
520 void *kgem_bo_map__debug(struct kgem *kgem, struct kgem_bo *bo);
521 void *kgem_bo_map__cpu(struct kgem *kgem, struct kgem_bo *bo);
522 void kgem_bo_sync__cpu(struct kgem *kgem, struct kgem_bo *bo);
523 void kgem_bo_sync__cpu_full(struct kgem *kgem, struct kgem_bo *bo, bool write);
524 uint32_t kgem_bo_flink(struct kgem *kgem, struct kgem_bo *bo);
525
526 bool kgem_bo_write(struct kgem *kgem, struct kgem_bo *bo,
527 const void *data, int length);
528
529 int kgem_bo_fenced_size(struct kgem *kgem, struct kgem_bo *bo);
530 void kgem_get_tile_size(struct kgem *kgem, int tiling, int pitch,
531 int *tile_width, int *tile_height, int *tile_size);
532
__kgem_buffer_size(struct kgem_bo * bo)533 static inline int __kgem_buffer_size(struct kgem_bo *bo)
534 {
535 assert(bo->proxy != NULL);
536 return bo->size.bytes;
537 }
538
__kgem_bo_size(struct kgem_bo * bo)539 static inline int __kgem_bo_size(struct kgem_bo *bo)
540 {
541 assert(bo->proxy == NULL);
542 return PAGE_SIZE * bo->size.pages.count;
543 }
544
__kgem_bo_num_pages(struct kgem_bo * bo)545 static inline int __kgem_bo_num_pages(struct kgem_bo *bo)
546 {
547 assert(bo->proxy == NULL);
548 return bo->size.pages.count;
549 }
550
kgem_bo_size(struct kgem_bo * bo)551 static inline int kgem_bo_size(struct kgem_bo *bo)
552 {
553 if (bo->proxy)
554 return __kgem_buffer_size(bo);
555 else
556 return __kgem_bo_size(bo);
557 }
558
kgem_bo_blt_pitch_is_ok(struct kgem * kgem,struct kgem_bo * bo)559 static inline bool kgem_bo_blt_pitch_is_ok(struct kgem *kgem,
560 struct kgem_bo *bo)
561 {
562 int pitch = bo->pitch;
563
564 if (kgem->gen >= 0100 && pitch & (1 << 4)) { /* bdw is broken */
565 DBG(("%s: can not blt to handle=%d, pitch=%d\n",
566 __FUNCTION__, bo->handle, pitch));
567 return false;
568 }
569
570 if (kgem->gen >= 040 && bo->tiling)
571 pitch /= 4;
572 if (pitch > MAXSHORT) {
573 DBG(("%s: can not blt to handle=%d, adjusted pitch=%d\n",
574 __FUNCTION__, bo->handle, pitch));
575 return false;
576 }
577
578 return true;
579 }
580
kgem_bo_can_blt(struct kgem * kgem,struct kgem_bo * bo)581 static inline bool kgem_bo_can_blt(struct kgem *kgem,
582 struct kgem_bo *bo)
583 {
584 assert(bo->refcnt);
585
586 if (bo->tiling == I915_TILING_Y && !kgem->can_blt_y) {
587 DBG(("%s: can not blt to handle=%d, tiling=Y\n",
588 __FUNCTION__, bo->handle));
589 return false;
590 }
591
592 if (kgem->gen >= 0100 && bo->proxy && bo->delta & 63) {
593 DBG(("%s: can not blt to handle=%d, delta=%d\n",
594 __FUNCTION__, bo->handle, bo->delta));
595 return false;
596 }
597
598 return kgem_bo_blt_pitch_is_ok(kgem, bo);
599 }
600
601 void __kgem_bcs_set_tiling(struct kgem *kgem,
602 struct kgem_bo *src,
603 struct kgem_bo *dst);
604
kgem_bcs_set_tiling(struct kgem * kgem,struct kgem_bo * src,struct kgem_bo * dst)605 inline static void kgem_bcs_set_tiling(struct kgem *kgem,
606 struct kgem_bo *src,
607 struct kgem_bo *dst)
608 {
609 assert(kgem->mode == KGEM_BLT);
610
611 if (!kgem->can_blt_y)
612 return;
613
614 __kgem_bcs_set_tiling(kgem, src, dst);
615 }
616
kgem_bo_is_snoop(struct kgem_bo * bo)617 static inline bool kgem_bo_is_snoop(struct kgem_bo *bo)
618 {
619 assert(bo->refcnt);
620 while (bo->proxy)
621 bo = bo->proxy;
622 return bo->snoop;
623 }
624
625 void kgem_bo_undo(struct kgem *kgem, struct kgem_bo *bo);
626 void kgem_bo_pair_undo(struct kgem *kgem, struct kgem_bo *a, struct kgem_bo *b);
627
628 bool __kgem_busy(struct kgem *kgem, int handle);
629
kgem_bo_mark_busy(struct kgem * kgem,struct kgem_bo * bo,int ring)630 static inline void kgem_bo_mark_busy(struct kgem *kgem, struct kgem_bo *bo, int ring)
631 {
632 assert(bo->refcnt);
633 bo->needs_flush = true;
634 if (bo->rq) {
635 bo->rq = MAKE_REQUEST(RQ(bo->rq), ring);
636 } else {
637 bo->rq = MAKE_REQUEST(kgem, ring);
638 list_add(&bo->request, &kgem->flushing);
639 kgem->need_retire = true;
640 }
641 }
642
__kgem_bo_clear_dirty(struct kgem_bo * bo)643 static inline void __kgem_bo_clear_dirty(struct kgem_bo *bo)
644 {
645 DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
646
647 bo->domain = DOMAIN_NONE;
648 bo->needs_flush = false;
649 bo->gtt_dirty = false;
650 }
651
__kgem_bo_clear_busy(struct kgem_bo * bo)652 inline static void __kgem_bo_clear_busy(struct kgem_bo *bo)
653 {
654 DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
655 bo->rq = NULL;
656 list_del(&bo->request);
657
658 __kgem_bo_clear_dirty(bo);
659 }
660
kgem_bo_is_busy(struct kgem_bo * bo)661 static inline bool kgem_bo_is_busy(struct kgem_bo *bo)
662 {
663 DBG(("%s: handle=%d, domain: %d exec? %d, rq? %d\n", __FUNCTION__,
664 bo->handle, bo->domain, bo->exec != NULL, bo->rq != NULL));
665 assert(bo->refcnt);
666 return bo->rq;
667 }
668
669 bool __kgem_retire_requests_upto(struct kgem *kgem, struct kgem_bo *bo);
__kgem_bo_is_busy(struct kgem * kgem,struct kgem_bo * bo)670 static inline bool __kgem_bo_is_busy(struct kgem *kgem, struct kgem_bo *bo)
671 {
672 DBG(("%s: handle=%d, domain: %d exec? %d, rq? %d\n", __FUNCTION__,
673 bo->handle, bo->domain, bo->exec != NULL, bo->rq != NULL));
674 assert(bo->refcnt);
675
676 if (bo->exec)
677 return true;
678
679 if (bo->rq == NULL)
680 return false;
681
682 if (__kgem_busy(kgem, bo->handle))
683 return true;
684
685 return __kgem_retire_requests_upto(kgem, bo);
686 }
687
kgem_bo_is_render(struct kgem_bo * bo)688 static inline bool kgem_bo_is_render(struct kgem_bo *bo)
689 {
690 DBG(("%s: handle=%d, tiling=%d, rq? %d [%d]\n", __FUNCTION__,
691 bo->handle, bo->tiling, bo->rq != NULL, (int)RQ_RING(bo->rq)));
692 assert(bo->refcnt);
693 if (bo->tiling >= I915_TILING_Y)
694 return true;
695
696 return bo->rq && RQ_RING(bo->rq) != KGEM_BLT;
697 }
698
kgem_bo_is_blt(struct kgem_bo * bo)699 static inline bool kgem_bo_is_blt(struct kgem_bo *bo)
700 {
701 DBG(("%s: handle=%d, rq? %d\n", __FUNCTION__,
702 bo->handle, bo->rq != NULL, (int)RQ_RING(bo->rq)));
703 assert(bo->refcnt);
704 return RQ_RING(bo->rq) == KGEM_BLT;
705 }
706
kgem_bo_mark_unreusable(struct kgem_bo * bo)707 static inline void kgem_bo_mark_unreusable(struct kgem_bo *bo)
708 {
709 assert(bo->refcnt);
710 while (bo->proxy) {
711 bo->flush = true;
712 bo = bo->proxy;
713 assert(bo->refcnt);
714 }
715 bo->flush = true;
716 bo->reusable = false;
717 }
718
kgem_bo_is_dirty(struct kgem_bo * bo)719 static inline bool kgem_bo_is_dirty(struct kgem_bo *bo)
720 {
721 if (bo == NULL)
722 return false;
723
724 assert(bo->refcnt);
725 return bo->gpu_dirty;
726 }
727
kgem_bo_unclean(struct kgem * kgem,struct kgem_bo * bo)728 static inline void kgem_bo_unclean(struct kgem *kgem, struct kgem_bo *bo)
729 {
730 /* The bo is outside of our control, so presume it is written to */
731 bo->needs_flush = true;
732 if (bo->rq == NULL)
733 bo->rq = (void *)kgem;
734
735 if (bo->domain != DOMAIN_GPU)
736 bo->domain = DOMAIN_NONE;
737 }
738
__kgem_bo_mark_dirty(struct kgem_bo * bo)739 static inline void __kgem_bo_mark_dirty(struct kgem_bo *bo)
740 {
741 DBG(("%s: handle=%d (proxy? %d)\n", __FUNCTION__,
742 bo->handle, bo->proxy != NULL));
743
744 assert(bo->refcnt);
745 assert(bo->exec);
746 assert(bo->rq);
747
748 bo->exec->flags |= LOCAL_EXEC_OBJECT_WRITE;
749 bo->needs_flush = bo->gpu_dirty = true;
750 list_move(&bo->request, &RQ(bo->rq)->buffers);
751 }
752
kgem_bo_mark_dirty(struct kgem_bo * bo)753 static inline void kgem_bo_mark_dirty(struct kgem_bo *bo)
754 {
755 assert(bo->refcnt);
756 do {
757 assert(bo->exec);
758 assert(bo->rq);
759
760 if (bo->gpu_dirty)
761 return;
762
763 __kgem_bo_mark_dirty(bo);
764 } while ((bo = bo->proxy));
765 }
766
kgem_bo_mapped(struct kgem * kgem,struct kgem_bo * bo)767 static inline bool kgem_bo_mapped(struct kgem *kgem, struct kgem_bo *bo)
768 {
769 DBG(("%s: handle=%d, map=%p:%p, tiling=%d, domain=%d\n",
770 __FUNCTION__, bo->handle, bo->map__gtt, bo->map__cpu, bo->tiling, bo->domain));
771 assert(bo->proxy == NULL);
772
773 if (bo->tiling == I915_TILING_NONE && (bo->domain == DOMAIN_CPU || kgem->has_llc))
774 return bo->map__cpu != NULL;
775
776 if (bo->tiling == I915_TILING_NONE && bo->map__wc)
777 return true;
778
779 return bo->map__gtt != NULL;
780 }
781
kgem_bo_can_map(struct kgem * kgem,struct kgem_bo * bo)782 static inline bool kgem_bo_can_map(struct kgem *kgem, struct kgem_bo *bo)
783 {
784 DBG(("%s: handle=%d, map=%p:%p:%p, tiling=%d, domain=%d, offset=%ld\n",
785 __FUNCTION__, bo->handle, bo->map__gtt, bo->map__wc, bo->map__cpu, bo->tiling, bo->domain, (long)bo->presumed_offset));
786
787 if (!bo->tiling && (kgem->has_llc || bo->domain == DOMAIN_CPU))
788 return true;
789
790 assert(bo->proxy == NULL);
791
792 if (bo->map__gtt != NULL)
793 return true;
794
795 if (kgem->gen == 021 && bo->tiling == I915_TILING_Y)
796 return false;
797
798 if (!bo->tiling && kgem->has_wc_mmap)
799 return true;
800
801 return __kgem_bo_num_pages(bo) <= kgem->aperture_mappable / 4;
802 }
803
kgem_bo_can_map__cpu(struct kgem * kgem,struct kgem_bo * bo,bool write)804 static inline bool kgem_bo_can_map__cpu(struct kgem *kgem,
805 struct kgem_bo *bo,
806 bool write)
807 {
808 DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
809 assert(bo->refcnt);
810
811 if (bo->purged || (bo->scanout && write)) {
812 DBG(("%s: no, writing to scanout? %d, or is stolen [inaccessible via CPU]? %d\n",
813 __FUNCTION__, bo->scanout && write, bo->purged));
814 return false;
815 }
816
817 if (kgem->has_llc) {
818 DBG(("%s: yes, has LLC and target is in LLC\n", __FUNCTION__));
819 return true;
820 }
821
822 DBG(("%s: non-LLC - CPU domain? %d, clean? %d\n",
823 __FUNCTION__, bo->domain == DOMAIN_CPU, !write || bo->exec == NULL));
824 if (bo->domain != DOMAIN_CPU)
825 return false;
826
827 return !write || bo->exec == NULL;
828 }
829
830 #define KGEM_BUFFER_WRITE 0x1
831 #define KGEM_BUFFER_INPLACE 0x2
832 #define KGEM_BUFFER_LAST 0x4
833
834 #define KGEM_BUFFER_WRITE_INPLACE (KGEM_BUFFER_WRITE | KGEM_BUFFER_INPLACE)
835
836 struct kgem_bo *kgem_create_buffer(struct kgem *kgem,
837 uint32_t size, uint32_t flags,
838 void **ret);
839 struct kgem_bo *kgem_create_buffer_2d(struct kgem *kgem,
840 int width, int height, int bpp,
841 uint32_t flags,
842 void **ret);
843 bool kgem_buffer_is_inplace(struct kgem_bo *bo);
844 void kgem_buffer_read_sync(struct kgem *kgem, struct kgem_bo *bo);
845
846 int kgem_is_wedged(struct kgem *kgem);
847 void kgem_throttle(struct kgem *kgem);
848 #define MAX_INACTIVE_TIME 10
849 bool kgem_expire_cache(struct kgem *kgem);
850 bool kgem_cleanup_cache(struct kgem *kgem);
851
852 void kgem_clean_scanout_cache(struct kgem *kgem);
853 void kgem_clean_large_cache(struct kgem *kgem);
854
855 #if HAS_DEBUG_FULL
856 void __kgem_batch_debug(struct kgem *kgem, uint32_t nbatch);
857 #else
__kgem_batch_debug(struct kgem * kgem,uint32_t nbatch)858 static inline void __kgem_batch_debug(struct kgem *kgem, uint32_t nbatch)
859 {
860 (void)kgem;
861 (void)nbatch;
862 }
863 #endif
864
865 static inline void
memcpy_to_tiled_x(struct kgem * kgem,const void * src,void * dst,int bpp,int32_t src_stride,int32_t dst_stride,int16_t src_x,int16_t src_y,int16_t dst_x,int16_t dst_y,uint16_t width,uint16_t height)866 memcpy_to_tiled_x(struct kgem *kgem,
867 const void *src, void *dst, int bpp,
868 int32_t src_stride, int32_t dst_stride,
869 int16_t src_x, int16_t src_y,
870 int16_t dst_x, int16_t dst_y,
871 uint16_t width, uint16_t height)
872 {
873 assert(kgem->memcpy_to_tiled_x);
874 assert(src_x >= 0 && src_y >= 0);
875 assert(dst_x >= 0 && dst_y >= 0);
876 assert(8*src_stride >= (src_x+width) * bpp);
877 assert(8*dst_stride >= (dst_x+width) * bpp);
878 return kgem->memcpy_to_tiled_x(src, dst, bpp,
879 src_stride, dst_stride,
880 src_x, src_y,
881 dst_x, dst_y,
882 width, height);
883 }
884
885 static inline void
memcpy_from_tiled_x(struct kgem * kgem,const void * src,void * dst,int bpp,int32_t src_stride,int32_t dst_stride,int16_t src_x,int16_t src_y,int16_t dst_x,int16_t dst_y,uint16_t width,uint16_t height)886 memcpy_from_tiled_x(struct kgem *kgem,
887 const void *src, void *dst, int bpp,
888 int32_t src_stride, int32_t dst_stride,
889 int16_t src_x, int16_t src_y,
890 int16_t dst_x, int16_t dst_y,
891 uint16_t width, uint16_t height)
892 {
893 assert(kgem->memcpy_from_tiled_x);
894 assert(src_x >= 0 && src_y >= 0);
895 assert(dst_x >= 0 && dst_y >= 0);
896 assert(8*src_stride >= (src_x+width) * bpp);
897 assert(8*dst_stride >= (dst_x+width) * bpp);
898 return kgem->memcpy_from_tiled_x(src, dst, bpp,
899 src_stride, dst_stride,
900 src_x, src_y,
901 dst_x, dst_y,
902 width, height);
903 }
904
905 void choose_memcpy_tiled_x(struct kgem *kgem, int swizzling, unsigned cpu);
906
907 #endif /* KGEM_H */
908