1 /*
2 * Copyright © 2017 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 /**
24 * @file crocus_resource.c
25 *
26 * Resources are images, buffers, and other objects used by the GPU.
27 *
28 * XXX: explain resources
29 */
30
31 #include <stdio.h>
32 #include <errno.h>
33 #include "pipe/p_defines.h"
34 #include "pipe/p_state.h"
35 #include "pipe/p_context.h"
36 #include "pipe/p_screen.h"
37 #include "util/os_memory.h"
38 #include "util/u_cpu_detect.h"
39 #include "util/u_inlines.h"
40 #include "util/format/u_format.h"
41 #include "util/u_threaded_context.h"
42 #include "util/u_transfer.h"
43 #include "util/u_transfer_helper.h"
44 #include "util/u_upload_mgr.h"
45 #include "util/ralloc.h"
46 #include "util/u_memory.h"
47 #include "crocus_batch.h"
48 #include "crocus_context.h"
49 #include "crocus_resource.h"
50 #include "crocus_screen.h"
51 #include "intel/dev/intel_debug.h"
52 #include "isl/isl.h"
53 #include "drm-uapi/drm_fourcc.h"
54 #include "drm-uapi/i915_drm.h"
55
56 enum modifier_priority {
57 MODIFIER_PRIORITY_INVALID = 0,
58 MODIFIER_PRIORITY_LINEAR,
59 MODIFIER_PRIORITY_X,
60 MODIFIER_PRIORITY_Y,
61 MODIFIER_PRIORITY_Y_CCS,
62 };
63
64 static const uint64_t priority_to_modifier[] = {
65 [MODIFIER_PRIORITY_INVALID] = DRM_FORMAT_MOD_INVALID,
66 [MODIFIER_PRIORITY_LINEAR] = DRM_FORMAT_MOD_LINEAR,
67 [MODIFIER_PRIORITY_X] = I915_FORMAT_MOD_X_TILED,
68 [MODIFIER_PRIORITY_Y] = I915_FORMAT_MOD_Y_TILED,
69 [MODIFIER_PRIORITY_Y_CCS] = I915_FORMAT_MOD_Y_TILED_CCS,
70 };
71
72 static bool
modifier_is_supported(const struct intel_device_info * devinfo,enum pipe_format pfmt,unsigned bind,uint64_t modifier)73 modifier_is_supported(const struct intel_device_info *devinfo,
74 enum pipe_format pfmt, unsigned bind,
75 uint64_t modifier)
76 {
77 /* XXX: do something real */
78 switch (modifier) {
79 case I915_FORMAT_MOD_Y_TILED_CCS:
80 return false;
81 case I915_FORMAT_MOD_Y_TILED:
82 if (bind & PIPE_BIND_SCANOUT)
83 return false;
84 return devinfo->ver >= 6;
85 case I915_FORMAT_MOD_X_TILED:
86 case DRM_FORMAT_MOD_LINEAR:
87 return true;
88 case DRM_FORMAT_MOD_INVALID:
89 default:
90 return false;
91 }
92 }
93
94 static uint64_t
select_best_modifier(struct intel_device_info * devinfo,const struct pipe_resource * templ,const uint64_t * modifiers,int count)95 select_best_modifier(struct intel_device_info *devinfo,
96 const struct pipe_resource *templ,
97 const uint64_t *modifiers,
98 int count)
99 {
100 enum modifier_priority prio = MODIFIER_PRIORITY_INVALID;
101
102 for (int i = 0; i < count; i++) {
103 if (!modifier_is_supported(devinfo, templ->format, templ->bind,
104 modifiers[i]))
105 continue;
106
107 switch (modifiers[i]) {
108 case I915_FORMAT_MOD_Y_TILED_CCS:
109 prio = MAX2(prio, MODIFIER_PRIORITY_Y_CCS);
110 break;
111 case I915_FORMAT_MOD_Y_TILED:
112 prio = MAX2(prio, MODIFIER_PRIORITY_Y);
113 break;
114 case I915_FORMAT_MOD_X_TILED:
115 prio = MAX2(prio, MODIFIER_PRIORITY_X);
116 break;
117 case DRM_FORMAT_MOD_LINEAR:
118 prio = MAX2(prio, MODIFIER_PRIORITY_LINEAR);
119 break;
120 case DRM_FORMAT_MOD_INVALID:
121 default:
122 break;
123 }
124 }
125
126 return priority_to_modifier[prio];
127 }
128
129 static enum isl_surf_dim
crocus_target_to_isl_surf_dim(enum pipe_texture_target target)130 crocus_target_to_isl_surf_dim(enum pipe_texture_target target)
131 {
132 switch (target) {
133 case PIPE_BUFFER:
134 case PIPE_TEXTURE_1D:
135 case PIPE_TEXTURE_1D_ARRAY:
136 return ISL_SURF_DIM_1D;
137 case PIPE_TEXTURE_2D:
138 case PIPE_TEXTURE_CUBE:
139 case PIPE_TEXTURE_RECT:
140 case PIPE_TEXTURE_2D_ARRAY:
141 case PIPE_TEXTURE_CUBE_ARRAY:
142 return ISL_SURF_DIM_2D;
143 case PIPE_TEXTURE_3D:
144 return ISL_SURF_DIM_3D;
145 case PIPE_MAX_TEXTURE_TYPES:
146 break;
147 }
148 unreachable("invalid texture type");
149 }
150
151 static isl_surf_usage_flags_t
pipe_bind_to_isl_usage(unsigned bindings)152 pipe_bind_to_isl_usage(unsigned bindings)
153 {
154 isl_surf_usage_flags_t usage = 0;
155
156 if (bindings & PIPE_BIND_RENDER_TARGET)
157 usage |= ISL_SURF_USAGE_RENDER_TARGET_BIT;
158
159 if (bindings & PIPE_BIND_SAMPLER_VIEW)
160 usage |= ISL_SURF_USAGE_TEXTURE_BIT;
161
162 if (bindings & (PIPE_BIND_SHADER_IMAGE | PIPE_BIND_SHADER_BUFFER))
163 usage |= ISL_SURF_USAGE_STORAGE_BIT;
164
165 if (bindings & PIPE_BIND_SCANOUT)
166 usage |= ISL_SURF_USAGE_DISPLAY_BIT;
167 return usage;
168 }
169
170 static bool
crocus_resource_configure_main(const struct crocus_screen * screen,struct crocus_resource * res,const struct pipe_resource * templ,uint64_t modifier,uint32_t row_pitch_B)171 crocus_resource_configure_main(const struct crocus_screen *screen,
172 struct crocus_resource *res,
173 const struct pipe_resource *templ,
174 uint64_t modifier, uint32_t row_pitch_B)
175 {
176 const struct intel_device_info *devinfo = &screen->devinfo;
177 const struct util_format_description *format_desc =
178 util_format_description(templ->format);
179 const bool has_depth = util_format_has_depth(format_desc);
180 isl_surf_usage_flags_t usage = pipe_bind_to_isl_usage(templ->bind);
181 isl_tiling_flags_t tiling_flags = ISL_TILING_ANY_MASK;
182
183 /* TODO: This used to be because there wasn't BLORP to handle Y-tiling. */
184 if (devinfo->ver < 6 && !util_format_is_depth_or_stencil(templ->format))
185 tiling_flags &= ~ISL_TILING_Y0_BIT;
186
187 if (modifier != DRM_FORMAT_MOD_INVALID) {
188 res->mod_info = isl_drm_modifier_get_info(modifier);
189
190 tiling_flags = 1 << res->mod_info->tiling;
191 } else {
192 if (templ->bind & PIPE_BIND_RENDER_TARGET && devinfo->ver < 6) {
193 modifier = I915_FORMAT_MOD_X_TILED;
194 res->mod_info = isl_drm_modifier_get_info(modifier);
195 tiling_flags = 1 << res->mod_info->tiling;
196 }
197 /* Use linear for staging buffers */
198 if (templ->usage == PIPE_USAGE_STAGING ||
199 templ->bind & (PIPE_BIND_LINEAR | PIPE_BIND_CURSOR) )
200 tiling_flags = ISL_TILING_LINEAR_BIT;
201 else if (templ->bind & PIPE_BIND_SCANOUT)
202 tiling_flags = screen->devinfo.has_tiling_uapi ?
203 ISL_TILING_X_BIT : ISL_TILING_LINEAR_BIT;
204 }
205
206 if (templ->target == PIPE_TEXTURE_CUBE ||
207 templ->target == PIPE_TEXTURE_CUBE_ARRAY)
208 usage |= ISL_SURF_USAGE_CUBE_BIT;
209
210 if (templ->usage != PIPE_USAGE_STAGING) {
211 if (templ->format == PIPE_FORMAT_S8_UINT)
212 usage |= ISL_SURF_USAGE_STENCIL_BIT;
213 else if (has_depth) {
214 /* combined DS only on gen4/5 */
215 if (devinfo->ver < 6) {
216 if (templ->format == PIPE_FORMAT_Z24X8_UNORM ||
217 templ->format == PIPE_FORMAT_Z24_UNORM_S8_UINT ||
218 templ->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT)
219 usage |= ISL_SURF_USAGE_STENCIL_BIT;
220 }
221 usage |= ISL_SURF_USAGE_DEPTH_BIT;
222 }
223
224 if (templ->format == PIPE_FORMAT_S8_UINT)
225 tiling_flags = ISL_TILING_W_BIT;
226 }
227
228 const enum isl_format format =
229 crocus_format_for_usage(&screen->devinfo, templ->format, usage).fmt;
230
231 if (row_pitch_B == 0 && templ->usage == PIPE_USAGE_STAGING &&
232 templ->target == PIPE_TEXTURE_2D &&
233 devinfo->ver < 6) {
234 /* align row pitch to 4 so we can keep using BLT engine */
235 row_pitch_B = util_format_get_stride(templ->format, templ->width0);
236 row_pitch_B = ALIGN(row_pitch_B, 4);
237 }
238
239 const struct isl_surf_init_info init_info = {
240 .dim = crocus_target_to_isl_surf_dim(templ->target),
241 .format = format,
242 .width = templ->width0,
243 .height = templ->height0,
244 .depth = templ->depth0,
245 .levels = templ->last_level + 1,
246 .array_len = templ->array_size,
247 .samples = MAX2(templ->nr_samples, 1),
248 .min_alignment_B = 0,
249 .row_pitch_B = row_pitch_B,
250 .usage = usage,
251 .tiling_flags = tiling_flags
252 };
253
254 if (!isl_surf_init_s(&screen->isl_dev, &res->surf, &init_info))
255 return false;
256
257 res->internal_format = templ->format;
258
259 return true;
260 }
261
262 static void
crocus_query_dmabuf_modifiers(struct pipe_screen * pscreen,enum pipe_format pfmt,int max,uint64_t * modifiers,unsigned int * external_only,int * count)263 crocus_query_dmabuf_modifiers(struct pipe_screen *pscreen,
264 enum pipe_format pfmt,
265 int max,
266 uint64_t *modifiers,
267 unsigned int *external_only,
268 int *count)
269 {
270 struct crocus_screen *screen = (void *) pscreen;
271 const struct intel_device_info *devinfo = &screen->devinfo;
272
273 uint64_t all_modifiers[] = {
274 DRM_FORMAT_MOD_LINEAR,
275 I915_FORMAT_MOD_X_TILED,
276 I915_FORMAT_MOD_Y_TILED,
277 I915_FORMAT_MOD_Y_TILED_CCS,
278 };
279
280 int supported_mods = 0;
281
282 for (int i = 0; i < ARRAY_SIZE(all_modifiers); i++) {
283 if (!modifier_is_supported(devinfo, pfmt, 0, all_modifiers[i]))
284 continue;
285
286 if (supported_mods < max) {
287 if (modifiers)
288 modifiers[supported_mods] = all_modifiers[i];
289
290 if (external_only)
291 external_only[supported_mods] = util_format_is_yuv(pfmt);
292 }
293
294 supported_mods++;
295 }
296
297 *count = supported_mods;
298 }
299
300 static struct pipe_resource *
crocus_resource_get_separate_stencil(struct pipe_resource * p_res)301 crocus_resource_get_separate_stencil(struct pipe_resource *p_res)
302 {
303 return _crocus_resource_get_separate_stencil(p_res);
304 }
305
306 static void
crocus_resource_set_separate_stencil(struct pipe_resource * p_res,struct pipe_resource * stencil)307 crocus_resource_set_separate_stencil(struct pipe_resource *p_res,
308 struct pipe_resource *stencil)
309 {
310 assert(util_format_has_depth(util_format_description(p_res->format)));
311 pipe_resource_reference(&p_res->next, stencil);
312 }
313
314 void
crocus_resource_disable_aux(struct crocus_resource * res)315 crocus_resource_disable_aux(struct crocus_resource *res)
316 {
317 crocus_bo_unreference(res->aux.bo);
318 free(res->aux.state);
319
320 res->aux.usage = ISL_AUX_USAGE_NONE;
321 res->aux.has_hiz = 0;
322 res->aux.surf.size_B = 0;
323 res->aux.surf.levels = 0;
324 res->aux.bo = NULL;
325 res->aux.state = NULL;
326 }
327
328 static void
crocus_resource_destroy(struct pipe_screen * screen,struct pipe_resource * resource)329 crocus_resource_destroy(struct pipe_screen *screen,
330 struct pipe_resource *resource)
331 {
332 struct crocus_resource *res = (struct crocus_resource *)resource;
333
334 if (resource->target == PIPE_BUFFER)
335 util_range_destroy(&res->valid_buffer_range);
336
337 if (res->shadow)
338 pipe_resource_reference((struct pipe_resource **)&res->shadow, NULL);
339 crocus_resource_disable_aux(res);
340
341 threaded_resource_deinit(resource);
342 crocus_bo_unreference(res->bo);
343 crocus_pscreen_unref(res->orig_screen);
344 free(res);
345 }
346
347 static struct crocus_resource *
crocus_alloc_resource(struct pipe_screen * pscreen,const struct pipe_resource * templ)348 crocus_alloc_resource(struct pipe_screen *pscreen,
349 const struct pipe_resource *templ)
350 {
351 struct crocus_resource *res = calloc(1, sizeof(struct crocus_resource));
352 if (!res)
353 return NULL;
354
355 res->base.b = *templ;
356 res->base.b.screen = pscreen;
357 res->orig_screen = crocus_pscreen_ref(pscreen);
358 pipe_reference_init(&res->base.b.reference, 1);
359 threaded_resource_init(&res->base.b);
360
361 if (templ->target == PIPE_BUFFER)
362 util_range_init(&res->valid_buffer_range);
363
364 return res;
365 }
366
367 unsigned
crocus_get_num_logical_layers(const struct crocus_resource * res,unsigned level)368 crocus_get_num_logical_layers(const struct crocus_resource *res, unsigned level)
369 {
370 if (res->surf.dim == ISL_SURF_DIM_3D)
371 return minify(res->surf.logical_level0_px.depth, level);
372 else
373 return res->surf.logical_level0_px.array_len;
374 }
375
376 static enum isl_aux_state **
create_aux_state_map(struct crocus_resource * res,enum isl_aux_state initial)377 create_aux_state_map(struct crocus_resource *res, enum isl_aux_state initial)
378 {
379 assert(res->aux.state == NULL);
380
381 uint32_t total_slices = 0;
382 for (uint32_t level = 0; level < res->surf.levels; level++)
383 total_slices += crocus_get_num_logical_layers(res, level);
384
385 const size_t per_level_array_size =
386 res->surf.levels * sizeof(enum isl_aux_state *);
387
388 /* We're going to allocate a single chunk of data for both the per-level
389 * reference array and the arrays of aux_state. This makes cleanup
390 * significantly easier.
391 */
392 const size_t total_size =
393 per_level_array_size + total_slices * sizeof(enum isl_aux_state);
394
395 void *data = malloc(total_size);
396 if (!data)
397 return NULL;
398
399 enum isl_aux_state **per_level_arr = data;
400 enum isl_aux_state *s = data + per_level_array_size;
401 for (uint32_t level = 0; level < res->surf.levels; level++) {
402 per_level_arr[level] = s;
403 const unsigned level_layers = crocus_get_num_logical_layers(res, level);
404 for (uint32_t a = 0; a < level_layers; a++)
405 *(s++) = initial;
406 }
407 assert((void *)s == data + total_size);
408
409 return per_level_arr;
410 }
411
412 /**
413 * Configure aux for the resource, but don't allocate it. For images which
414 * might be shared with modifiers, we must allocate the image and aux data in
415 * a single bo.
416 *
417 * Returns false on unexpected error (e.g. allocation failed, or invalid
418 * configuration result).
419 */
420 static bool
crocus_resource_configure_aux(struct crocus_screen * screen,struct crocus_resource * res,bool imported,uint64_t * aux_size_B,uint32_t * alloc_flags)421 crocus_resource_configure_aux(struct crocus_screen *screen,
422 struct crocus_resource *res, bool imported,
423 uint64_t *aux_size_B,
424 uint32_t *alloc_flags)
425 {
426 const struct intel_device_info *devinfo = &screen->devinfo;
427
428 /* Try to create the auxiliary surfaces allowed by the modifier or by
429 * the user if no modifier is specified.
430 */
431 assert(!res->mod_info || res->mod_info->aux_usage == ISL_AUX_USAGE_NONE);
432
433 const bool has_mcs = devinfo->ver >= 7 && !res->mod_info &&
434 isl_surf_get_mcs_surf(&screen->isl_dev, &res->surf, &res->aux.surf);
435
436 const bool has_hiz = devinfo->ver >= 6 && !res->mod_info &&
437 !INTEL_DEBUG(DEBUG_NO_HIZ) &&
438 isl_surf_get_hiz_surf(&screen->isl_dev, &res->surf, &res->aux.surf);
439
440 const bool has_ccs =
441 ((devinfo->ver >= 7 && !res->mod_info && !INTEL_DEBUG(DEBUG_NO_RBC)) ||
442 (res->mod_info && res->mod_info->aux_usage != ISL_AUX_USAGE_NONE)) &&
443 isl_surf_get_ccs_surf(&screen->isl_dev, &res->surf, NULL,
444 &res->aux.surf, 0);
445
446 /* Having more than one type of compression is impossible */
447 assert(has_ccs + has_mcs + has_hiz <= 1);
448
449 if (res->mod_info && has_ccs) {
450 res->aux.usage = res->mod_info->aux_usage;
451 } else if (has_mcs) {
452 res->aux.usage = ISL_AUX_USAGE_MCS;
453 } else if (has_hiz) {
454 res->aux.usage = ISL_AUX_USAGE_HIZ;
455 } else if (has_ccs) {
456 if (isl_format_supports_ccs_d(devinfo, res->surf.format))
457 res->aux.usage = ISL_AUX_USAGE_CCS_D;
458 }
459
460 enum isl_aux_state initial_state = ISL_AUX_STATE_AUX_INVALID;
461 *aux_size_B = 0;
462 *alloc_flags = 0;
463 assert(!res->aux.bo);
464
465 switch (res->aux.usage) {
466 case ISL_AUX_USAGE_NONE:
467 /* Having no aux buffer is only okay if there's no modifier with aux. */
468 res->aux.surf.levels = 0;
469 return !res->mod_info || res->mod_info->aux_usage == ISL_AUX_USAGE_NONE;
470 case ISL_AUX_USAGE_HIZ:
471 initial_state = ISL_AUX_STATE_AUX_INVALID;
472 break;
473 case ISL_AUX_USAGE_MCS:
474 /* The Ivybridge PRM, Vol 2 Part 1 p326 says:
475 *
476 * "When MCS buffer is enabled and bound to MSRT, it is required
477 * that it is cleared prior to any rendering."
478 *
479 * Since we only use the MCS buffer for rendering, we just clear it
480 * immediately on allocation. The clear value for MCS buffers is all
481 * 1's, so we simply memset it to 0xff.
482 */
483 initial_state = ISL_AUX_STATE_CLEAR;
484 break;
485 case ISL_AUX_USAGE_CCS_D:
486 /* When CCS_E is used, we need to ensure that the CCS starts off in
487 * a valid state. From the Sky Lake PRM, "MCS Buffer for Render
488 * Target(s)":
489 *
490 * "If Software wants to enable Color Compression without Fast
491 * clear, Software needs to initialize MCS with zeros."
492 *
493 * A CCS value of 0 indicates that the corresponding block is in the
494 * pass-through state which is what we want.
495 *
496 * For CCS_D, do the same thing. On Gen9+, this avoids having any
497 * undefined bits in the aux buffer.
498 */
499 if (imported)
500 initial_state =
501 isl_drm_modifier_get_default_aux_state(res->mod_info->modifier);
502 else
503 initial_state = ISL_AUX_STATE_PASS_THROUGH;
504 *alloc_flags |= BO_ALLOC_ZEROED;
505 break;
506 default:
507 unreachable("non-crocus aux");
508 }
509
510 /* Create the aux_state for the auxiliary buffer. */
511 res->aux.state = create_aux_state_map(res, initial_state);
512 if (!res->aux.state)
513 return false;
514
515 /* Increase the aux offset if the main and aux surfaces will share a BO. */
516 res->aux.offset =
517 !res->mod_info || res->mod_info->aux_usage == res->aux.usage ?
518 ALIGN(res->surf.size_B, res->aux.surf.alignment_B) : 0;
519 uint64_t size = res->aux.surf.size_B;
520
521 /* Allocate space in the buffer for storing the clear color. On modern
522 * platforms (gen > 9), we can read it directly from such buffer.
523 *
524 * On gen <= 9, we are going to store the clear color on the buffer
525 * anyways, and copy it back to the surface state during state emission.
526 *
527 * Also add some padding to make sure the fast clear color state buffer
528 * starts at a 4K alignment. We believe that 256B might be enough, but due
529 * to lack of testing we will leave this as 4K for now.
530 */
531 size = ALIGN(size, 4096);
532 *aux_size_B = size;
533
534 if (isl_aux_usage_has_hiz(res->aux.usage)) {
535 for (unsigned level = 0; level < res->surf.levels; ++level) {
536 uint32_t width = u_minify(res->surf.phys_level0_sa.width, level);
537 uint32_t height = u_minify(res->surf.phys_level0_sa.height, level);
538
539 /* Disable HiZ for LOD > 0 unless the width/height are 8x4 aligned.
540 * For LOD == 0, we can grow the dimensions to make it work.
541 */
542 if (devinfo->verx10 < 75 ||
543 (level == 0 || ((width & 7) == 0 && (height & 3) == 0)))
544 res->aux.has_hiz |= 1 << level;
545 }
546 }
547
548 return true;
549 }
550
551 /**
552 * Initialize the aux buffer contents.
553 *
554 * Returns false on unexpected error (e.g. mapping a BO failed).
555 */
556 static bool
crocus_resource_init_aux_buf(struct crocus_resource * res,uint32_t alloc_flags)557 crocus_resource_init_aux_buf(struct crocus_resource *res, uint32_t alloc_flags)
558 {
559 if (!(alloc_flags & BO_ALLOC_ZEROED)) {
560 void *map = crocus_bo_map(NULL, res->aux.bo, MAP_WRITE | MAP_RAW);
561
562 if (!map)
563 return false;
564
565 if (crocus_resource_get_aux_state(res, 0, 0) != ISL_AUX_STATE_AUX_INVALID) {
566 uint8_t memset_value = isl_aux_usage_has_mcs(res->aux.usage) ? 0xFF : 0;
567 memset((char*)map + res->aux.offset, memset_value,
568 res->aux.surf.size_B);
569 }
570
571 crocus_bo_unmap(res->aux.bo);
572 }
573
574 return true;
575 }
576
577 /**
578 * Allocate the initial aux surface for a resource based on aux.usage
579 *
580 * Returns false on unexpected error (e.g. allocation failed, or invalid
581 * configuration result).
582 */
583 static bool
crocus_resource_alloc_separate_aux(struct crocus_screen * screen,struct crocus_resource * res)584 crocus_resource_alloc_separate_aux(struct crocus_screen *screen,
585 struct crocus_resource *res)
586 {
587 uint32_t alloc_flags;
588 uint64_t size;
589 if (!crocus_resource_configure_aux(screen, res, false, &size, &alloc_flags))
590 return false;
591
592 if (size == 0)
593 return true;
594
595 /* Allocate the auxiliary buffer. ISL has stricter set of alignment rules
596 * the drm allocator. Therefore, one can pass the ISL dimensions in terms
597 * of bytes instead of trying to recalculate based on different format
598 * block sizes.
599 */
600 res->aux.bo = crocus_bo_alloc_tiled(screen->bufmgr, "aux buffer", size, 4096,
601 isl_tiling_to_i915_tiling(res->aux.surf.tiling),
602 res->aux.surf.row_pitch_B, alloc_flags);
603 if (!res->aux.bo) {
604 return false;
605 }
606
607 if (!crocus_resource_init_aux_buf(res, alloc_flags))
608 return false;
609
610 return true;
611 }
612
613 void
crocus_resource_finish_aux_import(struct pipe_screen * pscreen,struct crocus_resource * res)614 crocus_resource_finish_aux_import(struct pipe_screen *pscreen,
615 struct crocus_resource *res)
616 {
617 struct crocus_screen *screen = (struct crocus_screen *)pscreen;
618 assert(crocus_resource_unfinished_aux_import(res));
619 assert(!res->mod_info->supports_clear_color);
620
621 struct crocus_resource *aux_res = (void *) res->base.b.next;
622 assert(aux_res->aux.surf.row_pitch_B && aux_res->aux.offset &&
623 aux_res->aux.bo);
624
625 assert(res->bo == aux_res->aux.bo);
626 crocus_bo_reference(aux_res->aux.bo);
627 res->aux.bo = aux_res->aux.bo;
628
629 res->aux.offset = aux_res->aux.offset;
630
631 assert(res->bo->size >= (res->aux.offset + res->aux.surf.size_B));
632 assert(aux_res->aux.surf.row_pitch_B == res->aux.surf.row_pitch_B);
633
634 crocus_resource_destroy(&screen->base, res->base.b.next);
635 res->base.b.next = NULL;
636 }
637
638 static struct pipe_resource *
crocus_resource_create_for_buffer(struct pipe_screen * pscreen,const struct pipe_resource * templ)639 crocus_resource_create_for_buffer(struct pipe_screen *pscreen,
640 const struct pipe_resource *templ)
641 {
642 struct crocus_screen *screen = (struct crocus_screen *)pscreen;
643 struct crocus_resource *res = crocus_alloc_resource(pscreen, templ);
644
645 assert(templ->target == PIPE_BUFFER);
646 assert(templ->height0 <= 1);
647 assert(templ->depth0 <= 1);
648 assert(templ->format == PIPE_FORMAT_NONE ||
649 util_format_get_blocksize(templ->format) == 1);
650
651 res->internal_format = templ->format;
652 res->surf.tiling = ISL_TILING_LINEAR;
653
654 const char *name = templ->target == PIPE_BUFFER ? "buffer" : "miptree";
655
656 res->bo = crocus_bo_alloc(screen->bufmgr, name, templ->width0);
657 if (!res->bo) {
658 crocus_resource_destroy(pscreen, &res->base.b);
659 return NULL;
660 }
661
662 return &res->base.b;
663 }
664
665 static struct pipe_resource *
crocus_resource_create_with_modifiers(struct pipe_screen * pscreen,const struct pipe_resource * templ,const uint64_t * modifiers,int modifiers_count)666 crocus_resource_create_with_modifiers(struct pipe_screen *pscreen,
667 const struct pipe_resource *templ,
668 const uint64_t *modifiers,
669 int modifiers_count)
670 {
671 struct crocus_screen *screen = (struct crocus_screen *)pscreen;
672 struct intel_device_info *devinfo = &screen->devinfo;
673 struct crocus_resource *res = crocus_alloc_resource(pscreen, templ);
674
675 if (!res)
676 return NULL;
677
678 uint64_t modifier =
679 select_best_modifier(devinfo, templ, modifiers, modifiers_count);
680
681 if (modifier == DRM_FORMAT_MOD_INVALID && modifiers_count > 0) {
682 fprintf(stderr, "Unsupported modifier, resource creation failed.\n");
683 goto fail;
684 }
685
686 if (templ->usage == PIPE_USAGE_STAGING &&
687 templ->bind == PIPE_BIND_DEPTH_STENCIL &&
688 devinfo->ver < 6)
689 return NULL;
690
691 const bool isl_surf_created_successfully =
692 crocus_resource_configure_main(screen, res, templ, modifier, 0);
693 if (!isl_surf_created_successfully)
694 return NULL;
695
696 const char *name = "miptree";
697
698 unsigned int flags = 0;
699 if (templ->usage == PIPE_USAGE_STAGING)
700 flags |= BO_ALLOC_COHERENT;
701
702 uint64_t aux_size = 0;
703 uint32_t aux_preferred_alloc_flags;
704
705 if (!crocus_resource_configure_aux(screen, res, false, &aux_size,
706 &aux_preferred_alloc_flags)) {
707 goto fail;
708 }
709
710 /* Modifiers require the aux data to be in the same buffer as the main
711 * surface, but we combine them even when a modifiers is not being used.
712 */
713 const uint64_t bo_size =
714 MAX2(res->surf.size_B, res->aux.offset + aux_size);
715 uint32_t alignment = MAX2(4096, res->surf.alignment_B);
716 res->bo = crocus_bo_alloc_tiled(screen->bufmgr, name, bo_size, alignment,
717 isl_tiling_to_i915_tiling(res->surf.tiling),
718 res->surf.row_pitch_B, flags);
719
720 if (!res->bo)
721 goto fail;
722
723 if (aux_size > 0) {
724 res->aux.bo = res->bo;
725 crocus_bo_reference(res->aux.bo);
726 if (!crocus_resource_init_aux_buf(res, flags))
727 goto fail;
728 }
729
730 if (templ->format == PIPE_FORMAT_S8_UINT && !(templ->usage == PIPE_USAGE_STAGING) &&
731 devinfo->ver == 7 && (templ->bind & PIPE_BIND_SAMPLER_VIEW)) {
732 struct pipe_resource templ_shadow = (struct pipe_resource) {
733 .usage = 0,
734 .bind = PIPE_BIND_SAMPLER_VIEW,
735 .width0 = res->base.b.width0,
736 .height0 = res->base.b.height0,
737 .depth0 = res->base.b.depth0,
738 .last_level = res->base.b.last_level,
739 .nr_samples = res->base.b.nr_samples,
740 .nr_storage_samples = res->base.b.nr_storage_samples,
741 .array_size = res->base.b.array_size,
742 .format = PIPE_FORMAT_R8_UINT,
743 .target = res->base.b.target,
744 };
745 res->shadow = (struct crocus_resource *)screen->base.resource_create(&screen->base, &templ_shadow);
746 assert(res->shadow);
747 }
748
749 return &res->base.b;
750
751 fail:
752 fprintf(stderr, "XXX: resource creation failed\n");
753 crocus_resource_destroy(pscreen, &res->base.b);
754 return NULL;
755
756 }
757
758 static struct pipe_resource *
crocus_resource_create(struct pipe_screen * pscreen,const struct pipe_resource * templ)759 crocus_resource_create(struct pipe_screen *pscreen,
760 const struct pipe_resource *templ)
761 {
762 if (templ->target == PIPE_BUFFER)
763 return crocus_resource_create_for_buffer(pscreen, templ);
764 else
765 return crocus_resource_create_with_modifiers(pscreen, templ, NULL, 0);
766 }
767
768 static uint64_t
tiling_to_modifier(uint32_t tiling)769 tiling_to_modifier(uint32_t tiling)
770 {
771 static const uint64_t map[] = {
772 [I915_TILING_NONE] = DRM_FORMAT_MOD_LINEAR,
773 [I915_TILING_X] = I915_FORMAT_MOD_X_TILED,
774 [I915_TILING_Y] = I915_FORMAT_MOD_Y_TILED,
775 };
776
777 assert(tiling < ARRAY_SIZE(map));
778
779 return map[tiling];
780 }
781
782 static struct pipe_resource *
crocus_resource_from_user_memory(struct pipe_screen * pscreen,const struct pipe_resource * templ,void * user_memory)783 crocus_resource_from_user_memory(struct pipe_screen *pscreen,
784 const struct pipe_resource *templ,
785 void *user_memory)
786 {
787 struct crocus_screen *screen = (struct crocus_screen *)pscreen;
788 struct crocus_bufmgr *bufmgr = screen->bufmgr;
789 struct crocus_resource *res = crocus_alloc_resource(pscreen, templ);
790 if (!res)
791 return NULL;
792
793 assert(templ->target == PIPE_BUFFER);
794
795 res->internal_format = templ->format;
796 res->bo = crocus_bo_create_userptr(bufmgr, "user",
797 user_memory, templ->width0);
798 if (!res->bo) {
799 free(res);
800 return NULL;
801 }
802
803 util_range_add(&res->base.b, &res->valid_buffer_range, 0, templ->width0);
804
805 return &res->base.b;
806 }
807
808 static struct pipe_resource *
crocus_resource_from_handle(struct pipe_screen * pscreen,const struct pipe_resource * templ,struct winsys_handle * whandle,unsigned usage)809 crocus_resource_from_handle(struct pipe_screen *pscreen,
810 const struct pipe_resource *templ,
811 struct winsys_handle *whandle,
812 unsigned usage)
813 {
814 assert(templ->target != PIPE_BUFFER);
815
816 struct crocus_screen *screen = (struct crocus_screen *)pscreen;
817 struct crocus_bufmgr *bufmgr = screen->bufmgr;
818 struct crocus_resource *res = crocus_alloc_resource(pscreen, templ);
819
820 if (!res)
821 return NULL;
822
823 switch (whandle->type) {
824 case WINSYS_HANDLE_TYPE_FD:
825 res->bo = crocus_bo_import_dmabuf(bufmgr, whandle->handle,
826 whandle->modifier);
827 break;
828 case WINSYS_HANDLE_TYPE_SHARED:
829 res->bo = crocus_bo_gem_create_from_name(bufmgr, "winsys image",
830 whandle->handle);
831 break;
832 default:
833 unreachable("invalid winsys handle type");
834 }
835 if (!res->bo)
836 return NULL;
837
838 res->offset = whandle->offset;
839 res->external_format = whandle->format;
840
841 if (whandle->plane < util_format_get_num_planes(whandle->format)) {
842 const uint64_t modifier =
843 whandle->modifier != DRM_FORMAT_MOD_INVALID ?
844 whandle->modifier : tiling_to_modifier(res->bo->tiling_mode);
845
846 UNUSED const bool isl_surf_created_successfully =
847 crocus_resource_configure_main(screen, res, templ, modifier,
848 whandle->stride);
849 assert(isl_surf_created_successfully);
850 assert(res->bo->tiling_mode ==
851 isl_tiling_to_i915_tiling(res->surf.tiling));
852
853 // XXX: create_ccs_buf_for_image?
854 if (whandle->modifier == DRM_FORMAT_MOD_INVALID) {
855 if (!crocus_resource_alloc_separate_aux(screen, res))
856 goto fail;
857 } else {
858 if (res->mod_info->aux_usage != ISL_AUX_USAGE_NONE) {
859 uint32_t alloc_flags;
860 uint64_t size;
861 UNUSED bool ok = crocus_resource_configure_aux(screen, res, true, &size,
862 &alloc_flags);
863 assert(ok);
864 /* The gallium dri layer will create a separate plane resource
865 * for the aux image. crocus_resource_finish_aux_import will
866 * merge the separate aux parameters back into a single
867 * crocus_resource.
868 */
869 }
870 }
871 } else {
872 /* Save modifier import information to reconstruct later. After
873 * import, this will be available under a second image accessible
874 * from the main image with res->base.next. See
875 * crocus_resource_finish_aux_import.
876 */
877 res->aux.surf.row_pitch_B = whandle->stride;
878 res->aux.offset = whandle->offset;
879 res->aux.bo = res->bo;
880 res->bo = NULL;
881 }
882
883 return &res->base.b;
884
885 fail:
886 crocus_resource_destroy(pscreen, &res->base.b);
887 return NULL;
888 }
889
890 static struct pipe_resource *
crocus_resource_from_memobj(struct pipe_screen * pscreen,const struct pipe_resource * templ,struct pipe_memory_object * pmemobj,uint64_t offset)891 crocus_resource_from_memobj(struct pipe_screen *pscreen,
892 const struct pipe_resource *templ,
893 struct pipe_memory_object *pmemobj,
894 uint64_t offset)
895 {
896 struct crocus_screen *screen = (struct crocus_screen *)pscreen;
897 struct crocus_memory_object *memobj = (struct crocus_memory_object *)pmemobj;
898 struct crocus_resource *res = crocus_alloc_resource(pscreen, templ);
899
900 if (!res)
901 return NULL;
902
903 /* Disable Depth, and combined Depth+Stencil for now. */
904 if (util_format_has_depth(util_format_description(templ->format)))
905 return NULL;
906
907 if (templ->flags & PIPE_RESOURCE_FLAG_TEXTURING_MORE_LIKELY) {
908 UNUSED const bool isl_surf_created_successfully =
909 crocus_resource_configure_main(screen, res, templ, DRM_FORMAT_MOD_INVALID, 0);
910 assert(isl_surf_created_successfully);
911 }
912
913 res->bo = memobj->bo;
914 res->offset = offset;
915 res->external_format = memobj->format;
916
917 crocus_bo_reference(memobj->bo);
918
919 return &res->base.b;
920 }
921
922 static void
crocus_flush_resource(struct pipe_context * ctx,struct pipe_resource * resource)923 crocus_flush_resource(struct pipe_context *ctx, struct pipe_resource *resource)
924 {
925 struct crocus_context *ice = (struct crocus_context *)ctx;
926 struct crocus_resource *res = (void *) resource;
927 const struct isl_drm_modifier_info *mod = res->mod_info;
928
929 crocus_resource_prepare_access(ice, res,
930 0, INTEL_REMAINING_LEVELS,
931 0, INTEL_REMAINING_LAYERS,
932 mod ? mod->aux_usage : ISL_AUX_USAGE_NONE,
933 mod ? mod->supports_clear_color : false);
934 }
935
936 static void
crocus_resource_disable_aux_on_first_query(struct pipe_resource * resource,unsigned usage)937 crocus_resource_disable_aux_on_first_query(struct pipe_resource *resource,
938 unsigned usage)
939 {
940 struct crocus_resource *res = (struct crocus_resource *)resource;
941 bool mod_with_aux =
942 res->mod_info && res->mod_info->aux_usage != ISL_AUX_USAGE_NONE;
943
944 /* Disable aux usage if explicit flush not set and this is the first time
945 * we are dealing with this resource and the resource was not created with
946 * a modifier with aux.
947 */
948 if (!mod_with_aux &&
949 (!(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) && res->aux.usage != 0) &&
950 p_atomic_read(&resource->reference.count) == 1) {
951 crocus_resource_disable_aux(res);
952 }
953 }
954
955 static bool
crocus_resource_get_param(struct pipe_screen * pscreen,struct pipe_context * context,struct pipe_resource * resource,unsigned plane,unsigned layer,unsigned level,enum pipe_resource_param param,unsigned handle_usage,uint64_t * value)956 crocus_resource_get_param(struct pipe_screen *pscreen,
957 struct pipe_context *context,
958 struct pipe_resource *resource,
959 unsigned plane,
960 unsigned layer,
961 unsigned level,
962 enum pipe_resource_param param,
963 unsigned handle_usage,
964 uint64_t *value)
965 {
966 struct crocus_screen *screen = (struct crocus_screen *)pscreen;
967 struct crocus_resource *res = (struct crocus_resource *)resource;
968 bool mod_with_aux =
969 res->mod_info && res->mod_info->aux_usage != ISL_AUX_USAGE_NONE;
970 bool wants_aux = mod_with_aux && plane > 0;
971 bool result;
972 unsigned handle;
973
974 if (crocus_resource_unfinished_aux_import(res))
975 crocus_resource_finish_aux_import(pscreen, res);
976
977 struct crocus_bo *bo = wants_aux ? res->aux.bo : res->bo;
978
979 crocus_resource_disable_aux_on_first_query(resource, handle_usage);
980
981 switch (param) {
982 case PIPE_RESOURCE_PARAM_NPLANES:
983 if (mod_with_aux) {
984 *value = util_format_get_num_planes(res->external_format);
985 } else {
986 unsigned count = 0;
987 for (struct pipe_resource *cur = resource; cur; cur = cur->next)
988 count++;
989 *value = count;
990 }
991 return true;
992 case PIPE_RESOURCE_PARAM_STRIDE:
993 *value = wants_aux ? res->aux.surf.row_pitch_B : res->surf.row_pitch_B;
994 return true;
995 case PIPE_RESOURCE_PARAM_OFFSET:
996 *value = wants_aux ? res->aux.offset : 0;
997 return true;
998 case PIPE_RESOURCE_PARAM_MODIFIER:
999 *value = res->mod_info ? res->mod_info->modifier :
1000 tiling_to_modifier(isl_tiling_to_i915_tiling(res->surf.tiling));
1001 return true;
1002 case PIPE_RESOURCE_PARAM_HANDLE_TYPE_SHARED:
1003 result = crocus_bo_flink(bo, &handle) == 0;
1004 if (result)
1005 *value = handle;
1006 return result;
1007 case PIPE_RESOURCE_PARAM_HANDLE_TYPE_KMS: {
1008 /* Because we share the same drm file across multiple crocus_screen, when
1009 * we export a GEM handle we must make sure it is valid in the DRM file
1010 * descriptor the caller is using (this is the FD given at screen
1011 * creation).
1012 */
1013 uint32_t handle;
1014 if (crocus_bo_export_gem_handle_for_device(bo, screen->winsys_fd, &handle))
1015 return false;
1016 *value = handle;
1017 return true;
1018 }
1019 case PIPE_RESOURCE_PARAM_HANDLE_TYPE_FD:
1020 result = crocus_bo_export_dmabuf(bo, (int *) &handle) == 0;
1021 if (result)
1022 *value = handle;
1023 return result;
1024 default:
1025 return false;
1026 }
1027 }
1028
1029 static bool
crocus_resource_get_handle(struct pipe_screen * pscreen,struct pipe_context * ctx,struct pipe_resource * resource,struct winsys_handle * whandle,unsigned usage)1030 crocus_resource_get_handle(struct pipe_screen *pscreen,
1031 struct pipe_context *ctx,
1032 struct pipe_resource *resource,
1033 struct winsys_handle *whandle,
1034 unsigned usage)
1035 {
1036 struct crocus_screen *screen = (struct crocus_screen *) pscreen;
1037 struct crocus_resource *res = (struct crocus_resource *)resource;
1038 bool mod_with_aux =
1039 res->mod_info && res->mod_info->aux_usage != ISL_AUX_USAGE_NONE;
1040
1041 crocus_resource_disable_aux_on_first_query(resource, usage);
1042
1043 struct crocus_bo *bo;
1044 if (mod_with_aux && whandle->plane > 0) {
1045 assert(res->aux.bo);
1046 bo = res->aux.bo;
1047 whandle->stride = res->aux.surf.row_pitch_B;
1048 whandle->offset = res->aux.offset;
1049 } else {
1050 /* If this is a buffer, stride should be 0 - no need to special case */
1051 whandle->stride = res->surf.row_pitch_B;
1052 bo = res->bo;
1053 }
1054 whandle->format = res->external_format;
1055 whandle->modifier =
1056 res->mod_info ? res->mod_info->modifier
1057 : tiling_to_modifier(res->bo->tiling_mode);
1058
1059 #ifndef NDEBUG
1060 enum isl_aux_usage allowed_usage =
1061 res->mod_info ? res->mod_info->aux_usage : ISL_AUX_USAGE_NONE;
1062
1063 if (res->aux.usage != allowed_usage) {
1064 enum isl_aux_state aux_state = crocus_resource_get_aux_state(res, 0, 0);
1065 assert(aux_state == ISL_AUX_STATE_RESOLVED ||
1066 aux_state == ISL_AUX_STATE_PASS_THROUGH);
1067 }
1068 #endif
1069
1070 switch (whandle->type) {
1071 case WINSYS_HANDLE_TYPE_SHARED:
1072 return crocus_bo_flink(bo, &whandle->handle) == 0;
1073 case WINSYS_HANDLE_TYPE_KMS: {
1074 /* Because we share the same drm file across multiple crocus_screen, when
1075 * we export a GEM handle we must make sure it is valid in the DRM file
1076 * descriptor the caller is using (this is the FD given at screen
1077 * creation).
1078 */
1079 uint32_t handle;
1080 if (crocus_bo_export_gem_handle_for_device(bo, screen->winsys_fd, &handle))
1081 return false;
1082 whandle->handle = handle;
1083 return true;
1084 }
1085 case WINSYS_HANDLE_TYPE_FD:
1086 return crocus_bo_export_dmabuf(bo, (int *) &whandle->handle) == 0;
1087 }
1088
1089 return false;
1090 }
1091
1092 static bool
resource_is_busy(struct crocus_context * ice,struct crocus_resource * res)1093 resource_is_busy(struct crocus_context *ice,
1094 struct crocus_resource *res)
1095 {
1096 bool busy = crocus_bo_busy(res->bo);
1097
1098 for (int i = 0; i < ice->batch_count; i++)
1099 busy |= crocus_batch_references(&ice->batches[i], res->bo);
1100
1101 return busy;
1102 }
1103
1104 void
crocus_replace_buffer_storage(struct pipe_context * ctx,struct pipe_resource * p_dst,struct pipe_resource * p_src,unsigned num_rebinds,uint32_t rebind_mask,uint32_t delete_buffer_id)1105 crocus_replace_buffer_storage(struct pipe_context *ctx,
1106 struct pipe_resource *p_dst,
1107 struct pipe_resource *p_src,
1108 unsigned num_rebinds,
1109 uint32_t rebind_mask,
1110 uint32_t delete_buffer_id)
1111 {
1112 struct crocus_screen *screen = (void *) ctx->screen;
1113 struct crocus_context *ice = (void *) ctx;
1114 struct crocus_resource *dst = (void *) p_dst;
1115 struct crocus_resource *src = (void *) p_src;
1116
1117 assert(memcmp(&dst->surf, &src->surf, sizeof(dst->surf)) == 0);
1118
1119 struct crocus_bo *old_bo = dst->bo;
1120
1121 /* Swap out the backing storage */
1122 crocus_bo_reference(src->bo);
1123 dst->bo = src->bo;
1124
1125 /* Rebind the buffer, replacing any state referring to the old BO's
1126 * address, and marking state dirty so it's reemitted.
1127 */
1128 screen->vtbl.rebind_buffer(ice, dst);
1129
1130 crocus_bo_unreference(old_bo);
1131 }
1132
1133 static void
crocus_invalidate_resource(struct pipe_context * ctx,struct pipe_resource * resource)1134 crocus_invalidate_resource(struct pipe_context *ctx,
1135 struct pipe_resource *resource)
1136 {
1137 struct crocus_screen *screen = (void *) ctx->screen;
1138 struct crocus_context *ice = (void *) ctx;
1139 struct crocus_resource *res = (void *) resource;
1140
1141 if (resource->target != PIPE_BUFFER)
1142 return;
1143
1144 /* If it's already invalidated, don't bother doing anything. */
1145 if (res->valid_buffer_range.start > res->valid_buffer_range.end)
1146 return;
1147
1148 if (!resource_is_busy(ice, res)) {
1149 /* The resource is idle, so just mark that it contains no data and
1150 * keep using the same underlying buffer object.
1151 */
1152 util_range_set_empty(&res->valid_buffer_range);
1153 return;
1154 }
1155
1156 /* Otherwise, try and replace the backing storage with a new BO. */
1157
1158 /* We can't reallocate memory we didn't allocate in the first place. */
1159 if (res->bo->userptr)
1160 return;
1161
1162 struct crocus_bo *old_bo = res->bo;
1163 struct crocus_bo *new_bo =
1164 crocus_bo_alloc(screen->bufmgr, res->bo->name, resource->width0);
1165
1166 if (!new_bo)
1167 return;
1168
1169 /* Swap out the backing storage */
1170 res->bo = new_bo;
1171
1172 /* Rebind the buffer, replacing any state referring to the old BO's
1173 * address, and marking state dirty so it's reemitted.
1174 */
1175 screen->vtbl.rebind_buffer(ice, res);
1176
1177 util_range_set_empty(&res->valid_buffer_range);
1178
1179 crocus_bo_unreference(old_bo);
1180 }
1181
1182 static void
crocus_flush_staging_region(struct pipe_transfer * xfer,const struct pipe_box * flush_box)1183 crocus_flush_staging_region(struct pipe_transfer *xfer,
1184 const struct pipe_box *flush_box)
1185 {
1186 if (!(xfer->usage & PIPE_MAP_WRITE))
1187 return;
1188
1189 struct crocus_transfer *map = (void *) xfer;
1190
1191 struct pipe_box src_box = *flush_box;
1192
1193 /* Account for extra alignment padding in staging buffer */
1194 if (xfer->resource->target == PIPE_BUFFER)
1195 src_box.x += xfer->box.x % CROCUS_MAP_BUFFER_ALIGNMENT;
1196
1197 struct pipe_box dst_box = (struct pipe_box) {
1198 .x = xfer->box.x + flush_box->x,
1199 .y = xfer->box.y + flush_box->y,
1200 .z = xfer->box.z + flush_box->z,
1201 .width = flush_box->width,
1202 .height = flush_box->height,
1203 .depth = flush_box->depth,
1204 };
1205
1206 crocus_copy_region(map->blorp, map->batch, xfer->resource, xfer->level,
1207 dst_box.x, dst_box.y, dst_box.z, map->staging, 0,
1208 &src_box);
1209 }
1210
1211 static void
crocus_unmap_copy_region(struct crocus_transfer * map)1212 crocus_unmap_copy_region(struct crocus_transfer *map)
1213 {
1214 crocus_resource_destroy(map->staging->screen, map->staging);
1215
1216 map->ptr = NULL;
1217 }
1218
1219 static void
crocus_map_copy_region(struct crocus_transfer * map)1220 crocus_map_copy_region(struct crocus_transfer *map)
1221 {
1222 struct pipe_screen *pscreen = &map->batch->screen->base;
1223 struct pipe_transfer *xfer = &map->base.b;
1224 struct pipe_box *box = &xfer->box;
1225 struct crocus_resource *res = (void *) xfer->resource;
1226
1227 unsigned extra = xfer->resource->target == PIPE_BUFFER ?
1228 box->x % CROCUS_MAP_BUFFER_ALIGNMENT : 0;
1229
1230 struct pipe_resource templ = (struct pipe_resource) {
1231 .usage = PIPE_USAGE_STAGING,
1232 .width0 = box->width + extra,
1233 .height0 = box->height,
1234 .depth0 = 1,
1235 .nr_samples = xfer->resource->nr_samples,
1236 .nr_storage_samples = xfer->resource->nr_storage_samples,
1237 .array_size = box->depth,
1238 .format = res->internal_format,
1239 };
1240
1241 if (xfer->resource->target == PIPE_BUFFER)
1242 templ.target = PIPE_BUFFER;
1243 else if (templ.array_size > 1)
1244 templ.target = PIPE_TEXTURE_2D_ARRAY;
1245 else
1246 templ.target = PIPE_TEXTURE_2D;
1247
1248 map->staging = crocus_resource_create(pscreen, &templ);
1249 assert(map->staging);
1250
1251 if (templ.target != PIPE_BUFFER) {
1252 struct isl_surf *surf = &((struct crocus_resource *) map->staging)->surf;
1253 xfer->stride = isl_surf_get_row_pitch_B(surf);
1254 xfer->layer_stride = isl_surf_get_array_pitch(surf);
1255 }
1256
1257 if (!(xfer->usage & PIPE_MAP_DISCARD_RANGE)) {
1258 crocus_copy_region(map->blorp, map->batch, map->staging, 0, extra, 0, 0,
1259 xfer->resource, xfer->level, box);
1260 /* Ensure writes to the staging BO land before we map it below. */
1261 crocus_emit_pipe_control_flush(map->batch,
1262 "transfer read: flush before mapping",
1263 PIPE_CONTROL_RENDER_TARGET_FLUSH |
1264 PIPE_CONTROL_CS_STALL);
1265 }
1266
1267 struct crocus_bo *staging_bo = crocus_resource_bo(map->staging);
1268
1269 if (crocus_batch_references(map->batch, staging_bo))
1270 crocus_batch_flush(map->batch);
1271
1272 map->ptr =
1273 crocus_bo_map(map->dbg, staging_bo, xfer->usage & MAP_FLAGS) + extra;
1274
1275 map->unmap = crocus_unmap_copy_region;
1276 }
1277
1278 static void
get_image_offset_el(const struct isl_surf * surf,unsigned level,unsigned z,unsigned * out_x0_el,unsigned * out_y0_el)1279 get_image_offset_el(const struct isl_surf *surf, unsigned level, unsigned z,
1280 unsigned *out_x0_el, unsigned *out_y0_el)
1281 {
1282 ASSERTED uint32_t z0_el, a0_el;
1283 if (surf->dim == ISL_SURF_DIM_3D) {
1284 isl_surf_get_image_offset_el(surf, level, 0, z,
1285 out_x0_el, out_y0_el, &z0_el, &a0_el);
1286 } else {
1287 isl_surf_get_image_offset_el(surf, level, z, 0,
1288 out_x0_el, out_y0_el, &z0_el, &a0_el);
1289 }
1290 assert(z0_el == 0 && a0_el == 0);
1291 }
1292
1293 void
crocus_resource_get_image_offset(struct crocus_resource * res,uint32_t level,uint32_t z,uint32_t * x,uint32_t * y)1294 crocus_resource_get_image_offset(struct crocus_resource *res,
1295 uint32_t level, uint32_t z,
1296 uint32_t *x, uint32_t *y)
1297 {
1298 get_image_offset_el(&res->surf, level, z, x, y);
1299 }
1300
1301 /**
1302 * Get pointer offset into stencil buffer.
1303 *
1304 * The stencil buffer is W tiled. Since the GTT is incapable of W fencing, we
1305 * must decode the tile's layout in software.
1306 *
1307 * See
1308 * - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.2.1 W-Major Tile
1309 * Format.
1310 * - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.3 Tiling Algorithm
1311 *
1312 * Even though the returned offset is always positive, the return type is
1313 * signed due to
1314 * commit e8b1c6d6f55f5be3bef25084fdd8b6127517e137
1315 * mesa: Fix return type of _mesa_get_format_bytes() (#37351)
1316 */
1317 static intptr_t
s8_offset(uint32_t stride,uint32_t x,uint32_t y,bool swizzled)1318 s8_offset(uint32_t stride, uint32_t x, uint32_t y, bool swizzled)
1319 {
1320 uint32_t tile_size = 4096;
1321 uint32_t tile_width = 64;
1322 uint32_t tile_height = 64;
1323 uint32_t row_size = 64 * stride / 2; /* Two rows are interleaved. */
1324
1325 uint32_t tile_x = x / tile_width;
1326 uint32_t tile_y = y / tile_height;
1327
1328 /* The byte's address relative to the tile's base addres. */
1329 uint32_t byte_x = x % tile_width;
1330 uint32_t byte_y = y % tile_height;
1331
1332 uintptr_t u = tile_y * row_size
1333 + tile_x * tile_size
1334 + 512 * (byte_x / 8)
1335 + 64 * (byte_y / 8)
1336 + 32 * ((byte_y / 4) % 2)
1337 + 16 * ((byte_x / 4) % 2)
1338 + 8 * ((byte_y / 2) % 2)
1339 + 4 * ((byte_x / 2) % 2)
1340 + 2 * (byte_y % 2)
1341 + 1 * (byte_x % 2);
1342
1343 if (swizzled) {
1344 /* adjust for bit6 swizzling */
1345 if (((byte_x / 8) % 2) == 1) {
1346 if (((byte_y / 8) % 2) == 0) {
1347 u += 64;
1348 } else {
1349 u -= 64;
1350 }
1351 }
1352 }
1353
1354 return u;
1355 }
1356
1357 static void
crocus_unmap_s8(struct crocus_transfer * map)1358 crocus_unmap_s8(struct crocus_transfer *map)
1359 {
1360 struct pipe_transfer *xfer = &map->base.b;
1361 const struct pipe_box *box = &xfer->box;
1362 struct crocus_resource *res = (struct crocus_resource *) xfer->resource;
1363 struct isl_surf *surf = &res->surf;
1364
1365 if (xfer->usage & PIPE_MAP_WRITE) {
1366 uint8_t *untiled_s8_map = map->ptr;
1367 uint8_t *tiled_s8_map =
1368 crocus_bo_map(map->dbg, res->bo, (xfer->usage | MAP_RAW) & MAP_FLAGS);
1369
1370 for (int s = 0; s < box->depth; s++) {
1371 unsigned x0_el, y0_el;
1372 get_image_offset_el(surf, xfer->level, box->z + s, &x0_el, &y0_el);
1373
1374 for (uint32_t y = 0; y < box->height; y++) {
1375 for (uint32_t x = 0; x < box->width; x++) {
1376 ptrdiff_t offset = s8_offset(surf->row_pitch_B,
1377 x0_el + box->x + x,
1378 y0_el + box->y + y,
1379 map->has_swizzling);
1380 tiled_s8_map[offset] =
1381 untiled_s8_map[s * xfer->layer_stride + y * xfer->stride + x];
1382 }
1383 }
1384 }
1385 }
1386
1387 free(map->buffer);
1388 }
1389
1390 static void
crocus_map_s8(struct crocus_transfer * map)1391 crocus_map_s8(struct crocus_transfer *map)
1392 {
1393 struct pipe_transfer *xfer = &map->base.b;
1394 const struct pipe_box *box = &xfer->box;
1395 struct crocus_resource *res = (struct crocus_resource *) xfer->resource;
1396 struct isl_surf *surf = &res->surf;
1397
1398 xfer->stride = surf->row_pitch_B;
1399 xfer->layer_stride = xfer->stride * box->height;
1400
1401 /* The tiling and detiling functions require that the linear buffer has
1402 * a 16-byte alignment (that is, its `x0` is 16-byte aligned). Here we
1403 * over-allocate the linear buffer to get the proper alignment.
1404 */
1405 map->buffer = map->ptr = malloc(xfer->layer_stride * box->depth);
1406 assert(map->buffer);
1407
1408 /* One of either READ_BIT or WRITE_BIT or both is set. READ_BIT implies no
1409 * INVALIDATE_RANGE_BIT. WRITE_BIT needs the original values read in unless
1410 * invalidate is set, since we'll be writing the whole rectangle from our
1411 * temporary buffer back out.
1412 */
1413 if (!(xfer->usage & PIPE_MAP_DISCARD_RANGE)) {
1414 uint8_t *untiled_s8_map = map->ptr;
1415 uint8_t *tiled_s8_map =
1416 crocus_bo_map(map->dbg, res->bo, (xfer->usage | MAP_RAW) & MAP_FLAGS);
1417
1418 for (int s = 0; s < box->depth; s++) {
1419 unsigned x0_el, y0_el;
1420 get_image_offset_el(surf, xfer->level, box->z + s, &x0_el, &y0_el);
1421
1422 for (uint32_t y = 0; y < box->height; y++) {
1423 for (uint32_t x = 0; x < box->width; x++) {
1424 ptrdiff_t offset = s8_offset(surf->row_pitch_B,
1425 x0_el + box->x + x,
1426 y0_el + box->y + y,
1427 map->has_swizzling);
1428 untiled_s8_map[s * xfer->layer_stride + y * xfer->stride + x] =
1429 tiled_s8_map[offset];
1430 }
1431 }
1432 }
1433 }
1434
1435 map->unmap = crocus_unmap_s8;
1436 }
1437
1438 /* Compute extent parameters for use with tiled_memcpy functions.
1439 * xs are in units of bytes and ys are in units of strides.
1440 */
1441 static inline void
tile_extents(const struct isl_surf * surf,const struct pipe_box * box,unsigned level,int z,unsigned * x1_B,unsigned * x2_B,unsigned * y1_el,unsigned * y2_el)1442 tile_extents(const struct isl_surf *surf,
1443 const struct pipe_box *box,
1444 unsigned level, int z,
1445 unsigned *x1_B, unsigned *x2_B,
1446 unsigned *y1_el, unsigned *y2_el)
1447 {
1448 const struct isl_format_layout *fmtl = isl_format_get_layout(surf->format);
1449 const unsigned cpp = fmtl->bpb / 8;
1450
1451 assert(box->x % fmtl->bw == 0);
1452 assert(box->y % fmtl->bh == 0);
1453
1454 unsigned x0_el, y0_el;
1455 get_image_offset_el(surf, level, box->z + z, &x0_el, &y0_el);
1456
1457 *x1_B = (box->x / fmtl->bw + x0_el) * cpp;
1458 *y1_el = box->y / fmtl->bh + y0_el;
1459 *x2_B = (DIV_ROUND_UP(box->x + box->width, fmtl->bw) + x0_el) * cpp;
1460 *y2_el = DIV_ROUND_UP(box->y + box->height, fmtl->bh) + y0_el;
1461 }
1462
1463 static void
crocus_unmap_tiled_memcpy(struct crocus_transfer * map)1464 crocus_unmap_tiled_memcpy(struct crocus_transfer *map)
1465 {
1466 struct pipe_transfer *xfer = &map->base.b;
1467 const struct pipe_box *box = &xfer->box;
1468 struct crocus_resource *res = (struct crocus_resource *) xfer->resource;
1469 struct isl_surf *surf = &res->surf;
1470
1471 if (xfer->usage & PIPE_MAP_WRITE) {
1472 char *dst =
1473 crocus_bo_map(map->dbg, res->bo, (xfer->usage | MAP_RAW) & MAP_FLAGS);
1474
1475 for (int s = 0; s < box->depth; s++) {
1476 unsigned x1, x2, y1, y2;
1477 tile_extents(surf, box, xfer->level, s, &x1, &x2, &y1, &y2);
1478
1479 void *ptr = map->ptr + s * xfer->layer_stride;
1480
1481 isl_memcpy_linear_to_tiled(x1, x2, y1, y2, dst, ptr,
1482 surf->row_pitch_B, xfer->stride,
1483 map->has_swizzling,
1484 surf->tiling, ISL_MEMCPY);
1485 }
1486 }
1487 os_free_aligned(map->buffer);
1488 map->buffer = map->ptr = NULL;
1489 }
1490
1491 static void
crocus_map_tiled_memcpy(struct crocus_transfer * map)1492 crocus_map_tiled_memcpy(struct crocus_transfer *map)
1493 {
1494 struct pipe_transfer *xfer = &map->base.b;
1495 const struct pipe_box *box = &xfer->box;
1496 struct crocus_resource *res = (struct crocus_resource *) xfer->resource;
1497 struct isl_surf *surf = &res->surf;
1498
1499 xfer->stride = ALIGN(surf->row_pitch_B, 16);
1500 xfer->layer_stride = xfer->stride * box->height;
1501
1502 unsigned x1, x2, y1, y2;
1503 tile_extents(surf, box, xfer->level, 0, &x1, &x2, &y1, &y2);
1504
1505 /* The tiling and detiling functions require that the linear buffer has
1506 * a 16-byte alignment (that is, its `x0` is 16-byte aligned). Here we
1507 * over-allocate the linear buffer to get the proper alignment.
1508 */
1509 map->buffer =
1510 os_malloc_aligned(xfer->layer_stride * box->depth, 16);
1511 assert(map->buffer);
1512 map->ptr = (char *)map->buffer + (x1 & 0xf);
1513
1514 if (!(xfer->usage & PIPE_MAP_DISCARD_RANGE)) {
1515 char *src =
1516 crocus_bo_map(map->dbg, res->bo, (xfer->usage | MAP_RAW) & MAP_FLAGS);
1517
1518 for (int s = 0; s < box->depth; s++) {
1519 unsigned x1, x2, y1, y2;
1520 tile_extents(surf, box, xfer->level, s, &x1, &x2, &y1, &y2);
1521
1522 /* Use 's' rather than 'box->z' to rebase the first slice to 0. */
1523 void *ptr = map->ptr + s * xfer->layer_stride;
1524
1525 isl_memcpy_tiled_to_linear(x1, x2, y1, y2, ptr, src, xfer->stride,
1526 surf->row_pitch_B,
1527 map->has_swizzling,
1528 surf->tiling,
1529 #if defined(USE_SSE41)
1530 util_get_cpu_caps()->has_sse4_1 ? ISL_MEMCPY_STREAMING_LOAD :
1531 #endif
1532 ISL_MEMCPY);
1533 }
1534 }
1535
1536 map->unmap = crocus_unmap_tiled_memcpy;
1537 }
1538
1539 static void
crocus_map_direct(struct crocus_transfer * map)1540 crocus_map_direct(struct crocus_transfer *map)
1541 {
1542 struct pipe_transfer *xfer = &map->base.b;
1543 struct pipe_box *box = &xfer->box;
1544 struct crocus_resource *res = (struct crocus_resource *) xfer->resource;
1545
1546 void *ptr = crocus_bo_map(map->dbg, res->bo, xfer->usage & MAP_FLAGS);
1547
1548 if (res->base.b.target == PIPE_BUFFER) {
1549 xfer->stride = 0;
1550 xfer->layer_stride = 0;
1551
1552 map->ptr = ptr + box->x;
1553 } else {
1554 struct isl_surf *surf = &res->surf;
1555 const struct isl_format_layout *fmtl =
1556 isl_format_get_layout(surf->format);
1557 const unsigned cpp = fmtl->bpb / 8;
1558 unsigned x0_el, y0_el;
1559
1560 assert(box->x % fmtl->bw == 0);
1561 assert(box->y % fmtl->bh == 0);
1562 get_image_offset_el(surf, xfer->level, box->z, &x0_el, &y0_el);
1563
1564 x0_el += box->x / fmtl->bw;
1565 y0_el += box->y / fmtl->bh;
1566
1567 xfer->stride = isl_surf_get_row_pitch_B(surf);
1568 xfer->layer_stride = isl_surf_get_array_pitch(surf);
1569
1570 map->ptr = ptr + y0_el * xfer->stride + x0_el * cpp;
1571 }
1572 }
1573
1574 static bool
can_promote_to_async(const struct crocus_resource * res,const struct pipe_box * box,unsigned usage)1575 can_promote_to_async(const struct crocus_resource *res,
1576 const struct pipe_box *box,
1577 unsigned usage)
1578 {
1579 /* If we're writing to a section of the buffer that hasn't even been
1580 * initialized with useful data, then we can safely promote this write
1581 * to be unsynchronized. This helps the common pattern of appending data.
1582 */
1583 return res->base.b.target == PIPE_BUFFER && (usage & PIPE_MAP_WRITE) &&
1584 !(usage & TC_TRANSFER_MAP_NO_INFER_UNSYNCHRONIZED) &&
1585 !util_ranges_intersect(&res->valid_buffer_range, box->x,
1586 box->x + box->width);
1587 }
1588
1589 static void *
crocus_transfer_map(struct pipe_context * ctx,struct pipe_resource * resource,unsigned level,unsigned usage,const struct pipe_box * box,struct pipe_transfer ** ptransfer)1590 crocus_transfer_map(struct pipe_context *ctx,
1591 struct pipe_resource *resource,
1592 unsigned level,
1593 unsigned usage,
1594 const struct pipe_box *box,
1595 struct pipe_transfer **ptransfer)
1596 {
1597 struct crocus_context *ice = (struct crocus_context *)ctx;
1598 struct crocus_resource *res = (struct crocus_resource *)resource;
1599 struct isl_surf *surf = &res->surf;
1600 struct crocus_screen *screen = (struct crocus_screen *)ctx->screen;
1601
1602 if (usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE) {
1603 /* Replace the backing storage with a fresh buffer for non-async maps */
1604 if (!(usage & (PIPE_MAP_UNSYNCHRONIZED |
1605 TC_TRANSFER_MAP_NO_INVALIDATE)))
1606 crocus_invalidate_resource(ctx, resource);
1607
1608 /* If we can discard the whole resource, we can discard the range. */
1609 usage |= PIPE_MAP_DISCARD_RANGE;
1610 }
1611
1612 if (!(usage & PIPE_MAP_UNSYNCHRONIZED) &&
1613 can_promote_to_async(res, box, usage)) {
1614 usage |= PIPE_MAP_UNSYNCHRONIZED;
1615 }
1616
1617 bool map_would_stall = false;
1618
1619 if (!(usage & PIPE_MAP_UNSYNCHRONIZED)) {
1620 map_would_stall = resource_is_busy(ice, res) ||
1621 crocus_has_invalid_primary(res, level, 1, box->z, box->depth);
1622
1623
1624 if (map_would_stall && (usage & PIPE_MAP_DONTBLOCK) &&
1625 (usage & PIPE_MAP_DIRECTLY))
1626 return NULL;
1627 }
1628
1629 if (surf->tiling != ISL_TILING_LINEAR &&
1630 (usage & PIPE_MAP_DIRECTLY))
1631 return NULL;
1632
1633 struct crocus_transfer *map;
1634 if (usage & TC_TRANSFER_MAP_THREADED_UNSYNC)
1635 map = slab_alloc(&ice->transfer_pool_unsync);
1636 else
1637 map = slab_alloc(&ice->transfer_pool);
1638
1639 struct pipe_transfer *xfer = &map->base.b;
1640
1641 if (!map)
1642 return NULL;
1643
1644 memset(map, 0, sizeof(*map));
1645 map->dbg = &ice->dbg;
1646
1647 map->has_swizzling = ((struct crocus_screen *)ctx->screen)->has_swizzling;
1648 pipe_resource_reference(&xfer->resource, resource);
1649 xfer->level = level;
1650 xfer->usage = usage;
1651 xfer->box = *box;
1652 *ptransfer = xfer;
1653
1654 map->dest_had_defined_contents =
1655 util_ranges_intersect(&res->valid_buffer_range, box->x,
1656 box->x + box->width);
1657
1658 if (usage & PIPE_MAP_WRITE)
1659 util_range_add(&res->base.b, &res->valid_buffer_range, box->x, box->x + box->width);
1660
1661 /* Avoid using GPU copies for persistent/coherent buffers, as the idea
1662 * there is to access them simultaneously on the CPU & GPU. This also
1663 * avoids trying to use GPU copies for our u_upload_mgr buffers which
1664 * contain state we're constructing for a GPU draw call, which would
1665 * kill us with infinite stack recursion.
1666 */
1667 bool no_gpu = usage & (PIPE_MAP_PERSISTENT |
1668 PIPE_MAP_COHERENT |
1669 PIPE_MAP_DIRECTLY);
1670
1671 /* GPU copies are not useful for buffer reads. Instead of stalling to
1672 * read from the original buffer, we'd simply copy it to a temporary...
1673 * then stall (a bit longer) to read from that buffer.
1674 *
1675 * Images are less clear-cut. Color resolves are destructive, removing
1676 * the underlying compression, so we'd rather blit the data to a linear
1677 * temporary and map that, to avoid the resolve. (It might be better to
1678 * a tiled temporary and use the tiled_memcpy paths...)
1679 */
1680 if (!(usage & PIPE_MAP_DISCARD_RANGE) &&
1681 !crocus_has_invalid_primary(res, level, 1, box->z, box->depth))
1682 no_gpu = true;
1683
1684 const struct isl_format_layout *fmtl = isl_format_get_layout(surf->format);
1685 if (fmtl->txc == ISL_TXC_ASTC)
1686 no_gpu = true;
1687
1688 if (map_would_stall && !no_gpu) {
1689 /* If we need a synchronous mapping and the resource is busy, or needs
1690 * resolving, we copy to/from a linear temporary buffer using the GPU.
1691 */
1692 map->batch = &ice->batches[CROCUS_BATCH_RENDER];
1693 map->blorp = &ice->blorp;
1694 crocus_map_copy_region(map);
1695 } else {
1696 /* Otherwise we're free to map on the CPU. */
1697
1698 if (resource->target != PIPE_BUFFER) {
1699 crocus_resource_access_raw(ice, res,
1700 level, box->z, box->depth,
1701 usage & PIPE_MAP_WRITE);
1702 }
1703
1704 if (!(usage & PIPE_MAP_UNSYNCHRONIZED)) {
1705 for (int i = 0; i < ice->batch_count; i++) {
1706 if (crocus_batch_references(&ice->batches[i], res->bo))
1707 crocus_batch_flush(&ice->batches[i]);
1708 }
1709 }
1710
1711 if (surf->tiling == ISL_TILING_W) {
1712 /* TODO: Teach crocus_map_tiled_memcpy about W-tiling... */
1713 crocus_map_s8(map);
1714 } else if (surf->tiling != ISL_TILING_LINEAR && screen->devinfo.ver > 4) {
1715 crocus_map_tiled_memcpy(map);
1716 } else {
1717 crocus_map_direct(map);
1718 }
1719 }
1720
1721 return map->ptr;
1722 }
1723
1724 static void
crocus_transfer_flush_region(struct pipe_context * ctx,struct pipe_transfer * xfer,const struct pipe_box * box)1725 crocus_transfer_flush_region(struct pipe_context *ctx,
1726 struct pipe_transfer *xfer,
1727 const struct pipe_box *box)
1728 {
1729 struct crocus_context *ice = (struct crocus_context *)ctx;
1730 struct crocus_resource *res = (struct crocus_resource *) xfer->resource;
1731 struct crocus_transfer *map = (void *) xfer;
1732
1733 if (map->staging)
1734 crocus_flush_staging_region(xfer, box);
1735
1736 uint32_t history_flush = 0;
1737
1738 if (res->base.b.target == PIPE_BUFFER) {
1739 if (map->staging)
1740 history_flush |= PIPE_CONTROL_RENDER_TARGET_FLUSH;
1741
1742 if (map->dest_had_defined_contents)
1743 history_flush |= crocus_flush_bits_for_history(res);
1744
1745 util_range_add(&res->base.b, &res->valid_buffer_range, box->x, box->x + box->width);
1746 }
1747
1748 if (history_flush & ~PIPE_CONTROL_CS_STALL) {
1749 for (int i = 0; i < ice->batch_count; i++) {
1750 struct crocus_batch *batch = &ice->batches[i];
1751
1752 if (!batch->command.bo)
1753 continue;
1754 if (batch->contains_draw || batch->cache.render->entries) {
1755 crocus_batch_maybe_flush(batch, 24);
1756 crocus_emit_pipe_control_flush(batch,
1757 "cache history: transfer flush",
1758 history_flush);
1759 }
1760 }
1761 }
1762
1763 /* Make sure we flag constants dirty even if there's no need to emit
1764 * any PIPE_CONTROLs to a batch.
1765 */
1766 crocus_dirty_for_history(ice, res);
1767 }
1768
1769 static void
crocus_transfer_unmap(struct pipe_context * ctx,struct pipe_transfer * xfer)1770 crocus_transfer_unmap(struct pipe_context *ctx, struct pipe_transfer *xfer)
1771 {
1772 struct crocus_context *ice = (struct crocus_context *)ctx;
1773 struct crocus_transfer *map = (void *) xfer;
1774
1775 if (!(xfer->usage & (PIPE_MAP_FLUSH_EXPLICIT |
1776 PIPE_MAP_COHERENT))) {
1777 struct pipe_box flush_box = {
1778 .x = 0, .y = 0, .z = 0,
1779 .width = xfer->box.width,
1780 .height = xfer->box.height,
1781 .depth = xfer->box.depth,
1782 };
1783 crocus_transfer_flush_region(ctx, xfer, &flush_box);
1784 }
1785
1786 if (map->unmap)
1787 map->unmap(map);
1788
1789 pipe_resource_reference(&xfer->resource, NULL);
1790 /* transfer_unmap is always called from the driver thread, so we have to
1791 * use transfer_pool, not transfer_pool_unsync. Freeing an object into a
1792 * different pool is allowed, however.
1793 */
1794 slab_free(&ice->transfer_pool, map);
1795 }
1796
1797 /**
1798 * Mark state dirty that needs to be re-emitted when a resource is written.
1799 */
1800 void
crocus_dirty_for_history(struct crocus_context * ice,struct crocus_resource * res)1801 crocus_dirty_for_history(struct crocus_context *ice,
1802 struct crocus_resource *res)
1803 {
1804 uint64_t stage_dirty = 0ull;
1805
1806 if (res->bind_history & PIPE_BIND_CONSTANT_BUFFER) {
1807 stage_dirty |= ((uint64_t)res->bind_stages) << CROCUS_SHIFT_FOR_STAGE_DIRTY_CONSTANTS;
1808 }
1809
1810 ice->state.stage_dirty |= stage_dirty;
1811 }
1812
1813 /**
1814 * Produce a set of PIPE_CONTROL bits which ensure data written to a
1815 * resource becomes visible, and any stale read cache data is invalidated.
1816 */
1817 uint32_t
crocus_flush_bits_for_history(struct crocus_resource * res)1818 crocus_flush_bits_for_history(struct crocus_resource *res)
1819 {
1820 uint32_t flush = PIPE_CONTROL_CS_STALL;
1821
1822 if (res->bind_history & PIPE_BIND_CONSTANT_BUFFER) {
1823 flush |= PIPE_CONTROL_CONST_CACHE_INVALIDATE |
1824 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
1825 }
1826
1827 if (res->bind_history & PIPE_BIND_SAMPLER_VIEW)
1828 flush |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
1829
1830 if (res->bind_history & (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER))
1831 flush |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
1832
1833 if (res->bind_history & (PIPE_BIND_SHADER_BUFFER | PIPE_BIND_SHADER_IMAGE))
1834 flush |= PIPE_CONTROL_DATA_CACHE_FLUSH;
1835
1836 return flush;
1837 }
1838
1839 void
crocus_flush_and_dirty_for_history(struct crocus_context * ice,struct crocus_batch * batch,struct crocus_resource * res,uint32_t extra_flags,const char * reason)1840 crocus_flush_and_dirty_for_history(struct crocus_context *ice,
1841 struct crocus_batch *batch,
1842 struct crocus_resource *res,
1843 uint32_t extra_flags,
1844 const char *reason)
1845 {
1846 if (res->base.b.target != PIPE_BUFFER)
1847 return;
1848
1849 uint32_t flush = crocus_flush_bits_for_history(res) | extra_flags;
1850
1851 crocus_emit_pipe_control_flush(batch, reason, flush);
1852
1853 crocus_dirty_for_history(ice, res);
1854 }
1855
1856 bool
crocus_resource_set_clear_color(struct crocus_context * ice,struct crocus_resource * res,union isl_color_value color)1857 crocus_resource_set_clear_color(struct crocus_context *ice,
1858 struct crocus_resource *res,
1859 union isl_color_value color)
1860 {
1861 if (memcmp(&res->aux.clear_color, &color, sizeof(color)) != 0) {
1862 res->aux.clear_color = color;
1863 return true;
1864 }
1865
1866 return false;
1867 }
1868
1869 union isl_color_value
crocus_resource_get_clear_color(const struct crocus_resource * res)1870 crocus_resource_get_clear_color(const struct crocus_resource *res)
1871 {
1872 assert(res->aux.bo);
1873
1874 return res->aux.clear_color;
1875 }
1876
1877 static enum pipe_format
crocus_resource_get_internal_format(struct pipe_resource * p_res)1878 crocus_resource_get_internal_format(struct pipe_resource *p_res)
1879 {
1880 struct crocus_resource *res = (void *) p_res;
1881 return res->internal_format;
1882 }
1883
1884 static const struct u_transfer_vtbl transfer_vtbl = {
1885 .resource_create = crocus_resource_create,
1886 .resource_destroy = crocus_resource_destroy,
1887 .transfer_map = crocus_transfer_map,
1888 .transfer_unmap = crocus_transfer_unmap,
1889 .transfer_flush_region = crocus_transfer_flush_region,
1890 .get_internal_format = crocus_resource_get_internal_format,
1891 .set_stencil = crocus_resource_set_separate_stencil,
1892 .get_stencil = crocus_resource_get_separate_stencil,
1893 };
1894
1895 static bool
crocus_is_dmabuf_modifier_supported(struct pipe_screen * pscreen,uint64_t modifier,enum pipe_format pfmt,bool * external_only)1896 crocus_is_dmabuf_modifier_supported(struct pipe_screen *pscreen,
1897 uint64_t modifier, enum pipe_format pfmt,
1898 bool *external_only)
1899 {
1900 struct crocus_screen *screen = (void *) pscreen;
1901 const struct intel_device_info *devinfo = &screen->devinfo;
1902
1903 if (modifier_is_supported(devinfo, pfmt, 0, modifier)) {
1904 if (external_only)
1905 *external_only = false;
1906
1907 return true;
1908 }
1909
1910 return false;
1911 }
1912
1913 static unsigned int
crocus_get_dmabuf_modifier_planes(struct pipe_screen * pscreen,uint64_t modifier,enum pipe_format format)1914 crocus_get_dmabuf_modifier_planes(struct pipe_screen *pscreen, uint64_t modifier,
1915 enum pipe_format format)
1916 {
1917 return util_format_get_num_planes(format);
1918 }
1919
1920 static struct pipe_memory_object *
crocus_memobj_create_from_handle(struct pipe_screen * pscreen,struct winsys_handle * whandle,bool dedicated)1921 crocus_memobj_create_from_handle(struct pipe_screen *pscreen,
1922 struct winsys_handle *whandle,
1923 bool dedicated)
1924 {
1925 struct crocus_screen *screen = (struct crocus_screen *)pscreen;
1926 struct crocus_memory_object *memobj = CALLOC_STRUCT(crocus_memory_object);
1927 struct crocus_bo *bo;
1928 const struct isl_drm_modifier_info *mod_inf;
1929
1930 if (!memobj)
1931 return NULL;
1932
1933 switch (whandle->type) {
1934 case WINSYS_HANDLE_TYPE_SHARED:
1935 bo = crocus_bo_gem_create_from_name(screen->bufmgr, "winsys image",
1936 whandle->handle);
1937 break;
1938 case WINSYS_HANDLE_TYPE_FD:
1939 mod_inf = isl_drm_modifier_get_info(whandle->modifier);
1940 if (mod_inf) {
1941 bo = crocus_bo_import_dmabuf(screen->bufmgr, whandle->handle,
1942 whandle->modifier);
1943 } else {
1944 /* If we can't get information about the tiling from the
1945 * kernel we ignore it. We are going to set it when we
1946 * create the resource.
1947 */
1948 bo = crocus_bo_import_dmabuf_no_mods(screen->bufmgr,
1949 whandle->handle);
1950 }
1951
1952 break;
1953 default:
1954 unreachable("invalid winsys handle type");
1955 }
1956
1957 if (!bo) {
1958 free(memobj);
1959 return NULL;
1960 }
1961
1962 memobj->b.dedicated = dedicated;
1963 memobj->bo = bo;
1964 memobj->format = whandle->format;
1965 memobj->stride = whandle->stride;
1966
1967 return &memobj->b;
1968 }
1969
1970 static void
crocus_memobj_destroy(struct pipe_screen * pscreen,struct pipe_memory_object * pmemobj)1971 crocus_memobj_destroy(struct pipe_screen *pscreen,
1972 struct pipe_memory_object *pmemobj)
1973 {
1974 struct crocus_memory_object *memobj = (struct crocus_memory_object *)pmemobj;
1975
1976 crocus_bo_unreference(memobj->bo);
1977 free(memobj);
1978 }
1979
1980 void
crocus_init_screen_resource_functions(struct pipe_screen * pscreen)1981 crocus_init_screen_resource_functions(struct pipe_screen *pscreen)
1982 {
1983 struct crocus_screen *screen = (void *) pscreen;
1984 pscreen->query_dmabuf_modifiers = crocus_query_dmabuf_modifiers;
1985 pscreen->is_dmabuf_modifier_supported = crocus_is_dmabuf_modifier_supported;
1986 pscreen->get_dmabuf_modifier_planes = crocus_get_dmabuf_modifier_planes;
1987 pscreen->resource_create_with_modifiers =
1988 crocus_resource_create_with_modifiers;
1989 pscreen->resource_create = u_transfer_helper_resource_create;
1990 pscreen->resource_from_user_memory = crocus_resource_from_user_memory;
1991 pscreen->resource_from_handle = crocus_resource_from_handle;
1992 pscreen->resource_from_memobj = crocus_resource_from_memobj;
1993 pscreen->resource_get_handle = crocus_resource_get_handle;
1994 pscreen->resource_get_param = crocus_resource_get_param;
1995 pscreen->resource_destroy = u_transfer_helper_resource_destroy;
1996 pscreen->memobj_create_from_handle = crocus_memobj_create_from_handle;
1997 pscreen->memobj_destroy = crocus_memobj_destroy;
1998 pscreen->transfer_helper =
1999 u_transfer_helper_create(&transfer_vtbl, screen->devinfo.ver >= 6,
2000 screen->devinfo.ver >= 6, false, true);
2001 }
2002
2003 void
crocus_init_resource_functions(struct pipe_context * ctx)2004 crocus_init_resource_functions(struct pipe_context *ctx)
2005 {
2006 ctx->flush_resource = crocus_flush_resource;
2007 ctx->invalidate_resource = crocus_invalidate_resource;
2008 ctx->buffer_map = u_transfer_helper_transfer_map;
2009 ctx->texture_map = u_transfer_helper_transfer_map;
2010 ctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;
2011 ctx->buffer_unmap = u_transfer_helper_transfer_unmap;
2012 ctx->texture_unmap = u_transfer_helper_transfer_unmap;
2013 ctx->buffer_subdata = u_default_buffer_subdata;
2014 ctx->texture_subdata = u_default_texture_subdata;
2015 }
2016