1 /*
2 * Copyright © 2017 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 #include <stdio.h>
24 #include "pipe/p_defines.h"
25 #include "pipe/p_state.h"
26 #include "pipe/p_context.h"
27 #include "pipe/p_screen.h"
28 #include "util/format/u_format.h"
29 #include "util/u_inlines.h"
30 #include "util/ralloc.h"
31 #include "intel/blorp/blorp.h"
32 #include "iris_context.h"
33 #include "iris_resource.h"
34 #include "iris_screen.h"
35
36 /**
37 * Helper function for handling mirror image blits.
38 *
39 * If coord0 > coord1, swap them and return "true" (mirrored).
40 */
41 static bool
apply_mirror(float * coord0,float * coord1)42 apply_mirror(float *coord0, float *coord1)
43 {
44 if (*coord0 > *coord1) {
45 float tmp = *coord0;
46 *coord0 = *coord1;
47 *coord1 = tmp;
48 return true;
49 }
50 return false;
51 }
52
53 /**
54 * Compute the number of pixels to clip for each side of a rect
55 *
56 * \param x0 The rect's left coordinate
57 * \param y0 The rect's bottom coordinate
58 * \param x1 The rect's right coordinate
59 * \param y1 The rect's top coordinate
60 * \param min_x The clipping region's left coordinate
61 * \param min_y The clipping region's bottom coordinate
62 * \param max_x The clipping region's right coordinate
63 * \param max_y The clipping region's top coordinate
64 * \param clipped_x0 The number of pixels to clip from the left side
65 * \param clipped_y0 The number of pixels to clip from the bottom side
66 * \param clipped_x1 The number of pixels to clip from the right side
67 * \param clipped_y1 The number of pixels to clip from the top side
68 *
69 * \return false if we clip everything away, true otherwise
70 */
71 static inline bool
compute_pixels_clipped(float x0,float y0,float x1,float y1,float min_x,float min_y,float max_x,float max_y,float * clipped_x0,float * clipped_y0,float * clipped_x1,float * clipped_y1)72 compute_pixels_clipped(float x0, float y0, float x1, float y1,
73 float min_x, float min_y, float max_x, float max_y,
74 float *clipped_x0, float *clipped_y0,
75 float *clipped_x1, float *clipped_y1)
76 {
77 /* If we are going to clip everything away, stop. */
78 if (!(min_x <= max_x &&
79 min_y <= max_y &&
80 x0 <= max_x &&
81 y0 <= max_y &&
82 min_x <= x1 &&
83 min_y <= y1 &&
84 x0 <= x1 &&
85 y0 <= y1)) {
86 return false;
87 }
88
89 if (x0 < min_x)
90 *clipped_x0 = min_x - x0;
91 else
92 *clipped_x0 = 0;
93 if (max_x < x1)
94 *clipped_x1 = x1 - max_x;
95 else
96 *clipped_x1 = 0;
97
98 if (y0 < min_y)
99 *clipped_y0 = min_y - y0;
100 else
101 *clipped_y0 = 0;
102 if (max_y < y1)
103 *clipped_y1 = y1 - max_y;
104 else
105 *clipped_y1 = 0;
106
107 return true;
108 }
109
110 /**
111 * Clips a coordinate (left, right, top or bottom) for the src or dst rect
112 * (whichever requires the largest clip) and adjusts the coordinate
113 * for the other rect accordingly.
114 *
115 * \param mirror true if mirroring is required
116 * \param src the source rect coordinate (for example src_x0)
117 * \param dst0 the dst rect coordinate (for example dst_x0)
118 * \param dst1 the opposite dst rect coordinate (for example dst_x1)
119 * \param clipped_dst0 number of pixels to clip from the dst coordinate
120 * \param clipped_dst1 number of pixels to clip from the opposite dst coordinate
121 * \param scale the src vs dst scale involved for that coordinate
122 * \param is_left_or_bottom true if we are clipping the left or bottom sides
123 * of the rect.
124 */
125 static void
clip_coordinates(bool mirror,float * src,float * dst0,float * dst1,float clipped_dst0,float clipped_dst1,float scale,bool is_left_or_bottom)126 clip_coordinates(bool mirror,
127 float *src, float *dst0, float *dst1,
128 float clipped_dst0,
129 float clipped_dst1,
130 float scale,
131 bool is_left_or_bottom)
132 {
133 /* When clipping we need to add or subtract pixels from the original
134 * coordinates depending on whether we are acting on the left/bottom
135 * or right/top sides of the rect respectively. We assume we have to
136 * add them in the code below, and multiply by -1 when we should
137 * subtract.
138 */
139 int mult = is_left_or_bottom ? 1 : -1;
140
141 if (!mirror) {
142 *dst0 += clipped_dst0 * mult;
143 *src += clipped_dst0 * scale * mult;
144 } else {
145 *dst1 -= clipped_dst1 * mult;
146 *src += clipped_dst1 * scale * mult;
147 }
148 }
149
150 /**
151 * Apply a scissor rectangle to blit coordinates.
152 *
153 * Returns true if the blit was entirely scissored away.
154 */
155 static bool
apply_blit_scissor(const struct pipe_scissor_state * scissor,float * src_x0,float * src_y0,float * src_x1,float * src_y1,float * dst_x0,float * dst_y0,float * dst_x1,float * dst_y1,bool mirror_x,bool mirror_y)156 apply_blit_scissor(const struct pipe_scissor_state *scissor,
157 float *src_x0, float *src_y0,
158 float *src_x1, float *src_y1,
159 float *dst_x0, float *dst_y0,
160 float *dst_x1, float *dst_y1,
161 bool mirror_x, bool mirror_y)
162 {
163 float clip_dst_x0, clip_dst_x1, clip_dst_y0, clip_dst_y1;
164
165 /* Compute number of pixels to scissor away. */
166 if (!compute_pixels_clipped(*dst_x0, *dst_y0, *dst_x1, *dst_y1,
167 scissor->minx, scissor->miny,
168 scissor->maxx, scissor->maxy,
169 &clip_dst_x0, &clip_dst_y0,
170 &clip_dst_x1, &clip_dst_y1))
171 return true;
172
173 // XXX: comments assume source clipping, which we don't do
174
175 /* When clipping any of the two rects we need to adjust the coordinates
176 * in the other rect considering the scaling factor involved. To obtain
177 * the best precision we want to make sure that we only clip once per
178 * side to avoid accumulating errors due to the scaling adjustment.
179 *
180 * For example, if src_x0 and dst_x0 need both to be clipped we want to
181 * avoid the situation where we clip src_x0 first, then adjust dst_x0
182 * accordingly but then we realize that the resulting dst_x0 still needs
183 * to be clipped, so we clip dst_x0 and adjust src_x0 again. Because we are
184 * applying scaling factors to adjust the coordinates in each clipping
185 * pass we lose some precision and that can affect the results of the
186 * blorp blit operation slightly. What we want to do here is detect the
187 * rect that we should clip first for each side so that when we adjust
188 * the other rect we ensure the resulting coordinate does not need to be
189 * clipped again.
190 *
191 * The code below implements this by comparing the number of pixels that
192 * we need to clip for each side of both rects considering the scales
193 * involved. For example, clip_src_x0 represents the number of pixels
194 * to be clipped for the src rect's left side, so if clip_src_x0 = 5,
195 * clip_dst_x0 = 4 and scale_x = 2 it means that we are clipping more
196 * from the dst rect so we should clip dst_x0 only and adjust src_x0.
197 * This is because clipping 4 pixels in the dst is equivalent to
198 * clipping 4 * 2 = 8 > 5 in the src.
199 */
200
201 if (*src_x0 == *src_x1 || *src_y0 == *src_y1
202 || *dst_x0 == *dst_x1 || *dst_y0 == *dst_y1)
203 return true;
204
205 float scale_x = (float) (*src_x1 - *src_x0) / (*dst_x1 - *dst_x0);
206 float scale_y = (float) (*src_y1 - *src_y0) / (*dst_y1 - *dst_y0);
207
208 /* Clip left side */
209 clip_coordinates(mirror_x, src_x0, dst_x0, dst_x1,
210 clip_dst_x0, clip_dst_x1, scale_x, true);
211
212 /* Clip right side */
213 clip_coordinates(mirror_x, src_x1, dst_x1, dst_x0,
214 clip_dst_x1, clip_dst_x0, scale_x, false);
215
216 /* Clip bottom side */
217 clip_coordinates(mirror_y, src_y0, dst_y0, dst_y1,
218 clip_dst_y0, clip_dst_y1, scale_y, true);
219
220 /* Clip top side */
221 clip_coordinates(mirror_y, src_y1, dst_y1, dst_y0,
222 clip_dst_y1, clip_dst_y0, scale_y, false);
223
224 /* Check for invalid bounds
225 * Can't blit for 0-dimensions
226 */
227 return *src_x0 == *src_x1 || *src_y0 == *src_y1
228 || *dst_x0 == *dst_x1 || *dst_y0 == *dst_y1;
229 }
230
231 void
iris_blorp_surf_for_resource(struct isl_device * isl_dev,struct blorp_surf * surf,struct pipe_resource * p_res,enum isl_aux_usage aux_usage,unsigned level,bool is_render_target)232 iris_blorp_surf_for_resource(struct isl_device *isl_dev,
233 struct blorp_surf *surf,
234 struct pipe_resource *p_res,
235 enum isl_aux_usage aux_usage,
236 unsigned level,
237 bool is_render_target)
238 {
239 struct iris_resource *res = (void *) p_res;
240
241 *surf = (struct blorp_surf) {
242 .surf = &res->surf,
243 .addr = (struct blorp_address) {
244 .buffer = res->bo,
245 .offset = res->offset,
246 .reloc_flags = is_render_target ? EXEC_OBJECT_WRITE : 0,
247 .mocs = iris_mocs(res->bo, isl_dev,
248 is_render_target ? ISL_SURF_USAGE_RENDER_TARGET_BIT
249 : ISL_SURF_USAGE_TEXTURE_BIT),
250 },
251 .aux_usage = aux_usage,
252 };
253
254 if (aux_usage != ISL_AUX_USAGE_NONE) {
255 surf->aux_surf = &res->aux.surf;
256 surf->aux_addr = (struct blorp_address) {
257 .buffer = res->aux.bo,
258 .offset = res->aux.offset,
259 .reloc_flags = is_render_target ? EXEC_OBJECT_WRITE : 0,
260 .mocs = iris_mocs(res->bo, isl_dev, 0),
261 };
262 surf->clear_color = res->aux.clear_color;
263 surf->clear_color_addr = (struct blorp_address) {
264 .buffer = res->aux.clear_color_bo,
265 .offset = res->aux.clear_color_offset,
266 .reloc_flags = 0,
267 .mocs = iris_mocs(res->aux.clear_color_bo, isl_dev, 0),
268 };
269 }
270 }
271
272 static bool
is_astc(enum isl_format format)273 is_astc(enum isl_format format)
274 {
275 return format != ISL_FORMAT_UNSUPPORTED &&
276 isl_format_get_layout(format)->txc == ISL_TXC_ASTC;
277 }
278
279 static void
tex_cache_flush_hack(struct iris_batch * batch,enum isl_format view_format,enum isl_format surf_format)280 tex_cache_flush_hack(struct iris_batch *batch,
281 enum isl_format view_format,
282 enum isl_format surf_format)
283 {
284 const struct intel_device_info *devinfo = &batch->screen->devinfo;
285
286 /* The WaSamplerCacheFlushBetweenRedescribedSurfaceReads workaround says:
287 *
288 * "Currently Sampler assumes that a surface would not have two
289 * different format associate with it. It will not properly cache
290 * the different views in the MT cache, causing a data corruption."
291 *
292 * We may need to handle this for texture views in general someday, but
293 * for now we handle it here, as it hurts copies and blits particularly
294 * badly because they ofter reinterpret formats.
295 *
296 * If the BO hasn't been referenced yet this batch, we assume that the
297 * texture cache doesn't contain any relevant data nor need flushing.
298 *
299 * Icelake (Gfx11+) claims to fix this issue, but seems to still have
300 * issues with ASTC formats.
301 */
302 bool need_flush = devinfo->ver >= 11 ?
303 is_astc(surf_format) != is_astc(view_format) :
304 view_format != surf_format;
305 if (!need_flush)
306 return;
307
308 const char *reason =
309 "workaround: WaSamplerCacheFlushBetweenRedescribedSurfaceReads";
310
311 iris_emit_pipe_control_flush(batch, reason, PIPE_CONTROL_CS_STALL);
312 iris_emit_pipe_control_flush(batch, reason,
313 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
314 }
315
316 static struct iris_resource *
iris_resource_for_aspect(struct pipe_resource * p_res,unsigned pipe_mask)317 iris_resource_for_aspect(struct pipe_resource *p_res, unsigned pipe_mask)
318 {
319 if (pipe_mask == PIPE_MASK_S) {
320 struct iris_resource *junk, *s_res;
321 iris_get_depth_stencil_resources(p_res, &junk, &s_res);
322 return s_res;
323 } else {
324 return (struct iris_resource *)p_res;
325 }
326 }
327
328 static enum pipe_format
pipe_format_for_aspect(enum pipe_format format,unsigned pipe_mask)329 pipe_format_for_aspect(enum pipe_format format, unsigned pipe_mask)
330 {
331 if (pipe_mask == PIPE_MASK_S) {
332 return util_format_stencil_only(format);
333 } else if (pipe_mask == PIPE_MASK_Z) {
334 return util_format_get_depth_only(format);
335 } else {
336 return format;
337 }
338 }
339
340 static bool
clear_color_is_fully_zero(const struct iris_resource * res)341 clear_color_is_fully_zero(const struct iris_resource *res)
342 {
343 return !res->aux.clear_color_unknown &&
344 res->aux.clear_color.u32[0] == 0 &&
345 res->aux.clear_color.u32[1] == 0 &&
346 res->aux.clear_color.u32[2] == 0 &&
347 res->aux.clear_color.u32[3] == 0;
348 }
349
350 /**
351 * The pipe->blit() driver hook.
352 *
353 * This performs a blit between two surfaces, which copies data but may
354 * also perform format conversion, scaling, flipping, and so on.
355 */
356 static void
iris_blit(struct pipe_context * ctx,const struct pipe_blit_info * info)357 iris_blit(struct pipe_context *ctx, const struct pipe_blit_info *info)
358 {
359 struct iris_context *ice = (void *) ctx;
360 struct iris_screen *screen = (struct iris_screen *)ctx->screen;
361 const struct intel_device_info *devinfo = &screen->devinfo;
362 struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
363 enum blorp_batch_flags blorp_flags = 0;
364
365 /* We don't support color masking. */
366 assert((info->mask & PIPE_MASK_RGBA) == PIPE_MASK_RGBA ||
367 (info->mask & PIPE_MASK_RGBA) == 0);
368
369 if (info->render_condition_enable) {
370 if (ice->state.predicate == IRIS_PREDICATE_STATE_DONT_RENDER)
371 return;
372
373 if (ice->state.predicate == IRIS_PREDICATE_STATE_USE_BIT)
374 blorp_flags |= BLORP_BATCH_PREDICATE_ENABLE;
375 }
376
377 float src_x0 = info->src.box.x;
378 float src_x1 = info->src.box.x + info->src.box.width;
379 float src_y0 = info->src.box.y;
380 float src_y1 = info->src.box.y + info->src.box.height;
381 float dst_x0 = info->dst.box.x;
382 float dst_x1 = info->dst.box.x + info->dst.box.width;
383 float dst_y0 = info->dst.box.y;
384 float dst_y1 = info->dst.box.y + info->dst.box.height;
385 bool mirror_x = apply_mirror(&src_x0, &src_x1);
386 bool mirror_y = apply_mirror(&src_y0, &src_y1);
387 enum blorp_filter filter;
388
389 if (info->scissor_enable) {
390 bool noop = apply_blit_scissor(&info->scissor,
391 &src_x0, &src_y0, &src_x1, &src_y1,
392 &dst_x0, &dst_y0, &dst_x1, &dst_y1,
393 mirror_x, mirror_y);
394 if (noop)
395 return;
396 }
397
398 if (abs(info->dst.box.width) == abs(info->src.box.width) &&
399 abs(info->dst.box.height) == abs(info->src.box.height)) {
400 if (info->src.resource->nr_samples > 1 &&
401 info->dst.resource->nr_samples <= 1) {
402 /* The OpenGL ES 3.2 specification, section 16.2.1, says:
403 *
404 * "If the read framebuffer is multisampled (its effective
405 * value of SAMPLE_BUFFERS is one) and the draw framebuffer
406 * is not (its value of SAMPLE_BUFFERS is zero), the samples
407 * corresponding to each pixel location in the source are
408 * converted to a single sample before being written to the
409 * destination. The filter parameter is ignored. If the
410 * source formats are integer types or stencil values, a
411 * single sample’s value is selected for each pixel. If the
412 * source formats are floating-point or normalized types,
413 * the sample values for each pixel are resolved in an
414 * implementation-dependent manner. If the source formats
415 * are depth values, sample values are resolved in an
416 * implementation-dependent manner where the result will be
417 * between the minimum and maximum depth values in the pixel."
418 *
419 * When selecting a single sample, we always choose sample 0.
420 */
421 if (util_format_is_depth_or_stencil(info->src.format) ||
422 util_format_is_pure_integer(info->src.format)) {
423 filter = BLORP_FILTER_SAMPLE_0;
424 } else {
425 filter = BLORP_FILTER_AVERAGE;
426 }
427 } else {
428 /* The OpenGL 4.6 specification, section 18.3.1, says:
429 *
430 * "If the source and destination dimensions are identical,
431 * no filtering is applied."
432 *
433 * Using BLORP_FILTER_NONE will also handle the upsample case by
434 * replicating the one value in the source to all values in the
435 * destination.
436 */
437 filter = BLORP_FILTER_NONE;
438 }
439 } else if (info->filter == PIPE_TEX_FILTER_LINEAR) {
440 filter = BLORP_FILTER_BILINEAR;
441 } else {
442 filter = BLORP_FILTER_NEAREST;
443 }
444
445 struct blorp_batch blorp_batch;
446 blorp_batch_init(&ice->blorp, &blorp_batch, batch, blorp_flags);
447
448 float src_z_step = (float)info->src.box.depth / (float)info->dst.box.depth;
449
450 /* There is no interpolation to the pixel center during rendering, so
451 * add the 0.5 offset ourselves here.
452 */
453 float depth_center_offset = 0;
454 if (info->src.resource->target == PIPE_TEXTURE_3D)
455 depth_center_offset = 0.5 / info->dst.box.depth * info->src.box.depth;
456
457 /* Perform a blit for each aspect requested by the caller. PIPE_MASK_R is
458 * used to represent the color aspect. */
459 unsigned aspect_mask = info->mask & (PIPE_MASK_R | PIPE_MASK_ZS);
460 while (aspect_mask) {
461 unsigned aspect = 1 << u_bit_scan(&aspect_mask);
462
463 struct iris_resource *src_res =
464 iris_resource_for_aspect(info->src.resource, aspect);
465 struct iris_resource *dst_res =
466 iris_resource_for_aspect(info->dst.resource, aspect);
467
468 enum pipe_format src_pfmt =
469 pipe_format_for_aspect(info->src.format, aspect);
470 enum pipe_format dst_pfmt =
471 pipe_format_for_aspect(info->dst.format, aspect);
472
473 struct iris_format_info src_fmt =
474 iris_format_for_usage(devinfo, src_pfmt, ISL_SURF_USAGE_TEXTURE_BIT);
475 enum isl_aux_usage src_aux_usage =
476 iris_resource_texture_aux_usage(ice, src_res, src_fmt.fmt);
477
478 iris_resource_prepare_texture(ice, src_res, src_fmt.fmt,
479 info->src.level, 1, info->src.box.z,
480 info->src.box.depth);
481 iris_emit_buffer_barrier_for(batch, src_res->bo,
482 IRIS_DOMAIN_OTHER_READ);
483
484 struct iris_format_info dst_fmt =
485 iris_format_for_usage(devinfo, dst_pfmt,
486 ISL_SURF_USAGE_RENDER_TARGET_BIT);
487 enum isl_aux_usage dst_aux_usage =
488 iris_resource_render_aux_usage(ice, dst_res, info->dst.level,
489 dst_fmt.fmt, false);
490
491 struct blorp_surf src_surf, dst_surf;
492 iris_blorp_surf_for_resource(&screen->isl_dev, &src_surf,
493 &src_res->base.b, src_aux_usage,
494 info->src.level, false);
495 iris_blorp_surf_for_resource(&screen->isl_dev, &dst_surf,
496 &dst_res->base.b, dst_aux_usage,
497 info->dst.level, true);
498
499 iris_resource_prepare_render(ice, dst_res, info->dst.level,
500 info->dst.box.z, info->dst.box.depth,
501 dst_aux_usage);
502 iris_emit_buffer_barrier_for(batch, dst_res->bo,
503 IRIS_DOMAIN_RENDER_WRITE);
504
505 if (iris_batch_references(batch, src_res->bo))
506 tex_cache_flush_hack(batch, src_fmt.fmt, src_res->surf.format);
507
508 if (dst_res->base.b.target == PIPE_BUFFER) {
509 util_range_add(&dst_res->base.b, &dst_res->valid_buffer_range,
510 dst_x0, dst_x1);
511 }
512
513 for (int slice = 0; slice < info->dst.box.depth; slice++) {
514 unsigned dst_z = info->dst.box.z + slice;
515 float src_z = info->src.box.z + slice * src_z_step +
516 depth_center_offset;
517
518 iris_batch_maybe_flush(batch, 1500);
519 iris_batch_sync_region_start(batch);
520
521 blorp_blit(&blorp_batch,
522 &src_surf, info->src.level, src_z,
523 src_fmt.fmt, src_fmt.swizzle,
524 &dst_surf, info->dst.level, dst_z,
525 dst_fmt.fmt, dst_fmt.swizzle,
526 src_x0, src_y0, src_x1, src_y1,
527 dst_x0, dst_y0, dst_x1, dst_y1,
528 filter, mirror_x, mirror_y);
529
530 iris_batch_sync_region_end(batch);
531 }
532
533 tex_cache_flush_hack(batch, src_fmt.fmt, src_res->surf.format);
534
535 iris_resource_finish_render(ice, dst_res, info->dst.level,
536 info->dst.box.z, info->dst.box.depth,
537 dst_aux_usage);
538 }
539
540 blorp_batch_finish(&blorp_batch);
541
542 iris_flush_and_dirty_for_history(ice, batch, (struct iris_resource *)
543 info->dst.resource,
544 PIPE_CONTROL_RENDER_TARGET_FLUSH,
545 "cache history: post-blit");
546 }
547
548 static void
get_copy_region_aux_settings(struct iris_context * ice,struct iris_resource * res,unsigned level,enum isl_aux_usage * out_aux_usage,bool * out_clear_supported,bool is_render_target)549 get_copy_region_aux_settings(struct iris_context *ice,
550 struct iris_resource *res,
551 unsigned level,
552 enum isl_aux_usage *out_aux_usage,
553 bool *out_clear_supported,
554 bool is_render_target)
555 {
556 struct iris_screen *screen = (void *) ice->ctx.screen;
557 struct intel_device_info *devinfo = &screen->devinfo;
558
559 switch (res->aux.usage) {
560 case ISL_AUX_USAGE_HIZ:
561 case ISL_AUX_USAGE_HIZ_CCS:
562 case ISL_AUX_USAGE_HIZ_CCS_WT:
563 case ISL_AUX_USAGE_STC_CCS:
564 if (is_render_target) {
565 *out_aux_usage = iris_resource_render_aux_usage(ice, res, level,
566 res->surf.format,
567 false);
568 } else {
569 *out_aux_usage = iris_resource_texture_aux_usage(ice, res,
570 res->surf.format);
571 }
572 *out_clear_supported = isl_aux_usage_has_fast_clears(*out_aux_usage);
573 break;
574 case ISL_AUX_USAGE_MCS:
575 case ISL_AUX_USAGE_MCS_CCS:
576 if (!is_render_target &&
577 !iris_can_sample_mcs_with_clear(devinfo, res)) {
578 *out_aux_usage = res->aux.usage;
579 *out_clear_supported = false;
580 break;
581 }
582 FALLTHROUGH;
583 case ISL_AUX_USAGE_CCS_E:
584 case ISL_AUX_USAGE_GFX12_CCS_E:
585 *out_aux_usage = res->aux.usage;
586
587 /* blorp_copy may reinterpret the surface format and has limited support
588 * for adjusting the clear color, so clear support may only be enabled
589 * in some cases:
590 *
591 * - On gfx11+, the clear color is indirect and comes in two forms: a
592 * 32bpc representation used for rendering and a pixel representation
593 * used for sampling. blorp_copy doesn't change indirect clear colors,
594 * so clears are only supported in the sampling case.
595 *
596 * - A clear color of zeroes holds the same meaning regardless of the
597 * format. Although it could avoid more resolves, we don't use
598 * isl_color_value_is_zero because the surface format used by
599 * blorp_copy isn't guaranteed to access the same components as the
600 * original format (e.g. A8_UNORM/R8_UINT).
601 */
602 *out_clear_supported = (devinfo->ver >= 11 && !is_render_target) ||
603 clear_color_is_fully_zero(res);
604 break;
605 default:
606 *out_aux_usage = ISL_AUX_USAGE_NONE;
607 *out_clear_supported = false;
608 break;
609 }
610 }
611
612 /**
613 * Perform a GPU-based raw memory copy between compatible view classes.
614 *
615 * Does not perform any flushing - the new data may still be left in the
616 * render cache, and old data may remain in other caches.
617 *
618 * Wraps blorp_copy() and blorp_buffer_copy().
619 */
620 void
iris_copy_region(struct blorp_context * blorp,struct iris_batch * batch,struct pipe_resource * dst,unsigned dst_level,unsigned dstx,unsigned dsty,unsigned dstz,struct pipe_resource * src,unsigned src_level,const struct pipe_box * src_box)621 iris_copy_region(struct blorp_context *blorp,
622 struct iris_batch *batch,
623 struct pipe_resource *dst,
624 unsigned dst_level,
625 unsigned dstx, unsigned dsty, unsigned dstz,
626 struct pipe_resource *src,
627 unsigned src_level,
628 const struct pipe_box *src_box)
629 {
630 struct blorp_batch blorp_batch;
631 struct iris_context *ice = blorp->driver_ctx;
632 struct iris_screen *screen = (void *) ice->ctx.screen;
633 struct iris_resource *src_res = (void *) src;
634 struct iris_resource *dst_res = (void *) dst;
635
636 enum isl_aux_usage src_aux_usage, dst_aux_usage;
637 bool src_clear_supported, dst_clear_supported;
638 get_copy_region_aux_settings(ice, src_res, src_level, &src_aux_usage,
639 &src_clear_supported, false);
640 get_copy_region_aux_settings(ice, dst_res, dst_level, &dst_aux_usage,
641 &dst_clear_supported, true);
642
643 if (iris_batch_references(batch, src_res->bo))
644 tex_cache_flush_hack(batch, ISL_FORMAT_UNSUPPORTED, src_res->surf.format);
645
646 if (dst->target == PIPE_BUFFER)
647 util_range_add(&dst_res->base.b, &dst_res->valid_buffer_range, dstx, dstx + src_box->width);
648
649 if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) {
650 struct blorp_address src_addr = {
651 .buffer = iris_resource_bo(src), .offset = src_box->x,
652 .mocs = iris_mocs(src_res->bo, &screen->isl_dev,
653 ISL_SURF_USAGE_RENDER_TARGET_BIT),
654 };
655 struct blorp_address dst_addr = {
656 .buffer = iris_resource_bo(dst), .offset = dstx,
657 .reloc_flags = EXEC_OBJECT_WRITE,
658 .mocs = iris_mocs(dst_res->bo, &screen->isl_dev,
659 ISL_SURF_USAGE_TEXTURE_BIT),
660 };
661
662 iris_emit_buffer_barrier_for(batch, iris_resource_bo(src),
663 IRIS_DOMAIN_OTHER_READ);
664 iris_emit_buffer_barrier_for(batch, iris_resource_bo(dst),
665 IRIS_DOMAIN_RENDER_WRITE);
666
667 iris_batch_maybe_flush(batch, 1500);
668
669 iris_batch_sync_region_start(batch);
670 blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0);
671 blorp_buffer_copy(&blorp_batch, src_addr, dst_addr, src_box->width);
672 blorp_batch_finish(&blorp_batch);
673 iris_batch_sync_region_end(batch);
674 } else {
675 // XXX: what about one surface being a buffer and not the other?
676
677 struct blorp_surf src_surf, dst_surf;
678 iris_blorp_surf_for_resource(&screen->isl_dev, &src_surf,
679 src, src_aux_usage, src_level, false);
680 iris_blorp_surf_for_resource(&screen->isl_dev, &dst_surf,
681 dst, dst_aux_usage, dst_level, true);
682
683 iris_resource_prepare_access(ice, src_res, src_level, 1,
684 src_box->z, src_box->depth,
685 src_aux_usage, src_clear_supported);
686 iris_resource_prepare_access(ice, dst_res, dst_level, 1,
687 dstz, src_box->depth,
688 dst_aux_usage, dst_clear_supported);
689
690 iris_emit_buffer_barrier_for(batch, iris_resource_bo(src),
691 IRIS_DOMAIN_OTHER_READ);
692 iris_emit_buffer_barrier_for(batch, iris_resource_bo(dst),
693 IRIS_DOMAIN_RENDER_WRITE);
694
695 blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0);
696
697 for (int slice = 0; slice < src_box->depth; slice++) {
698 iris_batch_maybe_flush(batch, 1500);
699
700 iris_batch_sync_region_start(batch);
701 blorp_copy(&blorp_batch, &src_surf, src_level, src_box->z + slice,
702 &dst_surf, dst_level, dstz + slice,
703 src_box->x, src_box->y, dstx, dsty,
704 src_box->width, src_box->height);
705 iris_batch_sync_region_end(batch);
706 }
707 blorp_batch_finish(&blorp_batch);
708
709 iris_resource_finish_write(ice, dst_res, dst_level, dstz,
710 src_box->depth, dst_aux_usage);
711 }
712
713 tex_cache_flush_hack(batch, ISL_FORMAT_UNSUPPORTED, src_res->surf.format);
714 }
715
716 /**
717 * The pipe->resource_copy_region() driver hook.
718 *
719 * This implements ARB_copy_image semantics - a raw memory copy between
720 * compatible view classes.
721 */
722 static void
iris_resource_copy_region(struct pipe_context * ctx,struct pipe_resource * p_dst,unsigned dst_level,unsigned dstx,unsigned dsty,unsigned dstz,struct pipe_resource * p_src,unsigned src_level,const struct pipe_box * src_box)723 iris_resource_copy_region(struct pipe_context *ctx,
724 struct pipe_resource *p_dst,
725 unsigned dst_level,
726 unsigned dstx, unsigned dsty, unsigned dstz,
727 struct pipe_resource *p_src,
728 unsigned src_level,
729 const struct pipe_box *src_box)
730 {
731 struct iris_context *ice = (void *) ctx;
732 struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
733
734 iris_copy_region(&ice->blorp, batch, p_dst, dst_level, dstx, dsty, dstz,
735 p_src, src_level, src_box);
736
737 if (util_format_is_depth_and_stencil(p_dst->format) &&
738 util_format_has_stencil(util_format_description(p_src->format))) {
739 struct iris_resource *junk, *s_src_res, *s_dst_res;
740 iris_get_depth_stencil_resources(p_src, &junk, &s_src_res);
741 iris_get_depth_stencil_resources(p_dst, &junk, &s_dst_res);
742
743 iris_copy_region(&ice->blorp, batch, &s_dst_res->base.b, dst_level, dstx,
744 dsty, dstz, &s_src_res->base.b, src_level, src_box);
745 }
746
747 iris_flush_and_dirty_for_history(ice, batch, (struct iris_resource *)p_dst,
748 PIPE_CONTROL_RENDER_TARGET_FLUSH,
749 "cache history: post copy_region");
750 }
751
752 void
iris_init_blit_functions(struct pipe_context * ctx)753 iris_init_blit_functions(struct pipe_context *ctx)
754 {
755 ctx->blit = iris_blit;
756 ctx->resource_copy_region = iris_resource_copy_region;
757 }
758