1 /*
2 * Copyright © 2016 Dave Airlie
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include <assert.h>
25 #include <stdbool.h>
26
27 #include "nir/nir_builder.h"
28 #include "radv_meta.h"
29 #include "radv_private.h"
30 #include "sid.h"
31 #include "vk_format.h"
32
33 static nir_shader *
build_nir_vertex_shader(void)34 build_nir_vertex_shader(void)
35 {
36 const struct glsl_type *vec4 = glsl_vec4_type();
37 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_VERTEX, NULL, "meta_resolve_vs");
38
39 nir_variable *pos_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "gl_Position");
40 pos_out->data.location = VARYING_SLOT_POS;
41
42 nir_ssa_def *outvec = radv_meta_gen_rect_vertices(&b);
43
44 nir_store_var(&b, pos_out, outvec, 0xf);
45 return b.shader;
46 }
47
48 static nir_shader *
build_resolve_fragment_shader(struct radv_device * dev,bool is_integer,int samples)49 build_resolve_fragment_shader(struct radv_device *dev, bool is_integer, int samples)
50 {
51 const struct glsl_type *vec4 = glsl_vec4_type();
52 const struct glsl_type *sampler_type =
53 glsl_sampler_type(GLSL_SAMPLER_DIM_MS, false, false, GLSL_TYPE_FLOAT);
54
55 nir_builder b = nir_builder_init_simple_shader(
56 MESA_SHADER_FRAGMENT, NULL, "meta_resolve_fs-%d-%s", samples, is_integer ? "int" : "float");
57
58 nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform, sampler_type, "s_tex");
59 input_img->data.descriptor_set = 0;
60 input_img->data.binding = 0;
61
62 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
63 color_out->data.location = FRAG_RESULT_DATA0;
64
65 nir_ssa_def *pos_in = nir_channels(&b, nir_load_frag_coord(&b), 0x3);
66 nir_ssa_def *src_offset = nir_load_push_constant(&b, 2, 32, nir_imm_int(&b, 0), 0, 8);
67
68 nir_ssa_def *pos_int = nir_f2i32(&b, pos_in);
69
70 nir_ssa_def *img_coord = nir_channels(&b, nir_iadd(&b, pos_int, src_offset), 0x3);
71 nir_variable *color = nir_local_variable_create(b.impl, glsl_vec4_type(), "color");
72
73 radv_meta_build_resolve_shader_core(&b, is_integer, samples, input_img, color, img_coord);
74
75 nir_ssa_def *outval = nir_load_var(&b, color);
76 nir_store_var(&b, color_out, outval, 0xf);
77 return b.shader;
78 }
79
80 static VkResult
create_layout(struct radv_device * device)81 create_layout(struct radv_device *device)
82 {
83 VkResult result;
84 /*
85 * one descriptors for the image being sampled
86 */
87 VkDescriptorSetLayoutCreateInfo ds_create_info = {
88 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
89 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
90 .bindingCount = 1,
91 .pBindings = (VkDescriptorSetLayoutBinding[]){
92 {.binding = 0,
93 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
94 .descriptorCount = 1,
95 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
96 .pImmutableSamplers = NULL},
97 }};
98
99 result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device), &ds_create_info,
100 &device->meta_state.alloc,
101 &device->meta_state.resolve_fragment.ds_layout);
102 if (result != VK_SUCCESS)
103 goto fail;
104
105 VkPipelineLayoutCreateInfo pl_create_info = {
106 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
107 .setLayoutCount = 1,
108 .pSetLayouts = &device->meta_state.resolve_fragment.ds_layout,
109 .pushConstantRangeCount = 1,
110 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_FRAGMENT_BIT, 0, 8},
111 };
112
113 result = radv_CreatePipelineLayout(radv_device_to_handle(device), &pl_create_info,
114 &device->meta_state.alloc,
115 &device->meta_state.resolve_fragment.p_layout);
116 if (result != VK_SUCCESS)
117 goto fail;
118 return VK_SUCCESS;
119 fail:
120 return result;
121 }
122
123 static const VkPipelineVertexInputStateCreateInfo normal_vi_create_info = {
124 .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
125 .vertexBindingDescriptionCount = 0,
126 .vertexAttributeDescriptionCount = 0,
127 };
128
129 static VkResult
create_resolve_pipeline(struct radv_device * device,int samples_log2,VkFormat format)130 create_resolve_pipeline(struct radv_device *device, int samples_log2, VkFormat format)
131 {
132 mtx_lock(&device->meta_state.mtx);
133
134 unsigned fs_key = radv_format_meta_fs_key(device, format);
135 VkPipeline *pipeline = &device->meta_state.resolve_fragment.rc[samples_log2].pipeline[fs_key];
136 if (*pipeline) {
137 mtx_unlock(&device->meta_state.mtx);
138 return VK_SUCCESS;
139 }
140
141 VkResult result;
142 bool is_integer = false;
143 uint32_t samples = 1 << samples_log2;
144 const VkPipelineVertexInputStateCreateInfo *vi_create_info;
145 vi_create_info = &normal_vi_create_info;
146 if (vk_format_is_int(format))
147 is_integer = true;
148
149 nir_shader *fs = build_resolve_fragment_shader(device, is_integer, samples);
150 nir_shader *vs = build_nir_vertex_shader();
151
152 VkRenderPass *rp = &device->meta_state.resolve_fragment.rc[samples_log2].render_pass[fs_key][0];
153
154 assert(!*rp);
155
156 VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
157 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
158 .stage = VK_SHADER_STAGE_VERTEX_BIT,
159 .module = vk_shader_module_handle_from_nir(vs),
160 .pName = "main",
161 .pSpecializationInfo = NULL},
162 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
163 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
164 .module = vk_shader_module_handle_from_nir(fs),
165 .pName = "main",
166 .pSpecializationInfo = NULL},
167 };
168
169 for (unsigned dst_layout = 0; dst_layout < RADV_META_DST_LAYOUT_COUNT; ++dst_layout) {
170 VkImageLayout layout = radv_meta_dst_layout_to_layout(dst_layout);
171 result = radv_CreateRenderPass2(
172 radv_device_to_handle(device),
173 &(VkRenderPassCreateInfo2){
174 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO_2,
175 .attachmentCount = 1,
176 .pAttachments =
177 &(VkAttachmentDescription2){
178 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_DESCRIPTION_2,
179 .format = format,
180 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
181 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
182 .initialLayout = layout,
183 .finalLayout = layout,
184 },
185 .subpassCount = 1,
186 .pSubpasses =
187 &(VkSubpassDescription2){
188 .sType = VK_STRUCTURE_TYPE_SUBPASS_DESCRIPTION_2,
189 .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
190 .inputAttachmentCount = 0,
191 .colorAttachmentCount = 1,
192 .pColorAttachments =
193 &(VkAttachmentReference2){
194 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2,
195 .attachment = 0,
196 .layout = layout,
197 },
198 .pResolveAttachments = NULL,
199 .pDepthStencilAttachment =
200 &(VkAttachmentReference2){
201 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2,
202 .attachment = VK_ATTACHMENT_UNUSED,
203 .layout = VK_IMAGE_LAYOUT_GENERAL,
204 },
205 .preserveAttachmentCount = 0,
206 .pPreserveAttachments = NULL,
207 },
208 .dependencyCount = 2,
209 .pDependencies =
210 (VkSubpassDependency2[]){{.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2,
211 .srcSubpass = VK_SUBPASS_EXTERNAL,
212 .dstSubpass = 0,
213 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
214 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
215 .srcAccessMask = 0,
216 .dstAccessMask = 0,
217 .dependencyFlags = 0},
218 {.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2,
219 .srcSubpass = 0,
220 .dstSubpass = VK_SUBPASS_EXTERNAL,
221 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
222 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
223 .srcAccessMask = 0,
224 .dstAccessMask = 0,
225 .dependencyFlags = 0}},
226 },
227 &device->meta_state.alloc, rp + dst_layout);
228 }
229
230 const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
231 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
232 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
233 .pStages = pipeline_shader_stages,
234 .pVertexInputState = vi_create_info,
235 .pInputAssemblyState =
236 &(VkPipelineInputAssemblyStateCreateInfo){
237 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
238 .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
239 .primitiveRestartEnable = false,
240 },
241 .pViewportState =
242 &(VkPipelineViewportStateCreateInfo){
243 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
244 .viewportCount = 1,
245 .scissorCount = 1,
246 },
247 .pRasterizationState =
248 &(VkPipelineRasterizationStateCreateInfo){
249 .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
250 .rasterizerDiscardEnable = false,
251 .polygonMode = VK_POLYGON_MODE_FILL,
252 .cullMode = VK_CULL_MODE_NONE,
253 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE},
254 .pMultisampleState =
255 &(VkPipelineMultisampleStateCreateInfo){
256 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
257 .rasterizationSamples = 1,
258 .sampleShadingEnable = false,
259 .pSampleMask = (VkSampleMask[]){UINT32_MAX},
260 },
261 .pColorBlendState =
262 &(VkPipelineColorBlendStateCreateInfo){
263 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
264 .attachmentCount = 1,
265 .pAttachments =
266 (VkPipelineColorBlendAttachmentState[]){
267 {.colorWriteMask = VK_COLOR_COMPONENT_A_BIT | VK_COLOR_COMPONENT_R_BIT |
268 VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT},
269 }},
270 .pDynamicState =
271 &(VkPipelineDynamicStateCreateInfo){
272 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
273 .dynamicStateCount = 9,
274 .pDynamicStates =
275 (VkDynamicState[]){
276 VK_DYNAMIC_STATE_VIEWPORT,
277 VK_DYNAMIC_STATE_SCISSOR,
278 VK_DYNAMIC_STATE_LINE_WIDTH,
279 VK_DYNAMIC_STATE_DEPTH_BIAS,
280 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
281 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
282 VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
283 VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
284 VK_DYNAMIC_STATE_STENCIL_REFERENCE,
285 },
286 },
287 .flags = 0,
288 .layout = device->meta_state.resolve_fragment.p_layout,
289 .renderPass = *rp,
290 .subpass = 0,
291 };
292
293 const struct radv_graphics_pipeline_create_info radv_pipeline_info = {.use_rectlist = true};
294
295 result = radv_graphics_pipeline_create(
296 radv_device_to_handle(device), radv_pipeline_cache_to_handle(&device->meta_state.cache),
297 &vk_pipeline_info, &radv_pipeline_info, &device->meta_state.alloc, pipeline);
298 ralloc_free(vs);
299 ralloc_free(fs);
300
301 mtx_unlock(&device->meta_state.mtx);
302 return result;
303 }
304
305 enum { DEPTH_RESOLVE, STENCIL_RESOLVE };
306
307 static const char *
get_resolve_mode_str(VkResolveModeFlagBits resolve_mode)308 get_resolve_mode_str(VkResolveModeFlagBits resolve_mode)
309 {
310 switch (resolve_mode) {
311 case VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR:
312 return "zero";
313 case VK_RESOLVE_MODE_AVERAGE_BIT_KHR:
314 return "average";
315 case VK_RESOLVE_MODE_MIN_BIT_KHR:
316 return "min";
317 case VK_RESOLVE_MODE_MAX_BIT_KHR:
318 return "max";
319 default:
320 unreachable("invalid resolve mode");
321 }
322 }
323
324 static nir_shader *
build_depth_stencil_resolve_fragment_shader(struct radv_device * dev,int samples,int index,VkResolveModeFlagBits resolve_mode)325 build_depth_stencil_resolve_fragment_shader(struct radv_device *dev, int samples, int index,
326 VkResolveModeFlagBits resolve_mode)
327 {
328 const struct glsl_type *vec4 = glsl_vec4_type();
329 const struct glsl_type *sampler_type =
330 glsl_sampler_type(GLSL_SAMPLER_DIM_2D, false, false, GLSL_TYPE_FLOAT);
331
332 nir_builder b = nir_builder_init_simple_shader(
333 MESA_SHADER_FRAGMENT, NULL, "meta_resolve_fs_%s-%s-%d",
334 index == DEPTH_RESOLVE ? "depth" : "stencil", get_resolve_mode_str(resolve_mode), samples);
335
336 nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform, sampler_type, "s_tex");
337 input_img->data.descriptor_set = 0;
338 input_img->data.binding = 0;
339
340 nir_variable *fs_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_out");
341 fs_out->data.location = index == DEPTH_RESOLVE ? FRAG_RESULT_DEPTH : FRAG_RESULT_STENCIL;
342
343 nir_ssa_def *pos_in = nir_channels(&b, nir_load_frag_coord(&b), 0x3);
344
345 nir_ssa_def *pos_int = nir_f2i32(&b, pos_in);
346
347 nir_ssa_def *img_coord = nir_channels(&b, pos_int, 0x3);
348
349 nir_ssa_def *input_img_deref = &nir_build_deref_var(&b, input_img)->dest.ssa;
350
351 nir_alu_type type = index == DEPTH_RESOLVE ? nir_type_float32 : nir_type_uint32;
352
353 nir_tex_instr *tex = nir_tex_instr_create(b.shader, 3);
354 tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
355 tex->op = nir_texop_txf_ms;
356 tex->src[0].src_type = nir_tex_src_coord;
357 tex->src[0].src = nir_src_for_ssa(img_coord);
358 tex->src[1].src_type = nir_tex_src_ms_index;
359 tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0));
360 tex->src[2].src_type = nir_tex_src_texture_deref;
361 tex->src[2].src = nir_src_for_ssa(input_img_deref);
362 tex->dest_type = type;
363 tex->is_array = false;
364 tex->coord_components = 2;
365
366 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
367 nir_builder_instr_insert(&b, &tex->instr);
368
369 nir_ssa_def *outval = &tex->dest.ssa;
370
371 if (resolve_mode != VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR) {
372 for (int i = 1; i < samples; i++) {
373 nir_tex_instr *tex_add = nir_tex_instr_create(b.shader, 3);
374 tex_add->sampler_dim = GLSL_SAMPLER_DIM_MS;
375 tex_add->op = nir_texop_txf_ms;
376 tex_add->src[0].src_type = nir_tex_src_coord;
377 tex_add->src[0].src = nir_src_for_ssa(img_coord);
378 tex_add->src[1].src_type = nir_tex_src_ms_index;
379 tex_add->src[1].src = nir_src_for_ssa(nir_imm_int(&b, i));
380 tex_add->src[2].src_type = nir_tex_src_texture_deref;
381 tex_add->src[2].src = nir_src_for_ssa(input_img_deref);
382 tex_add->dest_type = type;
383 tex_add->is_array = false;
384 tex_add->coord_components = 2;
385
386 nir_ssa_dest_init(&tex_add->instr, &tex_add->dest, 4, 32, "tex");
387 nir_builder_instr_insert(&b, &tex_add->instr);
388
389 switch (resolve_mode) {
390 case VK_RESOLVE_MODE_AVERAGE_BIT_KHR:
391 assert(index == DEPTH_RESOLVE);
392 outval = nir_fadd(&b, outval, &tex_add->dest.ssa);
393 break;
394 case VK_RESOLVE_MODE_MIN_BIT_KHR:
395 if (index == DEPTH_RESOLVE)
396 outval = nir_fmin(&b, outval, &tex_add->dest.ssa);
397 else
398 outval = nir_umin(&b, outval, &tex_add->dest.ssa);
399 break;
400 case VK_RESOLVE_MODE_MAX_BIT_KHR:
401 if (index == DEPTH_RESOLVE)
402 outval = nir_fmax(&b, outval, &tex_add->dest.ssa);
403 else
404 outval = nir_umax(&b, outval, &tex_add->dest.ssa);
405 break;
406 default:
407 unreachable("invalid resolve mode");
408 }
409 }
410
411 if (resolve_mode == VK_RESOLVE_MODE_AVERAGE_BIT_KHR)
412 outval = nir_fdiv(&b, outval, nir_imm_float(&b, samples));
413 }
414
415 nir_store_var(&b, fs_out, outval, 0x1);
416
417 return b.shader;
418 }
419
420 static VkResult
create_depth_stencil_resolve_pipeline(struct radv_device * device,int samples_log2,int index,VkResolveModeFlagBits resolve_mode)421 create_depth_stencil_resolve_pipeline(struct radv_device *device, int samples_log2, int index,
422 VkResolveModeFlagBits resolve_mode)
423 {
424 VkRenderPass *render_pass;
425 VkPipeline *pipeline;
426 VkFormat src_format;
427 VkResult result;
428
429 mtx_lock(&device->meta_state.mtx);
430
431 switch (resolve_mode) {
432 case VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR:
433 if (index == DEPTH_RESOLVE)
434 pipeline = &device->meta_state.resolve_fragment.depth_zero_pipeline;
435 else
436 pipeline = &device->meta_state.resolve_fragment.stencil_zero_pipeline;
437 break;
438 case VK_RESOLVE_MODE_AVERAGE_BIT_KHR:
439 assert(index == DEPTH_RESOLVE);
440 pipeline = &device->meta_state.resolve_fragment.depth[samples_log2].average_pipeline;
441 break;
442 case VK_RESOLVE_MODE_MIN_BIT_KHR:
443 if (index == DEPTH_RESOLVE)
444 pipeline = &device->meta_state.resolve_fragment.depth[samples_log2].min_pipeline;
445 else
446 pipeline = &device->meta_state.resolve_fragment.stencil[samples_log2].min_pipeline;
447 break;
448 case VK_RESOLVE_MODE_MAX_BIT_KHR:
449 if (index == DEPTH_RESOLVE)
450 pipeline = &device->meta_state.resolve_fragment.depth[samples_log2].max_pipeline;
451 else
452 pipeline = &device->meta_state.resolve_fragment.stencil[samples_log2].max_pipeline;
453 break;
454 default:
455 unreachable("invalid resolve mode");
456 }
457
458 if (*pipeline) {
459 mtx_unlock(&device->meta_state.mtx);
460 return VK_SUCCESS;
461 }
462
463 uint32_t samples = 1 << samples_log2;
464 nir_shader *fs =
465 build_depth_stencil_resolve_fragment_shader(device, samples, index, resolve_mode);
466 nir_shader *vs = build_nir_vertex_shader();
467
468 VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
469 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
470 .stage = VK_SHADER_STAGE_VERTEX_BIT,
471 .module = vk_shader_module_handle_from_nir(vs),
472 .pName = "main",
473 .pSpecializationInfo = NULL},
474 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
475 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
476 .module = vk_shader_module_handle_from_nir(fs),
477 .pName = "main",
478 .pSpecializationInfo = NULL},
479 };
480
481 if (index == DEPTH_RESOLVE) {
482 src_format = VK_FORMAT_D32_SFLOAT;
483 render_pass = &device->meta_state.resolve_fragment.depth_render_pass;
484 } else {
485 render_pass = &device->meta_state.resolve_fragment.stencil_render_pass;
486 src_format = VK_FORMAT_S8_UINT;
487 }
488
489 if (!*render_pass) {
490 result = radv_CreateRenderPass2(
491 radv_device_to_handle(device),
492 &(VkRenderPassCreateInfo2){
493 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO_2,
494 .attachmentCount = 1,
495 .pAttachments =
496 &(VkAttachmentDescription2){
497 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_DESCRIPTION_2,
498 .format = src_format,
499 .loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE,
500 .storeOp = VK_ATTACHMENT_STORE_OP_DONT_CARE,
501 .stencilLoadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
502 .stencilStoreOp = VK_ATTACHMENT_STORE_OP_STORE,
503 .initialLayout = VK_IMAGE_LAYOUT_GENERAL,
504 .finalLayout = VK_IMAGE_LAYOUT_GENERAL,
505 },
506 .subpassCount = 1,
507 .pSubpasses =
508 &(VkSubpassDescription2){
509 .sType = VK_STRUCTURE_TYPE_SUBPASS_DESCRIPTION_2,
510 .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
511 .inputAttachmentCount = 0,
512 .colorAttachmentCount = 0,
513 .pColorAttachments = NULL,
514 .pResolveAttachments = NULL,
515 .pDepthStencilAttachment =
516 &(VkAttachmentReference2){
517 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2,
518 .attachment = 0,
519 .layout = VK_IMAGE_LAYOUT_GENERAL,
520 },
521 .preserveAttachmentCount = 0,
522 .pPreserveAttachments = NULL,
523 },
524 .dependencyCount = 2,
525 .pDependencies =
526 (VkSubpassDependency2[]){{.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2,
527 .srcSubpass = VK_SUBPASS_EXTERNAL,
528 .dstSubpass = 0,
529 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
530 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
531 .srcAccessMask = 0,
532 .dstAccessMask = 0,
533 .dependencyFlags = 0},
534 {.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2,
535 .srcSubpass = 0,
536 .dstSubpass = VK_SUBPASS_EXTERNAL,
537 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
538 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
539 .srcAccessMask = 0,
540 .dstAccessMask = 0,
541 .dependencyFlags = 0}},
542 },
543 &device->meta_state.alloc, render_pass);
544 }
545
546 VkStencilOp stencil_op = index == DEPTH_RESOLVE ? VK_STENCIL_OP_KEEP : VK_STENCIL_OP_REPLACE;
547
548 VkPipelineDepthStencilStateCreateInfo depth_stencil_state = {
549 .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
550 .depthTestEnable = true,
551 .depthWriteEnable = index == DEPTH_RESOLVE,
552 .stencilTestEnable = index == STENCIL_RESOLVE,
553 .depthCompareOp = VK_COMPARE_OP_ALWAYS,
554 .front =
555 {
556 .failOp = stencil_op,
557 .passOp = stencil_op,
558 .depthFailOp = stencil_op,
559 .compareOp = VK_COMPARE_OP_ALWAYS,
560 },
561 .back = {
562 .failOp = stencil_op,
563 .passOp = stencil_op,
564 .depthFailOp = stencil_op,
565 .compareOp = VK_COMPARE_OP_ALWAYS,
566 }};
567
568 const VkPipelineVertexInputStateCreateInfo *vi_create_info;
569 vi_create_info = &normal_vi_create_info;
570
571 const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
572 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
573 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
574 .pStages = pipeline_shader_stages,
575 .pVertexInputState = vi_create_info,
576 .pInputAssemblyState =
577 &(VkPipelineInputAssemblyStateCreateInfo){
578 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
579 .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
580 .primitiveRestartEnable = false,
581 },
582 .pViewportState =
583 &(VkPipelineViewportStateCreateInfo){
584 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
585 .viewportCount = 1,
586 .scissorCount = 1,
587 },
588 .pDepthStencilState = &depth_stencil_state,
589 .pRasterizationState =
590 &(VkPipelineRasterizationStateCreateInfo){
591 .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
592 .rasterizerDiscardEnable = false,
593 .polygonMode = VK_POLYGON_MODE_FILL,
594 .cullMode = VK_CULL_MODE_NONE,
595 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE},
596 .pMultisampleState =
597 &(VkPipelineMultisampleStateCreateInfo){
598 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
599 .rasterizationSamples = 1,
600 .sampleShadingEnable = false,
601 .pSampleMask = (VkSampleMask[]){UINT32_MAX},
602 },
603 .pColorBlendState =
604 &(VkPipelineColorBlendStateCreateInfo){
605 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
606 .attachmentCount = 0,
607 .pAttachments =
608 (VkPipelineColorBlendAttachmentState[]){
609 {.colorWriteMask = VK_COLOR_COMPONENT_A_BIT | VK_COLOR_COMPONENT_R_BIT |
610 VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT},
611 }},
612 .pDynamicState =
613 &(VkPipelineDynamicStateCreateInfo){
614 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
615 .dynamicStateCount = 9,
616 .pDynamicStates =
617 (VkDynamicState[]){
618 VK_DYNAMIC_STATE_VIEWPORT,
619 VK_DYNAMIC_STATE_SCISSOR,
620 VK_DYNAMIC_STATE_LINE_WIDTH,
621 VK_DYNAMIC_STATE_DEPTH_BIAS,
622 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
623 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
624 VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
625 VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
626 VK_DYNAMIC_STATE_STENCIL_REFERENCE,
627 },
628 },
629 .flags = 0,
630 .layout = device->meta_state.resolve_fragment.p_layout,
631 .renderPass = *render_pass,
632 .subpass = 0,
633 };
634
635 const struct radv_graphics_pipeline_create_info radv_pipeline_info = {.use_rectlist = true};
636
637 result = radv_graphics_pipeline_create(
638 radv_device_to_handle(device), radv_pipeline_cache_to_handle(&device->meta_state.cache),
639 &vk_pipeline_info, &radv_pipeline_info, &device->meta_state.alloc, pipeline);
640
641 ralloc_free(vs);
642 ralloc_free(fs);
643
644 mtx_unlock(&device->meta_state.mtx);
645 return result;
646 }
647
648 VkResult
radv_device_init_meta_resolve_fragment_state(struct radv_device * device,bool on_demand)649 radv_device_init_meta_resolve_fragment_state(struct radv_device *device, bool on_demand)
650 {
651 VkResult res;
652
653 res = create_layout(device);
654 if (res != VK_SUCCESS)
655 goto fail;
656
657 if (on_demand)
658 return VK_SUCCESS;
659
660 for (uint32_t i = 0; i < MAX_SAMPLES_LOG2; ++i) {
661 for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
662 res = create_resolve_pipeline(device, i, radv_fs_key_format_exemplars[j]);
663 if (res != VK_SUCCESS)
664 goto fail;
665 }
666
667 res = create_depth_stencil_resolve_pipeline(device, i, DEPTH_RESOLVE,
668 VK_RESOLVE_MODE_AVERAGE_BIT_KHR);
669 if (res != VK_SUCCESS)
670 goto fail;
671
672 res = create_depth_stencil_resolve_pipeline(device, i, DEPTH_RESOLVE,
673 VK_RESOLVE_MODE_MIN_BIT_KHR);
674 if (res != VK_SUCCESS)
675 goto fail;
676
677 res = create_depth_stencil_resolve_pipeline(device, i, DEPTH_RESOLVE,
678 VK_RESOLVE_MODE_MAX_BIT_KHR);
679 if (res != VK_SUCCESS)
680 goto fail;
681
682 res = create_depth_stencil_resolve_pipeline(device, i, STENCIL_RESOLVE,
683 VK_RESOLVE_MODE_MIN_BIT_KHR);
684 if (res != VK_SUCCESS)
685 goto fail;
686
687 res = create_depth_stencil_resolve_pipeline(device, i, STENCIL_RESOLVE,
688 VK_RESOLVE_MODE_MAX_BIT_KHR);
689 if (res != VK_SUCCESS)
690 goto fail;
691 }
692
693 res = create_depth_stencil_resolve_pipeline(device, 0, DEPTH_RESOLVE,
694 VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR);
695 if (res != VK_SUCCESS)
696 goto fail;
697
698 res = create_depth_stencil_resolve_pipeline(device, 0, STENCIL_RESOLVE,
699 VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR);
700 if (res != VK_SUCCESS)
701 goto fail;
702
703 return VK_SUCCESS;
704 fail:
705 radv_device_finish_meta_resolve_fragment_state(device);
706 return res;
707 }
708
709 void
radv_device_finish_meta_resolve_fragment_state(struct radv_device * device)710 radv_device_finish_meta_resolve_fragment_state(struct radv_device *device)
711 {
712 struct radv_meta_state *state = &device->meta_state;
713 for (uint32_t i = 0; i < MAX_SAMPLES_LOG2; ++i) {
714 for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
715 for (unsigned k = 0; k < RADV_META_DST_LAYOUT_COUNT; ++k) {
716 radv_DestroyRenderPass(radv_device_to_handle(device),
717 state->resolve_fragment.rc[i].render_pass[j][k], &state->alloc);
718 }
719 radv_DestroyPipeline(radv_device_to_handle(device),
720 state->resolve_fragment.rc[i].pipeline[j], &state->alloc);
721 }
722
723 radv_DestroyPipeline(radv_device_to_handle(device),
724 state->resolve_fragment.depth[i].average_pipeline, &state->alloc);
725
726 radv_DestroyPipeline(radv_device_to_handle(device),
727 state->resolve_fragment.depth[i].max_pipeline, &state->alloc);
728
729 radv_DestroyPipeline(radv_device_to_handle(device),
730 state->resolve_fragment.depth[i].min_pipeline, &state->alloc);
731
732 radv_DestroyPipeline(radv_device_to_handle(device),
733 state->resolve_fragment.stencil[i].max_pipeline, &state->alloc);
734
735 radv_DestroyPipeline(radv_device_to_handle(device),
736 state->resolve_fragment.stencil[i].min_pipeline, &state->alloc);
737 }
738
739 radv_DestroyRenderPass(radv_device_to_handle(device), state->resolve_fragment.depth_render_pass,
740 &state->alloc);
741 radv_DestroyRenderPass(radv_device_to_handle(device),
742 state->resolve_fragment.stencil_render_pass, &state->alloc);
743
744 radv_DestroyPipeline(radv_device_to_handle(device), state->resolve_fragment.depth_zero_pipeline,
745 &state->alloc);
746 radv_DestroyPipeline(radv_device_to_handle(device),
747 state->resolve_fragment.stencil_zero_pipeline, &state->alloc);
748
749 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device), state->resolve_fragment.ds_layout,
750 &state->alloc);
751 radv_DestroyPipelineLayout(radv_device_to_handle(device), state->resolve_fragment.p_layout,
752 &state->alloc);
753 }
754
755 static VkPipeline *
radv_get_resolve_pipeline(struct radv_cmd_buffer * cmd_buffer,struct radv_image_view * src_iview,struct radv_image_view * dst_iview)756 radv_get_resolve_pipeline(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview,
757 struct radv_image_view *dst_iview)
758 {
759 struct radv_device *device = cmd_buffer->device;
760 unsigned fs_key = radv_format_meta_fs_key(cmd_buffer->device, dst_iview->vk_format);
761 const uint32_t samples = src_iview->image->info.samples;
762 const uint32_t samples_log2 = ffs(samples) - 1;
763 VkPipeline *pipeline;
764
765 pipeline = &device->meta_state.resolve_fragment.rc[samples_log2].pipeline[fs_key];
766 if (!*pipeline) {
767 VkResult ret;
768
769 ret = create_resolve_pipeline(device, samples_log2, radv_fs_key_format_exemplars[fs_key]);
770 if (ret != VK_SUCCESS) {
771 cmd_buffer->record_result = ret;
772 return NULL;
773 }
774 }
775
776 return pipeline;
777 }
778
779 static void
emit_resolve(struct radv_cmd_buffer * cmd_buffer,struct radv_image_view * src_iview,struct radv_image_view * dest_iview,const VkOffset2D * src_offset,const VkOffset2D * dest_offset,const VkExtent2D * resolve_extent)780 emit_resolve(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview,
781 struct radv_image_view *dest_iview, const VkOffset2D *src_offset,
782 const VkOffset2D *dest_offset, const VkExtent2D *resolve_extent)
783 {
784 struct radv_device *device = cmd_buffer->device;
785 VkCommandBuffer cmd_buffer_h = radv_cmd_buffer_to_handle(cmd_buffer);
786 VkPipeline *pipeline;
787
788 radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
789 cmd_buffer->device->meta_state.resolve_fragment.p_layout,
790 0, /* set */
791 1, /* descriptorWriteCount */
792 (VkWriteDescriptorSet[]){
793 {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
794 .dstBinding = 0,
795 .dstArrayElement = 0,
796 .descriptorCount = 1,
797 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
798 .pImageInfo =
799 (VkDescriptorImageInfo[]){
800 {
801 .sampler = VK_NULL_HANDLE,
802 .imageView = radv_image_view_to_handle(src_iview),
803 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
804 },
805 }},
806 });
807
808 cmd_buffer->state.flush_bits |=
809 radv_dst_access_flush(cmd_buffer, VK_ACCESS_SHADER_READ_BIT, src_iview->image) |
810 radv_dst_access_flush(cmd_buffer, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, dest_iview->image);
811
812 unsigned push_constants[2] = {
813 src_offset->x - dest_offset->x,
814 src_offset->y - dest_offset->y,
815 };
816 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
817 device->meta_state.resolve_fragment.p_layout, VK_SHADER_STAGE_FRAGMENT_BIT,
818 0, 8, push_constants);
819
820 pipeline = radv_get_resolve_pipeline(cmd_buffer, src_iview, dest_iview);
821
822 radv_CmdBindPipeline(cmd_buffer_h, VK_PIPELINE_BIND_POINT_GRAPHICS, *pipeline);
823
824 radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
825 &(VkViewport){.x = dest_offset->x,
826 .y = dest_offset->y,
827 .width = resolve_extent->width,
828 .height = resolve_extent->height,
829 .minDepth = 0.0f,
830 .maxDepth = 1.0f});
831
832 radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
833 &(VkRect2D){
834 .offset = *dest_offset,
835 .extent = *resolve_extent,
836 });
837
838 radv_CmdDraw(cmd_buffer_h, 3, 1, 0, 0);
839 cmd_buffer->state.flush_bits |=
840 radv_src_access_flush(cmd_buffer, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, dest_iview->image);
841 }
842
843 static void
emit_depth_stencil_resolve(struct radv_cmd_buffer * cmd_buffer,struct radv_image_view * src_iview,struct radv_image_view * dst_iview,const VkExtent2D * resolve_extent,VkImageAspectFlags aspects,VkResolveModeFlagBits resolve_mode)844 emit_depth_stencil_resolve(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview,
845 struct radv_image_view *dst_iview, const VkExtent2D *resolve_extent,
846 VkImageAspectFlags aspects, VkResolveModeFlagBits resolve_mode)
847 {
848 struct radv_device *device = cmd_buffer->device;
849 const uint32_t samples = src_iview->image->info.samples;
850 const uint32_t samples_log2 = ffs(samples) - 1;
851 VkPipeline *pipeline;
852
853 radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
854 cmd_buffer->device->meta_state.resolve_fragment.p_layout,
855 0, /* set */
856 1, /* descriptorWriteCount */
857 (VkWriteDescriptorSet[]){
858 {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
859 .dstBinding = 0,
860 .dstArrayElement = 0,
861 .descriptorCount = 1,
862 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
863 .pImageInfo =
864 (VkDescriptorImageInfo[]){
865 {
866 .sampler = VK_NULL_HANDLE,
867 .imageView = radv_image_view_to_handle(src_iview),
868 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
869 },
870 }},
871 });
872
873 switch (resolve_mode) {
874 case VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR:
875 if (aspects == VK_IMAGE_ASPECT_DEPTH_BIT)
876 pipeline = &device->meta_state.resolve_fragment.depth_zero_pipeline;
877 else
878 pipeline = &device->meta_state.resolve_fragment.stencil_zero_pipeline;
879 break;
880 case VK_RESOLVE_MODE_AVERAGE_BIT_KHR:
881 assert(aspects == VK_IMAGE_ASPECT_DEPTH_BIT);
882 pipeline = &device->meta_state.resolve_fragment.depth[samples_log2].average_pipeline;
883 break;
884 case VK_RESOLVE_MODE_MIN_BIT_KHR:
885 if (aspects == VK_IMAGE_ASPECT_DEPTH_BIT)
886 pipeline = &device->meta_state.resolve_fragment.depth[samples_log2].min_pipeline;
887 else
888 pipeline = &device->meta_state.resolve_fragment.stencil[samples_log2].min_pipeline;
889 break;
890 case VK_RESOLVE_MODE_MAX_BIT_KHR:
891 if (aspects == VK_IMAGE_ASPECT_DEPTH_BIT)
892 pipeline = &device->meta_state.resolve_fragment.depth[samples_log2].max_pipeline;
893 else
894 pipeline = &device->meta_state.resolve_fragment.stencil[samples_log2].max_pipeline;
895 break;
896 default:
897 unreachable("invalid resolve mode");
898 }
899
900 if (!*pipeline) {
901 int index = aspects == VK_IMAGE_ASPECT_DEPTH_BIT ? DEPTH_RESOLVE : STENCIL_RESOLVE;
902 VkResult ret;
903
904 ret = create_depth_stencil_resolve_pipeline(device, samples_log2, index, resolve_mode);
905 if (ret != VK_SUCCESS) {
906 cmd_buffer->record_result = ret;
907 return;
908 }
909 }
910
911 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS,
912 *pipeline);
913
914 radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
915 &(VkViewport){.x = 0,
916 .y = 0,
917 .width = resolve_extent->width,
918 .height = resolve_extent->height,
919 .minDepth = 0.0f,
920 .maxDepth = 1.0f});
921
922 radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
923 &(VkRect2D){
924 .offset = (VkOffset2D){0, 0},
925 .extent = *resolve_extent,
926 });
927
928 radv_CmdDraw(radv_cmd_buffer_to_handle(cmd_buffer), 3, 1, 0, 0);
929 }
930
931 void
radv_meta_resolve_fragment_image(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,VkImageLayout src_image_layout,struct radv_image * dest_image,VkImageLayout dest_image_layout,const VkImageResolve2KHR * region)932 radv_meta_resolve_fragment_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
933 VkImageLayout src_image_layout, struct radv_image *dest_image,
934 VkImageLayout dest_image_layout, const VkImageResolve2KHR *region)
935 {
936 struct radv_device *device = cmd_buffer->device;
937 struct radv_meta_saved_state saved_state;
938 const uint32_t samples = src_image->info.samples;
939 const uint32_t samples_log2 = ffs(samples) - 1;
940 unsigned fs_key = radv_format_meta_fs_key(cmd_buffer->device, dest_image->vk_format);
941 unsigned dst_layout = radv_meta_dst_layout_from_layout(dest_image_layout);
942 VkRenderPass rp;
943
944 radv_decompress_resolve_src(cmd_buffer, src_image, src_image_layout, region);
945
946 if (!device->meta_state.resolve_fragment.rc[samples_log2].render_pass[fs_key][dst_layout]) {
947 VkResult ret =
948 create_resolve_pipeline(device, samples_log2, radv_fs_key_format_exemplars[fs_key]);
949 if (ret != VK_SUCCESS) {
950 cmd_buffer->record_result = ret;
951 return;
952 }
953 }
954
955 rp = device->meta_state.resolve_fragment.rc[samples_log2].render_pass[fs_key][dst_layout];
956
957 radv_meta_save(
958 &saved_state, cmd_buffer,
959 RADV_META_SAVE_GRAPHICS_PIPELINE | RADV_META_SAVE_CONSTANTS | RADV_META_SAVE_DESCRIPTORS);
960
961 assert(region->srcSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
962 assert(region->dstSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
963 assert(region->srcSubresource.layerCount == region->dstSubresource.layerCount);
964
965 const uint32_t src_base_layer =
966 radv_meta_get_iview_layer(src_image, ®ion->srcSubresource, ®ion->srcOffset);
967
968 const uint32_t dest_base_layer =
969 radv_meta_get_iview_layer(dest_image, ®ion->dstSubresource, ®ion->dstOffset);
970
971 const struct VkExtent3D extent = radv_sanitize_image_extent(src_image->type, region->extent);
972 const struct VkOffset3D srcOffset =
973 radv_sanitize_image_offset(src_image->type, region->srcOffset);
974 const struct VkOffset3D dstOffset =
975 radv_sanitize_image_offset(dest_image->type, region->dstOffset);
976
977 for (uint32_t layer = 0; layer < region->srcSubresource.layerCount; ++layer) {
978
979 struct radv_image_view src_iview;
980 radv_image_view_init(&src_iview, cmd_buffer->device,
981 &(VkImageViewCreateInfo){
982 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
983 .image = radv_image_to_handle(src_image),
984 .viewType = radv_meta_get_view_type(src_image),
985 .format = src_image->vk_format,
986 .subresourceRange =
987 {
988 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
989 .baseMipLevel = region->srcSubresource.mipLevel,
990 .levelCount = 1,
991 .baseArrayLayer = src_base_layer + layer,
992 .layerCount = 1,
993 },
994 },
995 NULL);
996
997 struct radv_image_view dest_iview;
998 radv_image_view_init(&dest_iview, cmd_buffer->device,
999 &(VkImageViewCreateInfo){
1000 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
1001 .image = radv_image_to_handle(dest_image),
1002 .viewType = radv_meta_get_view_type(dest_image),
1003 .format = dest_image->vk_format,
1004 .subresourceRange =
1005 {
1006 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
1007 .baseMipLevel = region->dstSubresource.mipLevel,
1008 .levelCount = 1,
1009 .baseArrayLayer = dest_base_layer + layer,
1010 .layerCount = 1,
1011 },
1012 },
1013 NULL);
1014
1015 VkFramebuffer fb;
1016 radv_CreateFramebuffer(
1017 radv_device_to_handle(cmd_buffer->device),
1018 &(VkFramebufferCreateInfo){.sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
1019 .attachmentCount = 1,
1020 .pAttachments =
1021 (VkImageView[]){
1022 radv_image_view_to_handle(&dest_iview),
1023 },
1024 .width = extent.width + dstOffset.x,
1025 .height = extent.height + dstOffset.y,
1026 .layers = 1},
1027 &cmd_buffer->pool->alloc, &fb);
1028
1029 radv_cmd_buffer_begin_render_pass(cmd_buffer,
1030 &(VkRenderPassBeginInfo){
1031 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
1032 .renderPass = rp,
1033 .framebuffer = fb,
1034 .renderArea =
1035 {
1036 .offset =
1037 {
1038 dstOffset.x,
1039 dstOffset.y,
1040 },
1041 .extent = {extent.width, extent.height},
1042 },
1043 .clearValueCount = 0,
1044 .pClearValues = NULL,
1045 },
1046 NULL);
1047
1048 radv_cmd_buffer_set_subpass(cmd_buffer, &cmd_buffer->state.pass->subpasses[0]);
1049
1050 emit_resolve(cmd_buffer, &src_iview, &dest_iview, &(VkOffset2D){srcOffset.x, srcOffset.y},
1051 &(VkOffset2D){dstOffset.x, dstOffset.y},
1052 &(VkExtent2D){extent.width, extent.height});
1053
1054 radv_cmd_buffer_end_render_pass(cmd_buffer);
1055
1056 radv_image_view_finish(&src_iview);
1057 radv_image_view_finish(&dest_iview);
1058 radv_DestroyFramebuffer(radv_device_to_handle(cmd_buffer->device), fb,
1059 &cmd_buffer->pool->alloc);
1060 }
1061
1062 radv_meta_restore(&saved_state, cmd_buffer);
1063 }
1064
1065 /**
1066 * Emit any needed resolves for the current subpass.
1067 */
1068 void
radv_cmd_buffer_resolve_subpass_fs(struct radv_cmd_buffer * cmd_buffer)1069 radv_cmd_buffer_resolve_subpass_fs(struct radv_cmd_buffer *cmd_buffer)
1070 {
1071 struct radv_framebuffer *fb = cmd_buffer->state.framebuffer;
1072 const struct radv_subpass *subpass = cmd_buffer->state.subpass;
1073 struct radv_meta_saved_state saved_state;
1074 struct radv_subpass_barrier barrier;
1075
1076 /* Resolves happen before the end-of-subpass barriers get executed,
1077 * so we have to make the attachment shader-readable */
1078 barrier.src_stage_mask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
1079 barrier.src_access_mask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
1080 barrier.dst_access_mask = VK_ACCESS_INPUT_ATTACHMENT_READ_BIT;
1081 radv_emit_subpass_barrier(cmd_buffer, &barrier);
1082
1083 radv_decompress_resolve_subpass_src(cmd_buffer);
1084
1085 radv_meta_save(
1086 &saved_state, cmd_buffer,
1087 RADV_META_SAVE_GRAPHICS_PIPELINE | RADV_META_SAVE_CONSTANTS | RADV_META_SAVE_DESCRIPTORS);
1088
1089 for (uint32_t i = 0; i < subpass->color_count; ++i) {
1090 struct radv_subpass_attachment src_att = subpass->color_attachments[i];
1091 struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i];
1092
1093 if (dest_att.attachment == VK_ATTACHMENT_UNUSED)
1094 continue;
1095
1096 struct radv_image_view *dest_iview = cmd_buffer->state.attachments[dest_att.attachment].iview;
1097 struct radv_image_view *src_iview = cmd_buffer->state.attachments[src_att.attachment].iview;
1098
1099 struct radv_subpass resolve_subpass = {
1100 .color_count = 1,
1101 .color_attachments = (struct radv_subpass_attachment[]){dest_att},
1102 .depth_stencil_attachment = NULL,
1103 };
1104
1105 radv_cmd_buffer_set_subpass(cmd_buffer, &resolve_subpass);
1106
1107 emit_resolve(cmd_buffer, src_iview, dest_iview, &(VkOffset2D){0, 0}, &(VkOffset2D){0, 0},
1108 &(VkExtent2D){fb->width, fb->height});
1109 }
1110
1111 radv_cmd_buffer_restore_subpass(cmd_buffer, subpass);
1112
1113 radv_meta_restore(&saved_state, cmd_buffer);
1114 }
1115
1116 /**
1117 * Depth/stencil resolves for the current subpass.
1118 */
1119 void
radv_depth_stencil_resolve_subpass_fs(struct radv_cmd_buffer * cmd_buffer,VkImageAspectFlags aspects,VkResolveModeFlagBits resolve_mode)1120 radv_depth_stencil_resolve_subpass_fs(struct radv_cmd_buffer *cmd_buffer,
1121 VkImageAspectFlags aspects,
1122 VkResolveModeFlagBits resolve_mode)
1123 {
1124 struct radv_framebuffer *fb = cmd_buffer->state.framebuffer;
1125 const struct radv_subpass *subpass = cmd_buffer->state.subpass;
1126 struct radv_meta_saved_state saved_state;
1127 struct radv_subpass_barrier barrier;
1128
1129 /* Resolves happen before the end-of-subpass barriers get executed,
1130 * so we have to make the attachment shader-readable */
1131 barrier.src_stage_mask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
1132 barrier.src_access_mask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
1133 barrier.dst_access_mask = VK_ACCESS_INPUT_ATTACHMENT_READ_BIT;
1134 radv_emit_subpass_barrier(cmd_buffer, &barrier);
1135
1136 struct radv_subpass_attachment src_att = *subpass->depth_stencil_attachment;
1137 struct radv_image_view *src_iview = cmd_buffer->state.attachments[src_att.attachment].iview;
1138 struct radv_image *src_image = src_iview->image;
1139
1140 VkImageResolve2KHR region = {0};
1141 region.sType = VK_STRUCTURE_TYPE_IMAGE_RESOLVE_2_KHR;
1142 region.srcSubresource.aspectMask = aspects;
1143 region.srcSubresource.mipLevel = 0;
1144 region.srcSubresource.baseArrayLayer = 0;
1145 region.srcSubresource.layerCount = 1;
1146
1147 radv_decompress_resolve_src(cmd_buffer, src_image, src_att.layout, ®ion);
1148
1149 radv_meta_save(&saved_state, cmd_buffer,
1150 RADV_META_SAVE_GRAPHICS_PIPELINE | RADV_META_SAVE_DESCRIPTORS);
1151
1152 struct radv_subpass_attachment dst_att = *subpass->ds_resolve_attachment;
1153 struct radv_image_view *dst_iview = cmd_buffer->state.attachments[dst_att.attachment].iview;
1154
1155 struct radv_subpass resolve_subpass = {
1156 .color_count = 0,
1157 .color_attachments = NULL,
1158 .depth_stencil_attachment = (struct radv_subpass_attachment *){&dst_att},
1159 };
1160
1161 radv_cmd_buffer_set_subpass(cmd_buffer, &resolve_subpass);
1162
1163 struct radv_image_view tsrc_iview;
1164 radv_image_view_init(&tsrc_iview, cmd_buffer->device,
1165 &(VkImageViewCreateInfo){
1166 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
1167 .image = radv_image_to_handle(src_image),
1168 .viewType = radv_meta_get_view_type(src_image),
1169 .format = src_iview->vk_format,
1170 .subresourceRange =
1171 {
1172 .aspectMask = aspects,
1173 .baseMipLevel = 0,
1174 .levelCount = 1,
1175 .baseArrayLayer = 0,
1176 .layerCount = 1,
1177 },
1178 },
1179 NULL);
1180
1181 emit_depth_stencil_resolve(cmd_buffer, &tsrc_iview, dst_iview,
1182 &(VkExtent2D){fb->width, fb->height}, aspects, resolve_mode);
1183
1184 radv_cmd_buffer_restore_subpass(cmd_buffer, subpass);
1185
1186 radv_image_view_finish(&tsrc_iview);
1187
1188 radv_meta_restore(&saved_state, cmd_buffer);
1189 }
1190