1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include <assert.h>
25 #include <stdbool.h>
26
27 #include "nir/nir_builder.h"
28 #include "radv_meta.h"
29 #include "radv_private.h"
30 #include "sid.h"
31 #include "vk_format.h"
32
33 /* emit 0, 0, 0, 1 */
34 static nir_shader *
build_nir_fs(void)35 build_nir_fs(void)
36 {
37 const struct glsl_type *vec4 = glsl_vec4_type();
38 nir_variable *f_color; /* vec4, fragment output color */
39
40 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, NULL, "meta_resolve_fs");
41
42 f_color = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
43 f_color->data.location = FRAG_RESULT_DATA0;
44 nir_store_var(&b, f_color, nir_imm_vec4(&b, 0.0, 0.0, 0.0, 1.0), 0xf);
45
46 return b.shader;
47 }
48
49 static VkResult
create_pass(struct radv_device * device,VkFormat vk_format,VkRenderPass * pass)50 create_pass(struct radv_device *device, VkFormat vk_format, VkRenderPass *pass)
51 {
52 VkResult result;
53 VkDevice device_h = radv_device_to_handle(device);
54 const VkAllocationCallbacks *alloc = &device->meta_state.alloc;
55 VkAttachmentDescription2 attachments[2];
56 int i;
57
58 for (i = 0; i < 2; i++) {
59 attachments[i].sType = VK_STRUCTURE_TYPE_ATTACHMENT_DESCRIPTION_2;
60 attachments[i].pNext = NULL;
61 attachments[i].format = vk_format;
62 attachments[i].samples = 1;
63 attachments[i].loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
64 attachments[i].storeOp = VK_ATTACHMENT_STORE_OP_STORE;
65 }
66 attachments[0].initialLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
67 attachments[0].finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
68 attachments[1].initialLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
69 attachments[1].finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
70
71 result = radv_CreateRenderPass2(
72 device_h,
73 &(VkRenderPassCreateInfo2){
74 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO_2,
75 .attachmentCount = 2,
76 .pAttachments = attachments,
77 .subpassCount = 1,
78 .pSubpasses =
79 &(VkSubpassDescription2){
80 .sType = VK_STRUCTURE_TYPE_SUBPASS_DESCRIPTION_2,
81 .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
82 .inputAttachmentCount = 0,
83 .colorAttachmentCount = 2,
84 .pColorAttachments =
85 (VkAttachmentReference2[]){
86 {
87 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2,
88 .attachment = 0,
89 .layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
90 },
91 {
92 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2,
93 .attachment = 1,
94 .layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
95 },
96 },
97 .pResolveAttachments = NULL,
98 .pDepthStencilAttachment =
99 &(VkAttachmentReference2){
100 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2,
101 .attachment = VK_ATTACHMENT_UNUSED,
102 },
103 .preserveAttachmentCount = 0,
104 .pPreserveAttachments = NULL,
105 },
106 .dependencyCount = 2,
107 .pDependencies =
108 (VkSubpassDependency2[]){{.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2,
109 .srcSubpass = VK_SUBPASS_EXTERNAL,
110 .dstSubpass = 0,
111 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
112 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
113 .srcAccessMask = 0,
114 .dstAccessMask = 0,
115 .dependencyFlags = 0},
116 {.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2,
117 .srcSubpass = 0,
118 .dstSubpass = VK_SUBPASS_EXTERNAL,
119 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
120 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
121 .srcAccessMask = 0,
122 .dstAccessMask = 0,
123 .dependencyFlags = 0}},
124 },
125 alloc, pass);
126
127 return result;
128 }
129
130 static VkResult
create_pipeline(struct radv_device * device,VkShaderModule vs_module_h,VkPipeline * pipeline,VkRenderPass pass)131 create_pipeline(struct radv_device *device, VkShaderModule vs_module_h, VkPipeline *pipeline,
132 VkRenderPass pass)
133 {
134 VkResult result;
135 VkDevice device_h = radv_device_to_handle(device);
136
137 nir_shader *fs_module = build_nir_fs();
138 if (!fs_module) {
139 /* XXX: Need more accurate error */
140 result = VK_ERROR_OUT_OF_HOST_MEMORY;
141 goto cleanup;
142 }
143
144 VkPipelineLayoutCreateInfo pl_create_info = {
145 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
146 .setLayoutCount = 0,
147 .pSetLayouts = NULL,
148 .pushConstantRangeCount = 0,
149 .pPushConstantRanges = NULL,
150 };
151
152 if (!device->meta_state.resolve.p_layout) {
153 result =
154 radv_CreatePipelineLayout(radv_device_to_handle(device), &pl_create_info,
155 &device->meta_state.alloc, &device->meta_state.resolve.p_layout);
156 if (result != VK_SUCCESS)
157 goto cleanup;
158 }
159
160 result = radv_graphics_pipeline_create(
161 device_h, radv_pipeline_cache_to_handle(&device->meta_state.cache),
162 &(VkGraphicsPipelineCreateInfo){
163 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
164 .stageCount = 2,
165 .pStages =
166 (VkPipelineShaderStageCreateInfo[]){
167 {
168 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
169 .stage = VK_SHADER_STAGE_VERTEX_BIT,
170 .module = vs_module_h,
171 .pName = "main",
172 },
173 {
174 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
175 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
176 .module = vk_shader_module_handle_from_nir(fs_module),
177 .pName = "main",
178 },
179 },
180 .pVertexInputState =
181 &(VkPipelineVertexInputStateCreateInfo){
182 .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
183 .vertexBindingDescriptionCount = 0,
184 .vertexAttributeDescriptionCount = 0,
185 },
186 .pInputAssemblyState =
187 &(VkPipelineInputAssemblyStateCreateInfo){
188 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
189 .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
190 .primitiveRestartEnable = false,
191 },
192 .pViewportState =
193 &(VkPipelineViewportStateCreateInfo){
194 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
195 .viewportCount = 1,
196 .scissorCount = 1,
197 },
198 .pRasterizationState =
199 &(VkPipelineRasterizationStateCreateInfo){
200 .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
201 .depthClampEnable = false,
202 .rasterizerDiscardEnable = false,
203 .polygonMode = VK_POLYGON_MODE_FILL,
204 .cullMode = VK_CULL_MODE_NONE,
205 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE,
206 },
207 .pMultisampleState =
208 &(VkPipelineMultisampleStateCreateInfo){
209 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
210 .rasterizationSamples = 1,
211 .sampleShadingEnable = false,
212 .pSampleMask = NULL,
213 .alphaToCoverageEnable = false,
214 .alphaToOneEnable = false,
215 },
216 .pColorBlendState =
217 &(VkPipelineColorBlendStateCreateInfo){
218 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
219 .logicOpEnable = false,
220 .attachmentCount = 2,
221 .pAttachments =
222 (VkPipelineColorBlendAttachmentState[]){
223 {
224 .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT |
225 VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT,
226 },
227 {
228 .colorWriteMask = 0,
229
230 }},
231 },
232 .pDynamicState =
233 &(VkPipelineDynamicStateCreateInfo){
234 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
235 .dynamicStateCount = 2,
236 .pDynamicStates =
237 (VkDynamicState[]){
238 VK_DYNAMIC_STATE_VIEWPORT,
239 VK_DYNAMIC_STATE_SCISSOR,
240 },
241 },
242 .layout = device->meta_state.resolve.p_layout,
243 .renderPass = pass,
244 .subpass = 0,
245 },
246 &(struct radv_graphics_pipeline_create_info){
247 .use_rectlist = true,
248 .custom_blend_mode = V_028808_CB_RESOLVE,
249 },
250 &device->meta_state.alloc, pipeline);
251 if (result != VK_SUCCESS)
252 goto cleanup;
253
254 goto cleanup;
255
256 cleanup:
257 ralloc_free(fs_module);
258 return result;
259 }
260
261 void
radv_device_finish_meta_resolve_state(struct radv_device * device)262 radv_device_finish_meta_resolve_state(struct radv_device *device)
263 {
264 struct radv_meta_state *state = &device->meta_state;
265
266 for (uint32_t j = 0; j < NUM_META_FS_KEYS; j++) {
267 radv_DestroyRenderPass(radv_device_to_handle(device), state->resolve.pass[j], &state->alloc);
268 radv_DestroyPipeline(radv_device_to_handle(device), state->resolve.pipeline[j],
269 &state->alloc);
270 }
271 radv_DestroyPipelineLayout(radv_device_to_handle(device), state->resolve.p_layout,
272 &state->alloc);
273 }
274
275 VkResult
radv_device_init_meta_resolve_state(struct radv_device * device,bool on_demand)276 radv_device_init_meta_resolve_state(struct radv_device *device, bool on_demand)
277 {
278 if (on_demand)
279 return VK_SUCCESS;
280
281 VkResult res = VK_SUCCESS;
282 struct radv_meta_state *state = &device->meta_state;
283 nir_shader *vs_module = radv_meta_build_nir_vs_generate_vertices();
284 if (!vs_module) {
285 /* XXX: Need more accurate error */
286 res = VK_ERROR_OUT_OF_HOST_MEMORY;
287 goto fail;
288 }
289
290 for (uint32_t i = 0; i < NUM_META_FS_KEYS; ++i) {
291 VkFormat format = radv_fs_key_format_exemplars[i];
292 unsigned fs_key = radv_format_meta_fs_key(device, format);
293 res = create_pass(device, format, &state->resolve.pass[fs_key]);
294 if (res != VK_SUCCESS)
295 goto fail;
296
297 VkShaderModule vs_module_h = vk_shader_module_handle_from_nir(vs_module);
298 res = create_pipeline(device, vs_module_h, &state->resolve.pipeline[fs_key],
299 state->resolve.pass[fs_key]);
300 if (res != VK_SUCCESS)
301 goto fail;
302 }
303
304 goto cleanup;
305
306 fail:
307 radv_device_finish_meta_resolve_state(device);
308
309 cleanup:
310 ralloc_free(vs_module);
311
312 return res;
313 }
314
315 static void
emit_resolve(struct radv_cmd_buffer * cmd_buffer,const struct radv_image * src_image,const struct radv_image * dst_image,VkFormat vk_format,const VkOffset2D * dest_offset,const VkExtent2D * resolve_extent)316 emit_resolve(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *src_image,
317 const struct radv_image *dst_image, VkFormat vk_format, const VkOffset2D *dest_offset,
318 const VkExtent2D *resolve_extent)
319 {
320 struct radv_device *device = cmd_buffer->device;
321 VkCommandBuffer cmd_buffer_h = radv_cmd_buffer_to_handle(cmd_buffer);
322 unsigned fs_key = radv_format_meta_fs_key(device, vk_format);
323
324 cmd_buffer->state.flush_bits |=
325 radv_src_access_flush(cmd_buffer, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, src_image) |
326 radv_dst_access_flush(cmd_buffer, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT, src_image) |
327 radv_dst_access_flush(cmd_buffer, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, dst_image);
328
329 radv_CmdBindPipeline(cmd_buffer_h, VK_PIPELINE_BIND_POINT_GRAPHICS,
330 device->meta_state.resolve.pipeline[fs_key]);
331
332 radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
333 &(VkViewport){.x = dest_offset->x,
334 .y = dest_offset->y,
335 .width = resolve_extent->width,
336 .height = resolve_extent->height,
337 .minDepth = 0.0f,
338 .maxDepth = 1.0f});
339
340 radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
341 &(VkRect2D){
342 .offset = *dest_offset,
343 .extent = *resolve_extent,
344 });
345
346 radv_CmdDraw(cmd_buffer_h, 3, 1, 0, 0);
347 cmd_buffer->state.flush_bits |=
348 radv_src_access_flush(cmd_buffer, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, dst_image);
349 }
350
351 enum radv_resolve_method {
352 RESOLVE_HW,
353 RESOLVE_COMPUTE,
354 RESOLVE_FRAGMENT,
355 };
356
357 static bool
image_hw_resolve_compat(const struct radv_device * device,struct radv_image * src_image,struct radv_image * dst_image)358 image_hw_resolve_compat(const struct radv_device *device, struct radv_image *src_image,
359 struct radv_image *dst_image)
360 {
361 if (device->physical_device->rad_info.chip_class >= GFX9) {
362 return dst_image->planes[0].surface.u.gfx9.swizzle_mode ==
363 src_image->planes[0].surface.u.gfx9.swizzle_mode;
364 } else {
365 return dst_image->planes[0].surface.micro_tile_mode ==
366 src_image->planes[0].surface.micro_tile_mode;
367 }
368 }
369
370 static void
radv_pick_resolve_method_images(struct radv_device * device,struct radv_image * src_image,VkFormat src_format,struct radv_image * dest_image,unsigned dest_level,VkImageLayout dest_image_layout,bool dest_render_loop,struct radv_cmd_buffer * cmd_buffer,enum radv_resolve_method * method)371 radv_pick_resolve_method_images(struct radv_device *device, struct radv_image *src_image,
372 VkFormat src_format, struct radv_image *dest_image,
373 unsigned dest_level, VkImageLayout dest_image_layout,
374 bool dest_render_loop, struct radv_cmd_buffer *cmd_buffer,
375 enum radv_resolve_method *method)
376
377 {
378 uint32_t queue_mask = radv_image_queue_family_mask(dest_image, cmd_buffer->queue_family_index,
379 cmd_buffer->queue_family_index);
380
381 if (vk_format_is_color(src_format)) {
382 /* Using the fragment resolve path is currently a hint to
383 * avoid decompressing DCC for partial resolves and
384 * re-initialize it after resolving using compute.
385 * TODO: Add support for layered and int to the fragment path.
386 */
387 if (radv_layout_dcc_compressed(device, dest_image, dest_level, dest_image_layout,
388 dest_render_loop, queue_mask)) {
389 *method = RESOLVE_FRAGMENT;
390 } else if (!image_hw_resolve_compat(device, src_image, dest_image)) {
391 /* The micro tile mode only needs to match for the HW
392 * resolve path which is the default path for non-DCC
393 * resolves.
394 */
395 *method = RESOLVE_COMPUTE;
396 }
397
398 if (src_format == VK_FORMAT_R16G16_UNORM || src_format == VK_FORMAT_R16G16_SNORM)
399 *method = RESOLVE_COMPUTE;
400 else if (vk_format_is_int(src_format))
401 *method = RESOLVE_COMPUTE;
402 else if (src_image->info.array_size > 1 || dest_image->info.array_size > 1)
403 *method = RESOLVE_COMPUTE;
404 } else {
405 if (src_image->info.array_size > 1 || dest_image->info.array_size > 1)
406 *method = RESOLVE_COMPUTE;
407 else
408 *method = RESOLVE_FRAGMENT;
409 }
410 }
411
412 static VkResult
build_resolve_pipeline(struct radv_device * device,unsigned fs_key)413 build_resolve_pipeline(struct radv_device *device, unsigned fs_key)
414 {
415 VkResult result = VK_SUCCESS;
416
417 if (device->meta_state.resolve.pipeline[fs_key])
418 return result;
419
420 mtx_lock(&device->meta_state.mtx);
421 if (device->meta_state.resolve.pipeline[fs_key]) {
422 mtx_unlock(&device->meta_state.mtx);
423 return result;
424 }
425
426 nir_shader *vs_module = radv_meta_build_nir_vs_generate_vertices();
427
428 result = create_pass(device, radv_fs_key_format_exemplars[fs_key],
429 &device->meta_state.resolve.pass[fs_key]);
430 if (result != VK_SUCCESS)
431 goto fail;
432
433 VkShaderModule vs_module_h = vk_shader_module_handle_from_nir(vs_module);
434 result = create_pipeline(device, vs_module_h, &device->meta_state.resolve.pipeline[fs_key],
435 device->meta_state.resolve.pass[fs_key]);
436
437 fail:
438 ralloc_free(vs_module);
439 mtx_unlock(&device->meta_state.mtx);
440 return result;
441 }
442
443 static void
radv_meta_resolve_hardware_image(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,VkImageLayout src_image_layout,struct radv_image * dst_image,VkImageLayout dst_image_layout,const VkImageResolve2KHR * region)444 radv_meta_resolve_hardware_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
445 VkImageLayout src_image_layout, struct radv_image *dst_image,
446 VkImageLayout dst_image_layout, const VkImageResolve2KHR *region)
447 {
448 struct radv_device *device = cmd_buffer->device;
449 struct radv_meta_saved_state saved_state;
450
451 radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_GRAPHICS_PIPELINE);
452
453 assert(src_image->info.samples > 1);
454 assert(dst_image->info.samples == 1);
455
456 unsigned fs_key = radv_format_meta_fs_key(device, dst_image->vk_format);
457
458 /* From the Vulkan 1.0 spec:
459 *
460 * - The aspectMask member of srcSubresource and dstSubresource must
461 * only contain VK_IMAGE_ASPECT_COLOR_BIT
462 *
463 * - The layerCount member of srcSubresource and dstSubresource must
464 * match
465 */
466 assert(region->srcSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
467 assert(region->dstSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
468 assert(region->srcSubresource.layerCount == region->dstSubresource.layerCount);
469
470 const uint32_t src_base_layer =
471 radv_meta_get_iview_layer(src_image, ®ion->srcSubresource, ®ion->srcOffset);
472
473 const uint32_t dst_base_layer =
474 radv_meta_get_iview_layer(dst_image, ®ion->dstSubresource, ®ion->dstOffset);
475
476 /**
477 * From Vulkan 1.0.6 spec: 18.6 Resolving Multisample Images
478 *
479 * extent is the size in texels of the source image to resolve in width,
480 * height and depth. 1D images use only x and width. 2D images use x, y,
481 * width and height. 3D images use x, y, z, width, height and depth.
482 *
483 * srcOffset and dstOffset select the initial x, y, and z offsets in
484 * texels of the sub-regions of the source and destination image data.
485 * extent is the size in texels of the source image to resolve in width,
486 * height and depth. 1D images use only x and width. 2D images use x, y,
487 * width and height. 3D images use x, y, z, width, height and depth.
488 */
489 const struct VkExtent3D extent = radv_sanitize_image_extent(src_image->type, region->extent);
490 const struct VkOffset3D dstOffset =
491 radv_sanitize_image_offset(dst_image->type, region->dstOffset);
492
493 uint32_t queue_mask = radv_image_queue_family_mask(dst_image, cmd_buffer->queue_family_index,
494 cmd_buffer->queue_family_index);
495
496 if (radv_layout_dcc_compressed(cmd_buffer->device, dst_image, region->dstSubresource.mipLevel,
497 dst_image_layout, false, queue_mask)) {
498 VkImageSubresourceRange range = {
499 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
500 .baseMipLevel = region->dstSubresource.mipLevel,
501 .levelCount = 1,
502 .baseArrayLayer = dst_base_layer,
503 .layerCount = region->dstSubresource.layerCount,
504 };
505
506 cmd_buffer->state.flush_bits |= radv_init_dcc(cmd_buffer, dst_image, &range, 0xffffffff);
507 }
508
509 for (uint32_t layer = 0; layer < region->srcSubresource.layerCount; ++layer) {
510
511 VkResult ret = build_resolve_pipeline(device, fs_key);
512 if (ret != VK_SUCCESS) {
513 cmd_buffer->record_result = ret;
514 break;
515 }
516
517 struct radv_image_view src_iview;
518 radv_image_view_init(&src_iview, cmd_buffer->device,
519 &(VkImageViewCreateInfo){
520 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
521 .image = radv_image_to_handle(src_image),
522 .viewType = radv_meta_get_view_type(src_image),
523 .format = src_image->vk_format,
524 .subresourceRange =
525 {
526 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
527 .baseMipLevel = region->srcSubresource.mipLevel,
528 .levelCount = 1,
529 .baseArrayLayer = src_base_layer + layer,
530 .layerCount = 1,
531 },
532 },
533 NULL);
534
535 struct radv_image_view dst_iview;
536 radv_image_view_init(&dst_iview, cmd_buffer->device,
537 &(VkImageViewCreateInfo){
538 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
539 .image = radv_image_to_handle(dst_image),
540 .viewType = radv_meta_get_view_type(dst_image),
541 .format = dst_image->vk_format,
542 .subresourceRange =
543 {
544 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
545 .baseMipLevel = region->dstSubresource.mipLevel,
546 .levelCount = 1,
547 .baseArrayLayer = dst_base_layer + layer,
548 .layerCount = 1,
549 },
550 },
551 NULL);
552
553 VkFramebuffer fb_h;
554 radv_CreateFramebuffer(
555 radv_device_to_handle(device),
556 &(VkFramebufferCreateInfo){
557 .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
558 .attachmentCount = 2,
559 .pAttachments =
560 (VkImageView[]){
561 radv_image_view_to_handle(&src_iview),
562 radv_image_view_to_handle(&dst_iview),
563 },
564 .width = radv_minify(dst_image->info.width, region->dstSubresource.mipLevel),
565 .height = radv_minify(dst_image->info.height, region->dstSubresource.mipLevel),
566 .layers = 1},
567 &cmd_buffer->pool->alloc, &fb_h);
568
569 radv_cmd_buffer_begin_render_pass(cmd_buffer,
570 &(VkRenderPassBeginInfo){
571 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
572 .renderPass = device->meta_state.resolve.pass[fs_key],
573 .framebuffer = fb_h,
574 .renderArea = {.offset =
575 {
576 dstOffset.x,
577 dstOffset.y,
578 },
579 .extent =
580 {
581 extent.width,
582 extent.height,
583 }},
584 .clearValueCount = 0,
585 .pClearValues = NULL,
586 },
587 NULL);
588
589 radv_cmd_buffer_set_subpass(cmd_buffer, &cmd_buffer->state.pass->subpasses[0]);
590
591 emit_resolve(cmd_buffer, src_image, dst_image, dst_iview.vk_format,
592 &(VkOffset2D){
593 .x = dstOffset.x,
594 .y = dstOffset.y,
595 },
596 &(VkExtent2D){
597 .width = extent.width,
598 .height = extent.height,
599 });
600
601 radv_cmd_buffer_end_render_pass(cmd_buffer);
602
603 radv_image_view_finish(&src_iview);
604 radv_image_view_finish(&dst_iview);
605 radv_DestroyFramebuffer(radv_device_to_handle(device), fb_h, &cmd_buffer->pool->alloc);
606 }
607
608 radv_meta_restore(&saved_state, cmd_buffer);
609 }
610
611 static void
resolve_image(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,VkImageLayout src_image_layout,struct radv_image * dst_image,VkImageLayout dst_image_layout,const VkImageResolve2KHR * region,enum radv_resolve_method resolve_method)612 resolve_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
613 VkImageLayout src_image_layout, struct radv_image *dst_image,
614 VkImageLayout dst_image_layout, const VkImageResolve2KHR *region,
615 enum radv_resolve_method resolve_method)
616 {
617 switch (resolve_method) {
618 case RESOLVE_HW:
619 radv_meta_resolve_hardware_image(cmd_buffer, src_image, src_image_layout, dst_image,
620 dst_image_layout, region);
621 break;
622 case RESOLVE_FRAGMENT:
623 radv_meta_resolve_fragment_image(cmd_buffer, src_image, src_image_layout, dst_image,
624 dst_image_layout, region);
625 break;
626 case RESOLVE_COMPUTE:
627 radv_meta_resolve_compute_image(cmd_buffer, src_image, src_image->vk_format, src_image_layout,
628 dst_image, dst_image->vk_format, dst_image_layout, region);
629 break;
630 default:
631 assert(!"Invalid resolve method selected");
632 }
633 }
634
635 void
radv_CmdResolveImage2KHR(VkCommandBuffer commandBuffer,const VkResolveImageInfo2KHR * pResolveImageInfo)636 radv_CmdResolveImage2KHR(VkCommandBuffer commandBuffer,
637 const VkResolveImageInfo2KHR *pResolveImageInfo)
638 {
639 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
640 RADV_FROM_HANDLE(radv_image, src_image, pResolveImageInfo->srcImage);
641 RADV_FROM_HANDLE(radv_image, dst_image, pResolveImageInfo->dstImage);
642 VkImageLayout src_image_layout = pResolveImageInfo->srcImageLayout;
643 VkImageLayout dst_image_layout = pResolveImageInfo->dstImageLayout;
644 enum radv_resolve_method resolve_method = RESOLVE_HW;
645 /* we can use the hw resolve only for single full resolves */
646 if (pResolveImageInfo->regionCount == 1) {
647 if (pResolveImageInfo->pRegions[0].srcOffset.x ||
648 pResolveImageInfo->pRegions[0].srcOffset.y || pResolveImageInfo->pRegions[0].srcOffset.z)
649 resolve_method = RESOLVE_COMPUTE;
650 if (pResolveImageInfo->pRegions[0].dstOffset.x ||
651 pResolveImageInfo->pRegions[0].dstOffset.y || pResolveImageInfo->pRegions[0].dstOffset.z)
652 resolve_method = RESOLVE_COMPUTE;
653
654 if (pResolveImageInfo->pRegions[0].extent.width != src_image->info.width ||
655 pResolveImageInfo->pRegions[0].extent.height != src_image->info.height ||
656 pResolveImageInfo->pRegions[0].extent.depth != src_image->info.depth)
657 resolve_method = RESOLVE_COMPUTE;
658 } else
659 resolve_method = RESOLVE_COMPUTE;
660
661 for (uint32_t r = 0; r < pResolveImageInfo->regionCount; r++) {
662 const VkImageResolve2KHR *region = &pResolveImageInfo->pRegions[r];
663
664 radv_pick_resolve_method_images(cmd_buffer->device, src_image, src_image->vk_format, dst_image,
665 region->dstSubresource.mipLevel, dst_image_layout, false,
666 cmd_buffer, &resolve_method);
667
668 resolve_image(cmd_buffer, src_image, src_image_layout, dst_image, dst_image_layout, region,
669 resolve_method);
670 }
671 }
672
673 static void
radv_cmd_buffer_resolve_subpass_hw(struct radv_cmd_buffer * cmd_buffer)674 radv_cmd_buffer_resolve_subpass_hw(struct radv_cmd_buffer *cmd_buffer)
675 {
676 struct radv_framebuffer *fb = cmd_buffer->state.framebuffer;
677 const struct radv_subpass *subpass = cmd_buffer->state.subpass;
678 struct radv_meta_saved_state saved_state;
679
680 radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_GRAPHICS_PIPELINE);
681
682 for (uint32_t i = 0; i < subpass->color_count; ++i) {
683 struct radv_subpass_attachment src_att = subpass->color_attachments[i];
684 struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i];
685
686 if (dest_att.attachment == VK_ATTACHMENT_UNUSED)
687 continue;
688
689 struct radv_image_view *src_iview = cmd_buffer->state.attachments[src_att.attachment].iview;
690 struct radv_image *src_img = src_iview->image;
691
692 struct radv_image_view *dest_iview = cmd_buffer->state.attachments[dest_att.attachment].iview;
693 struct radv_image *dst_img = dest_iview->image;
694 VkImageLayout dst_image_layout = cmd_buffer->state.attachments[dest_att.attachment].current_layout;
695
696 uint32_t queue_mask = radv_image_queue_family_mask(dst_img, cmd_buffer->queue_family_index,
697 cmd_buffer->queue_family_index);
698
699 if (radv_layout_dcc_compressed(cmd_buffer->device, dst_img, dest_iview->base_mip,
700 dst_image_layout, false, queue_mask)) {
701 VkImageSubresourceRange range = {
702 .aspectMask = dest_iview->aspect_mask,
703 .baseMipLevel = dest_iview->base_mip,
704 .levelCount = dest_iview->level_count,
705 .baseArrayLayer = dest_iview->base_layer,
706 .layerCount = dest_iview->layer_count,
707 };
708
709 cmd_buffer->state.flush_bits |= radv_init_dcc(cmd_buffer, dst_img, &range, 0xffffffff);
710 cmd_buffer->state.attachments[dest_att.attachment].current_layout =
711 VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
712 }
713
714 struct radv_subpass resolve_subpass = {
715 .color_count = 2,
716 .color_attachments = (struct radv_subpass_attachment[]){src_att, dest_att},
717 .depth_stencil_attachment = NULL,
718 };
719
720 radv_cmd_buffer_set_subpass(cmd_buffer, &resolve_subpass);
721
722 VkResult ret = build_resolve_pipeline(
723 cmd_buffer->device, radv_format_meta_fs_key(cmd_buffer->device, dest_iview->vk_format));
724 if (ret != VK_SUCCESS) {
725 cmd_buffer->record_result = ret;
726 continue;
727 }
728
729 emit_resolve(cmd_buffer, src_img, dst_img, dest_iview->vk_format, &(VkOffset2D){0, 0},
730 &(VkExtent2D){fb->width, fb->height});
731 }
732
733 radv_cmd_buffer_restore_subpass(cmd_buffer, subpass);
734
735 radv_meta_restore(&saved_state, cmd_buffer);
736 }
737
738 /**
739 * Emit any needed resolves for the current subpass.
740 */
741 void
radv_cmd_buffer_resolve_subpass(struct radv_cmd_buffer * cmd_buffer)742 radv_cmd_buffer_resolve_subpass(struct radv_cmd_buffer *cmd_buffer)
743 {
744 const struct radv_subpass *subpass = cmd_buffer->state.subpass;
745 enum radv_resolve_method resolve_method = RESOLVE_HW;
746
747 if (!subpass->has_color_resolve && !subpass->ds_resolve_attachment)
748 return;
749
750 radv_describe_begin_render_pass_resolve(cmd_buffer);
751
752 if (subpass->ds_resolve_attachment) {
753 struct radv_subpass_attachment src_att = *subpass->depth_stencil_attachment;
754 struct radv_subpass_attachment dst_att = *subpass->ds_resolve_attachment;
755 struct radv_image_view *src_iview = cmd_buffer->state.attachments[src_att.attachment].iview;
756 struct radv_image_view *dst_iview = cmd_buffer->state.attachments[dst_att.attachment].iview;
757
758 /* Make sure to not clear the depth/stencil attachment after resolves. */
759 cmd_buffer->state.attachments[dst_att.attachment].pending_clear_aspects = 0;
760
761 radv_pick_resolve_method_images(cmd_buffer->device, src_iview->image, src_iview->vk_format,
762 dst_iview->image, dst_iview->base_mip, dst_att.layout,
763 dst_att.in_render_loop, cmd_buffer, &resolve_method);
764
765 if ((src_iview->aspect_mask & VK_IMAGE_ASPECT_DEPTH_BIT) &&
766 subpass->depth_resolve_mode != VK_RESOLVE_MODE_NONE_KHR) {
767 if (resolve_method == RESOLVE_FRAGMENT) {
768 radv_depth_stencil_resolve_subpass_fs(cmd_buffer, VK_IMAGE_ASPECT_DEPTH_BIT,
769 subpass->depth_resolve_mode);
770 } else {
771 assert(resolve_method == RESOLVE_COMPUTE);
772 radv_depth_stencil_resolve_subpass_cs(cmd_buffer, VK_IMAGE_ASPECT_DEPTH_BIT,
773 subpass->depth_resolve_mode);
774 }
775 }
776
777 if ((src_iview->aspect_mask & VK_IMAGE_ASPECT_STENCIL_BIT) &&
778 subpass->stencil_resolve_mode != VK_RESOLVE_MODE_NONE_KHR) {
779 if (resolve_method == RESOLVE_FRAGMENT) {
780 radv_depth_stencil_resolve_subpass_fs(cmd_buffer, VK_IMAGE_ASPECT_STENCIL_BIT,
781 subpass->stencil_resolve_mode);
782 } else {
783 assert(resolve_method == RESOLVE_COMPUTE);
784 radv_depth_stencil_resolve_subpass_cs(cmd_buffer, VK_IMAGE_ASPECT_STENCIL_BIT,
785 subpass->stencil_resolve_mode);
786 }
787 }
788
789 /* From the Vulkan spec 1.2.165:
790 *
791 * "VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT specifies
792 * write access to a color, resolve, or depth/stencil
793 * resolve attachment during a render pass or via
794 * certain subpass load and store operations."
795 *
796 * Yes, it's counterintuitive but it makes sense because ds
797 * resolve operations happen late at the end of the subpass.
798 *
799 * That said, RADV is wrong because it executes the subpass
800 * end barrier *before* any subpass resolves instead of after.
801 *
802 * TODO: Fix this properly by executing subpass end barriers
803 * after subpass resolves.
804 */
805 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB;
806 if (radv_image_has_htile(dst_iview->image))
807 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB_META;
808 }
809
810 if (subpass->has_color_resolve) {
811 for (uint32_t i = 0; i < subpass->color_count; ++i) {
812 struct radv_subpass_attachment src_att = subpass->color_attachments[i];
813 struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i];
814
815 if (dest_att.attachment == VK_ATTACHMENT_UNUSED)
816 continue;
817
818 /* Make sure to not clear color attachments after resolves. */
819 cmd_buffer->state.attachments[dest_att.attachment].pending_clear_aspects = 0;
820
821 struct radv_image_view *dst_iview =
822 cmd_buffer->state.attachments[dest_att.attachment].iview;
823 struct radv_image *dst_img = dst_iview->image;
824 struct radv_image_view *src_iview =
825 cmd_buffer->state.attachments[src_att.attachment].iview;
826 struct radv_image *src_img = src_iview->image;
827
828 radv_pick_resolve_method_images(cmd_buffer->device, src_img, src_iview->vk_format, dst_img,
829 dst_iview->base_mip, dest_att.layout,
830 dest_att.in_render_loop, cmd_buffer, &resolve_method);
831
832 if (resolve_method == RESOLVE_FRAGMENT) {
833 break;
834 }
835 }
836
837 switch (resolve_method) {
838 case RESOLVE_HW:
839 radv_cmd_buffer_resolve_subpass_hw(cmd_buffer);
840 break;
841 case RESOLVE_COMPUTE:
842 radv_cmd_buffer_resolve_subpass_cs(cmd_buffer);
843 break;
844 case RESOLVE_FRAGMENT:
845 radv_cmd_buffer_resolve_subpass_fs(cmd_buffer);
846 break;
847 default:
848 unreachable("Invalid resolve method");
849 }
850 }
851
852 radv_describe_end_render_pass_resolve(cmd_buffer);
853 }
854
855 /**
856 * Decompress CMask/FMask before resolving a multisampled source image inside a
857 * subpass.
858 */
859 void
radv_decompress_resolve_subpass_src(struct radv_cmd_buffer * cmd_buffer)860 radv_decompress_resolve_subpass_src(struct radv_cmd_buffer *cmd_buffer)
861 {
862 const struct radv_subpass *subpass = cmd_buffer->state.subpass;
863 struct radv_framebuffer *fb = cmd_buffer->state.framebuffer;
864 uint32_t layer_count = fb->layers;
865
866 if (subpass->view_mask)
867 layer_count = util_last_bit(subpass->view_mask);
868
869 for (uint32_t i = 0; i < subpass->color_count; ++i) {
870 struct radv_subpass_attachment src_att = subpass->color_attachments[i];
871 struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i];
872
873 if (dest_att.attachment == VK_ATTACHMENT_UNUSED)
874 continue;
875
876 struct radv_image_view *src_iview = cmd_buffer->state.attachments[src_att.attachment].iview;
877 struct radv_image *src_image = src_iview->image;
878
879 VkImageResolve2KHR region = {0};
880 region.sType = VK_STRUCTURE_TYPE_IMAGE_RESOLVE_2_KHR;
881 region.srcSubresource.aspectMask = src_iview->aspect_mask;
882 region.srcSubresource.mipLevel = 0;
883 region.srcSubresource.baseArrayLayer = src_iview->base_layer;
884 region.srcSubresource.layerCount = layer_count;
885
886 radv_decompress_resolve_src(cmd_buffer, src_image, src_att.layout, ®ion);
887 }
888 }
889
890 static struct radv_sample_locations_state *
radv_get_resolve_sample_locations(struct radv_cmd_buffer * cmd_buffer)891 radv_get_resolve_sample_locations(struct radv_cmd_buffer *cmd_buffer)
892 {
893 struct radv_cmd_state *state = &cmd_buffer->state;
894 uint32_t subpass_id = radv_get_subpass_id(cmd_buffer);
895
896 for (uint32_t i = 0; i < state->num_subpass_sample_locs; i++) {
897 if (state->subpass_sample_locs[i].subpass_idx == subpass_id)
898 return &state->subpass_sample_locs[i].sample_location;
899 }
900
901 return NULL;
902 }
903
904 /**
905 * Decompress CMask/FMask before resolving a multisampled source image.
906 */
907 void
radv_decompress_resolve_src(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,VkImageLayout src_image_layout,const VkImageResolve2KHR * region)908 radv_decompress_resolve_src(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
909 VkImageLayout src_image_layout, const VkImageResolve2KHR *region)
910 {
911 const uint32_t src_base_layer =
912 radv_meta_get_iview_layer(src_image, ®ion->srcSubresource, ®ion->srcOffset);
913
914 VkImageMemoryBarrier barrier = {0};
915 barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
916 barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
917 barrier.oldLayout = src_image_layout;
918 barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
919 barrier.image = radv_image_to_handle(src_image);
920 barrier.subresourceRange = (VkImageSubresourceRange){
921 .aspectMask = region->srcSubresource.aspectMask,
922 .baseMipLevel = region->srcSubresource.mipLevel,
923 .levelCount = 1,
924 .baseArrayLayer = src_base_layer,
925 .layerCount = region->srcSubresource.layerCount,
926 };
927
928 if (src_image->flags & VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT) {
929 /* If the depth/stencil image uses different sample
930 * locations, we need them during HTILE decompressions.
931 */
932 struct radv_sample_locations_state *sample_locs =
933 radv_get_resolve_sample_locations(cmd_buffer);
934
935 barrier.pNext = &(VkSampleLocationsInfoEXT){
936 .sType = VK_STRUCTURE_TYPE_SAMPLE_LOCATIONS_INFO_EXT,
937 .sampleLocationsPerPixel = sample_locs->per_pixel,
938 .sampleLocationGridSize = sample_locs->grid_size,
939 .sampleLocationsCount = sample_locs->count,
940 .pSampleLocations = sample_locs->locations,
941 };
942 }
943
944 radv_CmdPipelineBarrier(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
945 VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, false, 0, NULL, 0, NULL, 1,
946 &barrier);
947 }
948