1 /*
2 * Copyright © 2016 Red Hat
3 *
4 * based on anv driver:
5 * Copyright © 2016 Intel Corporation
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the next
15 * paragraph) shall be included in all copies or substantial portions of the
16 * Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * IN THE SOFTWARE.
25 */
26
27 #include "nir/nir_builder.h"
28 #include "radv_meta.h"
29 #include "vk_format.h"
30
31 enum blit2d_src_type {
32 BLIT2D_SRC_TYPE_IMAGE,
33 BLIT2D_SRC_TYPE_IMAGE_3D,
34 BLIT2D_SRC_TYPE_BUFFER,
35 BLIT2D_NUM_SRC_TYPES,
36 };
37
38 static VkResult blit2d_init_color_pipeline(struct radv_device *device,
39 enum blit2d_src_type src_type, VkFormat format,
40 uint32_t log2_samples);
41
42 static VkResult blit2d_init_depth_only_pipeline(struct radv_device *device,
43 enum blit2d_src_type src_type,
44 uint32_t log2_samples);
45
46 static VkResult blit2d_init_stencil_only_pipeline(struct radv_device *device,
47 enum blit2d_src_type src_type,
48 uint32_t log2_samples);
49
50 static void
create_iview(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * surf,struct radv_image_view * iview,VkFormat depth_format,VkImageAspectFlagBits aspects)51 create_iview(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *surf,
52 struct radv_image_view *iview, VkFormat depth_format, VkImageAspectFlagBits aspects)
53 {
54 VkFormat format;
55 VkImageViewType view_type = cmd_buffer->device->physical_device->rad_info.chip_class < GFX9
56 ? VK_IMAGE_VIEW_TYPE_2D
57 : radv_meta_get_view_type(surf->image);
58
59 if (depth_format)
60 format = depth_format;
61 else
62 format = surf->format;
63
64 radv_image_view_init(iview, cmd_buffer->device,
65 &(VkImageViewCreateInfo){
66 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
67 .image = radv_image_to_handle(surf->image),
68 .viewType = view_type,
69 .format = format,
70 .subresourceRange = {.aspectMask = aspects,
71 .baseMipLevel = surf->level,
72 .levelCount = 1,
73 .baseArrayLayer = surf->layer,
74 .layerCount = 1},
75 },
76 NULL);
77 }
78
79 static void
create_bview(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_buffer * src,struct radv_buffer_view * bview,VkFormat depth_format)80 create_bview(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_buffer *src,
81 struct radv_buffer_view *bview, VkFormat depth_format)
82 {
83 VkFormat format;
84
85 if (depth_format)
86 format = depth_format;
87 else
88 format = src->format;
89 radv_buffer_view_init(bview, cmd_buffer->device,
90 &(VkBufferViewCreateInfo){
91 .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO,
92 .flags = 0,
93 .buffer = radv_buffer_to_handle(src->buffer),
94 .format = format,
95 .offset = src->offset,
96 .range = VK_WHOLE_SIZE,
97 });
98 }
99
100 struct blit2d_src_temps {
101 struct radv_image_view iview;
102 struct radv_buffer_view bview;
103 };
104
105 static void
blit2d_bind_src(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * src_img,struct radv_meta_blit2d_buffer * src_buf,struct blit2d_src_temps * tmp,enum blit2d_src_type src_type,VkFormat depth_format,VkImageAspectFlagBits aspects,uint32_t log2_samples)106 blit2d_bind_src(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src_img,
107 struct radv_meta_blit2d_buffer *src_buf, struct blit2d_src_temps *tmp,
108 enum blit2d_src_type src_type, VkFormat depth_format, VkImageAspectFlagBits aspects,
109 uint32_t log2_samples)
110 {
111 struct radv_device *device = cmd_buffer->device;
112
113 if (src_type == BLIT2D_SRC_TYPE_BUFFER) {
114 create_bview(cmd_buffer, src_buf, &tmp->bview, depth_format);
115
116 radv_meta_push_descriptor_set(
117 cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
118 device->meta_state.blit2d[log2_samples].p_layouts[src_type], 0, /* set */
119 1, /* descriptorWriteCount */
120 (VkWriteDescriptorSet[]){
121 {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
122 .dstBinding = 0,
123 .dstArrayElement = 0,
124 .descriptorCount = 1,
125 .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER,
126 .pTexelBufferView = (VkBufferView[]){radv_buffer_view_to_handle(&tmp->bview)}}});
127
128 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
129 device->meta_state.blit2d[log2_samples].p_layouts[src_type],
130 VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4, &src_buf->pitch);
131 } else {
132 create_iview(cmd_buffer, src_img, &tmp->iview, depth_format, aspects);
133
134 if (src_type == BLIT2D_SRC_TYPE_IMAGE_3D)
135 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
136 device->meta_state.blit2d[log2_samples].p_layouts[src_type],
137 VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4, &src_img->layer);
138
139 radv_meta_push_descriptor_set(
140 cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
141 device->meta_state.blit2d[log2_samples].p_layouts[src_type], 0, /* set */
142 1, /* descriptorWriteCount */
143 (VkWriteDescriptorSet[]){{.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
144 .dstBinding = 0,
145 .dstArrayElement = 0,
146 .descriptorCount = 1,
147 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
148 .pImageInfo = (VkDescriptorImageInfo[]){
149 {
150 .sampler = VK_NULL_HANDLE,
151 .imageView = radv_image_view_to_handle(&tmp->iview),
152 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
153 },
154 }}});
155 }
156 }
157
158 struct blit2d_dst_temps {
159 VkImage image;
160 struct radv_image_view iview;
161 VkFramebuffer fb;
162 };
163
164 static void
blit2d_bind_dst(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * dst,uint32_t width,uint32_t height,VkFormat depth_format,struct blit2d_dst_temps * tmp,VkImageAspectFlagBits aspects)165 blit2d_bind_dst(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *dst,
166 uint32_t width, uint32_t height, VkFormat depth_format,
167 struct blit2d_dst_temps *tmp, VkImageAspectFlagBits aspects)
168 {
169 create_iview(cmd_buffer, dst, &tmp->iview, depth_format, aspects);
170
171 radv_CreateFramebuffer(
172 radv_device_to_handle(cmd_buffer->device),
173 &(VkFramebufferCreateInfo){.sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
174 .attachmentCount = 1,
175 .pAttachments =
176 (VkImageView[]){
177 radv_image_view_to_handle(&tmp->iview),
178 },
179 .width = width,
180 .height = height,
181 .layers = 1},
182 &cmd_buffer->pool->alloc, &tmp->fb);
183 }
184
185 static void
bind_pipeline(struct radv_cmd_buffer * cmd_buffer,enum blit2d_src_type src_type,unsigned fs_key,uint32_t log2_samples)186 bind_pipeline(struct radv_cmd_buffer *cmd_buffer, enum blit2d_src_type src_type, unsigned fs_key,
187 uint32_t log2_samples)
188 {
189 VkPipeline pipeline =
190 cmd_buffer->device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key];
191
192 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS,
193 pipeline);
194 }
195
196 static void
bind_depth_pipeline(struct radv_cmd_buffer * cmd_buffer,enum blit2d_src_type src_type,uint32_t log2_samples)197 bind_depth_pipeline(struct radv_cmd_buffer *cmd_buffer, enum blit2d_src_type src_type,
198 uint32_t log2_samples)
199 {
200 VkPipeline pipeline =
201 cmd_buffer->device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type];
202
203 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS,
204 pipeline);
205 }
206
207 static void
bind_stencil_pipeline(struct radv_cmd_buffer * cmd_buffer,enum blit2d_src_type src_type,uint32_t log2_samples)208 bind_stencil_pipeline(struct radv_cmd_buffer *cmd_buffer, enum blit2d_src_type src_type,
209 uint32_t log2_samples)
210 {
211 VkPipeline pipeline =
212 cmd_buffer->device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type];
213
214 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS,
215 pipeline);
216 }
217
218 static void
radv_meta_blit2d_normal_dst(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * src_img,struct radv_meta_blit2d_buffer * src_buf,struct radv_meta_blit2d_surf * dst,unsigned num_rects,struct radv_meta_blit2d_rect * rects,enum blit2d_src_type src_type,uint32_t log2_samples)219 radv_meta_blit2d_normal_dst(struct radv_cmd_buffer *cmd_buffer,
220 struct radv_meta_blit2d_surf *src_img,
221 struct radv_meta_blit2d_buffer *src_buf,
222 struct radv_meta_blit2d_surf *dst, unsigned num_rects,
223 struct radv_meta_blit2d_rect *rects, enum blit2d_src_type src_type,
224 uint32_t log2_samples)
225 {
226 struct radv_device *device = cmd_buffer->device;
227
228 for (unsigned r = 0; r < num_rects; ++r) {
229 u_foreach_bit(i, dst->aspect_mask)
230 {
231 unsigned aspect_mask = 1u << i;
232 unsigned src_aspect_mask = aspect_mask;
233 VkFormat depth_format = 0;
234 if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT)
235 depth_format = vk_format_stencil_only(dst->image->vk_format);
236 else if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT)
237 depth_format = vk_format_depth_only(dst->image->vk_format);
238 else if (src_img)
239 src_aspect_mask = src_img->aspect_mask;
240
241 struct blit2d_src_temps src_temps;
242 blit2d_bind_src(cmd_buffer, src_img, src_buf, &src_temps, src_type, depth_format,
243 src_aspect_mask, log2_samples);
244
245 struct blit2d_dst_temps dst_temps;
246 blit2d_bind_dst(cmd_buffer, dst, rects[r].dst_x + rects[r].width,
247 rects[r].dst_y + rects[r].height, depth_format, &dst_temps, aspect_mask);
248
249 float vertex_push_constants[4] = {
250 rects[r].src_x,
251 rects[r].src_y,
252 rects[r].src_x + rects[r].width,
253 rects[r].src_y + rects[r].height,
254 };
255
256 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
257 device->meta_state.blit2d[log2_samples].p_layouts[src_type],
258 VK_SHADER_STAGE_VERTEX_BIT, 0, 16, vertex_push_constants);
259
260 if (aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT ||
261 aspect_mask == VK_IMAGE_ASPECT_PLANE_0_BIT ||
262 aspect_mask == VK_IMAGE_ASPECT_PLANE_1_BIT ||
263 aspect_mask == VK_IMAGE_ASPECT_PLANE_2_BIT) {
264 unsigned fs_key = radv_format_meta_fs_key(device, dst_temps.iview.vk_format);
265 unsigned dst_layout = radv_meta_dst_layout_from_layout(dst->current_layout);
266
267 if (device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key] ==
268 VK_NULL_HANDLE) {
269 VkResult ret = blit2d_init_color_pipeline(
270 device, src_type, radv_fs_key_format_exemplars[fs_key], log2_samples);
271 if (ret != VK_SUCCESS) {
272 cmd_buffer->record_result = ret;
273 goto fail_pipeline;
274 }
275 }
276
277 radv_cmd_buffer_begin_render_pass(
278 cmd_buffer,
279 &(VkRenderPassBeginInfo){
280 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
281 .renderPass = device->meta_state.blit2d_render_passes[fs_key][dst_layout],
282 .framebuffer = dst_temps.fb,
283 .renderArea =
284 {
285 .offset =
286 {
287 rects[r].dst_x,
288 rects[r].dst_y,
289 },
290 .extent = {rects[r].width, rects[r].height},
291 },
292 .clearValueCount = 0,
293 .pClearValues = NULL,
294 },
295 &(struct radv_extra_render_pass_begin_info){.disable_dcc =
296 dst->disable_compression});
297
298 radv_cmd_buffer_set_subpass(cmd_buffer, &cmd_buffer->state.pass->subpasses[0]);
299
300 bind_pipeline(cmd_buffer, src_type, fs_key, log2_samples);
301 } else if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT) {
302 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst->current_layout);
303
304 if (device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type] ==
305 VK_NULL_HANDLE) {
306 VkResult ret = blit2d_init_depth_only_pipeline(device, src_type, log2_samples);
307 if (ret != VK_SUCCESS) {
308 cmd_buffer->record_result = ret;
309 goto fail_pipeline;
310 }
311 }
312
313 radv_cmd_buffer_begin_render_pass(
314 cmd_buffer,
315 &(VkRenderPassBeginInfo){
316 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
317 .renderPass = device->meta_state.blit2d_depth_only_rp[ds_layout],
318 .framebuffer = dst_temps.fb,
319 .renderArea =
320 {
321 .offset =
322 {
323 rects[r].dst_x,
324 rects[r].dst_y,
325 },
326 .extent = {rects[r].width, rects[r].height},
327 },
328 .clearValueCount = 0,
329 .pClearValues = NULL,
330 },
331 NULL);
332
333 radv_cmd_buffer_set_subpass(cmd_buffer, &cmd_buffer->state.pass->subpasses[0]);
334
335 bind_depth_pipeline(cmd_buffer, src_type, log2_samples);
336
337 } else if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT) {
338 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst->current_layout);
339
340 if (device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type] ==
341 VK_NULL_HANDLE) {
342 VkResult ret = blit2d_init_stencil_only_pipeline(device, src_type, log2_samples);
343 if (ret != VK_SUCCESS) {
344 cmd_buffer->record_result = ret;
345 goto fail_pipeline;
346 }
347 }
348
349 radv_cmd_buffer_begin_render_pass(
350 cmd_buffer,
351 &(VkRenderPassBeginInfo){
352 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
353 .renderPass = device->meta_state.blit2d_stencil_only_rp[ds_layout],
354 .framebuffer = dst_temps.fb,
355 .renderArea =
356 {
357 .offset =
358 {
359 rects[r].dst_x,
360 rects[r].dst_y,
361 },
362 .extent = {rects[r].width, rects[r].height},
363 },
364 .clearValueCount = 0,
365 .pClearValues = NULL,
366 },
367 NULL);
368
369 radv_cmd_buffer_set_subpass(cmd_buffer, &cmd_buffer->state.pass->subpasses[0]);
370
371 bind_stencil_pipeline(cmd_buffer, src_type, log2_samples);
372 } else
373 unreachable("Processing blit2d with multiple aspects.");
374
375 radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
376 &(VkViewport){.x = rects[r].dst_x,
377 .y = rects[r].dst_y,
378 .width = rects[r].width,
379 .height = rects[r].height,
380 .minDepth = 0.0f,
381 .maxDepth = 1.0f});
382
383 radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
384 &(VkRect2D){
385 .offset = (VkOffset2D){rects[r].dst_x, rects[r].dst_y},
386 .extent = (VkExtent2D){rects[r].width, rects[r].height},
387 });
388
389 radv_CmdDraw(radv_cmd_buffer_to_handle(cmd_buffer), 3, 1, 0, 0);
390 radv_cmd_buffer_end_render_pass(cmd_buffer);
391
392 fail_pipeline:
393 /* At the point where we emit the draw call, all data from the
394 * descriptor sets, etc. has been used. We are free to delete it.
395 */
396 radv_DestroyFramebuffer(radv_device_to_handle(device), dst_temps.fb,
397 &cmd_buffer->pool->alloc);
398
399 if (src_type == BLIT2D_SRC_TYPE_BUFFER)
400 radv_buffer_view_finish(&src_temps.bview);
401 else
402 radv_image_view_finish(&dst_temps.iview);
403
404 radv_image_view_finish(&dst_temps.iview);
405 }
406 }
407 }
408
409 void
radv_meta_blit2d(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * src_img,struct radv_meta_blit2d_buffer * src_buf,struct radv_meta_blit2d_surf * dst,unsigned num_rects,struct radv_meta_blit2d_rect * rects)410 radv_meta_blit2d(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src_img,
411 struct radv_meta_blit2d_buffer *src_buf, struct radv_meta_blit2d_surf *dst,
412 unsigned num_rects, struct radv_meta_blit2d_rect *rects)
413 {
414 bool use_3d = cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9 &&
415 (src_img && src_img->image->type == VK_IMAGE_TYPE_3D);
416 enum blit2d_src_type src_type = src_buf ? BLIT2D_SRC_TYPE_BUFFER
417 : use_3d ? BLIT2D_SRC_TYPE_IMAGE_3D
418 : BLIT2D_SRC_TYPE_IMAGE;
419 radv_meta_blit2d_normal_dst(cmd_buffer, src_img, src_buf, dst, num_rects, rects, src_type,
420 src_img ? util_logbase2(src_img->image->info.samples) : 0);
421 }
422
423 static nir_shader *
build_nir_vertex_shader(void)424 build_nir_vertex_shader(void)
425 {
426 const struct glsl_type *vec4 = glsl_vec4_type();
427 const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
428 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_VERTEX, NULL, "meta_blit2d_vs");
429
430 nir_variable *pos_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "gl_Position");
431 pos_out->data.location = VARYING_SLOT_POS;
432
433 nir_variable *tex_pos_out = nir_variable_create(b.shader, nir_var_shader_out, vec2, "v_tex_pos");
434 tex_pos_out->data.location = VARYING_SLOT_VAR0;
435 tex_pos_out->data.interpolation = INTERP_MODE_SMOOTH;
436
437 nir_ssa_def *outvec = radv_meta_gen_rect_vertices(&b);
438 nir_store_var(&b, pos_out, outvec, 0xf);
439
440 nir_ssa_def *src_box = nir_load_push_constant(&b, 4, 32, nir_imm_int(&b, 0), .range = 16);
441 nir_ssa_def *vertex_id = nir_load_vertex_id_zero_base(&b);
442
443 /* vertex 0 - src_x, src_y */
444 /* vertex 1 - src_x, src_y+h */
445 /* vertex 2 - src_x+w, src_y */
446 /* so channel 0 is vertex_id != 2 ? src_x : src_x + w
447 channel 1 is vertex id != 1 ? src_y : src_y + w */
448
449 nir_ssa_def *c0cmp = nir_ine(&b, vertex_id, nir_imm_int(&b, 2));
450 nir_ssa_def *c1cmp = nir_ine(&b, vertex_id, nir_imm_int(&b, 1));
451
452 nir_ssa_def *comp[2];
453 comp[0] = nir_bcsel(&b, c0cmp, nir_channel(&b, src_box, 0), nir_channel(&b, src_box, 2));
454
455 comp[1] = nir_bcsel(&b, c1cmp, nir_channel(&b, src_box, 1), nir_channel(&b, src_box, 3));
456 nir_ssa_def *out_tex_vec = nir_vec(&b, comp, 2);
457 nir_store_var(&b, tex_pos_out, out_tex_vec, 0x3);
458 return b.shader;
459 }
460
461 typedef nir_ssa_def *(*texel_fetch_build_func)(struct nir_builder *, struct radv_device *,
462 nir_ssa_def *, bool, bool);
463
464 static nir_ssa_def *
build_nir_texel_fetch(struct nir_builder * b,struct radv_device * device,nir_ssa_def * tex_pos,bool is_3d,bool is_multisampled)465 build_nir_texel_fetch(struct nir_builder *b, struct radv_device *device, nir_ssa_def *tex_pos,
466 bool is_3d, bool is_multisampled)
467 {
468 enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D
469 : is_multisampled ? GLSL_SAMPLER_DIM_MS
470 : GLSL_SAMPLER_DIM_2D;
471 const struct glsl_type *sampler_type = glsl_sampler_type(dim, false, false, GLSL_TYPE_UINT);
472 nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform, sampler_type, "s_tex");
473 sampler->data.descriptor_set = 0;
474 sampler->data.binding = 0;
475
476 nir_ssa_def *tex_pos_3d = NULL;
477 nir_ssa_def *sample_idx = NULL;
478 if (is_3d) {
479 nir_ssa_def *layer =
480 nir_load_push_constant(b, 1, 32, nir_imm_int(b, 0), .base = 16, .range = 4);
481
482 nir_ssa_def *chans[3];
483 chans[0] = nir_channel(b, tex_pos, 0);
484 chans[1] = nir_channel(b, tex_pos, 1);
485 chans[2] = layer;
486 tex_pos_3d = nir_vec(b, chans, 3);
487 }
488 if (is_multisampled) {
489 sample_idx = nir_load_sample_id(b);
490 }
491
492 nir_ssa_def *tex_deref = &nir_build_deref_var(b, sampler)->dest.ssa;
493
494 nir_tex_instr *tex = nir_tex_instr_create(b->shader, is_multisampled ? 4 : 3);
495 tex->sampler_dim = dim;
496 tex->op = is_multisampled ? nir_texop_txf_ms : nir_texop_txf;
497 tex->src[0].src_type = nir_tex_src_coord;
498 tex->src[0].src = nir_src_for_ssa(is_3d ? tex_pos_3d : tex_pos);
499 tex->src[1].src_type = is_multisampled ? nir_tex_src_ms_index : nir_tex_src_lod;
500 tex->src[1].src = nir_src_for_ssa(is_multisampled ? sample_idx : nir_imm_int(b, 0));
501 tex->src[2].src_type = nir_tex_src_texture_deref;
502 tex->src[2].src = nir_src_for_ssa(tex_deref);
503 if (is_multisampled) {
504 tex->src[3].src_type = nir_tex_src_lod;
505 tex->src[3].src = nir_src_for_ssa(nir_imm_int(b, 0));
506 }
507 tex->dest_type = nir_type_uint32;
508 tex->is_array = false;
509 tex->coord_components = is_3d ? 3 : 2;
510
511 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
512 nir_builder_instr_insert(b, &tex->instr);
513
514 return &tex->dest.ssa;
515 }
516
517 static nir_ssa_def *
build_nir_buffer_fetch(struct nir_builder * b,struct radv_device * device,nir_ssa_def * tex_pos,bool is_3d,bool is_multisampled)518 build_nir_buffer_fetch(struct nir_builder *b, struct radv_device *device, nir_ssa_def *tex_pos,
519 bool is_3d, bool is_multisampled)
520 {
521 const struct glsl_type *sampler_type =
522 glsl_sampler_type(GLSL_SAMPLER_DIM_BUF, false, false, GLSL_TYPE_UINT);
523 nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform, sampler_type, "s_tex");
524 sampler->data.descriptor_set = 0;
525 sampler->data.binding = 0;
526
527 nir_ssa_def *width = nir_load_push_constant(b, 1, 32, nir_imm_int(b, 0), .base = 16, .range = 4);
528
529 nir_ssa_def *pos_x = nir_channel(b, tex_pos, 0);
530 nir_ssa_def *pos_y = nir_channel(b, tex_pos, 1);
531 pos_y = nir_imul(b, pos_y, width);
532 pos_x = nir_iadd(b, pos_x, pos_y);
533
534 nir_ssa_def *tex_deref = &nir_build_deref_var(b, sampler)->dest.ssa;
535
536 nir_tex_instr *tex = nir_tex_instr_create(b->shader, 2);
537 tex->sampler_dim = GLSL_SAMPLER_DIM_BUF;
538 tex->op = nir_texop_txf;
539 tex->src[0].src_type = nir_tex_src_coord;
540 tex->src[0].src = nir_src_for_ssa(pos_x);
541 tex->src[1].src_type = nir_tex_src_texture_deref;
542 tex->src[1].src = nir_src_for_ssa(tex_deref);
543 tex->dest_type = nir_type_uint32;
544 tex->is_array = false;
545 tex->coord_components = 1;
546
547 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
548 nir_builder_instr_insert(b, &tex->instr);
549
550 return &tex->dest.ssa;
551 }
552
553 static const VkPipelineVertexInputStateCreateInfo normal_vi_create_info = {
554 .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
555 .vertexBindingDescriptionCount = 0,
556 .vertexAttributeDescriptionCount = 0,
557 };
558
559 static nir_shader *
build_nir_copy_fragment_shader(struct radv_device * device,texel_fetch_build_func txf_func,const char * name,bool is_3d,bool is_multisampled)560 build_nir_copy_fragment_shader(struct radv_device *device, texel_fetch_build_func txf_func,
561 const char *name, bool is_3d, bool is_multisampled)
562 {
563 const struct glsl_type *vec4 = glsl_vec4_type();
564 const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
565 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, NULL, "%s", name);
566
567 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec2, "v_tex_pos");
568 tex_pos_in->data.location = VARYING_SLOT_VAR0;
569
570 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
571 color_out->data.location = FRAG_RESULT_DATA0;
572
573 nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
574 nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
575
576 nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
577 nir_store_var(&b, color_out, color, 0xf);
578
579 return b.shader;
580 }
581
582 static nir_shader *
build_nir_copy_fragment_shader_depth(struct radv_device * device,texel_fetch_build_func txf_func,const char * name,bool is_3d,bool is_multisampled)583 build_nir_copy_fragment_shader_depth(struct radv_device *device, texel_fetch_build_func txf_func,
584 const char *name, bool is_3d, bool is_multisampled)
585 {
586 const struct glsl_type *vec4 = glsl_vec4_type();
587 const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
588 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, NULL, "%s", name);
589
590 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec2, "v_tex_pos");
591 tex_pos_in->data.location = VARYING_SLOT_VAR0;
592
593 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
594 color_out->data.location = FRAG_RESULT_DEPTH;
595
596 nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
597 nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
598
599 nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
600 nir_store_var(&b, color_out, color, 0x1);
601
602 return b.shader;
603 }
604
605 static nir_shader *
build_nir_copy_fragment_shader_stencil(struct radv_device * device,texel_fetch_build_func txf_func,const char * name,bool is_3d,bool is_multisampled)606 build_nir_copy_fragment_shader_stencil(struct radv_device *device, texel_fetch_build_func txf_func,
607 const char *name, bool is_3d, bool is_multisampled)
608 {
609 const struct glsl_type *vec4 = glsl_vec4_type();
610 const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
611 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, NULL, "%s", name);
612
613 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec2, "v_tex_pos");
614 tex_pos_in->data.location = VARYING_SLOT_VAR0;
615
616 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
617 color_out->data.location = FRAG_RESULT_STENCIL;
618
619 nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
620 nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3);
621
622 nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
623 nir_store_var(&b, color_out, color, 0x1);
624
625 return b.shader;
626 }
627
628 void
radv_device_finish_meta_blit2d_state(struct radv_device * device)629 radv_device_finish_meta_blit2d_state(struct radv_device *device)
630 {
631 struct radv_meta_state *state = &device->meta_state;
632
633 for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
634 for (unsigned k = 0; k < RADV_META_DST_LAYOUT_COUNT; ++k) {
635 radv_DestroyRenderPass(radv_device_to_handle(device), state->blit2d_render_passes[j][k],
636 &state->alloc);
637 }
638 }
639
640 for (enum radv_blit_ds_layout j = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; j < RADV_BLIT_DS_LAYOUT_COUNT;
641 j++) {
642 radv_DestroyRenderPass(radv_device_to_handle(device), state->blit2d_depth_only_rp[j],
643 &state->alloc);
644 radv_DestroyRenderPass(radv_device_to_handle(device), state->blit2d_stencil_only_rp[j],
645 &state->alloc);
646 }
647
648 for (unsigned log2_samples = 0; log2_samples < MAX_SAMPLES_LOG2; ++log2_samples) {
649 for (unsigned src = 0; src < BLIT2D_NUM_SRC_TYPES; src++) {
650 radv_DestroyPipelineLayout(radv_device_to_handle(device),
651 state->blit2d[log2_samples].p_layouts[src], &state->alloc);
652 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
653 state->blit2d[log2_samples].ds_layouts[src],
654 &state->alloc);
655
656 for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
657 radv_DestroyPipeline(radv_device_to_handle(device),
658 state->blit2d[log2_samples].pipelines[src][j], &state->alloc);
659 }
660
661 radv_DestroyPipeline(radv_device_to_handle(device),
662 state->blit2d[log2_samples].depth_only_pipeline[src], &state->alloc);
663 radv_DestroyPipeline(radv_device_to_handle(device),
664 state->blit2d[log2_samples].stencil_only_pipeline[src],
665 &state->alloc);
666 }
667 }
668 }
669
670 static VkResult
blit2d_init_color_pipeline(struct radv_device * device,enum blit2d_src_type src_type,VkFormat format,uint32_t log2_samples)671 blit2d_init_color_pipeline(struct radv_device *device, enum blit2d_src_type src_type,
672 VkFormat format, uint32_t log2_samples)
673 {
674 VkResult result;
675 unsigned fs_key = radv_format_meta_fs_key(device, format);
676 const char *name;
677
678 mtx_lock(&device->meta_state.mtx);
679 if (device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key]) {
680 mtx_unlock(&device->meta_state.mtx);
681 return VK_SUCCESS;
682 }
683
684 texel_fetch_build_func src_func;
685 switch (src_type) {
686 case BLIT2D_SRC_TYPE_IMAGE:
687 src_func = build_nir_texel_fetch;
688 name = "meta_blit2d_image_fs";
689 break;
690 case BLIT2D_SRC_TYPE_IMAGE_3D:
691 src_func = build_nir_texel_fetch;
692 name = "meta_blit3d_image_fs";
693 break;
694 case BLIT2D_SRC_TYPE_BUFFER:
695 src_func = build_nir_buffer_fetch;
696 name = "meta_blit2d_buffer_fs";
697 break;
698 default:
699 unreachable("unknown blit src type\n");
700 break;
701 }
702
703 const VkPipelineVertexInputStateCreateInfo *vi_create_info;
704 nir_shader *fs = build_nir_copy_fragment_shader(
705 device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
706 nir_shader *vs = build_nir_vertex_shader();
707
708 vi_create_info = &normal_vi_create_info;
709
710 VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
711 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
712 .stage = VK_SHADER_STAGE_VERTEX_BIT,
713 .module = vk_shader_module_handle_from_nir(vs),
714 .pName = "main",
715 .pSpecializationInfo = NULL},
716 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
717 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
718 .module = vk_shader_module_handle_from_nir(fs),
719 .pName = "main",
720 .pSpecializationInfo = NULL},
721 };
722
723 for (unsigned dst_layout = 0; dst_layout < RADV_META_DST_LAYOUT_COUNT; ++dst_layout) {
724 if (!device->meta_state.blit2d_render_passes[fs_key][dst_layout]) {
725 VkImageLayout layout = radv_meta_dst_layout_to_layout(dst_layout);
726
727 result = radv_CreateRenderPass2(
728 radv_device_to_handle(device),
729 &(VkRenderPassCreateInfo2){
730 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO_2,
731 .attachmentCount = 1,
732 .pAttachments =
733 &(VkAttachmentDescription2){
734 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_DESCRIPTION_2,
735 .format = format,
736 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
737 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
738 .initialLayout = layout,
739 .finalLayout = layout,
740 },
741 .subpassCount = 1,
742 .pSubpasses =
743 &(VkSubpassDescription2){
744 .sType = VK_STRUCTURE_TYPE_SUBPASS_DESCRIPTION_2,
745 .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
746 .inputAttachmentCount = 0,
747 .colorAttachmentCount = 1,
748 .pColorAttachments =
749 &(VkAttachmentReference2){
750 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2,
751 .attachment = 0,
752 .layout = layout,
753 },
754 .pResolveAttachments = NULL,
755 .pDepthStencilAttachment =
756 &(VkAttachmentReference2){
757 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2,
758 .attachment = VK_ATTACHMENT_UNUSED,
759 .layout = layout,
760 },
761 .preserveAttachmentCount = 0,
762 .pPreserveAttachments = NULL,
763 },
764 .dependencyCount = 2,
765 .pDependencies =
766 (VkSubpassDependency2[]){{.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2,
767 .srcSubpass = VK_SUBPASS_EXTERNAL,
768 .dstSubpass = 0,
769 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
770 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
771 .srcAccessMask = 0,
772 .dstAccessMask = 0,
773 .dependencyFlags = 0},
774 {.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2,
775 .srcSubpass = 0,
776 .dstSubpass = VK_SUBPASS_EXTERNAL,
777 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
778 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
779 .srcAccessMask = 0,
780 .dstAccessMask = 0,
781 .dependencyFlags = 0}},
782 },
783 &device->meta_state.alloc,
784 &device->meta_state.blit2d_render_passes[fs_key][dst_layout]);
785 }
786 }
787
788 const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
789 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
790 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
791 .pStages = pipeline_shader_stages,
792 .pVertexInputState = vi_create_info,
793 .pInputAssemblyState =
794 &(VkPipelineInputAssemblyStateCreateInfo){
795 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
796 .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
797 .primitiveRestartEnable = false,
798 },
799 .pViewportState =
800 &(VkPipelineViewportStateCreateInfo){
801 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
802 .viewportCount = 1,
803 .scissorCount = 1,
804 },
805 .pRasterizationState =
806 &(VkPipelineRasterizationStateCreateInfo){
807 .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
808 .rasterizerDiscardEnable = false,
809 .polygonMode = VK_POLYGON_MODE_FILL,
810 .cullMode = VK_CULL_MODE_NONE,
811 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE},
812 .pMultisampleState =
813 &(VkPipelineMultisampleStateCreateInfo){
814 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
815 .rasterizationSamples = 1 << log2_samples,
816 .sampleShadingEnable = log2_samples > 1,
817 .minSampleShading = 1.0,
818 .pSampleMask = (VkSampleMask[]){UINT32_MAX},
819 },
820 .pColorBlendState =
821 &(VkPipelineColorBlendStateCreateInfo){
822 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
823 .attachmentCount = 1,
824 .pAttachments =
825 (VkPipelineColorBlendAttachmentState[]){
826 {.colorWriteMask = VK_COLOR_COMPONENT_A_BIT | VK_COLOR_COMPONENT_R_BIT |
827 VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT},
828 }},
829 .pDynamicState =
830 &(VkPipelineDynamicStateCreateInfo){
831 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
832 .dynamicStateCount = 9,
833 .pDynamicStates =
834 (VkDynamicState[]){
835 VK_DYNAMIC_STATE_VIEWPORT,
836 VK_DYNAMIC_STATE_SCISSOR,
837 VK_DYNAMIC_STATE_LINE_WIDTH,
838 VK_DYNAMIC_STATE_DEPTH_BIAS,
839 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
840 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
841 VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
842 VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
843 VK_DYNAMIC_STATE_STENCIL_REFERENCE,
844 },
845 },
846 .flags = 0,
847 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
848 .renderPass = device->meta_state.blit2d_render_passes[fs_key][0],
849 .subpass = 0,
850 };
851
852 const struct radv_graphics_pipeline_create_info radv_pipeline_info = {.use_rectlist = true};
853
854 result = radv_graphics_pipeline_create(
855 radv_device_to_handle(device), radv_pipeline_cache_to_handle(&device->meta_state.cache),
856 &vk_pipeline_info, &radv_pipeline_info, &device->meta_state.alloc,
857 &device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key]);
858
859 ralloc_free(vs);
860 ralloc_free(fs);
861
862 mtx_unlock(&device->meta_state.mtx);
863 return result;
864 }
865
866 static VkResult
blit2d_init_depth_only_pipeline(struct radv_device * device,enum blit2d_src_type src_type,uint32_t log2_samples)867 blit2d_init_depth_only_pipeline(struct radv_device *device, enum blit2d_src_type src_type,
868 uint32_t log2_samples)
869 {
870 VkResult result;
871 const char *name;
872
873 mtx_lock(&device->meta_state.mtx);
874 if (device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type]) {
875 mtx_unlock(&device->meta_state.mtx);
876 return VK_SUCCESS;
877 }
878
879 texel_fetch_build_func src_func;
880 switch (src_type) {
881 case BLIT2D_SRC_TYPE_IMAGE:
882 src_func = build_nir_texel_fetch;
883 name = "meta_blit2d_depth_image_fs";
884 break;
885 case BLIT2D_SRC_TYPE_IMAGE_3D:
886 src_func = build_nir_texel_fetch;
887 name = "meta_blit3d_depth_image_fs";
888 break;
889 case BLIT2D_SRC_TYPE_BUFFER:
890 src_func = build_nir_buffer_fetch;
891 name = "meta_blit2d_depth_buffer_fs";
892 break;
893 default:
894 unreachable("unknown blit src type\n");
895 break;
896 }
897
898 const VkPipelineVertexInputStateCreateInfo *vi_create_info;
899 nir_shader *fs = build_nir_copy_fragment_shader_depth(
900 device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
901 nir_shader *vs = build_nir_vertex_shader();
902
903 vi_create_info = &normal_vi_create_info;
904
905 VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
906 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
907 .stage = VK_SHADER_STAGE_VERTEX_BIT,
908 .module = vk_shader_module_handle_from_nir(vs),
909 .pName = "main",
910 .pSpecializationInfo = NULL},
911 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
912 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
913 .module = vk_shader_module_handle_from_nir(fs),
914 .pName = "main",
915 .pSpecializationInfo = NULL},
916 };
917
918 for (enum radv_blit_ds_layout ds_layout = RADV_BLIT_DS_LAYOUT_TILE_ENABLE;
919 ds_layout < RADV_BLIT_DS_LAYOUT_COUNT; ds_layout++) {
920 if (!device->meta_state.blit2d_depth_only_rp[ds_layout]) {
921 VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
922 result = radv_CreateRenderPass2(
923 radv_device_to_handle(device),
924 &(VkRenderPassCreateInfo2){
925 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO_2,
926 .attachmentCount = 1,
927 .pAttachments =
928 &(VkAttachmentDescription2){
929 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_DESCRIPTION_2,
930 .format = VK_FORMAT_D32_SFLOAT,
931 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
932 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
933 .initialLayout = layout,
934 .finalLayout = layout,
935 },
936 .subpassCount = 1,
937 .pSubpasses =
938 &(VkSubpassDescription2){
939 .sType = VK_STRUCTURE_TYPE_SUBPASS_DESCRIPTION_2,
940 .inputAttachmentCount = 0,
941 .colorAttachmentCount = 0,
942 .pColorAttachments = NULL,
943 .pResolveAttachments = NULL,
944 .pDepthStencilAttachment =
945 &(VkAttachmentReference2){
946 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2,
947 .attachment = 0,
948 .layout = layout,
949 },
950 .preserveAttachmentCount = 0,
951 .pPreserveAttachments = NULL,
952 },
953 .dependencyCount = 2,
954 .pDependencies =
955 (VkSubpassDependency2[]){{.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2,
956 .srcSubpass = VK_SUBPASS_EXTERNAL,
957 .dstSubpass = 0,
958 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
959 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
960 .srcAccessMask = 0,
961 .dstAccessMask = 0,
962 .dependencyFlags = 0},
963 {.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2,
964 .srcSubpass = 0,
965 .dstSubpass = VK_SUBPASS_EXTERNAL,
966 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
967 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
968 .srcAccessMask = 0,
969 .dstAccessMask = 0,
970 .dependencyFlags = 0}},
971 },
972 &device->meta_state.alloc, &device->meta_state.blit2d_depth_only_rp[ds_layout]);
973 }
974 }
975
976 const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
977 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
978 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
979 .pStages = pipeline_shader_stages,
980 .pVertexInputState = vi_create_info,
981 .pInputAssemblyState =
982 &(VkPipelineInputAssemblyStateCreateInfo){
983 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
984 .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
985 .primitiveRestartEnable = false,
986 },
987 .pViewportState =
988 &(VkPipelineViewportStateCreateInfo){
989 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
990 .viewportCount = 1,
991 .scissorCount = 1,
992 },
993 .pRasterizationState =
994 &(VkPipelineRasterizationStateCreateInfo){
995 .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
996 .rasterizerDiscardEnable = false,
997 .polygonMode = VK_POLYGON_MODE_FILL,
998 .cullMode = VK_CULL_MODE_NONE,
999 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE},
1000 .pMultisampleState =
1001 &(VkPipelineMultisampleStateCreateInfo){
1002 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
1003 .rasterizationSamples = 1 << log2_samples,
1004 .sampleShadingEnable = false,
1005 .pSampleMask = (VkSampleMask[]){UINT32_MAX},
1006 },
1007 .pColorBlendState =
1008 &(VkPipelineColorBlendStateCreateInfo){
1009 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
1010 .attachmentCount = 0,
1011 .pAttachments = NULL,
1012 },
1013 .pDepthStencilState =
1014 &(VkPipelineDepthStencilStateCreateInfo){
1015 .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
1016 .depthTestEnable = true,
1017 .depthWriteEnable = true,
1018 .depthCompareOp = VK_COMPARE_OP_ALWAYS,
1019 },
1020 .pDynamicState =
1021 &(VkPipelineDynamicStateCreateInfo){
1022 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
1023 .dynamicStateCount = 9,
1024 .pDynamicStates =
1025 (VkDynamicState[]){
1026 VK_DYNAMIC_STATE_VIEWPORT,
1027 VK_DYNAMIC_STATE_SCISSOR,
1028 VK_DYNAMIC_STATE_LINE_WIDTH,
1029 VK_DYNAMIC_STATE_DEPTH_BIAS,
1030 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
1031 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
1032 VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
1033 VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
1034 VK_DYNAMIC_STATE_STENCIL_REFERENCE,
1035 },
1036 },
1037 .flags = 0,
1038 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
1039 .renderPass = device->meta_state.blit2d_depth_only_rp[0],
1040 .subpass = 0,
1041 };
1042
1043 const struct radv_graphics_pipeline_create_info radv_pipeline_info = {.use_rectlist = true};
1044
1045 result = radv_graphics_pipeline_create(
1046 radv_device_to_handle(device), radv_pipeline_cache_to_handle(&device->meta_state.cache),
1047 &vk_pipeline_info, &radv_pipeline_info, &device->meta_state.alloc,
1048 &device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type]);
1049
1050 ralloc_free(vs);
1051 ralloc_free(fs);
1052
1053 mtx_unlock(&device->meta_state.mtx);
1054 return result;
1055 }
1056
1057 static VkResult
blit2d_init_stencil_only_pipeline(struct radv_device * device,enum blit2d_src_type src_type,uint32_t log2_samples)1058 blit2d_init_stencil_only_pipeline(struct radv_device *device, enum blit2d_src_type src_type,
1059 uint32_t log2_samples)
1060 {
1061 VkResult result;
1062 const char *name;
1063
1064 mtx_lock(&device->meta_state.mtx);
1065 if (device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type]) {
1066 mtx_unlock(&device->meta_state.mtx);
1067 return VK_SUCCESS;
1068 }
1069
1070 texel_fetch_build_func src_func;
1071 switch (src_type) {
1072 case BLIT2D_SRC_TYPE_IMAGE:
1073 src_func = build_nir_texel_fetch;
1074 name = "meta_blit2d_stencil_image_fs";
1075 break;
1076 case BLIT2D_SRC_TYPE_IMAGE_3D:
1077 src_func = build_nir_texel_fetch;
1078 name = "meta_blit3d_stencil_image_fs";
1079 break;
1080 case BLIT2D_SRC_TYPE_BUFFER:
1081 src_func = build_nir_buffer_fetch;
1082 name = "meta_blit2d_stencil_buffer_fs";
1083 break;
1084 default:
1085 unreachable("unknown blit src type\n");
1086 break;
1087 }
1088
1089 const VkPipelineVertexInputStateCreateInfo *vi_create_info;
1090 nir_shader *fs = build_nir_copy_fragment_shader_stencil(
1091 device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
1092 nir_shader *vs = build_nir_vertex_shader();
1093
1094 vi_create_info = &normal_vi_create_info;
1095
1096 VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
1097 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
1098 .stage = VK_SHADER_STAGE_VERTEX_BIT,
1099 .module = vk_shader_module_handle_from_nir(vs),
1100 .pName = "main",
1101 .pSpecializationInfo = NULL},
1102 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
1103 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
1104 .module = vk_shader_module_handle_from_nir(fs),
1105 .pName = "main",
1106 .pSpecializationInfo = NULL},
1107 };
1108
1109 for (enum radv_blit_ds_layout ds_layout = RADV_BLIT_DS_LAYOUT_TILE_ENABLE;
1110 ds_layout < RADV_BLIT_DS_LAYOUT_COUNT; ds_layout++) {
1111 if (!device->meta_state.blit2d_stencil_only_rp[ds_layout]) {
1112 VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
1113 result = radv_CreateRenderPass2(
1114 radv_device_to_handle(device),
1115 &(VkRenderPassCreateInfo2){
1116 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO_2,
1117 .attachmentCount = 1,
1118 .pAttachments =
1119 &(VkAttachmentDescription2){
1120 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_DESCRIPTION_2,
1121 .format = VK_FORMAT_S8_UINT,
1122 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
1123 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
1124 .initialLayout = layout,
1125 .finalLayout = layout,
1126 },
1127 .subpassCount = 1,
1128 .pSubpasses =
1129 &(VkSubpassDescription2){
1130 .sType = VK_STRUCTURE_TYPE_SUBPASS_DESCRIPTION_2,
1131 .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
1132 .inputAttachmentCount = 0,
1133 .colorAttachmentCount = 0,
1134 .pColorAttachments = NULL,
1135 .pResolveAttachments = NULL,
1136 .pDepthStencilAttachment =
1137 &(VkAttachmentReference2){
1138 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2,
1139 .attachment = 0,
1140 .layout = layout,
1141 },
1142 .preserveAttachmentCount = 0,
1143 .pPreserveAttachments = NULL,
1144 },
1145 .dependencyCount = 2,
1146 .pDependencies =
1147 (VkSubpassDependency2[]){{.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2,
1148 .srcSubpass = VK_SUBPASS_EXTERNAL,
1149 .dstSubpass = 0,
1150 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
1151 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
1152 .srcAccessMask = 0,
1153 .dstAccessMask = 0,
1154 .dependencyFlags = 0},
1155 {.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2,
1156 .srcSubpass = 0,
1157 .dstSubpass = VK_SUBPASS_EXTERNAL,
1158 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
1159 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
1160 .srcAccessMask = 0,
1161 .dstAccessMask = 0,
1162 .dependencyFlags = 0}},
1163 },
1164 &device->meta_state.alloc, &device->meta_state.blit2d_stencil_only_rp[ds_layout]);
1165 }
1166 }
1167
1168 const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
1169 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
1170 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
1171 .pStages = pipeline_shader_stages,
1172 .pVertexInputState = vi_create_info,
1173 .pInputAssemblyState =
1174 &(VkPipelineInputAssemblyStateCreateInfo){
1175 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
1176 .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
1177 .primitiveRestartEnable = false,
1178 },
1179 .pViewportState =
1180 &(VkPipelineViewportStateCreateInfo){
1181 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
1182 .viewportCount = 1,
1183 .scissorCount = 1,
1184 },
1185 .pRasterizationState =
1186 &(VkPipelineRasterizationStateCreateInfo){
1187 .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
1188 .rasterizerDiscardEnable = false,
1189 .polygonMode = VK_POLYGON_MODE_FILL,
1190 .cullMode = VK_CULL_MODE_NONE,
1191 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE},
1192 .pMultisampleState =
1193 &(VkPipelineMultisampleStateCreateInfo){
1194 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
1195 .rasterizationSamples = 1 << log2_samples,
1196 .sampleShadingEnable = false,
1197 .pSampleMask = (VkSampleMask[]){UINT32_MAX},
1198 },
1199 .pColorBlendState =
1200 &(VkPipelineColorBlendStateCreateInfo){
1201 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
1202 .attachmentCount = 0,
1203 .pAttachments = NULL,
1204 },
1205 .pDepthStencilState =
1206 &(VkPipelineDepthStencilStateCreateInfo){
1207 .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
1208 .depthTestEnable = false,
1209 .depthWriteEnable = false,
1210 .stencilTestEnable = true,
1211 .front = {.failOp = VK_STENCIL_OP_REPLACE,
1212 .passOp = VK_STENCIL_OP_REPLACE,
1213 .depthFailOp = VK_STENCIL_OP_REPLACE,
1214 .compareOp = VK_COMPARE_OP_ALWAYS,
1215 .compareMask = 0xff,
1216 .writeMask = 0xff,
1217 .reference = 0},
1218 .back = {.failOp = VK_STENCIL_OP_REPLACE,
1219 .passOp = VK_STENCIL_OP_REPLACE,
1220 .depthFailOp = VK_STENCIL_OP_REPLACE,
1221 .compareOp = VK_COMPARE_OP_ALWAYS,
1222 .compareMask = 0xff,
1223 .writeMask = 0xff,
1224 .reference = 0},
1225 .depthCompareOp = VK_COMPARE_OP_ALWAYS,
1226 },
1227 .pDynamicState =
1228 &(VkPipelineDynamicStateCreateInfo){
1229 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
1230 .dynamicStateCount = 6,
1231 .pDynamicStates =
1232 (VkDynamicState[]){
1233 VK_DYNAMIC_STATE_VIEWPORT,
1234 VK_DYNAMIC_STATE_SCISSOR,
1235 VK_DYNAMIC_STATE_LINE_WIDTH,
1236 VK_DYNAMIC_STATE_DEPTH_BIAS,
1237 VK_DYNAMIC_STATE_BLEND_CONSTANTS,
1238 VK_DYNAMIC_STATE_DEPTH_BOUNDS,
1239 },
1240 },
1241 .flags = 0,
1242 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type],
1243 .renderPass = device->meta_state.blit2d_stencil_only_rp[0],
1244 .subpass = 0,
1245 };
1246
1247 const struct radv_graphics_pipeline_create_info radv_pipeline_info = {.use_rectlist = true};
1248
1249 result = radv_graphics_pipeline_create(
1250 radv_device_to_handle(device), radv_pipeline_cache_to_handle(&device->meta_state.cache),
1251 &vk_pipeline_info, &radv_pipeline_info, &device->meta_state.alloc,
1252 &device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type]);
1253
1254 ralloc_free(vs);
1255 ralloc_free(fs);
1256
1257 mtx_unlock(&device->meta_state.mtx);
1258 return result;
1259 }
1260
1261 static VkResult
meta_blit2d_create_pipe_layout(struct radv_device * device,int idx,uint32_t log2_samples)1262 meta_blit2d_create_pipe_layout(struct radv_device *device, int idx, uint32_t log2_samples)
1263 {
1264 VkResult result;
1265 VkDescriptorType desc_type = (idx == BLIT2D_SRC_TYPE_BUFFER)
1266 ? VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER
1267 : VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE;
1268 const VkPushConstantRange push_constant_ranges[] = {
1269 {VK_SHADER_STAGE_VERTEX_BIT, 0, 16},
1270 {VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4},
1271 };
1272 int num_push_constant_range = (idx != BLIT2D_SRC_TYPE_IMAGE || log2_samples > 0) ? 2 : 1;
1273
1274 result = radv_CreateDescriptorSetLayout(
1275 radv_device_to_handle(device),
1276 &(VkDescriptorSetLayoutCreateInfo){
1277 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
1278 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
1279 .bindingCount = 1,
1280 .pBindings =
1281 (VkDescriptorSetLayoutBinding[]){
1282 {.binding = 0,
1283 .descriptorType = desc_type,
1284 .descriptorCount = 1,
1285 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
1286 .pImmutableSamplers = NULL},
1287 }},
1288 &device->meta_state.alloc, &device->meta_state.blit2d[log2_samples].ds_layouts[idx]);
1289 if (result != VK_SUCCESS)
1290 goto fail;
1291
1292 result = radv_CreatePipelineLayout(
1293 radv_device_to_handle(device),
1294 &(VkPipelineLayoutCreateInfo){
1295 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
1296 .setLayoutCount = 1,
1297 .pSetLayouts = &device->meta_state.blit2d[log2_samples].ds_layouts[idx],
1298 .pushConstantRangeCount = num_push_constant_range,
1299 .pPushConstantRanges = push_constant_ranges,
1300 },
1301 &device->meta_state.alloc, &device->meta_state.blit2d[log2_samples].p_layouts[idx]);
1302 if (result != VK_SUCCESS)
1303 goto fail;
1304 return VK_SUCCESS;
1305 fail:
1306 return result;
1307 }
1308
1309 VkResult
radv_device_init_meta_blit2d_state(struct radv_device * device,bool on_demand)1310 radv_device_init_meta_blit2d_state(struct radv_device *device, bool on_demand)
1311 {
1312 VkResult result;
1313 bool create_3d = device->physical_device->rad_info.chip_class >= GFX9;
1314
1315 for (unsigned log2_samples = 0; log2_samples < MAX_SAMPLES_LOG2; log2_samples++) {
1316 for (unsigned src = 0; src < BLIT2D_NUM_SRC_TYPES; src++) {
1317 if (src == BLIT2D_SRC_TYPE_IMAGE_3D && !create_3d)
1318 continue;
1319
1320 /* Don't need to handle copies between buffers and multisample images. */
1321 if (src == BLIT2D_SRC_TYPE_BUFFER && log2_samples > 0)
1322 continue;
1323
1324 /* There are no multisampled 3D images. */
1325 if (src == BLIT2D_SRC_TYPE_IMAGE_3D && log2_samples > 0)
1326 continue;
1327
1328 result = meta_blit2d_create_pipe_layout(device, src, log2_samples);
1329 if (result != VK_SUCCESS)
1330 goto fail;
1331
1332 if (on_demand)
1333 continue;
1334
1335 for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) {
1336 result = blit2d_init_color_pipeline(device, src, radv_fs_key_format_exemplars[j],
1337 log2_samples);
1338 if (result != VK_SUCCESS)
1339 goto fail;
1340 }
1341
1342 result = blit2d_init_depth_only_pipeline(device, src, log2_samples);
1343 if (result != VK_SUCCESS)
1344 goto fail;
1345
1346 result = blit2d_init_stencil_only_pipeline(device, src, log2_samples);
1347 if (result != VK_SUCCESS)
1348 goto fail;
1349 }
1350 }
1351
1352 return VK_SUCCESS;
1353
1354 fail:
1355 radv_device_finish_meta_blit2d_state(device);
1356 return result;
1357 }
1358