Lines Matching refs:cmd_buffer
59 static void radv_handle_image_transition(struct radv_cmd_buffer *cmd_buffer,
66 static void radv_set_rt_stack_size(struct radv_cmd_buffer *cmd_buffer, uint32_t size);
127 radv_bind_dynamic_state(struct radv_cmd_buffer *cmd_buffer, const struct radv_dynamic_state *src) in radv_bind_dynamic_state() argument
129 struct radv_dynamic_state *dest = &cmd_buffer->state.dynamic; in radv_bind_dynamic_state()
351 cmd_buffer->state.dirty |= dest_mask; in radv_bind_dynamic_state()
355 radv_bind_streamout_state(struct radv_cmd_buffer *cmd_buffer, struct radv_pipeline *pipeline) in radv_bind_streamout_state() argument
357 struct radv_streamout_state *so = &cmd_buffer->state.streamout; in radv_bind_streamout_state()
360 if (!pipeline->streamout_shader || cmd_buffer->device->physical_device->use_ngg_streamout) in radv_bind_streamout_state()
371 radv_cmd_buffer_uses_mec(struct radv_cmd_buffer *cmd_buffer) in radv_cmd_buffer_uses_mec() argument
373 return cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE && in radv_cmd_buffer_uses_mec()
374 cmd_buffer->device->physical_device->rad_info.chip_class >= GFX7; in radv_cmd_buffer_uses_mec()
393 radv_emit_write_data_packet(struct radv_cmd_buffer *cmd_buffer, unsigned engine_sel, uint64_t va, in radv_emit_write_data_packet() argument
396 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_emit_write_data_packet()
398 radeon_check_space(cmd_buffer->device->ws, cs, 4 + count); in radv_emit_write_data_packet()
408 radv_emit_clear_data(struct radv_cmd_buffer *cmd_buffer, unsigned engine_sel, uint64_t va, in radv_emit_clear_data() argument
413 radv_emit_write_data_packet(cmd_buffer, engine_sel, va, size / 4, zeroes); in radv_emit_clear_data()
417 radv_destroy_cmd_buffer(struct radv_cmd_buffer *cmd_buffer) in radv_destroy_cmd_buffer() argument
419 list_del(&cmd_buffer->pool_link); in radv_destroy_cmd_buffer()
421 list_for_each_entry_safe(struct radv_cmd_buffer_upload, up, &cmd_buffer->upload.list, list) in radv_destroy_cmd_buffer()
423 cmd_buffer->device->ws->buffer_destroy(cmd_buffer->device->ws, up->upload_bo); in radv_destroy_cmd_buffer()
428 if (cmd_buffer->upload.upload_bo) in radv_destroy_cmd_buffer()
429 cmd_buffer->device->ws->buffer_destroy(cmd_buffer->device->ws, cmd_buffer->upload.upload_bo); in radv_destroy_cmd_buffer()
431 if (cmd_buffer->cs) in radv_destroy_cmd_buffer()
432 cmd_buffer->device->ws->cs_destroy(cmd_buffer->cs); in radv_destroy_cmd_buffer()
435 free(cmd_buffer->descriptors[i].push_set.set.mapped_ptr); in radv_destroy_cmd_buffer()
436 vk_object_base_finish(&cmd_buffer->descriptors[i].push_set.set.base); in radv_destroy_cmd_buffer()
439 vk_object_base_finish(&cmd_buffer->meta_push_descriptors.base); in radv_destroy_cmd_buffer()
441 vk_command_buffer_finish(&cmd_buffer->vk); in radv_destroy_cmd_buffer()
442 vk_free(&cmd_buffer->pool->alloc, cmd_buffer); in radv_destroy_cmd_buffer()
449 struct radv_cmd_buffer *cmd_buffer; in radv_create_cmd_buffer() local
451 cmd_buffer = vk_zalloc(&pool->alloc, sizeof(*cmd_buffer), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); in radv_create_cmd_buffer()
452 if (cmd_buffer == NULL) in radv_create_cmd_buffer()
456 vk_command_buffer_init(&cmd_buffer->vk, &device->vk); in radv_create_cmd_buffer()
458 vk_free(&cmd_buffer->pool->alloc, cmd_buffer); in radv_create_cmd_buffer()
462 cmd_buffer->device = device; in radv_create_cmd_buffer()
463 cmd_buffer->pool = pool; in radv_create_cmd_buffer()
464 cmd_buffer->level = level; in radv_create_cmd_buffer()
466 list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers); in radv_create_cmd_buffer()
467 cmd_buffer->queue_family_index = pool->queue_family_index; in radv_create_cmd_buffer()
469 ring = radv_queue_family_to_ring(cmd_buffer->queue_family_index); in radv_create_cmd_buffer()
471 cmd_buffer->cs = device->ws->cs_create(device->ws, ring); in radv_create_cmd_buffer()
472 if (!cmd_buffer->cs) { in radv_create_cmd_buffer()
473 radv_destroy_cmd_buffer(cmd_buffer); in radv_create_cmd_buffer()
477 vk_object_base_init(&device->vk, &cmd_buffer->meta_push_descriptors.base, in radv_create_cmd_buffer()
481 vk_object_base_init(&device->vk, &cmd_buffer->descriptors[i].push_set.set.base, in radv_create_cmd_buffer()
484 *pCommandBuffer = radv_cmd_buffer_to_handle(cmd_buffer); in radv_create_cmd_buffer()
486 list_inithead(&cmd_buffer->upload.list); in radv_create_cmd_buffer()
492 radv_reset_cmd_buffer(struct radv_cmd_buffer *cmd_buffer) in radv_reset_cmd_buffer() argument
494 vk_command_buffer_reset(&cmd_buffer->vk); in radv_reset_cmd_buffer()
496 cmd_buffer->device->ws->cs_reset(cmd_buffer->cs); in radv_reset_cmd_buffer()
498 list_for_each_entry_safe(struct radv_cmd_buffer_upload, up, &cmd_buffer->upload.list, list) in radv_reset_cmd_buffer()
500 cmd_buffer->device->ws->buffer_destroy(cmd_buffer->device->ws, up->upload_bo); in radv_reset_cmd_buffer()
505 cmd_buffer->push_constant_stages = 0; in radv_reset_cmd_buffer()
506 cmd_buffer->scratch_size_per_wave_needed = 0; in radv_reset_cmd_buffer()
507 cmd_buffer->scratch_waves_wanted = 0; in radv_reset_cmd_buffer()
508 cmd_buffer->compute_scratch_size_per_wave_needed = 0; in radv_reset_cmd_buffer()
509 cmd_buffer->compute_scratch_waves_wanted = 0; in radv_reset_cmd_buffer()
510 cmd_buffer->esgs_ring_size_needed = 0; in radv_reset_cmd_buffer()
511 cmd_buffer->gsvs_ring_size_needed = 0; in radv_reset_cmd_buffer()
512 cmd_buffer->tess_rings_needed = false; in radv_reset_cmd_buffer()
513 cmd_buffer->gds_needed = false; in radv_reset_cmd_buffer()
514 cmd_buffer->gds_oa_needed = false; in radv_reset_cmd_buffer()
515 cmd_buffer->sample_positions_needed = false; in radv_reset_cmd_buffer()
517 if (cmd_buffer->upload.upload_bo) in radv_reset_cmd_buffer()
518 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, cmd_buffer->upload.upload_bo); in radv_reset_cmd_buffer()
519 cmd_buffer->upload.offset = 0; in radv_reset_cmd_buffer()
521 cmd_buffer->record_result = VK_SUCCESS; in radv_reset_cmd_buffer()
523 memset(cmd_buffer->vertex_bindings, 0, sizeof(cmd_buffer->vertex_bindings)); in radv_reset_cmd_buffer()
526 cmd_buffer->descriptors[i].dirty = 0; in radv_reset_cmd_buffer()
527 cmd_buffer->descriptors[i].valid = 0; in radv_reset_cmd_buffer()
528 cmd_buffer->descriptors[i].push_dirty = false; in radv_reset_cmd_buffer()
531 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9 && in radv_reset_cmd_buffer()
532 cmd_buffer->queue_family_index == RADV_QUEUE_GENERAL) { in radv_reset_cmd_buffer()
533 unsigned num_db = cmd_buffer->device->physical_device->rad_info.max_render_backends; in radv_reset_cmd_buffer()
537 radv_cmd_buffer_upload_alloc(cmd_buffer, 8, &fence_offset, &fence_ptr); in radv_reset_cmd_buffer()
540 cmd_buffer->gfx9_fence_va = radv_buffer_get_va(cmd_buffer->upload.upload_bo); in radv_reset_cmd_buffer()
541 cmd_buffer->gfx9_fence_va += fence_offset; in radv_reset_cmd_buffer()
543 radv_emit_clear_data(cmd_buffer, V_370_PFP, cmd_buffer->gfx9_fence_va, 8); in radv_reset_cmd_buffer()
545 if (cmd_buffer->device->physical_device->rad_info.chip_class == GFX9) { in radv_reset_cmd_buffer()
547 radv_cmd_buffer_upload_alloc(cmd_buffer, 16 * num_db, &eop_bug_offset, &fence_ptr); in radv_reset_cmd_buffer()
549 cmd_buffer->gfx9_eop_bug_va = radv_buffer_get_va(cmd_buffer->upload.upload_bo); in radv_reset_cmd_buffer()
550 cmd_buffer->gfx9_eop_bug_va += eop_bug_offset; in radv_reset_cmd_buffer()
552 radv_emit_clear_data(cmd_buffer, V_370_PFP, cmd_buffer->gfx9_eop_bug_va, 16 * num_db); in radv_reset_cmd_buffer()
556 cmd_buffer->status = RADV_CMD_BUFFER_STATUS_INITIAL; in radv_reset_cmd_buffer()
558 return cmd_buffer->record_result; in radv_reset_cmd_buffer()
562 radv_cmd_buffer_resize_upload_buf(struct radv_cmd_buffer *cmd_buffer, uint64_t min_needed) in radv_cmd_buffer_resize_upload_buf() argument
567 struct radv_device *device = cmd_buffer->device; in radv_cmd_buffer_resize_upload_buf()
570 new_size = MAX2(new_size, 2 * cmd_buffer->upload.size); in radv_cmd_buffer_resize_upload_buf()
579 cmd_buffer->record_result = result; in radv_cmd_buffer_resize_upload_buf()
583 radv_cs_add_buffer(device->ws, cmd_buffer->cs, bo); in radv_cmd_buffer_resize_upload_buf()
584 if (cmd_buffer->upload.upload_bo) { in radv_cmd_buffer_resize_upload_buf()
588 cmd_buffer->record_result = VK_ERROR_OUT_OF_HOST_MEMORY; in radv_cmd_buffer_resize_upload_buf()
593 memcpy(upload, &cmd_buffer->upload, sizeof(*upload)); in radv_cmd_buffer_resize_upload_buf()
594 list_add(&upload->list, &cmd_buffer->upload.list); in radv_cmd_buffer_resize_upload_buf()
597 cmd_buffer->upload.upload_bo = bo; in radv_cmd_buffer_resize_upload_buf()
598 cmd_buffer->upload.size = new_size; in radv_cmd_buffer_resize_upload_buf()
599 cmd_buffer->upload.offset = 0; in radv_cmd_buffer_resize_upload_buf()
600 cmd_buffer->upload.map = device->ws->buffer_map(cmd_buffer->upload.upload_bo); in radv_cmd_buffer_resize_upload_buf()
602 if (!cmd_buffer->upload.map) { in radv_cmd_buffer_resize_upload_buf()
603 cmd_buffer->record_result = VK_ERROR_OUT_OF_DEVICE_MEMORY; in radv_cmd_buffer_resize_upload_buf()
611 radv_cmd_buffer_upload_alloc(struct radv_cmd_buffer *cmd_buffer, unsigned size, in radv_cmd_buffer_upload_alloc() argument
616 struct radeon_info *rad_info = &cmd_buffer->device->physical_device->rad_info; in radv_cmd_buffer_upload_alloc()
621 unsigned offset = cmd_buffer->upload.offset; in radv_cmd_buffer_upload_alloc()
627 if (offset + size > cmd_buffer->upload.size) { in radv_cmd_buffer_upload_alloc()
628 if (!radv_cmd_buffer_resize_upload_buf(cmd_buffer, size)) in radv_cmd_buffer_upload_alloc()
634 *ptr = cmd_buffer->upload.map + offset; in radv_cmd_buffer_upload_alloc()
636 cmd_buffer->upload.offset = offset + size; in radv_cmd_buffer_upload_alloc()
641 radv_cmd_buffer_upload_data(struct radv_cmd_buffer *cmd_buffer, unsigned size, const void *data, in radv_cmd_buffer_upload_data() argument
646 if (!radv_cmd_buffer_upload_alloc(cmd_buffer, size, out_offset, (void **)&ptr)) in radv_cmd_buffer_upload_data()
656 radv_cmd_buffer_trace_emit(struct radv_cmd_buffer *cmd_buffer) in radv_cmd_buffer_trace_emit() argument
658 struct radv_device *device = cmd_buffer->device; in radv_cmd_buffer_trace_emit()
659 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_cmd_buffer_trace_emit()
663 if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY) in radv_cmd_buffer_trace_emit()
666 ++cmd_buffer->state.trace_id; in radv_cmd_buffer_trace_emit()
667 radv_emit_write_data_packet(cmd_buffer, V_370_ME, va, 1, &cmd_buffer->state.trace_id); in radv_cmd_buffer_trace_emit()
669 radeon_check_space(cmd_buffer->device->ws, cs, 2); in radv_cmd_buffer_trace_emit()
672 radeon_emit(cs, AC_ENCODE_TRACE_POINT(cmd_buffer->state.trace_id)); in radv_cmd_buffer_trace_emit()
676 radv_cmd_buffer_after_draw(struct radv_cmd_buffer *cmd_buffer, enum radv_cmd_flush_bits flags) in radv_cmd_buffer_after_draw() argument
678 if (unlikely(cmd_buffer->device->thread_trace.bo)) { in radv_cmd_buffer_after_draw()
679 radeon_emit(cmd_buffer->cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); in radv_cmd_buffer_after_draw()
680 radeon_emit(cmd_buffer->cs, EVENT_TYPE(V_028A90_THREAD_TRACE_MARKER) | EVENT_INDEX(0)); in radv_cmd_buffer_after_draw()
683 if (cmd_buffer->device->instance->debug_flags & RADV_DEBUG_SYNC_SHADERS) { in radv_cmd_buffer_after_draw()
687 radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 4); in radv_cmd_buffer_after_draw()
690 si_cs_emit_cache_flush(cmd_buffer->cs, in radv_cmd_buffer_after_draw()
691 cmd_buffer->device->physical_device->rad_info.chip_class, in radv_cmd_buffer_after_draw()
692 &cmd_buffer->gfx9_fence_idx, cmd_buffer->gfx9_fence_va, in radv_cmd_buffer_after_draw()
693 radv_cmd_buffer_uses_mec(cmd_buffer), flags, &sqtt_flush_bits, in radv_cmd_buffer_after_draw()
694 cmd_buffer->gfx9_eop_bug_va); in radv_cmd_buffer_after_draw()
697 if (unlikely(cmd_buffer->device->trace_bo)) in radv_cmd_buffer_after_draw()
698 radv_cmd_buffer_trace_emit(cmd_buffer); in radv_cmd_buffer_after_draw()
702 radv_save_pipeline(struct radv_cmd_buffer *cmd_buffer, struct radv_pipeline *pipeline) in radv_save_pipeline() argument
704 struct radv_device *device = cmd_buffer->device; in radv_save_pipeline()
711 ring = radv_queue_family_to_ring(cmd_buffer->queue_family_index); in radv_save_pipeline()
728 radv_emit_write_data_packet(cmd_buffer, V_370_ME, va, 2, data); in radv_save_pipeline()
732 radv_save_vertex_descriptors(struct radv_cmd_buffer *cmd_buffer, uint64_t vb_ptr) in radv_save_vertex_descriptors() argument
734 struct radv_device *device = cmd_buffer->device; in radv_save_vertex_descriptors()
744 radv_emit_write_data_packet(cmd_buffer, V_370_ME, va, 2, data); in radv_save_vertex_descriptors()
748 radv_set_descriptor_set(struct radv_cmd_buffer *cmd_buffer, VkPipelineBindPoint bind_point, in radv_set_descriptor_set() argument
752 radv_get_descriptors_state(cmd_buffer, bind_point); in radv_set_descriptor_set()
761 radv_save_descriptors(struct radv_cmd_buffer *cmd_buffer, VkPipelineBindPoint bind_point) in radv_save_descriptors() argument
764 radv_get_descriptors_state(cmd_buffer, bind_point); in radv_save_descriptors()
765 struct radv_device *device = cmd_buffer->device; in radv_save_descriptors()
777 radv_emit_write_data_packet(cmd_buffer, V_370_ME, va, MAX_SETS * 2, data); in radv_save_descriptors()
788 radv_emit_userdata_address(struct radv_cmd_buffer *cmd_buffer, struct radv_pipeline *pipeline, in radv_emit_userdata_address() argument
798 radv_emit_shader_pointer(cmd_buffer->device, cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, va, in radv_emit_userdata_address()
803 radv_emit_descriptor_pointers(struct radv_cmd_buffer *cmd_buffer, struct radv_pipeline *pipeline, in radv_emit_descriptor_pointers() argument
807 struct radv_device *device = cmd_buffer->device; in radv_emit_descriptor_pointers()
808 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_emit_descriptor_pointers()
890 radv_compute_centroid_priority(struct radv_cmd_buffer *cmd_buffer, VkOffset2D *sample_locs, in radv_compute_centroid_priority() argument
928 radv_emit_sample_locations(struct radv_cmd_buffer *cmd_buffer) in radv_emit_sample_locations() argument
930 struct radv_sample_locations_state *sample_location = &cmd_buffer->state.dynamic.sample_location; in radv_emit_sample_locations()
932 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_emit_sample_locations()
938 if (!cmd_buffer->state.dynamic.sample_location.count) in radv_emit_sample_locations()
953 centroid_priority = radv_compute_centroid_priority(cmd_buffer, sample_locs[0], num_samples); in radv_emit_sample_locations()
1006 cmd_buffer->state.context_roll_without_scissor_emitted = true; in radv_emit_sample_locations()
1010 radv_emit_inline_push_consts(struct radv_cmd_buffer *cmd_buffer, struct radv_pipeline *pipeline, in radv_emit_inline_push_consts() argument
1018 radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 2 + loc->num_sgprs); in radv_emit_inline_push_consts()
1020 radeon_set_sh_reg_seq(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, loc->num_sgprs); in radv_emit_inline_push_consts()
1021 radeon_emit_array(cmd_buffer->cs, values, loc->num_sgprs); in radv_emit_inline_push_consts()
1025 radv_update_multisample_state(struct radv_cmd_buffer *cmd_buffer, struct radv_pipeline *pipeline) in radv_update_multisample_state() argument
1028 struct radv_pipeline *old_pipeline = cmd_buffer->state.emitted_pipeline; in radv_update_multisample_state()
1031 cmd_buffer->sample_positions_needed = true; in radv_update_multisample_state()
1036 radv_emit_default_sample_locations(cmd_buffer->cs, num_samples); in radv_update_multisample_state()
1038 cmd_buffer->state.context_roll_without_scissor_emitted = true; in radv_update_multisample_state()
1042 radv_update_binning_state(struct radv_cmd_buffer *cmd_buffer, struct radv_pipeline *pipeline) in radv_update_binning_state() argument
1044 const struct radv_pipeline *old_pipeline = cmd_buffer->state.emitted_pipeline; in radv_update_binning_state()
1055 if (cmd_buffer->device->physical_device->rad_info.family == CHIP_VEGA12 || in radv_update_binning_state()
1056 cmd_buffer->device->physical_device->rad_info.family == CHIP_VEGA20 || in radv_update_binning_state()
1057 cmd_buffer->device->physical_device->rad_info.family == CHIP_RAVEN2 || in radv_update_binning_state()
1058 cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10) { in radv_update_binning_state()
1064 radeon_set_context_reg(cmd_buffer->cs, R_028C44_PA_SC_BINNER_CNTL_0, in radv_update_binning_state()
1068 cmd_buffer->state.context_roll_without_scissor_emitted = true; in radv_update_binning_state()
1072 radv_emit_shader_prefetch(struct radv_cmd_buffer *cmd_buffer, struct radv_shader_variant *shader) in radv_emit_shader_prefetch() argument
1081 si_cp_dma_prefetch(cmd_buffer, va, shader->code_size); in radv_emit_shader_prefetch()
1085 radv_emit_prefetch_L2(struct radv_cmd_buffer *cmd_buffer, struct radv_pipeline *pipeline, in radv_emit_prefetch_L2() argument
1088 struct radv_cmd_state *state = &cmd_buffer->state; in radv_emit_prefetch_L2()
1098 radv_emit_shader_prefetch(cmd_buffer, pipeline->shaders[MESA_SHADER_VERTEX]); in radv_emit_prefetch_L2()
1101 si_cp_dma_prefetch(cmd_buffer, state->vb_va, pipeline->vb_desc_alloc_size); in radv_emit_prefetch_L2()
1104 radv_emit_shader_prefetch(cmd_buffer, pipeline->shaders[MESA_SHADER_TESS_CTRL]); in radv_emit_prefetch_L2()
1107 radv_emit_shader_prefetch(cmd_buffer, pipeline->shaders[MESA_SHADER_TESS_EVAL]); in radv_emit_prefetch_L2()
1110 radv_emit_shader_prefetch(cmd_buffer, pipeline->shaders[MESA_SHADER_GEOMETRY]); in radv_emit_prefetch_L2()
1112 radv_emit_shader_prefetch(cmd_buffer, pipeline->gs_copy_shader); in radv_emit_prefetch_L2()
1116 radv_emit_shader_prefetch(cmd_buffer, pipeline->shaders[MESA_SHADER_FRAGMENT]); in radv_emit_prefetch_L2()
1122 radv_emit_rbplus_state(struct radv_cmd_buffer *cmd_buffer) in radv_emit_rbplus_state() argument
1124 if (!cmd_buffer->device->physical_device->rad_info.rbplus_allowed) in radv_emit_rbplus_state()
1127 struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; in radv_emit_rbplus_state()
1128 const struct radv_subpass *subpass = cmd_buffer->state.subpass; in radv_emit_rbplus_state()
1134 if (!cmd_buffer->state.attachments || !subpass) in radv_emit_rbplus_state()
1146 struct radv_color_buffer_info *cb = &cmd_buffer->state.attachments[idx].cb; in radv_emit_rbplus_state()
1263 if (sx_ps_downconvert == cmd_buffer->state.last_sx_ps_downconvert && in radv_emit_rbplus_state()
1264 sx_blend_opt_epsilon == cmd_buffer->state.last_sx_blend_opt_epsilon && in radv_emit_rbplus_state()
1265 sx_blend_opt_control == cmd_buffer->state.last_sx_blend_opt_control) in radv_emit_rbplus_state()
1268 radeon_set_context_reg_seq(cmd_buffer->cs, R_028754_SX_PS_DOWNCONVERT, 3); in radv_emit_rbplus_state()
1269 radeon_emit(cmd_buffer->cs, sx_ps_downconvert); in radv_emit_rbplus_state()
1270 radeon_emit(cmd_buffer->cs, sx_blend_opt_epsilon); in radv_emit_rbplus_state()
1271 radeon_emit(cmd_buffer->cs, sx_blend_opt_control); in radv_emit_rbplus_state()
1273 cmd_buffer->state.context_roll_without_scissor_emitted = true; in radv_emit_rbplus_state()
1275 cmd_buffer->state.last_sx_ps_downconvert = sx_ps_downconvert; in radv_emit_rbplus_state()
1276 cmd_buffer->state.last_sx_blend_opt_epsilon = sx_blend_opt_epsilon; in radv_emit_rbplus_state()
1277 cmd_buffer->state.last_sx_blend_opt_control = sx_blend_opt_control; in radv_emit_rbplus_state()
1281 radv_emit_batch_break_on_new_ps(struct radv_cmd_buffer *cmd_buffer) in radv_emit_batch_break_on_new_ps() argument
1283 if (!cmd_buffer->device->pbb_allowed) in radv_emit_batch_break_on_new_ps()
1287 radv_get_binning_settings(cmd_buffer->device->physical_device); in radv_emit_batch_break_on_new_ps()
1289 (!cmd_buffer->state.emitted_pipeline || in radv_emit_batch_break_on_new_ps()
1290 cmd_buffer->state.emitted_pipeline->shaders[MESA_SHADER_FRAGMENT] != in radv_emit_batch_break_on_new_ps()
1291 cmd_buffer->state.pipeline->shaders[MESA_SHADER_FRAGMENT]) && in radv_emit_batch_break_on_new_ps()
1294 (cmd_buffer->state.dirty & RADV_CMD_DIRTY_DYNAMIC_COLOR_WRITE_ENABLE) && in radv_emit_batch_break_on_new_ps()
1300 radeon_emit(cmd_buffer->cs, PKT3(PKT3_EVENT_WRITE, 0, 0)); in radv_emit_batch_break_on_new_ps()
1301 radeon_emit(cmd_buffer->cs, EVENT_TYPE(V_028A90_BREAK_BATCH) | EVENT_INDEX(0)); in radv_emit_batch_break_on_new_ps()
1305 radv_emit_graphics_pipeline(struct radv_cmd_buffer *cmd_buffer) in radv_emit_graphics_pipeline() argument
1307 struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; in radv_emit_graphics_pipeline()
1309 if (!pipeline || cmd_buffer->state.emitted_pipeline == pipeline) in radv_emit_graphics_pipeline()
1312 radv_update_multisample_state(cmd_buffer, pipeline); in radv_emit_graphics_pipeline()
1313 radv_update_binning_state(cmd_buffer, pipeline); in radv_emit_graphics_pipeline()
1315 cmd_buffer->scratch_size_per_wave_needed = in radv_emit_graphics_pipeline()
1316 MAX2(cmd_buffer->scratch_size_per_wave_needed, pipeline->scratch_bytes_per_wave); in radv_emit_graphics_pipeline()
1317 cmd_buffer->scratch_waves_wanted = MAX2(cmd_buffer->scratch_waves_wanted, pipeline->max_waves); in radv_emit_graphics_pipeline()
1319 if (!cmd_buffer->state.emitted_pipeline || in radv_emit_graphics_pipeline()
1320 cmd_buffer->state.emitted_pipeline->graphics.can_use_guardband != in radv_emit_graphics_pipeline()
1322 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_SCISSOR; in radv_emit_graphics_pipeline()
1324 if (!cmd_buffer->state.emitted_pipeline || in radv_emit_graphics_pipeline()
1325 cmd_buffer->state.emitted_pipeline->graphics.pa_su_sc_mode_cntl != in radv_emit_graphics_pipeline()
1327 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_CULL_MODE | in radv_emit_graphics_pipeline()
1331 if (!cmd_buffer->state.emitted_pipeline || in radv_emit_graphics_pipeline()
1332 cmd_buffer->state.emitted_pipeline->graphics.pa_cl_clip_cntl != in radv_emit_graphics_pipeline()
1334 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_RASTERIZER_DISCARD_ENABLE; in radv_emit_graphics_pipeline()
1336 if (!cmd_buffer->state.emitted_pipeline || in radv_emit_graphics_pipeline()
1337 cmd_buffer->state.emitted_pipeline->graphics.cb_color_control != in radv_emit_graphics_pipeline()
1339 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_LOGIC_OP; in radv_emit_graphics_pipeline()
1341 if (!cmd_buffer->state.emitted_pipeline) in radv_emit_graphics_pipeline()
1342 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_PRIMITIVE_TOPOLOGY | in radv_emit_graphics_pipeline()
1347 if (!cmd_buffer->state.emitted_pipeline || in radv_emit_graphics_pipeline()
1348 cmd_buffer->state.emitted_pipeline->graphics.db_depth_control != in radv_emit_graphics_pipeline()
1350 cmd_buffer->state.dirty |= in radv_emit_graphics_pipeline()
1355 if (!cmd_buffer->state.emitted_pipeline) in radv_emit_graphics_pipeline()
1356 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_STENCIL_OP; in radv_emit_graphics_pipeline()
1358 if (!cmd_buffer->state.emitted_pipeline || in radv_emit_graphics_pipeline()
1359 cmd_buffer->state.emitted_pipeline->graphics.cb_target_mask != in radv_emit_graphics_pipeline()
1361 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_COLOR_WRITE_ENABLE; in radv_emit_graphics_pipeline()
1364 radeon_emit_array(cmd_buffer->cs, pipeline->cs.buf, pipeline->cs.cdw); in radv_emit_graphics_pipeline()
1368 !cmd_buffer->state.last_nggc_settings) { in radv_emit_graphics_pipeline()
1374 radeon_set_sh_reg(cmd_buffer->cs, R_00B22C_SPI_SHADER_PGM_RSRC2_GS, in radv_emit_graphics_pipeline()
1379 if (!cmd_buffer->state.emitted_pipeline || in radv_emit_graphics_pipeline()
1380 cmd_buffer->state.emitted_pipeline->ctx_cs.cdw != pipeline->ctx_cs.cdw || in radv_emit_graphics_pipeline()
1381 cmd_buffer->state.emitted_pipeline->ctx_cs_hash != pipeline->ctx_cs_hash || in radv_emit_graphics_pipeline()
1382 memcmp(cmd_buffer->state.emitted_pipeline->ctx_cs.buf, pipeline->ctx_cs.buf, in radv_emit_graphics_pipeline()
1384 radeon_emit_array(cmd_buffer->cs, pipeline->ctx_cs.buf, pipeline->ctx_cs.cdw); in radv_emit_graphics_pipeline()
1385 cmd_buffer->state.context_roll_without_scissor_emitted = true; in radv_emit_graphics_pipeline()
1388 radv_emit_batch_break_on_new_ps(cmd_buffer); in radv_emit_graphics_pipeline()
1394 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, pipeline->shaders[i]->bo); in radv_emit_graphics_pipeline()
1398 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, pipeline->gs_copy_shader->bo); in radv_emit_graphics_pipeline()
1400 if (unlikely(cmd_buffer->device->trace_bo)) in radv_emit_graphics_pipeline()
1401 radv_save_pipeline(cmd_buffer, pipeline); in radv_emit_graphics_pipeline()
1403 cmd_buffer->state.emitted_pipeline = pipeline; in radv_emit_graphics_pipeline()
1405 cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_PIPELINE; in radv_emit_graphics_pipeline()
1409 radv_emit_viewport(struct radv_cmd_buffer *cmd_buffer) in radv_emit_viewport() argument
1411 const struct radv_viewport_state *viewport = &cmd_buffer->state.dynamic.viewport; in radv_emit_viewport()
1416 radeon_set_context_reg_seq(cmd_buffer->cs, R_02843C_PA_CL_VPORT_XSCALE, count * 6); in radv_emit_viewport()
1419 radeon_emit(cmd_buffer->cs, fui(viewport->xform[i].scale[0])); in radv_emit_viewport()
1420 radeon_emit(cmd_buffer->cs, fui(viewport->xform[i].translate[0])); in radv_emit_viewport()
1421 radeon_emit(cmd_buffer->cs, fui(viewport->xform[i].scale[1])); in radv_emit_viewport()
1422 radeon_emit(cmd_buffer->cs, fui(viewport->xform[i].translate[1])); in radv_emit_viewport()
1423 radeon_emit(cmd_buffer->cs, fui(viewport->xform[i].scale[2])); in radv_emit_viewport()
1424 radeon_emit(cmd_buffer->cs, fui(viewport->xform[i].translate[2])); in radv_emit_viewport()
1427 radeon_set_context_reg_seq(cmd_buffer->cs, R_0282D0_PA_SC_VPORT_ZMIN_0, count * 2); in radv_emit_viewport()
1431 radeon_emit(cmd_buffer->cs, fui(zmin)); in radv_emit_viewport()
1432 radeon_emit(cmd_buffer->cs, fui(zmax)); in radv_emit_viewport()
1437 radv_emit_scissor(struct radv_cmd_buffer *cmd_buffer) in radv_emit_scissor() argument
1439 uint32_t count = cmd_buffer->state.dynamic.scissor.count; in radv_emit_scissor()
1441 si_write_scissors(cmd_buffer->cs, 0, count, cmd_buffer->state.dynamic.scissor.scissors, in radv_emit_scissor()
1442 cmd_buffer->state.dynamic.viewport.viewports, in radv_emit_scissor()
1443 cmd_buffer->state.emitted_pipeline->graphics.can_use_guardband); in radv_emit_scissor()
1445 cmd_buffer->state.context_roll_without_scissor_emitted = false; in radv_emit_scissor()
1449 radv_emit_discard_rectangle(struct radv_cmd_buffer *cmd_buffer) in radv_emit_discard_rectangle() argument
1451 if (!cmd_buffer->state.dynamic.discard_rectangle.count) in radv_emit_discard_rectangle()
1454 radeon_set_context_reg_seq(cmd_buffer->cs, R_028210_PA_SC_CLIPRECT_0_TL, in radv_emit_discard_rectangle()
1455 cmd_buffer->state.dynamic.discard_rectangle.count * 2); in radv_emit_discard_rectangle()
1456 for (unsigned i = 0; i < cmd_buffer->state.dynamic.discard_rectangle.count; ++i) { in radv_emit_discard_rectangle()
1457 VkRect2D rect = cmd_buffer->state.dynamic.discard_rectangle.rectangles[i]; in radv_emit_discard_rectangle()
1458 radeon_emit(cmd_buffer->cs, S_028210_TL_X(rect.offset.x) | S_028210_TL_Y(rect.offset.y)); in radv_emit_discard_rectangle()
1459 radeon_emit(cmd_buffer->cs, S_028214_BR_X(rect.offset.x + rect.extent.width) | in radv_emit_discard_rectangle()
1465 radv_emit_line_width(struct radv_cmd_buffer *cmd_buffer) in radv_emit_line_width() argument
1467 unsigned width = cmd_buffer->state.dynamic.line_width * 8; in radv_emit_line_width()
1469 radeon_set_context_reg(cmd_buffer->cs, R_028A08_PA_SU_LINE_CNTL, in radv_emit_line_width()
1474 radv_emit_blend_constants(struct radv_cmd_buffer *cmd_buffer) in radv_emit_blend_constants() argument
1476 struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; in radv_emit_blend_constants()
1478 radeon_set_context_reg_seq(cmd_buffer->cs, R_028414_CB_BLEND_RED, 4); in radv_emit_blend_constants()
1479 radeon_emit_array(cmd_buffer->cs, (uint32_t *)d->blend_constants, 4); in radv_emit_blend_constants()
1483 radv_emit_stencil(struct radv_cmd_buffer *cmd_buffer) in radv_emit_stencil() argument
1485 struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; in radv_emit_stencil()
1487 radeon_set_context_reg_seq(cmd_buffer->cs, R_028430_DB_STENCILREFMASK, 2); in radv_emit_stencil()
1488 radeon_emit(cmd_buffer->cs, S_028430_STENCILTESTVAL(d->stencil_reference.front) | in radv_emit_stencil()
1492 radeon_emit(cmd_buffer->cs, S_028434_STENCILTESTVAL_BF(d->stencil_reference.back) | in radv_emit_stencil()
1499 radv_emit_depth_bounds(struct radv_cmd_buffer *cmd_buffer) in radv_emit_depth_bounds() argument
1501 struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; in radv_emit_depth_bounds()
1503 radeon_set_context_reg_seq(cmd_buffer->cs, R_028020_DB_DEPTH_BOUNDS_MIN, 2); in radv_emit_depth_bounds()
1504 radeon_emit(cmd_buffer->cs, fui(d->depth_bounds.min)); in radv_emit_depth_bounds()
1505 radeon_emit(cmd_buffer->cs, fui(d->depth_bounds.max)); in radv_emit_depth_bounds()
1509 radv_emit_depth_bias(struct radv_cmd_buffer *cmd_buffer) in radv_emit_depth_bias() argument
1511 struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; in radv_emit_depth_bias()
1514 radeon_set_context_reg_seq(cmd_buffer->cs, R_028B7C_PA_SU_POLY_OFFSET_CLAMP, 5); in radv_emit_depth_bias()
1515 radeon_emit(cmd_buffer->cs, fui(d->depth_bias.clamp)); /* CLAMP */ in radv_emit_depth_bias()
1516 radeon_emit(cmd_buffer->cs, slope); /* FRONT SCALE */ in radv_emit_depth_bias()
1517 radeon_emit(cmd_buffer->cs, fui(d->depth_bias.bias)); /* FRONT OFFSET */ in radv_emit_depth_bias()
1518 radeon_emit(cmd_buffer->cs, slope); /* BACK SCALE */ in radv_emit_depth_bias()
1519 radeon_emit(cmd_buffer->cs, fui(d->depth_bias.bias)); /* BACK OFFSET */ in radv_emit_depth_bias()
1523 radv_emit_line_stipple(struct radv_cmd_buffer *cmd_buffer) in radv_emit_line_stipple() argument
1525 struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; in radv_emit_line_stipple()
1531 radeon_set_context_reg(cmd_buffer->cs, R_028A0C_PA_SC_LINE_STIPPLE, in radv_emit_line_stipple()
1538 radv_emit_culling(struct radv_cmd_buffer *cmd_buffer, uint64_t states) in radv_emit_culling() argument
1540 unsigned pa_su_sc_mode_cntl = cmd_buffer->state.pipeline->graphics.pa_su_sc_mode_cntl; in radv_emit_culling()
1541 struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; in radv_emit_culling()
1557 radeon_set_context_reg(cmd_buffer->cs, R_028814_PA_SU_SC_MODE_CNTL, pa_su_sc_mode_cntl); in radv_emit_culling()
1561 radv_emit_primitive_topology(struct radv_cmd_buffer *cmd_buffer) in radv_emit_primitive_topology() argument
1563 struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; in radv_emit_primitive_topology()
1565 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX7) { in radv_emit_primitive_topology()
1566 radeon_set_uconfig_reg_idx(cmd_buffer->device->physical_device, cmd_buffer->cs, in radv_emit_primitive_topology()
1569 radeon_set_config_reg(cmd_buffer->cs, R_008958_VGT_PRIMITIVE_TYPE, d->primitive_topology); in radv_emit_primitive_topology()
1574 radv_emit_depth_control(struct radv_cmd_buffer *cmd_buffer, uint64_t states) in radv_emit_depth_control() argument
1576 unsigned db_depth_control = cmd_buffer->state.pipeline->graphics.db_depth_control; in radv_emit_depth_control()
1577 struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; in radv_emit_depth_control()
1597 radeon_set_context_reg(cmd_buffer->cs, R_028800_DB_DEPTH_CONTROL, db_depth_control); in radv_emit_depth_control()
1601 radv_emit_stencil_control(struct radv_cmd_buffer *cmd_buffer) in radv_emit_stencil_control() argument
1603 struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; in radv_emit_stencil_control()
1606 cmd_buffer->cs, R_02842C_DB_STENCIL_CONTROL, in radv_emit_stencil_control()
1616 radv_emit_fragment_shading_rate(struct radv_cmd_buffer *cmd_buffer) in radv_emit_fragment_shading_rate() argument
1618 struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; in radv_emit_fragment_shading_rate()
1619 const struct radv_subpass *subpass = cmd_buffer->state.subpass; in radv_emit_fragment_shading_rate()
1620 struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; in radv_emit_fragment_shading_rate()
1656 radeon_set_uconfig_reg(cmd_buffer->cs, R_03098C_GE_VRS_RATE, in radv_emit_fragment_shading_rate()
1669 radeon_set_context_reg(cmd_buffer->cs, R_028848_PA_CL_VRS_CNTL, pa_cl_vrs_cntl); in radv_emit_fragment_shading_rate()
1673 radv_emit_primitive_restart_enable(struct radv_cmd_buffer *cmd_buffer) in radv_emit_primitive_restart_enable() argument
1675 struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; in radv_emit_primitive_restart_enable()
1677 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9) { in radv_emit_primitive_restart_enable()
1678 radeon_set_uconfig_reg(cmd_buffer->cs, R_03092C_VGT_MULTI_PRIM_IB_RESET_EN, in radv_emit_primitive_restart_enable()
1681 radeon_set_context_reg(cmd_buffer->cs, R_028A94_VGT_MULTI_PRIM_IB_RESET_EN, in radv_emit_primitive_restart_enable()
1687 radv_emit_rasterizer_discard_enable(struct radv_cmd_buffer *cmd_buffer) in radv_emit_rasterizer_discard_enable() argument
1689 unsigned pa_cl_clip_cntl = cmd_buffer->state.pipeline->graphics.pa_cl_clip_cntl; in radv_emit_rasterizer_discard_enable()
1690 struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; in radv_emit_rasterizer_discard_enable()
1695 radeon_set_context_reg(cmd_buffer->cs, R_028810_PA_CL_CLIP_CNTL, pa_cl_clip_cntl); in radv_emit_rasterizer_discard_enable()
1699 radv_emit_logic_op(struct radv_cmd_buffer *cmd_buffer) in radv_emit_logic_op() argument
1701 unsigned cb_color_control = cmd_buffer->state.pipeline->graphics.cb_color_control; in radv_emit_logic_op()
1702 struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; in radv_emit_logic_op()
1707 radeon_set_context_reg(cmd_buffer->cs, R_028808_CB_COLOR_CONTROL, cb_color_control); in radv_emit_logic_op()
1711 radv_emit_color_write_enable(struct radv_cmd_buffer *cmd_buffer) in radv_emit_color_write_enable() argument
1713 struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; in radv_emit_color_write_enable()
1714 struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; in radv_emit_color_write_enable()
1716 radeon_set_context_reg(cmd_buffer->cs, R_028238_CB_TARGET_MASK, in radv_emit_color_write_enable()
1721 radv_emit_fb_color_state(struct radv_cmd_buffer *cmd_buffer, int index, in radv_emit_fb_color_state() argument
1725 bool is_vi = cmd_buffer->device->physical_device->rad_info.chip_class >= GFX8; in radv_emit_fb_color_state()
1730 cmd_buffer->device, image, iview->base_mip, layout, in_render_loop, in radv_emit_fb_color_state()
1731 radv_image_queue_family_mask(image, cmd_buffer->queue_family_index, in radv_emit_fb_color_state()
1732 cmd_buffer->queue_family_index)) || in radv_emit_fb_color_state()
1738 cmd_buffer->device, image, layout, in radv_emit_fb_color_state()
1739 radv_image_queue_family_mask(image, cmd_buffer->queue_family_index, in radv_emit_fb_color_state()
1740 cmd_buffer->queue_family_index))) { in radv_emit_fb_color_state()
1744 if (radv_image_is_tc_compat_cmask(image) && (radv_is_fmask_decompress_pipeline(cmd_buffer) || in radv_emit_fb_color_state()
1745 radv_is_dcc_decompress_pipeline(cmd_buffer))) { in radv_emit_fb_color_state()
1752 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10) { in radv_emit_fb_color_state()
1753 radeon_set_context_reg_seq(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, 11); in radv_emit_fb_color_state()
1754 radeon_emit(cmd_buffer->cs, cb->cb_color_base); in radv_emit_fb_color_state()
1755 radeon_emit(cmd_buffer->cs, 0); in radv_emit_fb_color_state()
1756 radeon_emit(cmd_buffer->cs, 0); in radv_emit_fb_color_state()
1757 radeon_emit(cmd_buffer->cs, cb->cb_color_view); in radv_emit_fb_color_state()
1758 radeon_emit(cmd_buffer->cs, cb_color_info); in radv_emit_fb_color_state()
1759 radeon_emit(cmd_buffer->cs, cb->cb_color_attrib); in radv_emit_fb_color_state()
1760 radeon_emit(cmd_buffer->cs, cb->cb_dcc_control); in radv_emit_fb_color_state()
1761 radeon_emit(cmd_buffer->cs, cb->cb_color_cmask); in radv_emit_fb_color_state()
1762 radeon_emit(cmd_buffer->cs, 0); in radv_emit_fb_color_state()
1763 radeon_emit(cmd_buffer->cs, cb->cb_color_fmask); in radv_emit_fb_color_state()
1764 radeon_emit(cmd_buffer->cs, 0); in radv_emit_fb_color_state()
1766 …radeon_set_context_reg(cmd_buffer->cs, R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, cb->cb_dcc_base… in radv_emit_fb_color_state()
1768 radeon_set_context_reg(cmd_buffer->cs, R_028E40_CB_COLOR0_BASE_EXT + index * 4, in radv_emit_fb_color_state()
1770 radeon_set_context_reg(cmd_buffer->cs, R_028E60_CB_COLOR0_CMASK_BASE_EXT + index * 4, in radv_emit_fb_color_state()
1772 radeon_set_context_reg(cmd_buffer->cs, R_028E80_CB_COLOR0_FMASK_BASE_EXT + index * 4, in radv_emit_fb_color_state()
1774 radeon_set_context_reg(cmd_buffer->cs, R_028EA0_CB_COLOR0_DCC_BASE_EXT + index * 4, in radv_emit_fb_color_state()
1776 radeon_set_context_reg(cmd_buffer->cs, R_028EC0_CB_COLOR0_ATTRIB2 + index * 4, in radv_emit_fb_color_state()
1778 radeon_set_context_reg(cmd_buffer->cs, R_028EE0_CB_COLOR0_ATTRIB3 + index * 4, in radv_emit_fb_color_state()
1780 } else if (cmd_buffer->device->physical_device->rad_info.chip_class == GFX9) { in radv_emit_fb_color_state()
1781 radeon_set_context_reg_seq(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, 11); in radv_emit_fb_color_state()
1782 radeon_emit(cmd_buffer->cs, cb->cb_color_base); in radv_emit_fb_color_state()
1783 radeon_emit(cmd_buffer->cs, S_028C64_BASE_256B(cb->cb_color_base >> 32)); in radv_emit_fb_color_state()
1784 radeon_emit(cmd_buffer->cs, cb->cb_color_attrib2); in radv_emit_fb_color_state()
1785 radeon_emit(cmd_buffer->cs, cb->cb_color_view); in radv_emit_fb_color_state()
1786 radeon_emit(cmd_buffer->cs, cb_color_info); in radv_emit_fb_color_state()
1787 radeon_emit(cmd_buffer->cs, cb->cb_color_attrib); in radv_emit_fb_color_state()
1788 radeon_emit(cmd_buffer->cs, cb->cb_dcc_control); in radv_emit_fb_color_state()
1789 radeon_emit(cmd_buffer->cs, cb->cb_color_cmask); in radv_emit_fb_color_state()
1790 radeon_emit(cmd_buffer->cs, S_028C80_BASE_256B(cb->cb_color_cmask >> 32)); in radv_emit_fb_color_state()
1791 radeon_emit(cmd_buffer->cs, cb->cb_color_fmask); in radv_emit_fb_color_state()
1792 radeon_emit(cmd_buffer->cs, S_028C88_BASE_256B(cb->cb_color_fmask >> 32)); in radv_emit_fb_color_state()
1794 radeon_set_context_reg_seq(cmd_buffer->cs, R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, 2); in radv_emit_fb_color_state()
1795 radeon_emit(cmd_buffer->cs, cb->cb_dcc_base); in radv_emit_fb_color_state()
1796 radeon_emit(cmd_buffer->cs, S_028C98_BASE_256B(cb->cb_dcc_base >> 32)); in radv_emit_fb_color_state()
1798 radeon_set_context_reg(cmd_buffer->cs, R_0287A0_CB_MRT0_EPITCH + index * 4, in radv_emit_fb_color_state()
1801 radeon_set_context_reg_seq(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, 11); in radv_emit_fb_color_state()
1802 radeon_emit(cmd_buffer->cs, cb->cb_color_base); in radv_emit_fb_color_state()
1803 radeon_emit(cmd_buffer->cs, cb->cb_color_pitch); in radv_emit_fb_color_state()
1804 radeon_emit(cmd_buffer->cs, cb->cb_color_slice); in radv_emit_fb_color_state()
1805 radeon_emit(cmd_buffer->cs, cb->cb_color_view); in radv_emit_fb_color_state()
1806 radeon_emit(cmd_buffer->cs, cb_color_info); in radv_emit_fb_color_state()
1807 radeon_emit(cmd_buffer->cs, cb->cb_color_attrib); in radv_emit_fb_color_state()
1808 radeon_emit(cmd_buffer->cs, cb->cb_dcc_control); in radv_emit_fb_color_state()
1809 radeon_emit(cmd_buffer->cs, cb->cb_color_cmask); in radv_emit_fb_color_state()
1810 radeon_emit(cmd_buffer->cs, cb->cb_color_cmask_slice); in radv_emit_fb_color_state()
1811 radeon_emit(cmd_buffer->cs, cb->cb_color_fmask); in radv_emit_fb_color_state()
1812 radeon_emit(cmd_buffer->cs, cb->cb_color_fmask_slice); in radv_emit_fb_color_state()
1815 radeon_set_context_reg(cmd_buffer->cs, R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, in radv_emit_fb_color_state()
1830 radv_update_dcc_metadata(cmd_buffer, image, &range, true); in radv_emit_fb_color_state()
1835 radv_update_zrange_precision(struct radv_cmd_buffer *cmd_buffer, struct radv_ds_buffer_info *ds, in radv_update_zrange_precision() argument
1843 if (!cmd_buffer->device->physical_device->rad_info.has_tc_compat_zrange_bug || in radv_update_zrange_precision()
1848 cmd_buffer->device, image, layout, in_render_loop, in radv_update_zrange_precision()
1849 radv_image_queue_family_mask(image, cmd_buffer->queue_family_index, in radv_update_zrange_precision()
1850 cmd_buffer->queue_family_index))) { in radv_update_zrange_precision()
1856 if (cmd_buffer->device->physical_device->rad_info.chip_class == GFX9) { in radv_update_zrange_precision()
1869 radeon_emit(cmd_buffer->cs, PKT3(PKT3_COND_EXEC, 3, 0)); in radv_update_zrange_precision()
1870 radeon_emit(cmd_buffer->cs, va); in radv_update_zrange_precision()
1871 radeon_emit(cmd_buffer->cs, va >> 32); in radv_update_zrange_precision()
1872 radeon_emit(cmd_buffer->cs, 0); in radv_update_zrange_precision()
1873 radeon_emit(cmd_buffer->cs, 3); /* SET_CONTEXT_REG size */ in radv_update_zrange_precision()
1876 radeon_set_context_reg(cmd_buffer->cs, db_z_info_reg, db_z_info); in radv_update_zrange_precision()
1880 radv_emit_fb_ds_state(struct radv_cmd_buffer *cmd_buffer, struct radv_ds_buffer_info *ds, in radv_emit_fb_ds_state() argument
1888 cmd_buffer->device, image, layout, in_render_loop, in radv_emit_fb_ds_state()
1889 radv_image_queue_family_mask(image, cmd_buffer->queue_family_index, in radv_emit_fb_ds_state()
1890 cmd_buffer->queue_family_index))) { in radv_emit_fb_ds_state()
1895 radeon_set_context_reg(cmd_buffer->cs, R_028008_DB_DEPTH_VIEW, ds->db_depth_view); in radv_emit_fb_ds_state()
1896 radeon_set_context_reg(cmd_buffer->cs, R_028ABC_DB_HTILE_SURFACE, ds->db_htile_surface); in radv_emit_fb_ds_state()
1898 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10) { in radv_emit_fb_ds_state()
1899 radeon_set_context_reg(cmd_buffer->cs, R_028014_DB_HTILE_DATA_BASE, ds->db_htile_data_base); in radv_emit_fb_ds_state()
1900 radeon_set_context_reg(cmd_buffer->cs, R_02801C_DB_DEPTH_SIZE_XY, ds->db_depth_size); in radv_emit_fb_ds_state()
1902 radeon_set_context_reg_seq(cmd_buffer->cs, R_02803C_DB_DEPTH_INFO, 7); in radv_emit_fb_ds_state()
1903 radeon_emit(cmd_buffer->cs, S_02803C_RESOURCE_LEVEL(1)); in radv_emit_fb_ds_state()
1904 radeon_emit(cmd_buffer->cs, db_z_info); in radv_emit_fb_ds_state()
1905 radeon_emit(cmd_buffer->cs, db_stencil_info); in radv_emit_fb_ds_state()
1906 radeon_emit(cmd_buffer->cs, ds->db_z_read_base); in radv_emit_fb_ds_state()
1907 radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base); in radv_emit_fb_ds_state()
1908 radeon_emit(cmd_buffer->cs, ds->db_z_read_base); in radv_emit_fb_ds_state()
1909 radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base); in radv_emit_fb_ds_state()
1911 radeon_set_context_reg_seq(cmd_buffer->cs, R_028068_DB_Z_READ_BASE_HI, 5); in radv_emit_fb_ds_state()
1912 radeon_emit(cmd_buffer->cs, ds->db_z_read_base >> 32); in radv_emit_fb_ds_state()
1913 radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base >> 32); in radv_emit_fb_ds_state()
1914 radeon_emit(cmd_buffer->cs, ds->db_z_read_base >> 32); in radv_emit_fb_ds_state()
1915 radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base >> 32); in radv_emit_fb_ds_state()
1916 radeon_emit(cmd_buffer->cs, ds->db_htile_data_base >> 32); in radv_emit_fb_ds_state()
1917 } else if (cmd_buffer->device->physical_device->rad_info.chip_class == GFX9) { in radv_emit_fb_ds_state()
1918 radeon_set_context_reg_seq(cmd_buffer->cs, R_028014_DB_HTILE_DATA_BASE, 3); in radv_emit_fb_ds_state()
1919 radeon_emit(cmd_buffer->cs, ds->db_htile_data_base); in radv_emit_fb_ds_state()
1920 radeon_emit(cmd_buffer->cs, S_028018_BASE_HI(ds->db_htile_data_base >> 32)); in radv_emit_fb_ds_state()
1921 radeon_emit(cmd_buffer->cs, ds->db_depth_size); in radv_emit_fb_ds_state()
1923 radeon_set_context_reg_seq(cmd_buffer->cs, R_028038_DB_Z_INFO, 10); in radv_emit_fb_ds_state()
1924 radeon_emit(cmd_buffer->cs, db_z_info); /* DB_Z_INFO */ in radv_emit_fb_ds_state()
1925 radeon_emit(cmd_buffer->cs, db_stencil_info); /* DB_STENCIL_INFO */ in radv_emit_fb_ds_state()
1926 radeon_emit(cmd_buffer->cs, ds->db_z_read_base); /* DB_Z_READ_BASE */ in radv_emit_fb_ds_state()
1927 radeon_emit(cmd_buffer->cs, in radv_emit_fb_ds_state()
1929 radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base); /* DB_STENCIL_READ_BASE */ in radv_emit_fb_ds_state()
1930 radeon_emit(cmd_buffer->cs, in radv_emit_fb_ds_state()
1932 radeon_emit(cmd_buffer->cs, ds->db_z_write_base); /* DB_Z_WRITE_BASE */ in radv_emit_fb_ds_state()
1933 radeon_emit(cmd_buffer->cs, in radv_emit_fb_ds_state()
1935 radeon_emit(cmd_buffer->cs, ds->db_stencil_write_base); /* DB_STENCIL_WRITE_BASE */ in radv_emit_fb_ds_state()
1936 radeon_emit(cmd_buffer->cs, in radv_emit_fb_ds_state()
1939 radeon_set_context_reg_seq(cmd_buffer->cs, R_028068_DB_Z_INFO2, 2); in radv_emit_fb_ds_state()
1940 radeon_emit(cmd_buffer->cs, ds->db_z_info2); in radv_emit_fb_ds_state()
1941 radeon_emit(cmd_buffer->cs, ds->db_stencil_info2); in radv_emit_fb_ds_state()
1943 radeon_set_context_reg(cmd_buffer->cs, R_028014_DB_HTILE_DATA_BASE, ds->db_htile_data_base); in radv_emit_fb_ds_state()
1945 radeon_set_context_reg_seq(cmd_buffer->cs, R_02803C_DB_DEPTH_INFO, 9); in radv_emit_fb_ds_state()
1946 radeon_emit(cmd_buffer->cs, ds->db_depth_info); /* R_02803C_DB_DEPTH_INFO */ in radv_emit_fb_ds_state()
1947 radeon_emit(cmd_buffer->cs, db_z_info); /* R_028040_DB_Z_INFO */ in radv_emit_fb_ds_state()
1948 radeon_emit(cmd_buffer->cs, db_stencil_info); /* R_028044_DB_STENCIL_INFO */ in radv_emit_fb_ds_state()
1949 radeon_emit(cmd_buffer->cs, ds->db_z_read_base); /* R_028048_DB_Z_READ_BASE */ in radv_emit_fb_ds_state()
1950 radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base); /* R_02804C_DB_STENCIL_READ_BASE */ in radv_emit_fb_ds_state()
1951 radeon_emit(cmd_buffer->cs, ds->db_z_write_base); /* R_028050_DB_Z_WRITE_BASE */ in radv_emit_fb_ds_state()
1952 radeon_emit(cmd_buffer->cs, ds->db_stencil_write_base); /* R_028054_DB_STENCIL_WRITE_BASE */ in radv_emit_fb_ds_state()
1953 radeon_emit(cmd_buffer->cs, ds->db_depth_size); /* R_028058_DB_DEPTH_SIZE */ in radv_emit_fb_ds_state()
1954 radeon_emit(cmd_buffer->cs, ds->db_depth_slice); /* R_02805C_DB_DEPTH_SLICE */ in radv_emit_fb_ds_state()
1958 radv_update_zrange_precision(cmd_buffer, ds, iview, layout, in_render_loop, true); in radv_emit_fb_ds_state()
1960 radeon_set_context_reg(cmd_buffer->cs, R_028B78_PA_SU_POLY_OFFSET_DB_FMT_CNTL, in radv_emit_fb_ds_state()
1969 radv_update_bound_fast_clear_ds(struct radv_cmd_buffer *cmd_buffer, in radv_update_bound_fast_clear_ds() argument
1973 const struct radv_subpass *subpass = cmd_buffer->state.subpass; in radv_update_bound_fast_clear_ds()
1975 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_update_bound_fast_clear_ds()
1978 if (!cmd_buffer->state.attachments || !subpass) in radv_update_bound_fast_clear_ds()
1985 if (cmd_buffer->state.attachments[att_idx].iview->image != image) in radv_update_bound_fast_clear_ds()
2006 radv_update_zrange_precision(cmd_buffer, &cmd_buffer->state.attachments[att_idx].ds, iview, in radv_update_bound_fast_clear_ds()
2010 cmd_buffer->state.context_roll_without_scissor_emitted = true; in radv_update_bound_fast_clear_ds()
2017 radv_set_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, in radv_set_ds_clear_metadata() argument
2021 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_set_ds_clear_metadata()
2028 radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + 2 * level_count, cmd_buffer->state.predicating)); in radv_set_ds_clear_metadata()
2051 radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, cmd_buffer->state.predicating)); in radv_set_ds_clear_metadata()
2065 radv_set_tc_compat_zrange_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, in radv_set_tc_compat_zrange_metadata() argument
2068 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_set_tc_compat_zrange_metadata()
2070 if (!cmd_buffer->device->physical_device->rad_info.has_tc_compat_zrange_bug) in radv_set_tc_compat_zrange_metadata()
2076 radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + level_count, cmd_buffer->state.predicating)); in radv_set_tc_compat_zrange_metadata()
2086 radv_update_tc_compat_zrange_metadata(struct radv_cmd_buffer *cmd_buffer, in radv_update_tc_compat_zrange_metadata() argument
2104 radv_set_tc_compat_zrange_metadata(cmd_buffer, iview->image, &range, cond_val); in radv_update_tc_compat_zrange_metadata()
2111 radv_update_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer, in radv_update_ds_clear_metadata() argument
2126 radv_set_ds_clear_metadata(cmd_buffer, iview->image, &range, ds_clear_value, aspects); in radv_update_ds_clear_metadata()
2129 radv_update_tc_compat_zrange_metadata(cmd_buffer, iview, ds_clear_value); in radv_update_ds_clear_metadata()
2132 radv_update_bound_fast_clear_ds(cmd_buffer, iview, ds_clear_value, aspects); in radv_update_ds_clear_metadata()
2139 radv_load_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer, const struct radv_image_view *iview) in radv_load_ds_clear_metadata() argument
2141 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_load_ds_clear_metadata()
2160 if (cmd_buffer->device->physical_device->rad_info.has_load_ctx_reg_pkt) { in radv_load_ds_clear_metadata()
2186 radv_update_fce_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, in radv_update_fce_metadata() argument
2197 radeon_emit(cmd_buffer->cs, PKT3(PKT3_WRITE_DATA, 2 + count, 0)); in radv_update_fce_metadata()
2198 radeon_emit(cmd_buffer->cs, in radv_update_fce_metadata()
2200 radeon_emit(cmd_buffer->cs, va); in radv_update_fce_metadata()
2201 radeon_emit(cmd_buffer->cs, va >> 32); in radv_update_fce_metadata()
2204 radeon_emit(cmd_buffer->cs, pred_val); in radv_update_fce_metadata()
2205 radeon_emit(cmd_buffer->cs, pred_val >> 32); in radv_update_fce_metadata()
2213 radv_update_dcc_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, in radv_update_dcc_metadata() argument
2226 radeon_emit(cmd_buffer->cs, PKT3(PKT3_WRITE_DATA, 2 + count, 0)); in radv_update_dcc_metadata()
2227 radeon_emit(cmd_buffer->cs, in radv_update_dcc_metadata()
2229 radeon_emit(cmd_buffer->cs, va); in radv_update_dcc_metadata()
2230 radeon_emit(cmd_buffer->cs, va >> 32); in radv_update_dcc_metadata()
2233 radeon_emit(cmd_buffer->cs, pred_val); in radv_update_dcc_metadata()
2234 radeon_emit(cmd_buffer->cs, pred_val >> 32); in radv_update_dcc_metadata()
2242 radv_update_bound_fast_clear_color(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, in radv_update_bound_fast_clear_color() argument
2245 const struct radv_subpass *subpass = cmd_buffer->state.subpass; in radv_update_bound_fast_clear_color()
2246 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_update_bound_fast_clear_color()
2249 if (!cmd_buffer->state.attachments || !subpass) in radv_update_bound_fast_clear_color()
2256 if (cmd_buffer->state.attachments[att_idx].iview->image != image) in radv_update_bound_fast_clear_color()
2263 cmd_buffer->state.context_roll_without_scissor_emitted = true; in radv_update_bound_fast_clear_color()
2270 radv_set_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, in radv_set_color_clear_metadata() argument
2273 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_set_color_clear_metadata()
2282 radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + count, cmd_buffer->state.predicating)); in radv_set_color_clear_metadata()
2301 radv_update_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer, in radv_update_color_clear_metadata() argument
2322 radv_set_color_clear_metadata(cmd_buffer, image, &range, color_values); in radv_update_color_clear_metadata()
2324 radv_update_bound_fast_clear_color(cmd_buffer, image, cb_idx, color_values); in radv_update_color_clear_metadata()
2331 radv_load_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *iview, in radv_load_color_clear_metadata() argument
2334 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_load_color_clear_metadata()
2345 radv_update_bound_fast_clear_color(cmd_buffer, image, cb_idx, color_values); in radv_load_color_clear_metadata()
2352 if (cmd_buffer->device->physical_device->rad_info.has_load_ctx_reg_pkt) { in radv_load_color_clear_metadata()
2353 radeon_emit(cs, PKT3(PKT3_LOAD_CONTEXT_REG_INDEX, 3, cmd_buffer->state.predicating)); in radv_load_color_clear_metadata()
2359 radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, cmd_buffer->state.predicating)); in radv_load_color_clear_metadata()
2367 radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating)); in radv_load_color_clear_metadata()
2379 radv_emit_fb_mip_change_flush(struct radv_cmd_buffer *cmd_buffer) in radv_emit_fb_mip_change_flush() argument
2381 struct radv_framebuffer *framebuffer = cmd_buffer->state.framebuffer; in radv_emit_fb_mip_change_flush()
2382 const struct radv_subpass *subpass = cmd_buffer->state.subpass; in radv_emit_fb_mip_change_flush()
2386 if (cmd_buffer->device->physical_device->rad_info.chip_class < GFX9) in radv_emit_fb_mip_change_flush()
2397 struct radv_image_view *iview = cmd_buffer->state.attachments[idx].iview; in radv_emit_fb_mip_change_flush()
2401 radv_dcc_enabled(iview->image, cmd_buffer->state.cb_mip[i])) && in radv_emit_fb_mip_change_flush()
2402 cmd_buffer->state.cb_mip[i] != iview->base_mip) in radv_emit_fb_mip_change_flush()
2405 cmd_buffer->state.cb_mip[i] = iview->base_mip; in radv_emit_fb_mip_change_flush()
2409 cmd_buffer->state.flush_bits |= in radv_emit_fb_mip_change_flush()
2419 radv_emit_mip_change_flush_default(struct radv_cmd_buffer *cmd_buffer) in radv_emit_mip_change_flush_default() argument
2422 if (cmd_buffer->device->physical_device->rad_info.chip_class < GFX9) in radv_emit_mip_change_flush_default()
2427 if (cmd_buffer->state.cb_mip[i]) { in radv_emit_mip_change_flush_default()
2434 cmd_buffer->state.flush_bits |= in radv_emit_mip_change_flush_default()
2438 memset(cmd_buffer->state.cb_mip, 0, sizeof(cmd_buffer->state.cb_mip)); in radv_emit_mip_change_flush_default()
2442 radv_cmd_buffer_get_vrs_image(struct radv_cmd_buffer *cmd_buffer) in radv_cmd_buffer_get_vrs_image() argument
2444 struct radv_device *device = cmd_buffer->device; in radv_cmd_buffer_get_vrs_image()
2452 cmd_buffer->record_result = result; in radv_cmd_buffer_get_vrs_image()
2461 radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer) in radv_emit_framebuffer_state() argument
2464 struct radv_framebuffer *framebuffer = cmd_buffer->state.framebuffer; in radv_emit_framebuffer_state()
2465 const struct radv_subpass *subpass = cmd_buffer->state.subpass; in radv_emit_framebuffer_state()
2474 radeon_set_context_reg(cmd_buffer->cs, R_028C70_CB_COLOR0_INFO + i * 0x3C, in radv_emit_framebuffer_state()
2480 struct radv_image_view *iview = cmd_buffer->state.attachments[idx].iview; in radv_emit_framebuffer_state()
2484 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, iview->image->bo); in radv_emit_framebuffer_state()
2488 radv_emit_fb_color_state(cmd_buffer, i, &cmd_buffer->state.attachments[idx].cb, iview, layout, in radv_emit_framebuffer_state()
2489 in_render_loop, cmd_buffer->state.attachments[idx].disable_dcc); in radv_emit_framebuffer_state()
2491 radv_load_color_clear_metadata(cmd_buffer, iview, i); in radv_emit_framebuffer_state()
2498 struct radv_image_view *iview = cmd_buffer->state.attachments[idx].iview; in radv_emit_framebuffer_state()
2499 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, in radv_emit_framebuffer_state()
2500 cmd_buffer->state.attachments[idx].iview->image->bo); in radv_emit_framebuffer_state()
2502 radv_emit_fb_ds_state(cmd_buffer, &cmd_buffer->state.attachments[idx].ds, iview, layout, in radv_emit_framebuffer_state()
2506 cmd_buffer->device, iview->image, layout, in_render_loop, in radv_emit_framebuffer_state()
2507 radv_image_queue_family_mask(iview->image, cmd_buffer->queue_family_index, in radv_emit_framebuffer_state()
2508 cmd_buffer->queue_family_index))) { in radv_emit_framebuffer_state()
2512 radv_load_ds_clear_metadata(cmd_buffer, iview); in radv_emit_framebuffer_state()
2514 } else if (subpass->vrs_attachment && cmd_buffer->device->vrs.image) { in radv_emit_framebuffer_state()
2519 struct radv_buffer *htile_buffer = cmd_buffer->device->vrs.buffer; in radv_emit_framebuffer_state()
2520 struct radv_image *image = cmd_buffer->device->vrs.image; in radv_emit_framebuffer_state()
2524 radv_image_view_init(&iview, cmd_buffer->device, in radv_emit_framebuffer_state()
2543 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, htile_buffer->bo); in radv_emit_framebuffer_state()
2545 radv_emit_fb_ds_state(cmd_buffer, &ds, &iview, layout, false); in radv_emit_framebuffer_state()
2549 if (cmd_buffer->device->physical_device->rad_info.chip_class == GFX9) in radv_emit_framebuffer_state()
2550 radeon_set_context_reg_seq(cmd_buffer->cs, R_028038_DB_Z_INFO, 2); in radv_emit_framebuffer_state()
2552 radeon_set_context_reg_seq(cmd_buffer->cs, R_028040_DB_Z_INFO, 2); in radv_emit_framebuffer_state()
2554 radeon_emit(cmd_buffer->cs, S_028040_FORMAT(V_028040_Z_INVALID)); /* DB_Z_INFO */ in radv_emit_framebuffer_state()
2555 radeon_emit(cmd_buffer->cs, S_028044_FORMAT(V_028044_STENCIL_INVALID)); /* DB_STENCIL_INFO */ in radv_emit_framebuffer_state()
2557 radeon_set_context_reg(cmd_buffer->cs, R_028208_PA_SC_WINDOW_SCISSOR_BR, in radv_emit_framebuffer_state()
2560 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX8) { in radv_emit_framebuffer_state()
2562 cmd_buffer->device->physical_device->rad_info.has_dcc_constant_encode; in radv_emit_framebuffer_state()
2563 enum chip_class chip_class = cmd_buffer->device->physical_device->rad_info.chip_class; in radv_emit_framebuffer_state()
2566 radeon_set_context_reg(cmd_buffer->cs, R_028424_CB_DCC_CONTROL, in radv_emit_framebuffer_state()
2572 cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_FRAMEBUFFER; in radv_emit_framebuffer_state()
2576 radv_emit_index_buffer(struct radv_cmd_buffer *cmd_buffer, bool indirect) in radv_emit_index_buffer() argument
2578 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_emit_index_buffer()
2579 struct radv_cmd_state *state = &cmd_buffer->state; in radv_emit_index_buffer()
2582 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9) { in radv_emit_index_buffer()
2583 radeon_set_uconfig_reg_idx(cmd_buffer->device->physical_device, cs, in radv_emit_index_buffer()
2605 cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_INDEX_BUFFER; in radv_emit_index_buffer()
2609 radv_set_db_count_control(struct radv_cmd_buffer *cmd_buffer) in radv_set_db_count_control() argument
2611 bool has_perfect_queries = cmd_buffer->state.perfect_occlusion_queries_enabled; in radv_set_db_count_control()
2612 struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; in radv_set_db_count_control()
2616 if (!cmd_buffer->state.active_occlusion_queries) { in radv_set_db_count_control()
2617 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX7) { in radv_set_db_count_control()
2625 radeon_set_context_reg(cmd_buffer->cs, R_028A4C_PA_SC_MODE_CNTL_1, pa_sc_mode_cntl_1); in radv_set_db_count_control()
2630 const struct radv_subpass *subpass = cmd_buffer->state.subpass; in radv_set_db_count_control()
2633 cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10 && has_perfect_queries; in radv_set_db_count_control()
2635 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX7) { in radv_set_db_count_control()
2653 radeon_set_context_reg(cmd_buffer->cs, R_028A4C_PA_SC_MODE_CNTL_1, pa_sc_mode_cntl_1); in radv_set_db_count_control()
2660 radeon_set_context_reg(cmd_buffer->cs, R_028004_DB_COUNT_CONTROL, db_count_control); in radv_set_db_count_control()
2662 cmd_buffer->state.context_roll_without_scissor_emitted = true; in radv_set_db_count_control()
2737 lookup_vs_prolog(struct radv_cmd_buffer *cmd_buffer, struct radv_shader_variant *vs_shader, in lookup_vs_prolog() argument
2743 const struct radv_vs_input_state *state = &cmd_buffer->state.dynamic_vs_input; in lookup_vs_prolog()
2744 const struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; in lookup_vs_prolog()
2745 struct radv_device *device = cmd_buffer->device; in lookup_vs_prolog()
2753 …const uint32_t misaligned_mask = chip == GFX6 || chip >= GFX10 ? cmd_buffer->state.vbo_misaligned_… in lookup_vs_prolog()
2832 if (cmd_buffer->state.emitted_vs_prolog && in lookup_vs_prolog()
2833 cmd_buffer->state.emitted_vs_prolog_key_hash == hash && in lookup_vs_prolog()
2834 radv_cmp_vs_prolog(key_words, cmd_buffer->state.emitted_vs_prolog_key)) in lookup_vs_prolog()
2835 return cmd_buffer->state.emitted_vs_prolog; in lookup_vs_prolog()
2869 emit_prolog_regs(struct radv_cmd_buffer *cmd_buffer, struct radv_shader_variant *vs_shader, in emit_prolog_regs() argument
2873 if (cmd_buffer->state.emitted_vs_prolog == prolog && !pipeline_is_dirty) in emit_prolog_regs()
2876 enum chip_class chip = cmd_buffer->device->physical_device->rad_info.chip_class; in emit_prolog_regs()
2877 struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; in emit_prolog_regs()
2880 assert(cmd_buffer->state.emitted_pipeline == cmd_buffer->state.pipeline); in emit_prolog_regs()
2908 radeon_set_sh_reg_seq(cmd_buffer->cs, pgm_lo_reg, 2); in emit_prolog_regs()
2909 radeon_emit(cmd_buffer->cs, prolog_va >> 8); in emit_prolog_regs()
2910 radeon_emit(cmd_buffer->cs, S_00B124_MEM_BASE(prolog_va >> 40)); in emit_prolog_regs()
2913 radeon_set_sh_reg(cmd_buffer->cs, rsrc1_reg, rsrc1); in emit_prolog_regs()
2917 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, prolog->bo); in emit_prolog_regs()
2921 emit_prolog_inputs(struct radv_cmd_buffer *cmd_buffer, struct radv_shader_variant *vs_shader, in emit_prolog_inputs() argument
2925 if (!nontrivial_divisors && !pipeline_is_dirty && cmd_buffer->state.emitted_vs_prolog && in emit_prolog_inputs()
2926 !cmd_buffer->state.emitted_vs_prolog->nontrivial_divisors) in emit_prolog_inputs()
2929 struct radv_vs_input_state *state = &cmd_buffer->state.dynamic_vs_input; in emit_prolog_inputs()
2936 if (!radv_cmd_buffer_upload_alloc(cmd_buffer, size, &inputs_offset, (void **)&inputs)) in emit_prolog_inputs()
2958 input_va = radv_buffer_get_va(cmd_buffer->upload.upload_bo) + inputs_offset; in emit_prolog_inputs()
2963 uint32_t base_reg = cmd_buffer->state.pipeline->user_data_0[MESA_SHADER_VERTEX]; in emit_prolog_inputs()
2966 radv_emit_shader_pointer(cmd_buffer->device, cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, in emit_prolog_inputs()
2971 radv_emit_vertex_input(struct radv_cmd_buffer *cmd_buffer, bool pipeline_is_dirty) in radv_emit_vertex_input() argument
2973 struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; in radv_emit_vertex_input()
2981 lookup_vs_prolog(cmd_buffer, vs_shader, &nontrivial_divisors); in radv_emit_vertex_input()
2983 cmd_buffer->record_result = VK_ERROR_OUT_OF_HOST_MEMORY; in radv_emit_vertex_input()
2986 emit_prolog_regs(cmd_buffer, vs_shader, prolog, pipeline_is_dirty); in radv_emit_vertex_input()
2987 emit_prolog_inputs(cmd_buffer, vs_shader, nontrivial_divisors, pipeline_is_dirty); in radv_emit_vertex_input()
2989 cmd_buffer->state.emitted_vs_prolog = prolog; in radv_emit_vertex_input()
2993 radv_cmd_buffer_flush_dynamic_state(struct radv_cmd_buffer *cmd_buffer, bool pipeline_is_dirty) in radv_cmd_buffer_flush_dynamic_state() argument
2996 cmd_buffer->state.dirty & cmd_buffer->state.emitted_pipeline->graphics.needed_dynamic_state; in radv_cmd_buffer_flush_dynamic_state()
2999 radv_emit_viewport(cmd_buffer); in radv_cmd_buffer_flush_dynamic_state()
3002 !cmd_buffer->device->physical_device->rad_info.has_gfx9_scissor_bug) in radv_cmd_buffer_flush_dynamic_state()
3003 radv_emit_scissor(cmd_buffer); in radv_cmd_buffer_flush_dynamic_state()
3006 radv_emit_line_width(cmd_buffer); in radv_cmd_buffer_flush_dynamic_state()
3009 radv_emit_blend_constants(cmd_buffer); in radv_cmd_buffer_flush_dynamic_state()
3014 radv_emit_stencil(cmd_buffer); in radv_cmd_buffer_flush_dynamic_state()
3017 radv_emit_depth_bounds(cmd_buffer); in radv_cmd_buffer_flush_dynamic_state()
3020 radv_emit_depth_bias(cmd_buffer); in radv_cmd_buffer_flush_dynamic_state()
3023 radv_emit_discard_rectangle(cmd_buffer); in radv_cmd_buffer_flush_dynamic_state()
3026 radv_emit_sample_locations(cmd_buffer); in radv_cmd_buffer_flush_dynamic_state()
3029 radv_emit_line_stipple(cmd_buffer); in radv_cmd_buffer_flush_dynamic_state()
3033 radv_emit_culling(cmd_buffer, states); in radv_cmd_buffer_flush_dynamic_state()
3036 radv_emit_primitive_topology(cmd_buffer); in radv_cmd_buffer_flush_dynamic_state()
3042 radv_emit_depth_control(cmd_buffer, states); in radv_cmd_buffer_flush_dynamic_state()
3045 radv_emit_stencil_control(cmd_buffer); in radv_cmd_buffer_flush_dynamic_state()
3048 radv_emit_fragment_shading_rate(cmd_buffer); in radv_cmd_buffer_flush_dynamic_state()
3051 radv_emit_primitive_restart_enable(cmd_buffer); in radv_cmd_buffer_flush_dynamic_state()
3054 radv_emit_rasterizer_discard_enable(cmd_buffer); in radv_cmd_buffer_flush_dynamic_state()
3057 radv_emit_logic_op(cmd_buffer); in radv_cmd_buffer_flush_dynamic_state()
3060 radv_emit_color_write_enable(cmd_buffer); in radv_cmd_buffer_flush_dynamic_state()
3063 radv_emit_vertex_input(cmd_buffer, pipeline_is_dirty); in radv_cmd_buffer_flush_dynamic_state()
3065 cmd_buffer->state.dirty &= ~states; in radv_cmd_buffer_flush_dynamic_state()
3069 radv_flush_push_descriptors(struct radv_cmd_buffer *cmd_buffer, VkPipelineBindPoint bind_point) in radv_flush_push_descriptors() argument
3072 radv_get_descriptors_state(cmd_buffer, bind_point); in radv_flush_push_descriptors()
3076 if (!radv_cmd_buffer_upload_data(cmd_buffer, set->header.size, set->header.mapped_ptr, in radv_flush_push_descriptors()
3080 set->header.va = radv_buffer_get_va(cmd_buffer->upload.upload_bo); in radv_flush_push_descriptors()
3085 radv_flush_indirect_descriptor_sets(struct radv_cmd_buffer *cmd_buffer, in radv_flush_indirect_descriptor_sets() argument
3089 radv_get_descriptors_state(cmd_buffer, bind_point); in radv_flush_indirect_descriptor_sets()
3094 if (!radv_cmd_buffer_upload_alloc(cmd_buffer, size, &offset, &ptr)) in radv_flush_indirect_descriptor_sets()
3106 uint64_t va = radv_buffer_get_va(cmd_buffer->upload.upload_bo); in radv_flush_indirect_descriptor_sets()
3111 radv_emit_userdata_address(cmd_buffer, pipeline, MESA_SHADER_VERTEX, in radv_flush_indirect_descriptor_sets()
3115 radv_emit_userdata_address(cmd_buffer, pipeline, MESA_SHADER_FRAGMENT, in radv_flush_indirect_descriptor_sets()
3119 radv_emit_userdata_address(cmd_buffer, pipeline, MESA_SHADER_GEOMETRY, in radv_flush_indirect_descriptor_sets()
3123 radv_emit_userdata_address(cmd_buffer, pipeline, MESA_SHADER_TESS_CTRL, in radv_flush_indirect_descriptor_sets()
3127 radv_emit_userdata_address(cmd_buffer, pipeline, MESA_SHADER_TESS_EVAL, in radv_flush_indirect_descriptor_sets()
3130 radv_emit_userdata_address(cmd_buffer, pipeline, MESA_SHADER_COMPUTE, in radv_flush_indirect_descriptor_sets()
3136 radv_flush_descriptors(struct radv_cmd_buffer *cmd_buffer, VkShaderStageFlags stages, in radv_flush_descriptors() argument
3140 radv_get_descriptors_state(cmd_buffer, bind_point); in radv_flush_descriptors()
3147 radv_flush_push_descriptors(cmd_buffer, bind_point); in radv_flush_descriptors()
3152 radv_flush_indirect_descriptor_sets(cmd_buffer, pipeline, bind_point); in radv_flush_descriptors()
3155 radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, MAX_SETS * MESA_SHADER_STAGES * 4); in radv_flush_descriptors()
3159 radv_emit_descriptor_pointers(cmd_buffer, pipeline, descriptors_state, in radv_flush_descriptors()
3164 if (!cmd_buffer->state.pipeline->shaders[stage]) in radv_flush_descriptors()
3167 radv_emit_descriptor_pointers(cmd_buffer, pipeline, descriptors_state, stage); in radv_flush_descriptors()
3175 assert(cmd_buffer->cs->cdw <= cdw_max); in radv_flush_descriptors()
3177 if (unlikely(cmd_buffer->device->trace_bo)) in radv_flush_descriptors()
3178 radv_save_descriptors(cmd_buffer, bind_point); in radv_flush_descriptors()
3190 radv_flush_constants(struct radv_cmd_buffer *cmd_buffer, VkShaderStageFlags stages, in radv_flush_constants() argument
3194 radv_get_descriptors_state(cmd_buffer, bind_point); in radv_flush_constants()
3203 stages &= cmd_buffer->push_constant_stages; in radv_flush_constants()
3232 radv_emit_inline_push_consts(cmd_buffer, pipeline, stage, AC_UD_INLINE_PUSH_CONSTANTS, in radv_flush_constants()
3233 (uint32_t *)&cmd_buffer->push_constants[base * 4]); in radv_flush_constants()
3238 … cmd_buffer, pipeline->push_constant_size + 16 * pipeline->dynamic_offset_count, &offset, in radv_flush_constants()
3242 memcpy(ptr, cmd_buffer->push_constants, pipeline->push_constant_size); in radv_flush_constants()
3246 va = radv_buffer_get_va(cmd_buffer->upload.upload_bo); in radv_flush_constants()
3250 radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, MESA_SHADER_STAGES * 4); in radv_flush_constants()
3259 radv_emit_userdata_address(cmd_buffer, pipeline, stage, AC_UD_PUSH_CONSTANTS, va); in radv_flush_constants()
3264 assert(cmd_buffer->cs->cdw <= cdw_max); in radv_flush_constants()
3267 cmd_buffer->push_constant_stages &= ~stages; in radv_flush_constants()
3268 cmd_buffer->push_constant_stages |= dirty_stages; in radv_flush_constants()
3305 radv_flush_vertex_descriptors(struct radv_cmd_buffer *cmd_buffer, bool pipeline_is_dirty) in radv_flush_vertex_descriptors() argument
3307 if ((pipeline_is_dirty || (cmd_buffer->state.dirty & RADV_CMD_DIRTY_VERTEX_BUFFER)) && in radv_flush_vertex_descriptors()
3308 cmd_buffer->state.pipeline->vb_desc_usage_mask) { in radv_flush_vertex_descriptors()
3309 struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; in radv_flush_vertex_descriptors()
3311 enum chip_class chip = cmd_buffer->device->physical_device->rad_info.chip_class; in radv_flush_vertex_descriptors()
3318 vs_shader->info.vs.dynamic_inputs ? &cmd_buffer->state.dynamic_vs_input : NULL; in radv_flush_vertex_descriptors()
3321 … if (!radv_cmd_buffer_upload_alloc(cmd_buffer, pipeline->vb_desc_alloc_size, &vb_offset, &vb_ptr)) in radv_flush_vertex_descriptors()
3331 vs_state ? cmd_buffer->state.dynamic_vs_input.bindings[i] in radv_flush_vertex_descriptors()
3333 struct radv_buffer *buffer = cmd_buffer->vertex_bindings[binding].buffer; in radv_flush_vertex_descriptors()
3375 offset = cmd_buffer->vertex_bindings[binding].offset; in radv_flush_vertex_descriptors()
3380 if (cmd_buffer->vertex_bindings[binding].size) { in radv_flush_vertex_descriptors()
3381 num_records = cmd_buffer->vertex_bindings[binding].size; in radv_flush_vertex_descriptors()
3387 stride = cmd_buffer->vertex_bindings[binding].stride; in radv_flush_vertex_descriptors()
3449 va = radv_buffer_get_va(cmd_buffer->upload.upload_bo); in radv_flush_vertex_descriptors()
3452 radv_emit_userdata_address(cmd_buffer, pipeline, MESA_SHADER_VERTEX, AC_UD_VS_VERTEX_BUFFERS, in radv_flush_vertex_descriptors()
3455 cmd_buffer->state.vb_va = va; in radv_flush_vertex_descriptors()
3456 cmd_buffer->state.prefetch_L2_mask |= RADV_PREFETCH_VBO_DESCRIPTORS; in radv_flush_vertex_descriptors()
3458 if (unlikely(cmd_buffer->device->trace_bo)) in radv_flush_vertex_descriptors()
3459 radv_save_vertex_descriptors(cmd_buffer, (uintptr_t)vb_ptr); in radv_flush_vertex_descriptors()
3461 cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_VERTEX_BUFFER; in radv_flush_vertex_descriptors()
3465 radv_emit_streamout_buffers(struct radv_cmd_buffer *cmd_buffer, uint64_t va) in radv_emit_streamout_buffers() argument
3467 struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; in radv_emit_streamout_buffers()
3481 radv_emit_shader_pointer(cmd_buffer->device, cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, va, in radv_emit_streamout_buffers()
3490 radv_emit_shader_pointer(cmd_buffer->device, cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, in radv_emit_streamout_buffers()
3497 radv_flush_streamout_descriptors(struct radv_cmd_buffer *cmd_buffer) in radv_flush_streamout_descriptors() argument
3499 if (cmd_buffer->state.dirty & RADV_CMD_DIRTY_STREAMOUT_BUFFER) { in radv_flush_streamout_descriptors()
3500 struct radv_streamout_binding *sb = cmd_buffer->streamout_bindings; in radv_flush_streamout_descriptors()
3501 struct radv_streamout_state *so = &cmd_buffer->state.streamout; in radv_flush_streamout_descriptors()
3507 if (!radv_cmd_buffer_upload_alloc(cmd_buffer, MAX_SO_BUFFERS * 16, &so_offset, &so_ptr)) in radv_flush_streamout_descriptors()
3533 if (cmd_buffer->device->physical_device->use_ngg_streamout) in radv_flush_streamout_descriptors()
3540 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10) { in radv_flush_streamout_descriptors()
3553 va = radv_buffer_get_va(cmd_buffer->upload.upload_bo); in radv_flush_streamout_descriptors()
3556 radv_emit_streamout_buffers(cmd_buffer, va); in radv_flush_streamout_descriptors()
3559 cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_STREAMOUT_BUFFER; in radv_flush_streamout_descriptors()
3563 radv_flush_ngg_gs_state(struct radv_cmd_buffer *cmd_buffer) in radv_flush_ngg_gs_state() argument
3565 struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; in radv_flush_ngg_gs_state()
3577 if (cmd_buffer->state.active_pipeline_gds_queries || in radv_flush_ngg_gs_state()
3578 (cmd_buffer->state.inherited_pipeline_statistics & in radv_flush_ngg_gs_state()
3586 radeon_set_sh_reg(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, ngg_gs_state); in radv_flush_ngg_gs_state()
3590 radv_upload_graphics_shader_descriptors(struct radv_cmd_buffer *cmd_buffer, bool pipeline_is_dirty) in radv_upload_graphics_shader_descriptors() argument
3592 radv_flush_vertex_descriptors(cmd_buffer, pipeline_is_dirty); in radv_upload_graphics_shader_descriptors()
3593 radv_flush_streamout_descriptors(cmd_buffer); in radv_upload_graphics_shader_descriptors()
3594 radv_flush_descriptors(cmd_buffer, VK_SHADER_STAGE_ALL_GRAPHICS, cmd_buffer->state.pipeline, in radv_upload_graphics_shader_descriptors()
3596 radv_flush_constants(cmd_buffer, VK_SHADER_STAGE_ALL_GRAPHICS, cmd_buffer->state.pipeline, in radv_upload_graphics_shader_descriptors()
3598 radv_flush_ngg_gs_state(cmd_buffer); in radv_upload_graphics_shader_descriptors()
3643 radv_get_primitive_reset_index(struct radv_cmd_buffer *cmd_buffer) in radv_get_primitive_reset_index() argument
3645 switch (cmd_buffer->state.index_type) { in radv_get_primitive_reset_index()
3658 si_emit_ia_multi_vgt_param(struct radv_cmd_buffer *cmd_buffer, bool instanced_draw, in si_emit_ia_multi_vgt_param() argument
3662 struct radeon_info *info = &cmd_buffer->device->physical_device->rad_info; in si_emit_ia_multi_vgt_param()
3663 struct radv_cmd_state *state = &cmd_buffer->state; in si_emit_ia_multi_vgt_param()
3666 struct radeon_cmdbuf *cs = cmd_buffer->cs; in si_emit_ia_multi_vgt_param()
3670 si_get_ia_multi_vgt_param(cmd_buffer, instanced_draw, indirect_draw, count_from_stream_output, in si_emit_ia_multi_vgt_param()
3675 radeon_set_uconfig_reg_idx(cmd_buffer->device->physical_device, cs, in si_emit_ia_multi_vgt_param()
3687 radv_emit_draw_registers(struct radv_cmd_buffer *cmd_buffer, const struct radv_draw_info *draw_info) in radv_emit_draw_registers() argument
3689 struct radeon_info *info = &cmd_buffer->device->physical_device->rad_info; in radv_emit_draw_registers()
3690 struct radv_cmd_state *state = &cmd_buffer->state; in radv_emit_draw_registers()
3691 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_emit_draw_registers()
3695 si_emit_ia_multi_vgt_param(cmd_buffer, draw_info->instance_count > 1, draw_info->indirect, in radv_emit_draw_registers()
3701 uint32_t primitive_reset_index = radv_get_primitive_reset_index(cmd_buffer); in radv_emit_draw_registers()
3724 radv_cs_add_buffer(cmd_buffer->device->ws, cs, draw_info->strmout_buffer->bo); in radv_emit_draw_registers()
3729 radv_stage_flush(struct radv_cmd_buffer *cmd_buffer, VkPipelineStageFlags src_stage_mask) in radv_stage_flush() argument
3736 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH; in radv_stage_flush()
3744 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_PS_PARTIAL_FLUSH; in radv_stage_flush()
3752 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_VS_PARTIAL_FLUSH; in radv_stage_flush()
3794 radv_src_access_flush(struct radv_cmd_buffer *cmd_buffer, VkAccessFlags src_flags, in radv_src_access_flush() argument
3873 radv_dst_access_flush(struct radv_cmd_buffer *cmd_buffer, VkAccessFlags dst_flags, in radv_dst_access_flush() argument
3896 can_skip_buffer_l2_flushes(cmd_buffer->device) && !cmd_buffer->state.rb_noncoherent_dirty; in radv_dst_access_flush()
3923 if (!cmd_buffer->device->physical_device->use_llvm && !image) in radv_dst_access_flush()
3933 if (cmd_buffer->device->physical_device->rad_info.chip_class < GFX9) in radv_dst_access_flush()
3975 radv_emit_subpass_barrier(struct radv_cmd_buffer *cmd_buffer, const struct radv_subpass_barrier *ba… in radv_emit_subpass_barrier() argument
3977 struct radv_framebuffer *fb = cmd_buffer->state.framebuffer; in radv_emit_subpass_barrier()
3980 cmd_buffer->state.flush_bits |= in radv_emit_subpass_barrier()
3981 radv_src_access_flush(cmd_buffer, barrier->src_access_mask, fb->attachments[i]->image); in radv_emit_subpass_barrier()
3984 cmd_buffer->state.flush_bits |= in radv_emit_subpass_barrier()
3985 radv_src_access_flush(cmd_buffer, barrier->src_access_mask, NULL); in radv_emit_subpass_barrier()
3988 radv_stage_flush(cmd_buffer, barrier->src_stage_mask); in radv_emit_subpass_barrier()
3992 cmd_buffer->state.flush_bits |= in radv_emit_subpass_barrier()
3993 radv_dst_access_flush(cmd_buffer, barrier->dst_access_mask, fb->attachments[i]->image); in radv_emit_subpass_barrier()
3996 cmd_buffer->state.flush_bits |= in radv_emit_subpass_barrier()
3997 radv_dst_access_flush(cmd_buffer, barrier->dst_access_mask, NULL); in radv_emit_subpass_barrier()
4002 radv_get_subpass_id(struct radv_cmd_buffer *cmd_buffer) in radv_get_subpass_id() argument
4004 struct radv_cmd_state *state = &cmd_buffer->state; in radv_get_subpass_id()
4015 radv_get_attachment_sample_locations(struct radv_cmd_buffer *cmd_buffer, uint32_t att_idx, in radv_get_attachment_sample_locations() argument
4018 struct radv_cmd_state *state = &cmd_buffer->state; in radv_get_attachment_sample_locations()
4019 uint32_t subpass_id = radv_get_subpass_id(cmd_buffer); in radv_get_attachment_sample_locations()
4055 radv_handle_subpass_image_transition(struct radv_cmd_buffer *cmd_buffer, in radv_handle_subpass_image_transition() argument
4059 struct radv_image_view *view = cmd_buffer->state.attachments[idx].iview; in radv_handle_subpass_image_transition()
4066 range.layerCount = cmd_buffer->state.framebuffer->layers; in radv_handle_subpass_image_transition()
4068 if (cmd_buffer->state.subpass->view_mask) { in radv_handle_subpass_image_transition()
4076 range.layerCount = util_last_bit(cmd_buffer->state.subpass->view_mask); in radv_handle_subpass_image_transition()
4082 sample_locs = radv_get_attachment_sample_locations(cmd_buffer, idx, begin_subpass); in radv_handle_subpass_image_transition()
4086 if ((cmd_buffer->state.attachments[idx].current_layout != in radv_handle_subpass_image_transition()
4087 cmd_buffer->state.attachments[idx].current_stencil_layout) || in radv_handle_subpass_image_transition()
4099 radv_handle_image_transition(cmd_buffer, view->image, in radv_handle_subpass_image_transition()
4100 cmd_buffer->state.attachments[idx].current_layout, in radv_handle_subpass_image_transition()
4101 cmd_buffer->state.attachments[idx].current_in_render_loop, in radv_handle_subpass_image_transition()
4107 cmd_buffer, view->image, cmd_buffer->state.attachments[idx].current_stencil_layout, in radv_handle_subpass_image_transition()
4108 cmd_buffer->state.attachments[idx].current_in_render_loop, att.stencil_layout, in radv_handle_subpass_image_transition()
4111 radv_handle_image_transition(cmd_buffer, view->image, in radv_handle_subpass_image_transition()
4112 cmd_buffer->state.attachments[idx].current_layout, in radv_handle_subpass_image_transition()
4113 cmd_buffer->state.attachments[idx].current_in_render_loop, in radv_handle_subpass_image_transition()
4117 cmd_buffer->state.attachments[idx].current_layout = att.layout; in radv_handle_subpass_image_transition()
4118 cmd_buffer->state.attachments[idx].current_stencil_layout = att.stencil_layout; in radv_handle_subpass_image_transition()
4119 cmd_buffer->state.attachments[idx].current_in_render_loop = att.in_render_loop; in radv_handle_subpass_image_transition()
4123 radv_cmd_buffer_set_subpass(struct radv_cmd_buffer *cmd_buffer, const struct radv_subpass *subpass) in radv_cmd_buffer_set_subpass() argument
4125 cmd_buffer->state.subpass = subpass; in radv_cmd_buffer_set_subpass()
4127 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_FRAMEBUFFER; in radv_cmd_buffer_set_subpass()
4131 radv_cmd_state_setup_sample_locations(struct radv_cmd_buffer *cmd_buffer, in radv_cmd_state_setup_sample_locations() argument
4137 struct radv_cmd_state *state = &cmd_buffer->state; in radv_cmd_state_setup_sample_locations()
4148 struct radv_image *image = cmd_buffer->state.attachments[att_idx].iview->image; in radv_cmd_state_setup_sample_locations()
4175 vk_alloc(&cmd_buffer->pool->alloc, in radv_cmd_state_setup_sample_locations()
4179 cmd_buffer->record_result = VK_ERROR_OUT_OF_HOST_MEMORY; in radv_cmd_state_setup_sample_locations()
4180 return cmd_buffer->record_result; in radv_cmd_state_setup_sample_locations()
4205 radv_cmd_state_setup_attachments(struct radv_cmd_buffer *cmd_buffer, struct radv_render_pass *pass, in radv_cmd_state_setup_attachments() argument
4209 struct radv_cmd_state *state = &cmd_buffer->state; in radv_cmd_state_setup_attachments()
4222 vk_alloc(&cmd_buffer->pool->alloc, pass->attachment_count * sizeof(state->attachments[0]), 8, in radv_cmd_state_setup_attachments()
4225 cmd_buffer->record_result = VK_ERROR_OUT_OF_HOST_MEMORY; in radv_cmd_state_setup_attachments()
4226 return cmd_buffer->record_result; in radv_cmd_state_setup_attachments()
4276 radv_initialise_ds_surface(cmd_buffer->device, &state->attachments[i].ds, iview); in radv_cmd_state_setup_attachments()
4278 radv_initialise_color_surface(cmd_buffer->device, &state->attachments[i].cb, iview); in radv_cmd_state_setup_attachments()
4298 struct radv_cmd_buffer *cmd_buffer = in radv_AllocateCommandBuffers() local
4301 list_del(&cmd_buffer->pool_link); in radv_AllocateCommandBuffers()
4302 list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers); in radv_AllocateCommandBuffers()
4304 result = radv_reset_cmd_buffer(cmd_buffer); in radv_AllocateCommandBuffers()
4305 cmd_buffer->level = pAllocateInfo->level; in radv_AllocateCommandBuffers()
4306 vk_command_buffer_finish(&cmd_buffer->vk); in radv_AllocateCommandBuffers()
4308 vk_command_buffer_init(&cmd_buffer->vk, &device->vk); in radv_AllocateCommandBuffers()
4312 pCommandBuffers[i] = radv_cmd_buffer_to_handle(cmd_buffer); in radv_AllocateCommandBuffers()
4343 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, pCommandBuffers[i]); in radv_FreeCommandBuffers()
4345 if (cmd_buffer) { in radv_FreeCommandBuffers()
4346 if (cmd_buffer->pool) { in radv_FreeCommandBuffers()
4347 list_del(&cmd_buffer->pool_link); in radv_FreeCommandBuffers()
4348 list_addtail(&cmd_buffer->pool_link, &cmd_buffer->pool->free_cmd_buffers); in radv_FreeCommandBuffers()
4350 radv_destroy_cmd_buffer(cmd_buffer); in radv_FreeCommandBuffers()
4358 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_ResetCommandBuffer()
4359 return radv_reset_cmd_buffer(cmd_buffer); in radv_ResetCommandBuffer()
4365 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_BeginCommandBuffer()
4368 if (cmd_buffer->status != RADV_CMD_BUFFER_STATUS_INITIAL) { in radv_BeginCommandBuffer()
4372 result = radv_reset_cmd_buffer(cmd_buffer); in radv_BeginCommandBuffer()
4377 memset(&cmd_buffer->state, 0, sizeof(cmd_buffer->state)); in radv_BeginCommandBuffer()
4378 cmd_buffer->state.last_primitive_reset_en = -1; in radv_BeginCommandBuffer()
4379 cmd_buffer->state.last_index_type = -1; in radv_BeginCommandBuffer()
4380 cmd_buffer->state.last_num_instances = -1; in radv_BeginCommandBuffer()
4381 cmd_buffer->state.last_vertex_offset = -1; in radv_BeginCommandBuffer()
4382 cmd_buffer->state.last_first_instance = -1; in radv_BeginCommandBuffer()
4383 cmd_buffer->state.last_drawid = -1; in radv_BeginCommandBuffer()
4384 cmd_buffer->state.predication_type = -1; in radv_BeginCommandBuffer()
4385 cmd_buffer->state.last_sx_ps_downconvert = -1; in radv_BeginCommandBuffer()
4386 cmd_buffer->state.last_sx_blend_opt_epsilon = -1; in radv_BeginCommandBuffer()
4387 cmd_buffer->state.last_sx_blend_opt_control = -1; in radv_BeginCommandBuffer()
4388 cmd_buffer->state.last_nggc_settings = -1; in radv_BeginCommandBuffer()
4389 cmd_buffer->state.last_nggc_settings_sgpr_idx = -1; in radv_BeginCommandBuffer()
4390 cmd_buffer->usage_flags = pBeginInfo->flags; in radv_BeginCommandBuffer()
4392 if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY && in radv_BeginCommandBuffer()
4395 cmd_buffer->state.framebuffer = in radv_BeginCommandBuffer()
4397 cmd_buffer->state.pass = in radv_BeginCommandBuffer()
4401 &cmd_buffer->state.pass->subpasses[pBeginInfo->pInheritanceInfo->subpass]; in radv_BeginCommandBuffer()
4403 if (cmd_buffer->state.framebuffer) { in radv_BeginCommandBuffer()
4404 result = radv_cmd_state_setup_attachments(cmd_buffer, cmd_buffer->state.pass, NULL, NULL); in radv_BeginCommandBuffer()
4409 cmd_buffer->state.inherited_pipeline_statistics = in radv_BeginCommandBuffer()
4412 radv_cmd_buffer_set_subpass(cmd_buffer, subpass); in radv_BeginCommandBuffer()
4415 if (unlikely(cmd_buffer->device->trace_bo)) in radv_BeginCommandBuffer()
4416 radv_cmd_buffer_trace_emit(cmd_buffer); in radv_BeginCommandBuffer()
4418 radv_describe_begin_cmd_buffer(cmd_buffer); in radv_BeginCommandBuffer()
4420 cmd_buffer->status = RADV_CMD_BUFFER_STATUS_RECORDING; in radv_BeginCommandBuffer()
4440 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdBindVertexBuffers2EXT()
4441 struct radv_vertex_binding *vb = cmd_buffer->vertex_bindings; in radv_CmdBindVertexBuffers2EXT()
4442 struct radv_vs_input_state *state = &cmd_buffer->state.dynamic_vs_input; in radv_CmdBindVertexBuffers2EXT()
4449 cmd_buffer->state.vbo_misaligned_mask = state->misaligned_mask; in radv_CmdBindVertexBuffers2EXT()
4450 enum chip_class chip = cmd_buffer->device->physical_device->rad_info.chip_class; in radv_CmdBindVertexBuffers2EXT()
4471 cmd_buffer->state.vbo_misaligned_mask &= ~bit; in radv_CmdBindVertexBuffers2EXT()
4472 cmd_buffer->state.vbo_bound_mask &= ~bit; in radv_CmdBindVertexBuffers2EXT()
4474 cmd_buffer->state.vbo_bound_mask |= bit; in radv_CmdBindVertexBuffers2EXT()
4477 cmd_buffer->state.vbo_misaligned_mask |= bit; in radv_CmdBindVertexBuffers2EXT()
4479 cmd_buffer->state.vbo_misaligned_mask &= ~bit; in radv_CmdBindVertexBuffers2EXT()
4483 cmd_buffer->state.vbo_misaligned_mask |= bit; in radv_CmdBindVertexBuffers2EXT()
4491 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, vb[idx].buffer->bo); in radv_CmdBindVertexBuffers2EXT()
4500 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_VERTEX_BUFFER | in radv_CmdBindVertexBuffers2EXT()
4538 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdBindIndexBuffer()
4541 if (cmd_buffer->state.index_buffer == index_buffer && cmd_buffer->state.index_offset == offset && in radv_CmdBindIndexBuffer()
4542 cmd_buffer->state.index_type == indexType) { in radv_CmdBindIndexBuffer()
4547 cmd_buffer->state.index_buffer = index_buffer; in radv_CmdBindIndexBuffer()
4548 cmd_buffer->state.index_offset = offset; in radv_CmdBindIndexBuffer()
4549 cmd_buffer->state.index_type = vk_to_index_type(indexType); in radv_CmdBindIndexBuffer()
4550 cmd_buffer->state.index_va = radv_buffer_get_va(index_buffer->bo); in radv_CmdBindIndexBuffer()
4551 cmd_buffer->state.index_va += index_buffer->offset + offset; in radv_CmdBindIndexBuffer()
4554 cmd_buffer->state.max_index_count = (index_buffer->size - offset) / index_size; in radv_CmdBindIndexBuffer()
4555 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_INDEX_BUFFER; in radv_CmdBindIndexBuffer()
4556 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, index_buffer->bo); in radv_CmdBindIndexBuffer()
4560 radv_bind_descriptor_set(struct radv_cmd_buffer *cmd_buffer, VkPipelineBindPoint bind_point, in radv_bind_descriptor_set() argument
4563 struct radeon_winsys *ws = cmd_buffer->device->ws; in radv_bind_descriptor_set()
4565 radv_set_descriptor_set(cmd_buffer, bind_point, set, idx); in radv_bind_descriptor_set()
4569 if (!cmd_buffer->device->use_global_bo_list) { in radv_bind_descriptor_set()
4572 radv_cs_add_buffer(ws, cmd_buffer->cs, set->descriptors[j]); in radv_bind_descriptor_set()
4576 radv_cs_add_buffer(ws, cmd_buffer->cs, set->header.bo); in radv_bind_descriptor_set()
4585 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdBindDescriptorSets()
4590 cmd_buffer->device->instance->debug_flags & RADV_DEBUG_NO_DYNAMIC_BOUNDS; in radv_CmdBindDescriptorSets()
4592 radv_get_descriptors_state(cmd_buffer, pipelineBindPoint); in radv_CmdBindDescriptorSets()
4602 radv_bind_descriptor_set(cmd_buffer, pipelineBindPoint, set, set_idx); in radv_CmdBindDescriptorSets()
4622 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10) { in radv_CmdBindDescriptorSets()
4631 cmd_buffer->push_constant_stages |= layout->set[set_idx].dynamic_offset_stages; in radv_CmdBindDescriptorSets()
4637 radv_init_push_descriptor_set(struct radv_cmd_buffer *cmd_buffer, struct radv_descriptor_set *set, in radv_init_push_descriptor_set() argument
4642 radv_get_descriptors_state(cmd_buffer, bind_point); in radv_init_push_descriptor_set()
4656 cmd_buffer->record_result = VK_ERROR_OUT_OF_HOST_MEMORY; in radv_init_push_descriptor_set()
4667 radv_meta_push_descriptor_set(struct radv_cmd_buffer *cmd_buffer, in radv_meta_push_descriptor_set() argument
4674 (struct radv_descriptor_set *)&cmd_buffer->meta_push_descriptors; in radv_meta_push_descriptor_set()
4683 if (!radv_cmd_buffer_upload_alloc(cmd_buffer, push_set->header.size, &bo_offset, in radv_meta_push_descriptor_set()
4687 push_set->header.va = radv_buffer_get_va(cmd_buffer->upload.upload_bo); in radv_meta_push_descriptor_set()
4690 radv_update_descriptor_sets(cmd_buffer->device, cmd_buffer, in radv_meta_push_descriptor_set()
4694 radv_set_descriptor_set(cmd_buffer, pipelineBindPoint, push_set, set); in radv_meta_push_descriptor_set()
4702 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdPushDescriptorSetKHR()
4705 radv_get_descriptors_state(cmd_buffer, pipelineBindPoint); in radv_CmdPushDescriptorSetKHR()
4711 if (!radv_init_push_descriptor_set(cmd_buffer, push_set, layout->set[set].layout, in radv_CmdPushDescriptorSetKHR()
4723 radv_update_descriptor_sets(cmd_buffer->device, cmd_buffer, in radv_CmdPushDescriptorSetKHR()
4727 radv_set_descriptor_set(cmd_buffer, pipelineBindPoint, push_set, set); in radv_CmdPushDescriptorSetKHR()
4736 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdPushDescriptorSetWithTemplateKHR()
4740 radv_get_descriptors_state(cmd_buffer, templ->bind_point); in radv_CmdPushDescriptorSetWithTemplateKHR()
4746 if (!radv_init_push_descriptor_set(cmd_buffer, push_set, layout->set[set].layout, in radv_CmdPushDescriptorSetWithTemplateKHR()
4750 radv_update_descriptor_set_with_template(cmd_buffer->device, cmd_buffer, push_set, in radv_CmdPushDescriptorSetWithTemplateKHR()
4753 radv_set_descriptor_set(cmd_buffer, templ->bind_point, push_set, set); in radv_CmdPushDescriptorSetWithTemplateKHR()
4762 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdPushConstants()
4763 memcpy(cmd_buffer->push_constants + offset, pValues, size); in radv_CmdPushConstants()
4764 cmd_buffer->push_constant_stages |= stageFlags; in radv_CmdPushConstants()
4770 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_EndCommandBuffer()
4772 radv_emit_mip_change_flush_default(cmd_buffer); in radv_EndCommandBuffer()
4774 if (cmd_buffer->queue_family_index != RADV_QUEUE_TRANSFER) { in radv_EndCommandBuffer()
4775 if (cmd_buffer->device->physical_device->rad_info.chip_class == GFX6) in radv_EndCommandBuffer()
4776 cmd_buffer->state.flush_bits |= in radv_EndCommandBuffer()
4782 cmd_buffer->state.flush_bits |= cmd_buffer->active_query_flush_bits; in radv_EndCommandBuffer()
4787 if (cmd_buffer->state.rb_noncoherent_dirty && can_skip_buffer_l2_flushes(cmd_buffer->device)) in radv_EndCommandBuffer()
4788 cmd_buffer->state.flush_bits |= radv_src_access_flush( in radv_EndCommandBuffer()
4789 cmd_buffer, in radv_EndCommandBuffer()
4797 if (cmd_buffer->gds_needed) in radv_EndCommandBuffer()
4798 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_PS_PARTIAL_FLUSH; in radv_EndCommandBuffer()
4800 si_emit_cache_flush(cmd_buffer); in radv_EndCommandBuffer()
4806 si_cp_dma_wait_for_idle(cmd_buffer); in radv_EndCommandBuffer()
4808 radv_describe_end_cmd_buffer(cmd_buffer); in radv_EndCommandBuffer()
4810 vk_free(&cmd_buffer->pool->alloc, cmd_buffer->state.attachments); in radv_EndCommandBuffer()
4811 vk_free(&cmd_buffer->pool->alloc, cmd_buffer->state.subpass_sample_locs); in radv_EndCommandBuffer()
4813 VkResult result = cmd_buffer->device->ws->cs_finalize(cmd_buffer->cs); in radv_EndCommandBuffer()
4815 return vk_error(cmd_buffer, result); in radv_EndCommandBuffer()
4817 cmd_buffer->status = RADV_CMD_BUFFER_STATUS_EXECUTABLE; in radv_EndCommandBuffer()
4819 return cmd_buffer->record_result; in radv_EndCommandBuffer()
4823 radv_emit_compute_pipeline(struct radv_cmd_buffer *cmd_buffer, struct radv_pipeline *pipeline) in radv_emit_compute_pipeline() argument
4825 if (!pipeline || pipeline == cmd_buffer->state.emitted_compute_pipeline) in radv_emit_compute_pipeline()
4830 cmd_buffer->state.emitted_compute_pipeline = pipeline; in radv_emit_compute_pipeline()
4832 radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, pipeline->cs.cdw); in radv_emit_compute_pipeline()
4833 radeon_emit_array(cmd_buffer->cs, pipeline->cs.buf, pipeline->cs.cdw); in radv_emit_compute_pipeline()
4835 cmd_buffer->compute_scratch_size_per_wave_needed = in radv_emit_compute_pipeline()
4836 MAX2(cmd_buffer->compute_scratch_size_per_wave_needed, pipeline->scratch_bytes_per_wave); in radv_emit_compute_pipeline()
4837 cmd_buffer->compute_scratch_waves_wanted = in radv_emit_compute_pipeline()
4838 MAX2(cmd_buffer->compute_scratch_waves_wanted, pipeline->max_waves); in radv_emit_compute_pipeline()
4840 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, in radv_emit_compute_pipeline()
4843 if (unlikely(cmd_buffer->device->trace_bo)) in radv_emit_compute_pipeline()
4844 radv_save_pipeline(cmd_buffer, pipeline); in radv_emit_compute_pipeline()
4848 radv_mark_descriptor_sets_dirty(struct radv_cmd_buffer *cmd_buffer, VkPipelineBindPoint bind_point) in radv_mark_descriptor_sets_dirty() argument
4851 radv_get_descriptors_state(cmd_buffer, bind_point); in radv_mark_descriptor_sets_dirty()
4860 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdBindPipeline()
4865 if (cmd_buffer->state.compute_pipeline == pipeline) in radv_CmdBindPipeline()
4867 radv_mark_descriptor_sets_dirty(cmd_buffer, pipelineBindPoint); in radv_CmdBindPipeline()
4869 cmd_buffer->state.compute_pipeline = pipeline; in radv_CmdBindPipeline()
4870 cmd_buffer->push_constant_stages |= VK_SHADER_STAGE_COMPUTE_BIT; in radv_CmdBindPipeline()
4873 if (cmd_buffer->state.rt_pipeline == pipeline) in radv_CmdBindPipeline()
4875 radv_mark_descriptor_sets_dirty(cmd_buffer, pipelineBindPoint); in radv_CmdBindPipeline()
4877 cmd_buffer->state.rt_pipeline = pipeline; in radv_CmdBindPipeline()
4878 cmd_buffer->push_constant_stages |= in radv_CmdBindPipeline()
4882 radv_set_rt_stack_size(cmd_buffer, cmd_buffer->state.rt_stack_size); in radv_CmdBindPipeline()
4885 if (cmd_buffer->state.pipeline == pipeline) in radv_CmdBindPipeline()
4887 radv_mark_descriptor_sets_dirty(cmd_buffer, pipelineBindPoint); in radv_CmdBindPipeline()
4890 !pipeline || !cmd_buffer->state.pipeline || in radv_CmdBindPipeline()
4891 cmd_buffer->state.pipeline->graphics.vtx_emit_num != pipeline->graphics.vtx_emit_num || in radv_CmdBindPipeline()
4892 cmd_buffer->state.pipeline->graphics.vtx_base_sgpr != pipeline->graphics.vtx_base_sgpr; in radv_CmdBindPipeline()
4893 cmd_buffer->state.pipeline = pipeline; in radv_CmdBindPipeline()
4897 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_PIPELINE | RADV_CMD_DIRTY_DYNAMIC_VERTEX_INPUT; in radv_CmdBindPipeline()
4898 cmd_buffer->push_constant_stages |= pipeline->active_stages; in radv_CmdBindPipeline()
4902 cmd_buffer->state.last_first_instance = -1; in radv_CmdBindPipeline()
4903 cmd_buffer->state.last_vertex_offset = -1; in radv_CmdBindPipeline()
4904 cmd_buffer->state.last_drawid = -1; in radv_CmdBindPipeline()
4908 cmd_buffer->state.prefetch_L2_mask |= RADV_PREFETCH_SHADERS; in radv_CmdBindPipeline()
4910 if (cmd_buffer->device->physical_device->rad_info.has_vgt_flush_ngg_legacy_bug && in radv_CmdBindPipeline()
4911 cmd_buffer->state.emitted_pipeline && in radv_CmdBindPipeline()
4912 cmd_buffer->state.emitted_pipeline->graphics.is_ngg && in radv_CmdBindPipeline()
4913 !cmd_buffer->state.pipeline->graphics.is_ngg) { in radv_CmdBindPipeline()
4919 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_VGT_FLUSH; in radv_CmdBindPipeline()
4922 radv_bind_dynamic_state(cmd_buffer, &pipeline->dynamic_state); in radv_CmdBindPipeline()
4923 radv_bind_streamout_state(cmd_buffer, pipeline); in radv_CmdBindPipeline()
4925 if (pipeline->graphics.esgs_ring_size > cmd_buffer->esgs_ring_size_needed) in radv_CmdBindPipeline()
4926 cmd_buffer->esgs_ring_size_needed = pipeline->graphics.esgs_ring_size; in radv_CmdBindPipeline()
4927 if (pipeline->graphics.gsvs_ring_size > cmd_buffer->gsvs_ring_size_needed) in radv_CmdBindPipeline()
4928 cmd_buffer->gsvs_ring_size_needed = pipeline->graphics.gsvs_ring_size; in radv_CmdBindPipeline()
4931 cmd_buffer->tess_rings_needed = true; in radv_CmdBindPipeline()
4943 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetViewport()
4944 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetViewport()
4974 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetScissor()
4975 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetScissor()
4999 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetLineWidth()
5001 if (cmd_buffer->state.dynamic.line_width == lineWidth) in radv_CmdSetLineWidth()
5004 cmd_buffer->state.dynamic.line_width = lineWidth; in radv_CmdSetLineWidth()
5005 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_LINE_WIDTH; in radv_CmdSetLineWidth()
5012 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetDepthBias()
5013 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetDepthBias()
5031 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetBlendConstants()
5032 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetBlendConstants()
5045 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetDepthBounds()
5046 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetDepthBounds()
5063 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetStencilCompareMask()
5064 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetStencilCompareMask()
5085 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetStencilWriteMask()
5086 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetStencilWriteMask()
5107 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetStencilReference()
5108 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetStencilReference()
5118 cmd_buffer->state.dynamic.stencil_reference.front = reference; in radv_CmdSetStencilReference()
5120 cmd_buffer->state.dynamic.stencil_reference.back = reference; in radv_CmdSetStencilReference()
5122 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE; in radv_CmdSetStencilReference()
5129 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetDiscardRectangleEXT()
5130 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetDiscardRectangleEXT()
5151 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetSampleLocationsEXT()
5152 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetSampleLocationsEXT()
5169 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetLineStippleEXT()
5170 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetLineStippleEXT()
5185 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetCullModeEXT()
5186 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetCullModeEXT()
5199 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetFrontFaceEXT()
5200 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetFrontFaceEXT()
5214 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetPrimitiveTopologyEXT()
5215 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetPrimitiveTopologyEXT()
5244 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetDepthTestEnableEXT()
5245 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetDepthTestEnableEXT()
5258 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetDepthWriteEnableEXT()
5259 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetDepthWriteEnableEXT()
5272 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetDepthCompareOpEXT()
5273 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetDepthCompareOpEXT()
5286 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetDepthBoundsTestEnableEXT()
5287 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetDepthBoundsTestEnableEXT()
5300 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetStencilTestEnableEXT()
5301 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetStencilTestEnableEXT()
5316 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetStencilOpEXT()
5317 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetStencilOpEXT()
5352 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetFragmentShadingRateKHR()
5353 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetFragmentShadingRateKHR()
5371 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetDepthBiasEnableEXT()
5372 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetDepthBiasEnableEXT()
5385 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetPrimitiveRestartEnableEXT()
5386 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetPrimitiveRestartEnableEXT()
5400 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetRasterizerDiscardEnableEXT()
5401 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetRasterizerDiscardEnableEXT()
5420 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetLogicOpEXT()
5421 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetLogicOpEXT()
5436 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetColorWriteEnableEXT()
5437 struct radv_cmd_state *state = &cmd_buffer->state; in radv_CmdSetColorWriteEnableEXT()
5460 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetVertexInputEXT()
5461 struct radv_vs_input_state *state = &cmd_buffer->state.dynamic_vs_input; in radv_CmdSetVertexInputEXT()
5467 cmd_buffer->state.vbo_misaligned_mask = 0; in radv_CmdSetVertexInputEXT()
5471 enum chip_class chip = cmd_buffer->device->physical_device->rad_info.chip_class; in radv_CmdSetVertexInputEXT()
5489 cmd_buffer->vertex_bindings[attrib->binding].stride = binding->stride; in radv_CmdSetVertexInputEXT()
5492 radv_translate_vertex_format(cmd_buffer->device->physical_device, attrib->format, format_desc, in radv_CmdSetVertexInputEXT()
5502 struct radv_vertex_binding *vb = cmd_buffer->vertex_bindings; in radv_CmdSetVertexInputEXT()
5506 if (cmd_buffer->state.vbo_bound_mask & bit) in radv_CmdSetVertexInputEXT()
5507 cmd_buffer->state.vbo_misaligned_mask |= bit; in radv_CmdSetVertexInputEXT()
5510 if (cmd_buffer->state.vbo_bound_mask & bit && in radv_CmdSetVertexInputEXT()
5512 cmd_buffer->state.vbo_misaligned_mask |= bit; in radv_CmdSetVertexInputEXT()
5525 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_VERTEX_BUFFER | in radv_CmdSetVertexInputEXT()
5681 list_for_each_entry_safe(struct radv_cmd_buffer, cmd_buffer, &pool->cmd_buffers, pool_link) in radv_DestroyCommandPool()
5683 radv_destroy_cmd_buffer(cmd_buffer); in radv_DestroyCommandPool()
5686 list_for_each_entry_safe(struct radv_cmd_buffer, cmd_buffer, &pool->free_cmd_buffers, pool_link) in radv_DestroyCommandPool()
5688 radv_destroy_cmd_buffer(cmd_buffer); in radv_DestroyCommandPool()
5701 list_for_each_entry(struct radv_cmd_buffer, cmd_buffer, &pool->cmd_buffers, pool_link) in radv_ResetCommandPool()
5703 result = radv_reset_cmd_buffer(cmd_buffer); in radv_ResetCommandPool()
5719 list_for_each_entry_safe(struct radv_cmd_buffer, cmd_buffer, &pool->free_cmd_buffers, pool_link) in radv_TrimCommandPool()
5721 radv_destroy_cmd_buffer(cmd_buffer); in radv_TrimCommandPool()
5726 radv_cmd_buffer_begin_subpass(struct radv_cmd_buffer *cmd_buffer, uint32_t subpass_id) in radv_cmd_buffer_begin_subpass() argument
5728 struct radv_cmd_state *state = &cmd_buffer->state; in radv_cmd_buffer_begin_subpass()
5731 ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 4096); in radv_cmd_buffer_begin_subpass()
5733 radv_emit_subpass_barrier(cmd_buffer, &subpass->start_barrier); in radv_cmd_buffer_begin_subpass()
5735 radv_cmd_buffer_set_subpass(cmd_buffer, subpass); in radv_cmd_buffer_begin_subpass()
5737 radv_describe_barrier_start(cmd_buffer, RGP_BARRIER_EXTERNAL_RENDER_PASS_SYNC); in radv_cmd_buffer_begin_subpass()
5744 radv_handle_subpass_image_transition(cmd_buffer, subpass->attachments[i], true); in radv_cmd_buffer_begin_subpass()
5749 struct radv_image_view *vrs_iview = cmd_buffer->state.attachments[idx].iview; in radv_cmd_buffer_begin_subpass()
5756 struct radv_image_view *ds_iview = cmd_buffer->state.attachments[ds_idx].iview; in radv_cmd_buffer_begin_subpass()
5769 … radv_buffer_init(&htile_buffer, cmd_buffer->device, ds_image->bo, htile_size, htile_offset); in radv_cmd_buffer_begin_subpass()
5772 radv_copy_vrs_htile(cmd_buffer, vrs_iview->image, &extent, ds_image, &htile_buffer, true); in radv_cmd_buffer_begin_subpass()
5779 struct radv_framebuffer *fb = cmd_buffer->state.framebuffer; in radv_cmd_buffer_begin_subpass()
5780 struct radv_image *ds_image = radv_cmd_buffer_get_vrs_image(cmd_buffer); in radv_cmd_buffer_begin_subpass()
5784 struct radv_buffer *htile_buffer = cmd_buffer->device->vrs.buffer; in radv_cmd_buffer_begin_subpass()
5792 … radv_copy_vrs_htile(cmd_buffer, vrs_iview->image, &extent, ds_image, htile_buffer, false); in radv_cmd_buffer_begin_subpass()
5797 radv_describe_barrier_end(cmd_buffer); in radv_cmd_buffer_begin_subpass()
5799 radv_cmd_buffer_clear_subpass(cmd_buffer); in radv_cmd_buffer_begin_subpass()
5801 assert(cmd_buffer->cs->cdw <= cdw_max); in radv_cmd_buffer_begin_subpass()
5805 radv_mark_noncoherent_rb(struct radv_cmd_buffer *cmd_buffer) in radv_mark_noncoherent_rb() argument
5807 const struct radv_subpass *subpass = cmd_buffer->state.subpass; in radv_mark_noncoherent_rb()
5810 if (!cmd_buffer->state.attachments) { in radv_mark_noncoherent_rb()
5811 cmd_buffer->state.rb_noncoherent_dirty = true; in radv_mark_noncoherent_rb()
5819 if (!cmd_buffer->state.attachments[a].iview->image->l2_coherent) { in radv_mark_noncoherent_rb()
5820 cmd_buffer->state.rb_noncoherent_dirty = true; in radv_mark_noncoherent_rb()
5825 !cmd_buffer->state.attachments[subpass->depth_stencil_attachment->attachment] in radv_mark_noncoherent_rb()
5827 cmd_buffer->state.rb_noncoherent_dirty = true; in radv_mark_noncoherent_rb()
5831 radv_cmd_buffer_restore_subpass(struct radv_cmd_buffer *cmd_buffer, in radv_cmd_buffer_restore_subpass() argument
5834 radv_mark_noncoherent_rb(cmd_buffer); in radv_cmd_buffer_restore_subpass()
5835 radv_cmd_buffer_set_subpass(cmd_buffer, subpass); in radv_cmd_buffer_restore_subpass()
5839 radv_cmd_buffer_end_subpass(struct radv_cmd_buffer *cmd_buffer) in radv_cmd_buffer_end_subpass() argument
5841 struct radv_cmd_state *state = &cmd_buffer->state; in radv_cmd_buffer_end_subpass()
5843 uint32_t subpass_id = radv_get_subpass_id(cmd_buffer); in radv_cmd_buffer_end_subpass()
5845 radv_cmd_buffer_resolve_subpass(cmd_buffer); in radv_cmd_buffer_end_subpass()
5847 radv_describe_barrier_start(cmd_buffer, RGP_BARRIER_EXTERNAL_RENDER_PASS_SYNC); in radv_cmd_buffer_end_subpass()
5860 radv_handle_subpass_image_transition(cmd_buffer, att, false); in radv_cmd_buffer_end_subpass()
5863 radv_describe_barrier_end(cmd_buffer); in radv_cmd_buffer_end_subpass()
5867 radv_cmd_buffer_begin_render_pass(struct radv_cmd_buffer *cmd_buffer, in radv_cmd_buffer_begin_render_pass() argument
5875 cmd_buffer->state.framebuffer = framebuffer; in radv_cmd_buffer_begin_render_pass()
5876 cmd_buffer->state.pass = pass; in radv_cmd_buffer_begin_render_pass()
5877 cmd_buffer->state.render_area = pRenderPassBegin->renderArea; in radv_cmd_buffer_begin_render_pass()
5879 result = radv_cmd_state_setup_attachments(cmd_buffer, pass, pRenderPassBegin, extra_info); in radv_cmd_buffer_begin_render_pass()
5883 result = radv_cmd_state_setup_sample_locations(cmd_buffer, pass, pRenderPassBegin); in radv_cmd_buffer_begin_render_pass()
5893 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdBeginRenderPass2()
5895 radv_cmd_buffer_begin_render_pass(cmd_buffer, pRenderPassBeginInfo, NULL); in radv_CmdBeginRenderPass2()
5897 radv_cmd_buffer_begin_subpass(cmd_buffer, 0); in radv_CmdBeginRenderPass2()
5904 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdNextSubpass2()
5906 radv_mark_noncoherent_rb(cmd_buffer); in radv_CmdNextSubpass2()
5908 uint32_t prev_subpass = radv_get_subpass_id(cmd_buffer); in radv_CmdNextSubpass2()
5909 radv_cmd_buffer_end_subpass(cmd_buffer); in radv_CmdNextSubpass2()
5910 radv_cmd_buffer_begin_subpass(cmd_buffer, prev_subpass + 1); in radv_CmdNextSubpass2()
5914 radv_emit_view_index(struct radv_cmd_buffer *cmd_buffer, unsigned index) in radv_emit_view_index() argument
5916 struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; in radv_emit_view_index()
5925 radeon_set_sh_reg(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, index); in radv_emit_view_index()
5932 radeon_set_sh_reg(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, index); in radv_emit_view_index()
5938 radv_cs_emit_draw_packet(struct radv_cmd_buffer *cmd_buffer, uint32_t vertex_count, in radv_cs_emit_draw_packet() argument
5941 radeon_emit(cmd_buffer->cs, PKT3(PKT3_DRAW_INDEX_AUTO, 1, cmd_buffer->state.predicating)); in radv_cs_emit_draw_packet()
5942 radeon_emit(cmd_buffer->cs, vertex_count); in radv_cs_emit_draw_packet()
5943 radeon_emit(cmd_buffer->cs, V_0287F0_DI_SRC_SEL_AUTO_INDEX | use_opaque); in radv_cs_emit_draw_packet()
5954 radv_cs_emit_draw_indexed_packet(struct radv_cmd_buffer *cmd_buffer, uint64_t index_va, in radv_cs_emit_draw_indexed_packet() argument
5957 radeon_emit(cmd_buffer->cs, PKT3(PKT3_DRAW_INDEX_2, 4, cmd_buffer->state.predicating)); in radv_cs_emit_draw_indexed_packet()
5958 radeon_emit(cmd_buffer->cs, max_index_count); in radv_cs_emit_draw_indexed_packet()
5959 radeon_emit(cmd_buffer->cs, index_va); in radv_cs_emit_draw_indexed_packet()
5960 radeon_emit(cmd_buffer->cs, index_va >> 32); in radv_cs_emit_draw_indexed_packet()
5961 radeon_emit(cmd_buffer->cs, index_count); in radv_cs_emit_draw_indexed_packet()
5966 radeon_emit(cmd_buffer->cs, V_0287F0_DI_SRC_SEL_DMA | S_0287F0_NOT_EOP(not_eop)); in radv_cs_emit_draw_indexed_packet()
5971 radv_cs_emit_indirect_draw_packet(struct radv_cmd_buffer *cmd_buffer, bool indexed, in radv_cs_emit_indirect_draw_packet() argument
5974 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_cs_emit_indirect_draw_packet()
5976 bool draw_id_enable = cmd_buffer->state.pipeline->graphics.uses_drawid; in radv_cs_emit_indirect_draw_packet()
5977 uint32_t base_reg = cmd_buffer->state.pipeline->graphics.vtx_base_sgpr; in radv_cs_emit_indirect_draw_packet()
5979 bool predicating = cmd_buffer->state.predicating; in radv_cs_emit_indirect_draw_packet()
5983 cmd_buffer->state.last_first_instance = -1; in radv_cs_emit_indirect_draw_packet()
5984 cmd_buffer->state.last_num_instances = -1; in radv_cs_emit_indirect_draw_packet()
5985 cmd_buffer->state.last_drawid = -1; in radv_cs_emit_indirect_draw_packet()
5986 cmd_buffer->state.last_vertex_offset = -1; in radv_cs_emit_indirect_draw_packet()
5989 if (cmd_buffer->state.pipeline->graphics.uses_baseinstance) in radv_cs_emit_indirect_draw_packet()
6015 cmd_buffer->state.uses_draw_indirect_multi = true; in radv_cs_emit_indirect_draw_packet()
6020 radv_emit_userdata_vertex_internal(struct radv_cmd_buffer *cmd_buffer, in radv_emit_userdata_vertex_internal() argument
6023 struct radv_cmd_state *state = &cmd_buffer->state; in radv_emit_userdata_vertex_internal()
6024 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_emit_userdata_vertex_internal()
6043 radv_emit_userdata_vertex(struct radv_cmd_buffer *cmd_buffer, const struct radv_draw_info *info, in radv_emit_userdata_vertex() argument
6046 const struct radv_cmd_state *state = &cmd_buffer->state; in radv_emit_userdata_vertex()
6054 radv_emit_userdata_vertex_internal(cmd_buffer, info, vertex_offset); in radv_emit_userdata_vertex()
6056 radv_emit_userdata_vertex_internal(cmd_buffer, info, vertex_offset); in radv_emit_userdata_vertex()
6058 radv_emit_userdata_vertex_internal(cmd_buffer, info, vertex_offset); in radv_emit_userdata_vertex()
6063 radv_emit_userdata_vertex_drawid(struct radv_cmd_buffer *cmd_buffer, uint32_t vertex_offset, uint32… in radv_emit_userdata_vertex_drawid() argument
6065 struct radv_cmd_state *state = &cmd_buffer->state; in radv_emit_userdata_vertex_drawid()
6066 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_emit_userdata_vertex_drawid()
6076 radv_emit_draw_packets_indexed(struct radv_cmd_buffer *cmd_buffer, in radv_emit_draw_packets_indexed() argument
6083 struct radv_cmd_state *state = &cmd_buffer->state; in radv_emit_draw_packets_indexed()
6084 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_emit_draw_packets_indexed()
6088 …const bool can_eop = !uses_drawid && cmd_buffer->device->physical_device->rad_info.chip_class >= G… in radv_emit_draw_packets_indexed()
6092 radv_emit_userdata_vertex(cmd_buffer, info, *vertexOffset); in radv_emit_draw_packets_indexed()
6098 cmd_buffer->device->physical_device->rad_info.has_zero_index_buffer_bug) in radv_emit_draw_packets_indexed()
6107 …radv_cs_emit_draw_indexed_packet(cmd_buffer, index_va, remaining_indexes, draw->indexCount, false); in radv_emit_draw_packets_indexed()
6110 radv_emit_view_index(cmd_buffer, view); in radv_emit_draw_packets_indexed()
6112 …radv_cs_emit_draw_indexed_packet(cmd_buffer, index_va, remaining_indexes, draw->indexCount, false); in radv_emit_draw_packets_indexed()
6122 cmd_buffer->device->physical_device->rad_info.has_zero_index_buffer_bug) in radv_emit_draw_packets_indexed()
6127 radv_emit_userdata_vertex_drawid(cmd_buffer, draw->vertexOffset, i); in radv_emit_draw_packets_indexed()
6131 radv_emit_userdata_vertex(cmd_buffer, info, draw->vertexOffset); in radv_emit_draw_packets_indexed()
6136 …radv_cs_emit_draw_indexed_packet(cmd_buffer, index_va, remaining_indexes, draw->indexCount, false); in radv_emit_draw_packets_indexed()
6139 radv_emit_view_index(cmd_buffer, view); in radv_emit_draw_packets_indexed()
6141 …radv_cs_emit_draw_indexed_packet(cmd_buffer, index_va, remaining_indexes, draw->indexCount, false); in radv_emit_draw_packets_indexed()
6151 if (cmd_buffer->device->physical_device->rad_info.chip_class == GFX10) { in radv_emit_draw_packets_indexed()
6163 radv_emit_userdata_vertex(cmd_buffer, info, *vertexOffset); in radv_emit_draw_packets_indexed()
6169 cmd_buffer->device->physical_device->rad_info.has_zero_index_buffer_bug) in radv_emit_draw_packets_indexed()
6175 …radv_cs_emit_draw_indexed_packet(cmd_buffer, index_va, remaining_indexes, draw->indexCount, can_eo… in radv_emit_draw_packets_indexed()
6178 radv_emit_view_index(cmd_buffer, view); in radv_emit_draw_packets_indexed()
6180 …radv_cs_emit_draw_indexed_packet(cmd_buffer, index_va, remaining_indexes, draw->indexCount, false); in radv_emit_draw_packets_indexed()
6190 cmd_buffer->device->physical_device->rad_info.has_zero_index_buffer_bug) in radv_emit_draw_packets_indexed()
6195 radv_emit_userdata_vertex(cmd_buffer, info, draw->vertexOffset); in radv_emit_draw_packets_indexed()
6200 …radv_cs_emit_draw_indexed_packet(cmd_buffer, index_va, remaining_indexes, draw->indexCount, can_eo… in radv_emit_draw_packets_indexed()
6203 radv_emit_view_index(cmd_buffer, view); in radv_emit_draw_packets_indexed()
6205 …radv_cs_emit_draw_indexed_packet(cmd_buffer, index_va, remaining_indexes, draw->indexCount, false); in radv_emit_draw_packets_indexed()
6217 radv_emit_direct_draw_packets(struct radv_cmd_buffer *cmd_buffer, const struct radv_draw_info *info, in radv_emit_direct_draw_packets() argument
6222 const uint32_t view_mask = cmd_buffer->state.subpass->view_mask; in radv_emit_direct_draw_packets()
6223 const bool uses_drawid = cmd_buffer->state.pipeline->graphics.uses_drawid; in radv_emit_direct_draw_packets()
6228 radv_emit_userdata_vertex(cmd_buffer, info, draw->firstVertex); in radv_emit_direct_draw_packets()
6230 radv_emit_userdata_vertex_drawid(cmd_buffer, draw->firstVertex, uses_drawid ? i : 0); in radv_emit_direct_draw_packets()
6233 radv_cs_emit_draw_packet(cmd_buffer, draw->vertexCount, use_opaque); in radv_emit_direct_draw_packets()
6236 radv_emit_view_index(cmd_buffer, view); in radv_emit_direct_draw_packets()
6237 radv_cs_emit_draw_packet(cmd_buffer, draw->vertexCount, use_opaque); in radv_emit_direct_draw_packets()
6243 struct radv_cmd_state *state = &cmd_buffer->state; in radv_emit_direct_draw_packets()
6251 radv_emit_indirect_draw_packets(struct radv_cmd_buffer *cmd_buffer, in radv_emit_indirect_draw_packets() argument
6254 const struct radv_cmd_state *state = &cmd_buffer->state; in radv_emit_indirect_draw_packets()
6255 struct radeon_winsys *ws = cmd_buffer->device->ws; in radv_emit_indirect_draw_packets()
6256 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_emit_indirect_draw_packets()
6276 radv_cs_emit_indirect_draw_packet(cmd_buffer, info->indexed, info->count, count_va, in radv_emit_indirect_draw_packets()
6281 radv_emit_view_index(cmd_buffer, i); in radv_emit_indirect_draw_packets()
6283 radv_cs_emit_indirect_draw_packet(cmd_buffer, info->indexed, info->count, count_va, in radv_emit_indirect_draw_packets()
6306 radv_need_late_scissor_emission(struct radv_cmd_buffer *cmd_buffer, in radv_need_late_scissor_emission() argument
6309 struct radv_cmd_state *state = &cmd_buffer->state; in radv_need_late_scissor_emission()
6311 if (!cmd_buffer->device->physical_device->rad_info.has_gfx9_scissor_bug) in radv_need_late_scissor_emission()
6314 if (cmd_buffer->state.context_roll_without_scissor_emitted || info->strmout_buffer) in radv_need_late_scissor_emission()
6318 cmd_buffer->state.pipeline->graphics.needed_dynamic_state | ~RADV_CMD_DIRTY_DYNAMIC_ALL; in radv_need_late_scissor_emission()
6327 if (cmd_buffer->state.dirty & used_states) in radv_need_late_scissor_emission()
6330 uint32_t primitive_reset_index = radv_get_primitive_reset_index(cmd_buffer); in radv_need_late_scissor_emission()
6361 radv_get_ngg_culling_settings(struct radv_cmd_buffer *cmd_buffer, bool vp_y_inverted) in radv_get_ngg_culling_settings() argument
6363 const struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; in radv_get_ngg_culling_settings()
6364 const struct radv_dynamic_state *d = &cmd_buffer->state.dynamic; in radv_get_ngg_culling_settings()
6368 G_028810_DX_RASTERIZATION_KILL(cmd_buffer->state.pipeline->graphics.pa_cl_clip_cntl)) in radv_get_ngg_culling_settings()
6371 uint32_t pa_su_sc_mode_cntl = cmd_buffer->state.pipeline->graphics.pa_su_sc_mode_cntl; in radv_get_ngg_culling_settings()
6412 radv_emit_ngg_culling_state(struct radv_cmd_buffer *cmd_buffer, const struct radv_draw_info *draw_i… in radv_emit_ngg_culling_state() argument
6414 struct radv_pipeline *pipeline = cmd_buffer->state.pipeline; in radv_emit_ngg_culling_state()
6418 if (!nggc_supported && !cmd_buffer->state.last_nggc_settings) { in radv_emit_ngg_culling_state()
6422 cmd_buffer->state.last_nggc_settings_sgpr_idx = -1; in radv_emit_ngg_culling_state()
6431 cmd_buffer->state.dirty & in radv_emit_ngg_culling_state()
6443 if (!dirty && skip == cmd_buffer->state.last_nggc_skip) in radv_emit_ngg_culling_state()
6447 cmd_buffer->state.last_nggc_skip = skip; in radv_emit_ngg_culling_state()
6458 memcpy(vp_scale, cmd_buffer->state.dynamic.viewport.xform[0].scale, 2 * sizeof(float)); in radv_emit_ngg_culling_state()
6459 memcpy(vp_translate, cmd_buffer->state.dynamic.viewport.xform[0].translate, 2 * sizeof(float)); in radv_emit_ngg_culling_state()
6464 ? radv_get_ngg_culling_settings(cmd_buffer, vp_y_inverted) in radv_emit_ngg_culling_state()
6468 (cmd_buffer->state.dirty & RADV_CMD_DIRTY_DYNAMIC_VIEWPORT || in radv_emit_ngg_culling_state()
6469 cmd_buffer->state.last_nggc_settings_sgpr_idx != nggc_sgpr_idx || in radv_emit_ngg_culling_state()
6470 !cmd_buffer->state.last_nggc_settings); in radv_emit_ngg_culling_state()
6488 radeon_set_sh_reg_seq(cmd_buffer->cs, base_reg + vp_sgpr_idx * 4, 4); in radv_emit_ngg_culling_state()
6489 radeon_emit_array(cmd_buffer->cs, vp_reg_values, 4); in radv_emit_ngg_culling_state()
6493 (cmd_buffer->state.last_nggc_settings != nggc_settings || in radv_emit_ngg_culling_state()
6494 cmd_buffer->state.last_nggc_settings_sgpr_idx != nggc_sgpr_idx); in radv_emit_ngg_culling_state()
6501 radeon_set_sh_reg(cmd_buffer->cs, base_reg + nggc_sgpr_idx * 4, nggc_settings); in radv_emit_ngg_culling_state()
6507 if (!!cmd_buffer->state.last_nggc_settings != !!nggc_settings) { in radv_emit_ngg_culling_state()
6519 if (!(cmd_buffer->state.dirty & RADV_CMD_DIRTY_PIPELINE) || in radv_emit_ngg_culling_state()
6520 cmd_buffer->state.emitted_pipeline == pipeline) { in radv_emit_ngg_culling_state()
6521 radeon_set_sh_reg(cmd_buffer->cs, R_00B22C_SPI_SHADER_PGM_RSRC2_GS, rsrc2); in radv_emit_ngg_culling_state()
6525 cmd_buffer->state.last_nggc_settings = nggc_settings; in radv_emit_ngg_culling_state()
6526 cmd_buffer->state.last_nggc_settings_sgpr_idx = nggc_sgpr_idx; in radv_emit_ngg_culling_state()
6530 radv_emit_all_graphics_states(struct radv_cmd_buffer *cmd_buffer, const struct radv_draw_info *info, in radv_emit_all_graphics_states() argument
6535 if ((cmd_buffer->state.dirty & RADV_CMD_DIRTY_FRAMEBUFFER) || in radv_emit_all_graphics_states()
6536 cmd_buffer->state.emitted_pipeline != cmd_buffer->state.pipeline) in radv_emit_all_graphics_states()
6537 radv_emit_rbplus_state(cmd_buffer); in radv_emit_all_graphics_states()
6539 if (cmd_buffer->device->physical_device->use_ngg_culling && in radv_emit_all_graphics_states()
6540 cmd_buffer->state.pipeline->graphics.is_ngg) in radv_emit_all_graphics_states()
6541 radv_emit_ngg_culling_state(cmd_buffer, info); in radv_emit_all_graphics_states()
6543 if (cmd_buffer->state.dirty & RADV_CMD_DIRTY_PIPELINE) in radv_emit_all_graphics_states()
6544 radv_emit_graphics_pipeline(cmd_buffer); in radv_emit_all_graphics_states()
6549 late_scissor_emission = radv_need_late_scissor_emission(cmd_buffer, info); in radv_emit_all_graphics_states()
6551 if (cmd_buffer->state.dirty & RADV_CMD_DIRTY_FRAMEBUFFER) in radv_emit_all_graphics_states()
6552 radv_emit_framebuffer_state(cmd_buffer); in radv_emit_all_graphics_states()
6555 if (cmd_buffer->state.dirty & RADV_CMD_DIRTY_INDEX_BUFFER) in radv_emit_all_graphics_states()
6556 radv_emit_index_buffer(cmd_buffer, info->indirect); in radv_emit_all_graphics_states()
6562 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX7) { in radv_emit_all_graphics_states()
6563 cmd_buffer->state.last_index_type = -1; in radv_emit_all_graphics_states()
6564 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_INDEX_BUFFER; in radv_emit_all_graphics_states()
6568 radv_cmd_buffer_flush_dynamic_state(cmd_buffer, pipeline_is_dirty); in radv_emit_all_graphics_states()
6570 radv_emit_draw_registers(cmd_buffer, info); in radv_emit_all_graphics_states()
6573 radv_emit_scissor(cmd_buffer); in radv_emit_all_graphics_states()
6578 radv_before_draw(struct radv_cmd_buffer *cmd_buffer, const struct radv_draw_info *info, uint32_t dr… in radv_before_draw() argument
6580 const bool has_prefetch = cmd_buffer->device->physical_device->rad_info.chip_class >= GFX7; in radv_before_draw()
6581 const bool pipeline_is_dirty = (cmd_buffer->state.dirty & RADV_CMD_DIRTY_PIPELINE) && in radv_before_draw()
6582 cmd_buffer->state.pipeline != cmd_buffer->state.emitted_pipeline; in radv_before_draw()
6585 radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 4096 + 128 * (drawCount - 1)); in radv_before_draw()
6601 if (cmd_buffer->state.dirty & RADV_CMD_DIRTY_FRAMEBUFFER) in radv_before_draw()
6602 radv_emit_fb_mip_change_flush(cmd_buffer); in radv_before_draw()
6607 if (cmd_buffer->state.flush_bits & in radv_before_draw()
6617 radv_emit_all_graphics_states(cmd_buffer, info, pipeline_is_dirty); in radv_before_draw()
6618 si_emit_cache_flush(cmd_buffer); in radv_before_draw()
6621 radv_upload_graphics_shader_descriptors(cmd_buffer, pipeline_is_dirty); in radv_before_draw()
6626 si_emit_cache_flush(cmd_buffer); in radv_before_draw()
6628 if (has_prefetch && cmd_buffer->state.prefetch_L2_mask) { in radv_before_draw()
6632 radv_emit_prefetch_L2(cmd_buffer, cmd_buffer->state.pipeline, true); in radv_before_draw()
6635 radv_upload_graphics_shader_descriptors(cmd_buffer, pipeline_is_dirty); in radv_before_draw()
6637 radv_emit_all_graphics_states(cmd_buffer, info, pipeline_is_dirty); in radv_before_draw()
6640 radv_describe_draw(cmd_buffer); in radv_before_draw()
6642 struct radv_cmd_state *state = &cmd_buffer->state; in radv_before_draw()
6643 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_before_draw()
6651 assert(cmd_buffer->cs->cdw <= cdw_max); in radv_before_draw()
6657 radv_after_draw(struct radv_cmd_buffer *cmd_buffer) in radv_after_draw() argument
6659 const struct radeon_info *rad_info = &cmd_buffer->device->physical_device->rad_info; in radv_after_draw()
6660 bool has_prefetch = cmd_buffer->device->physical_device->rad_info.chip_class >= GFX7; in radv_after_draw()
6665 if (has_prefetch && cmd_buffer->state.prefetch_L2_mask) { in radv_after_draw()
6666 radv_emit_prefetch_L2(cmd_buffer, cmd_buffer->state.pipeline, false); in radv_after_draw()
6672 if (cmd_buffer->state.streamout.streamout_enabled && in radv_after_draw()
6675 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_VGT_STREAMOUT_SYNC; in radv_after_draw()
6678 radv_cmd_buffer_after_draw(cmd_buffer, RADV_CMD_FLAG_PS_PARTIAL_FLUSH); in radv_after_draw()
6685 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdDraw()
6695 if (!radv_before_draw(cmd_buffer, &info, 1)) in radv_CmdDraw()
6698 radv_emit_direct_draw_packets(cmd_buffer, &info, 1, &minfo, 0, 0); in radv_CmdDraw()
6699 radv_after_draw(cmd_buffer); in radv_CmdDraw()
6706 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdDrawMultiEXT()
6719 if (!radv_before_draw(cmd_buffer, &info, drawCount)) in radv_CmdDrawMultiEXT()
6721 radv_emit_direct_draw_packets(cmd_buffer, &info, drawCount, pVertexInfo, 0, stride); in radv_CmdDrawMultiEXT()
6722 radv_after_draw(cmd_buffer); in radv_CmdDrawMultiEXT()
6729 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdDrawIndexed()
6739 if (!radv_before_draw(cmd_buffer, &info, 1)) in radv_CmdDrawIndexed()
6742 radv_emit_draw_packets_indexed(cmd_buffer, &info, 1, &minfo, 0, NULL); in radv_CmdDrawIndexed()
6743 radv_after_draw(cmd_buffer); in radv_CmdDrawIndexed()
6749 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdDrawMultiIndexedEXT()
6763 if (!radv_before_draw(cmd_buffer, &info, drawCount)) in radv_CmdDrawMultiIndexedEXT()
6765 radv_emit_draw_packets_indexed(cmd_buffer, &info, drawCount, pIndexInfo, stride, pVertexOffset); in radv_CmdDrawMultiIndexedEXT()
6766 radv_after_draw(cmd_buffer); in radv_CmdDrawMultiIndexedEXT()
6773 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdDrawIndirect()
6786 if (!radv_before_draw(cmd_buffer, &info, 1)) in radv_CmdDrawIndirect()
6788 radv_emit_indirect_draw_packets(cmd_buffer, &info); in radv_CmdDrawIndirect()
6789 radv_after_draw(cmd_buffer); in radv_CmdDrawIndirect()
6796 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdDrawIndexedIndirect()
6809 if (!radv_before_draw(cmd_buffer, &info, 1)) in radv_CmdDrawIndexedIndirect()
6811 radv_emit_indirect_draw_packets(cmd_buffer, &info); in radv_CmdDrawIndexedIndirect()
6812 radv_after_draw(cmd_buffer); in radv_CmdDrawIndexedIndirect()
6820 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdDrawIndirectCount()
6835 if (!radv_before_draw(cmd_buffer, &info, 1)) in radv_CmdDrawIndirectCount()
6837 radv_emit_indirect_draw_packets(cmd_buffer, &info); in radv_CmdDrawIndirectCount()
6838 radv_after_draw(cmd_buffer); in radv_CmdDrawIndirectCount()
6847 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdDrawIndexedIndirectCount()
6862 if (!radv_before_draw(cmd_buffer, &info, 1)) in radv_CmdDrawIndexedIndirectCount()
6864 radv_emit_indirect_draw_packets(cmd_buffer, &info); in radv_CmdDrawIndexedIndirectCount()
6865 radv_after_draw(cmd_buffer); in radv_CmdDrawIndexedIndirectCount()
6892 radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer, struct radv_pipeline *pipeline, in radv_emit_dispatch_packets() argument
6896 unsigned dispatch_initiator = cmd_buffer->device->dispatch_initiator; in radv_emit_dispatch_packets()
6897 struct radeon_winsys *ws = cmd_buffer->device->ws; in radv_emit_dispatch_packets()
6898 bool predicating = cmd_buffer->state.predicating; in radv_emit_dispatch_packets()
6899 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_emit_dispatch_packets()
6902 radv_describe_dispatch(cmd_buffer, info->blocks[0], info->blocks[1], info->blocks[2]); in radv_emit_dispatch_packets()
6909 assert(cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10); in radv_emit_dispatch_packets()
6928 if (radv_cmd_buffer_uses_mec(cmd_buffer)) { in radv_emit_dispatch_packets()
7007 assert(cmd_buffer->cs->cdw <= cdw_max); in radv_emit_dispatch_packets()
7011 radv_upload_compute_shader_descriptors(struct radv_cmd_buffer *cmd_buffer, in radv_upload_compute_shader_descriptors() argument
7015 radv_flush_descriptors(cmd_buffer, VK_SHADER_STAGE_COMPUTE_BIT, pipeline, bind_point); in radv_upload_compute_shader_descriptors()
7016 radv_flush_constants(cmd_buffer, in radv_upload_compute_shader_descriptors()
7024 radv_dispatch(struct radv_cmd_buffer *cmd_buffer, const struct radv_dispatch_info *info, in radv_dispatch() argument
7027 bool has_prefetch = cmd_buffer->device->physical_device->rad_info.chip_class >= GFX7; in radv_dispatch()
7028 bool pipeline_is_dirty = pipeline && pipeline != cmd_buffer->state.emitted_compute_pipeline; in radv_dispatch()
7029 bool cs_regalloc_hang = cmd_buffer->device->physical_device->rad_info.has_cs_regalloc_hang_bug && in radv_dispatch()
7033 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_PS_PARTIAL_FLUSH | in radv_dispatch()
7036 if (cmd_buffer->state.flush_bits & in radv_dispatch()
7046 radv_emit_compute_pipeline(cmd_buffer, pipeline); in radv_dispatch()
7047 si_emit_cache_flush(cmd_buffer); in radv_dispatch()
7050 radv_upload_compute_shader_descriptors(cmd_buffer, pipeline, bind_point); in radv_dispatch()
7052 radv_emit_dispatch_packets(cmd_buffer, pipeline, info); in radv_dispatch()
7060 radv_emit_shader_prefetch(cmd_buffer, pipeline->shaders[MESA_SHADER_COMPUTE]); in radv_dispatch()
7066 si_emit_cache_flush(cmd_buffer); in radv_dispatch()
7069 radv_emit_shader_prefetch(cmd_buffer, pipeline->shaders[MESA_SHADER_COMPUTE]); in radv_dispatch()
7072 radv_upload_compute_shader_descriptors(cmd_buffer, pipeline, bind_point); in radv_dispatch()
7074 radv_emit_compute_pipeline(cmd_buffer, pipeline); in radv_dispatch()
7075 radv_emit_dispatch_packets(cmd_buffer, pipeline, info); in radv_dispatch()
7086 radv_mark_descriptor_sets_dirty(cmd_buffer, bind_point == VK_PIPELINE_BIND_POINT_COMPUTE in radv_dispatch()
7092 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH; in radv_dispatch()
7094 radv_cmd_buffer_after_draw(cmd_buffer, RADV_CMD_FLAG_CS_PARTIAL_FLUSH); in radv_dispatch()
7098 radv_compute_dispatch(struct radv_cmd_buffer *cmd_buffer, const struct radv_dispatch_info *info) in radv_compute_dispatch() argument
7100 radv_dispatch(cmd_buffer, info, cmd_buffer->state.compute_pipeline, in radv_compute_dispatch()
7108 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdDispatchBase()
7118 radv_compute_dispatch(cmd_buffer, &info); in radv_CmdDispatchBase()
7130 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdDispatchIndirect()
7137 radv_compute_dispatch(cmd_buffer, &info); in radv_CmdDispatchIndirect()
7141 radv_unaligned_dispatch(struct radv_cmd_buffer *cmd_buffer, uint32_t x, uint32_t y, uint32_t z) in radv_unaligned_dispatch() argument
7150 radv_compute_dispatch(cmd_buffer, &info); in radv_unaligned_dispatch()
7154 radv_indirect_dispatch(struct radv_cmd_buffer *cmd_buffer, struct radeon_winsys_bo *bo, uint64_t va) in radv_indirect_dispatch() argument
7161 radv_compute_dispatch(cmd_buffer, &info); in radv_indirect_dispatch()
7165 radv_rt_dispatch(struct radv_cmd_buffer *cmd_buffer, const struct radv_dispatch_info *info) in radv_rt_dispatch() argument
7167 radv_dispatch(cmd_buffer, info, cmd_buffer->state.rt_pipeline, in radv_rt_dispatch()
7172 radv_rt_bind_tables(struct radv_cmd_buffer *cmd_buffer, in radv_rt_bind_tables() argument
7175 struct radv_pipeline *pipeline = cmd_buffer->state.rt_pipeline; in radv_rt_bind_tables()
7181 if (!radv_cmd_buffer_upload_alloc(cmd_buffer, 64, &offset, &ptr)) in radv_rt_bind_tables()
7192 uint64_t va = radv_buffer_get_va(cmd_buffer->upload.upload_bo) + offset; in radv_rt_bind_tables()
7199 radv_emit_shader_pointer(cmd_buffer->device, cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, va, in radv_rt_bind_tables()
7212 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdTraceRaysKHR()
7227 if (!radv_rt_bind_tables(cmd_buffer, tables)) { in radv_CmdTraceRaysKHR()
7232 cmd_buffer->state.rt_pipeline, MESA_SHADER_COMPUTE, AC_UD_CS_RAY_LAUNCH_SIZE); in radv_CmdTraceRaysKHR()
7237 radeon_set_sh_reg_seq(cmd_buffer->cs, R_00B900_COMPUTE_USER_DATA_0 + loc->sgpr_idx * 4, 3); in radv_CmdTraceRaysKHR()
7238 radeon_emit(cmd_buffer->cs, width); in radv_CmdTraceRaysKHR()
7239 radeon_emit(cmd_buffer->cs, height); in radv_CmdTraceRaysKHR()
7240 radeon_emit(cmd_buffer->cs, depth); in radv_CmdTraceRaysKHR()
7243 radv_rt_dispatch(cmd_buffer, &info); in radv_CmdTraceRaysKHR()
7247 radv_set_rt_stack_size(struct radv_cmd_buffer *cmd_buffer, uint32_t size) in radv_set_rt_stack_size() argument
7252 if (cmd_buffer->state.rt_pipeline) { in radv_set_rt_stack_size()
7253 scratch_bytes_per_wave = cmd_buffer->state.rt_pipeline->scratch_bytes_per_wave; in radv_set_rt_stack_size()
7254 wave_size = cmd_buffer->state.rt_pipeline->shaders[MESA_SHADER_COMPUTE]->info.wave_size; in radv_set_rt_stack_size()
7260 cmd_buffer->compute_scratch_size_per_wave_needed = in radv_set_rt_stack_size()
7261 MAX2(cmd_buffer->compute_scratch_size_per_wave_needed, scratch_bytes_per_wave); in radv_set_rt_stack_size()
7267 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetRayTracingPipelineStackSizeKHR()
7269 radv_set_rt_stack_size(cmd_buffer, size); in radv_CmdSetRayTracingPipelineStackSizeKHR()
7270 cmd_buffer->state.rt_stack_size = size; in radv_CmdSetRayTracingPipelineStackSizeKHR()
7274 radv_cmd_buffer_end_render_pass(struct radv_cmd_buffer *cmd_buffer) in radv_cmd_buffer_end_render_pass() argument
7276 vk_free(&cmd_buffer->pool->alloc, cmd_buffer->state.attachments); in radv_cmd_buffer_end_render_pass()
7277 vk_free(&cmd_buffer->pool->alloc, cmd_buffer->state.subpass_sample_locs); in radv_cmd_buffer_end_render_pass()
7279 cmd_buffer->state.pass = NULL; in radv_cmd_buffer_end_render_pass()
7280 cmd_buffer->state.subpass = NULL; in radv_cmd_buffer_end_render_pass()
7281 cmd_buffer->state.attachments = NULL; in radv_cmd_buffer_end_render_pass()
7282 cmd_buffer->state.framebuffer = NULL; in radv_cmd_buffer_end_render_pass()
7283 cmd_buffer->state.subpass_sample_locs = NULL; in radv_cmd_buffer_end_render_pass()
7289 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdEndRenderPass2()
7291 radv_mark_noncoherent_rb(cmd_buffer); in radv_CmdEndRenderPass2()
7293 radv_emit_subpass_barrier(cmd_buffer, &cmd_buffer->state.pass->end_barrier); in radv_CmdEndRenderPass2()
7295 radv_cmd_buffer_end_subpass(cmd_buffer); in radv_CmdEndRenderPass2()
7297 radv_cmd_buffer_end_render_pass(cmd_buffer); in radv_CmdEndRenderPass2()
7308 radv_initialize_htile(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, in radv_initialize_htile() argument
7311 struct radv_cmd_state *state = &cmd_buffer->state; in radv_initialize_htile()
7312 uint32_t htile_value = radv_get_htile_initial_value(cmd_buffer->device, image); in radv_initialize_htile()
7317 radv_describe_layout_transition(cmd_buffer, &barrier); in radv_initialize_htile()
7322 radv_src_access_flush(cmd_buffer, VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, image); in radv_initialize_htile()
7329 state->flush_bits |= radv_dst_access_flush(cmd_buffer, VK_ACCESS_SHADER_READ_BIT, image); in radv_initialize_htile()
7332 state->flush_bits |= radv_clear_htile(cmd_buffer, image, range, htile_value); in radv_initialize_htile()
7334 radv_set_ds_clear_metadata(cmd_buffer, image, range, value, range->aspectMask); in radv_initialize_htile()
7342 radv_set_tc_compat_zrange_metadata(cmd_buffer, image, range, 0); in radv_initialize_htile()
7347 radv_handle_depth_image_transition(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, in radv_handle_depth_image_transition() argument
7354 struct radv_device *device = cmd_buffer->device; in radv_handle_depth_image_transition()
7360 radv_initialize_htile(cmd_buffer, image, range); in radv_handle_depth_image_transition()
7365 radv_initialize_htile(cmd_buffer, image, range); in radv_handle_depth_image_transition()
7370 cmd_buffer->state.flush_bits |= in radv_handle_depth_image_transition()
7373 radv_expand_depth_stencil(cmd_buffer, image, range, sample_locs); in radv_handle_depth_image_transition()
7375 cmd_buffer->state.flush_bits |= in radv_handle_depth_image_transition()
7381 radv_init_cmask(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, in radv_init_cmask() argument
7387 radv_describe_layout_transition(cmd_buffer, &barrier); in radv_init_cmask()
7389 return radv_clear_cmask(cmd_buffer, image, range, value); in radv_init_cmask()
7393 radv_init_fmask(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, in radv_init_fmask() argument
7402 radv_describe_layout_transition(cmd_buffer, &barrier); in radv_init_fmask()
7404 return radv_clear_fmask(cmd_buffer, image, range, value); in radv_init_fmask()
7408 radv_init_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, in radv_init_dcc() argument
7416 radv_describe_layout_transition(cmd_buffer, &barrier); in radv_init_dcc()
7418 flush_bits |= radv_clear_dcc(cmd_buffer, image, range, value); in radv_init_dcc()
7420 if (cmd_buffer->device->physical_device->rad_info.chip_class == GFX8) { in radv_init_dcc()
7439 flush_bits |= radv_fill_buffer(cmd_buffer, image, image->bo, in radv_init_dcc()
7452 radv_init_color_image_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, in radv_init_color_image_metadata() argument
7463 cmd_buffer->state.flush_bits |= in radv_init_color_image_metadata()
7464 radv_src_access_flush(cmd_buffer, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, image); in radv_init_color_image_metadata()
7469 if (cmd_buffer->device->physical_device->rad_info.chip_class == GFX9) { in radv_init_color_image_metadata()
7473 radv_layout_can_fast_clear(cmd_buffer->device, image, range->baseMipLevel, dst_layout, in radv_init_color_image_metadata()
7486 flush_bits |= radv_init_cmask(cmd_buffer, image, range, value); in radv_init_color_image_metadata()
7490 flush_bits |= radv_init_fmask(cmd_buffer, image, range); in radv_init_color_image_metadata()
7496 if (radv_layout_dcc_compressed(cmd_buffer->device, image, range->baseMipLevel, in radv_init_color_image_metadata()
7501 flush_bits |= radv_init_dcc(cmd_buffer, image, range, value); in radv_init_color_image_metadata()
7505 radv_update_fce_metadata(cmd_buffer, image, range, false); in radv_init_color_image_metadata()
7508 radv_set_color_clear_metadata(cmd_buffer, image, range, color_values); in radv_init_color_image_metadata()
7511 cmd_buffer->state.flush_bits |= flush_bits; in radv_init_color_image_metadata()
7515 radv_retile_transition(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, in radv_retile_transition() argument
7521 radv_retile_dcc(cmd_buffer, image); in radv_retile_transition()
7535 radv_handle_color_image_transition(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, in radv_handle_color_image_transition() argument
7548 radv_init_color_image_metadata(cmd_buffer, image, src_layout, src_render_loop, dst_layout, in radv_handle_color_image_transition()
7552 radv_retile_transition(cmd_buffer, image, src_layout, dst_layout, dst_queue_mask); in radv_handle_color_image_transition()
7558 cmd_buffer->state.flush_bits |= radv_init_dcc(cmd_buffer, image, range, 0xffffffffu); in radv_handle_color_image_transition()
7559 } else if (radv_layout_dcc_compressed(cmd_buffer->device, image, range->baseMipLevel, in radv_handle_color_image_transition()
7561 !radv_layout_dcc_compressed(cmd_buffer->device, image, range->baseMipLevel, in radv_handle_color_image_transition()
7563 radv_decompress_dcc(cmd_buffer, image, range); in radv_handle_color_image_transition()
7565 } else if (radv_layout_can_fast_clear(cmd_buffer->device, image, range->baseMipLevel, in radv_handle_color_image_transition()
7567 !radv_layout_can_fast_clear(cmd_buffer->device, image, range->baseMipLevel, in radv_handle_color_image_transition()
7569 radv_fast_clear_flush_image_inplace(cmd_buffer, image, range); in radv_handle_color_image_transition()
7574 radv_retile_transition(cmd_buffer, image, src_layout, dst_layout, dst_queue_mask); in radv_handle_color_image_transition()
7576 if (radv_layout_can_fast_clear(cmd_buffer->device, image, range->baseMipLevel, in radv_handle_color_image_transition()
7578 !radv_layout_can_fast_clear(cmd_buffer->device, image, range->baseMipLevel, in radv_handle_color_image_transition()
7580 radv_fast_clear_flush_image_inplace(cmd_buffer, image, range); in radv_handle_color_image_transition()
7588 radv_layout_fmask_compressed(cmd_buffer->device, image, src_layout, src_queue_mask) && in radv_handle_color_image_transition()
7589 !radv_layout_fmask_compressed(cmd_buffer->device, image, dst_layout, dst_queue_mask)) { in radv_handle_color_image_transition()
7591 !radv_image_use_dcc_image_stores(cmd_buffer->device, image) && !dcc_decompressed) { in radv_handle_color_image_transition()
7597 radv_decompress_dcc(cmd_buffer, image, range); in radv_handle_color_image_transition()
7602 radv_fast_clear_flush_image_inplace(cmd_buffer, image, range); in radv_handle_color_image_transition()
7607 radv_describe_layout_transition(cmd_buffer, &barrier); in radv_handle_color_image_transition()
7609 radv_expand_fmask_image_inplace(cmd_buffer, image, range); in radv_handle_color_image_transition()
7614 radv_handle_image_transition(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, in radv_handle_image_transition() argument
7625 assert(src_family == cmd_buffer->queue_family_index || in radv_handle_image_transition()
7626 dst_family == cmd_buffer->queue_family_index); in radv_handle_image_transition()
7631 if (cmd_buffer->queue_family_index == RADV_QUEUE_TRANSFER) in radv_handle_image_transition()
7634 if (cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE && in radv_handle_image_transition()
7640 radv_image_queue_family_mask(image, src_family, cmd_buffer->queue_family_index); in radv_handle_image_transition()
7642 radv_image_queue_family_mask(image, dst_family, cmd_buffer->queue_family_index); in radv_handle_image_transition()
7648 radv_handle_depth_image_transition(cmd_buffer, image, src_layout, src_render_loop, dst_layout, in radv_handle_image_transition()
7652 radv_handle_color_image_transition(cmd_buffer, image, src_layout, src_render_loop, dst_layout, in radv_handle_image_transition()
7666 radv_barrier(struct radv_cmd_buffer *cmd_buffer, uint32_t memoryBarrierCount, in radv_barrier() argument
7671 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_barrier()
7675 if (cmd_buffer->state.subpass) in radv_barrier()
7676 radv_mark_noncoherent_rb(cmd_buffer); in radv_barrier()
7678 radv_describe_barrier_start(cmd_buffer, info->reason); in radv_barrier()
7684 radv_cs_add_buffer(cmd_buffer->device->ws, cs, event->bo); in radv_barrier()
7686 ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 7); in radv_barrier()
7689 assert(cmd_buffer->cs->cdw <= cdw_max); in radv_barrier()
7693 src_flush_bits |= radv_src_access_flush(cmd_buffer, pMemoryBarriers[i].srcAccessMask, NULL); in radv_barrier()
7694 dst_flush_bits |= radv_dst_access_flush(cmd_buffer, pMemoryBarriers[i].dstAccessMask, NULL); in radv_barrier()
7699 radv_src_access_flush(cmd_buffer, pBufferMemoryBarriers[i].srcAccessMask, NULL); in radv_barrier()
7701 radv_dst_access_flush(cmd_buffer, pBufferMemoryBarriers[i].dstAccessMask, NULL); in radv_barrier()
7708 radv_src_access_flush(cmd_buffer, pImageMemoryBarriers[i].srcAccessMask, image); in radv_barrier()
7710 radv_dst_access_flush(cmd_buffer, pImageMemoryBarriers[i].dstAccessMask, image); in radv_barrier()
7725 radv_stage_flush(cmd_buffer, info->srcStageMask); in radv_barrier()
7726 cmd_buffer->state.flush_bits |= src_flush_bits; in radv_barrier()
7745 cmd_buffer, image, pImageMemoryBarriers[i].oldLayout, in radv_barrier()
7757 si_cp_dma_wait_for_idle(cmd_buffer); in radv_barrier()
7759 cmd_buffer->state.flush_bits |= dst_flush_bits; in radv_barrier()
7761 radv_describe_barrier_end(cmd_buffer); in radv_barrier()
7773 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdPipelineBarrier()
7782 radv_barrier(cmd_buffer, memoryBarrierCount, pMemoryBarriers, bufferMemoryBarrierCount, in radv_CmdPipelineBarrier()
7787 write_event(struct radv_cmd_buffer *cmd_buffer, struct radv_event *event, in write_event() argument
7790 struct radeon_cmdbuf *cs = cmd_buffer->cs; in write_event()
7793 si_emit_cache_flush(cmd_buffer); in write_event()
7795 radv_cs_add_buffer(cmd_buffer->device->ws, cs, event->bo); in write_event()
7797 ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 28); in write_event()
7822 si_cp_dma_wait_for_idle(cmd_buffer); in write_event()
7852 si_cs_emit_write_event_eop(cs, cmd_buffer->device->physical_device->rad_info.chip_class, in write_event()
7853 radv_cmd_buffer_uses_mec(cmd_buffer), event_type, 0, in write_event()
7855 cmd_buffer->gfx9_eop_bug_va); in write_event()
7858 assert(cmd_buffer->cs->cdw <= cdw_max); in write_event()
7864 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdSetEvent()
7867 write_event(cmd_buffer, event, stageMask, 1); in radv_CmdSetEvent()
7873 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdResetEvent()
7876 write_event(cmd_buffer, event, stageMask, 0); in radv_CmdResetEvent()
7888 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdWaitEvents()
7896 radv_barrier(cmd_buffer, memoryBarrierCount, pMemoryBarriers, bufferMemoryBarrierCount, in radv_CmdWaitEvents()
7912 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdBeginConditionalRenderingEXT()
7914 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_CmdBeginConditionalRenderingEXT()
7930 si_emit_cache_flush(cmd_buffer); in radv_CmdBeginConditionalRenderingEXT()
7932 if (cmd_buffer->queue_family_index == RADV_QUEUE_GENERAL && in radv_CmdBeginConditionalRenderingEXT()
7933 !cmd_buffer->device->physical_device->rad_info.has_32bit_predication) { in radv_CmdBeginConditionalRenderingEXT()
7963 radv_cmd_buffer_upload_data(cmd_buffer, 8, &pred_value, &pred_offset); in radv_CmdBeginConditionalRenderingEXT()
7965 pred_va = radv_buffer_get_va(cmd_buffer->upload.upload_bo) + pred_offset; in radv_CmdBeginConditionalRenderingEXT()
7983 si_emit_set_predication_state(cmd_buffer, draw_visible, pred_op, va); in radv_CmdBeginConditionalRenderingEXT()
7984 cmd_buffer->state.predicating = true; in radv_CmdBeginConditionalRenderingEXT()
7987 cmd_buffer->state.predication_type = draw_visible; in radv_CmdBeginConditionalRenderingEXT()
7988 cmd_buffer->state.predication_op = pred_op; in radv_CmdBeginConditionalRenderingEXT()
7989 cmd_buffer->state.predication_va = va; in radv_CmdBeginConditionalRenderingEXT()
7995 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdEndConditionalRenderingEXT()
7998 si_emit_set_predication_state(cmd_buffer, false, 0, 0); in radv_CmdEndConditionalRenderingEXT()
7999 cmd_buffer->state.predicating = false; in radv_CmdEndConditionalRenderingEXT()
8002 cmd_buffer->state.predication_type = -1; in radv_CmdEndConditionalRenderingEXT()
8003 cmd_buffer->state.predication_op = 0; in radv_CmdEndConditionalRenderingEXT()
8004 cmd_buffer->state.predication_va = 0; in radv_CmdEndConditionalRenderingEXT()
8013 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdBindTransformFeedbackBuffersEXT()
8014 struct radv_streamout_binding *sb = cmd_buffer->streamout_bindings; in radv_CmdBindTransformFeedbackBuffersEXT()
8030 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, sb[idx].buffer->bo); in radv_CmdBindTransformFeedbackBuffersEXT()
8035 cmd_buffer->state.streamout.enabled_mask |= enabled_mask; in radv_CmdBindTransformFeedbackBuffersEXT()
8037 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_STREAMOUT_BUFFER; in radv_CmdBindTransformFeedbackBuffersEXT()
8041 radv_emit_streamout_enable(struct radv_cmd_buffer *cmd_buffer) in radv_emit_streamout_enable() argument
8043 struct radv_streamout_state *so = &cmd_buffer->state.streamout; in radv_emit_streamout_enable()
8044 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_emit_streamout_enable()
8053 cmd_buffer->state.context_roll_without_scissor_emitted = true; in radv_emit_streamout_enable()
8057 radv_set_streamout_enable(struct radv_cmd_buffer *cmd_buffer, bool enable) in radv_set_streamout_enable() argument
8059 struct radv_streamout_state *so = &cmd_buffer->state.streamout; in radv_set_streamout_enable()
8068 if (!cmd_buffer->device->physical_device->use_ngg_streamout && in radv_set_streamout_enable()
8071 radv_emit_streamout_enable(cmd_buffer); in radv_set_streamout_enable()
8073 if (cmd_buffer->device->physical_device->use_ngg_streamout) { in radv_set_streamout_enable()
8074 cmd_buffer->gds_needed = true; in radv_set_streamout_enable()
8075 cmd_buffer->gds_oa_needed = true; in radv_set_streamout_enable()
8080 radv_flush_vgt_streamout(struct radv_cmd_buffer *cmd_buffer) in radv_flush_vgt_streamout() argument
8082 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_flush_vgt_streamout()
8086 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX7) { in radv_flush_vgt_streamout()
8108 radv_emit_streamout_begin(struct radv_cmd_buffer *cmd_buffer, uint32_t firstCounterBuffer, in radv_emit_streamout_begin() argument
8113 struct radv_streamout_binding *sb = cmd_buffer->streamout_bindings; in radv_emit_streamout_begin()
8114 struct radv_streamout_state *so = &cmd_buffer->state.streamout; in radv_emit_streamout_begin()
8115 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_emit_streamout_begin()
8117 radv_flush_vgt_streamout(cmd_buffer); in radv_emit_streamout_begin()
8134 cmd_buffer->state.context_roll_without_scissor_emitted = true; in radv_emit_streamout_begin()
8156 radv_cs_add_buffer(cmd_buffer->device->ws, cs, buffer->bo); in radv_emit_streamout_begin()
8169 radv_set_streamout_enable(cmd_buffer, true); in radv_emit_streamout_begin()
8173 gfx10_emit_streamout_begin(struct radv_cmd_buffer *cmd_buffer, uint32_t firstCounterBuffer, in gfx10_emit_streamout_begin() argument
8177 struct radv_streamout_state *so = &cmd_buffer->state.streamout; in gfx10_emit_streamout_begin()
8179 struct radeon_cmdbuf *cs = cmd_buffer->cs; in gfx10_emit_streamout_begin()
8181 assert(cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10); in gfx10_emit_streamout_begin()
8189 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_VS_PARTIAL_FLUSH; in gfx10_emit_streamout_begin()
8190 si_emit_cache_flush(cmd_buffer); in gfx10_emit_streamout_begin()
8212 radv_cs_add_buffer(cmd_buffer->device->ws, cs, buffer->bo); in gfx10_emit_streamout_begin()
8225 radv_set_streamout_enable(cmd_buffer, true); in gfx10_emit_streamout_begin()
8233 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdBeginTransformFeedbackEXT()
8235 if (cmd_buffer->device->physical_device->use_ngg_streamout) { in radv_CmdBeginTransformFeedbackEXT()
8236 gfx10_emit_streamout_begin(cmd_buffer, firstCounterBuffer, counterBufferCount, in radv_CmdBeginTransformFeedbackEXT()
8239 radv_emit_streamout_begin(cmd_buffer, firstCounterBuffer, counterBufferCount, pCounterBuffers, in radv_CmdBeginTransformFeedbackEXT()
8245 radv_emit_streamout_end(struct radv_cmd_buffer *cmd_buffer, uint32_t firstCounterBuffer, in radv_emit_streamout_end() argument
8249 struct radv_streamout_state *so = &cmd_buffer->state.streamout; in radv_emit_streamout_end()
8250 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_emit_streamout_end()
8252 radv_flush_vgt_streamout(cmd_buffer); in radv_emit_streamout_end()
8281 radv_cs_add_buffer(cmd_buffer->device->ws, cs, buffer->bo); in radv_emit_streamout_end()
8291 cmd_buffer->state.context_roll_without_scissor_emitted = true; in radv_emit_streamout_end()
8294 radv_set_streamout_enable(cmd_buffer, false); in radv_emit_streamout_end()
8298 gfx10_emit_streamout_end(struct radv_cmd_buffer *cmd_buffer, uint32_t firstCounterBuffer, in gfx10_emit_streamout_end() argument
8302 struct radv_streamout_state *so = &cmd_buffer->state.streamout; in gfx10_emit_streamout_end()
8303 struct radeon_cmdbuf *cs = cmd_buffer->cs; in gfx10_emit_streamout_end()
8305 assert(cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10); in gfx10_emit_streamout_end()
8325 si_cs_emit_write_event_eop(cs, cmd_buffer->device->physical_device->rad_info.chip_class, in gfx10_emit_streamout_end()
8326 radv_cmd_buffer_uses_mec(cmd_buffer), V_028A90_PS_DONE, 0, in gfx10_emit_streamout_end()
8329 radv_cs_add_buffer(cmd_buffer->device->ws, cs, buffer->bo); in gfx10_emit_streamout_end()
8333 radv_set_streamout_enable(cmd_buffer, false); in gfx10_emit_streamout_end()
8341 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdEndTransformFeedbackEXT()
8343 if (cmd_buffer->device->physical_device->use_ngg_streamout) { in radv_CmdEndTransformFeedbackEXT()
8344 gfx10_emit_streamout_end(cmd_buffer, firstCounterBuffer, counterBufferCount, pCounterBuffers, in radv_CmdEndTransformFeedbackEXT()
8347 radv_emit_streamout_end(cmd_buffer, firstCounterBuffer, counterBufferCount, pCounterBuffers, in radv_CmdEndTransformFeedbackEXT()
8358 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdDrawIndirectByteCountEXT()
8371 if (!radv_before_draw(cmd_buffer, &info, 1)) in radv_CmdDrawIndirectByteCountEXT()
8374 radv_emit_direct_draw_packets(cmd_buffer, &info, 1, &minfo, S_0287F0_USE_OPAQUE(1), 0); in radv_CmdDrawIndirectByteCountEXT()
8375 radv_after_draw(cmd_buffer); in radv_CmdDrawIndirectByteCountEXT()
8383 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); in radv_CmdWriteBufferMarkerAMD()
8385 struct radeon_cmdbuf *cs = cmd_buffer->cs; in radv_CmdWriteBufferMarkerAMD()
8388 si_emit_cache_flush(cmd_buffer); in radv_CmdWriteBufferMarkerAMD()
8390 ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 12); in radv_CmdWriteBufferMarkerAMD()
8401 si_cs_emit_write_event_eop(cs, cmd_buffer->device->physical_device->rad_info.chip_class, in radv_CmdWriteBufferMarkerAMD()
8402 radv_cmd_buffer_uses_mec(cmd_buffer), V_028A90_BOTTOM_OF_PIPE_TS, in radv_CmdWriteBufferMarkerAMD()
8404 cmd_buffer->gfx9_eop_bug_va); in radv_CmdWriteBufferMarkerAMD()
8407 assert(cmd_buffer->cs->cdw <= cdw_max); in radv_CmdWriteBufferMarkerAMD()