1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include <assert.h>
25
26 #include "anv_private.h"
27 #include "anv_measure.h"
28
29 /* These are defined in anv_private.h and blorp_genX_exec.h */
30 #undef __gen_address_type
31 #undef __gen_user_data
32 #undef __gen_combine_address
33
34 #include "common/intel_l3_config.h"
35 #include "blorp/blorp_genX_exec.h"
36
blorp_measure_start(struct blorp_batch * _batch,const struct blorp_params * params)37 static void blorp_measure_start(struct blorp_batch *_batch,
38 const struct blorp_params *params)
39 {
40 struct anv_cmd_buffer *cmd_buffer = _batch->driver_batch;
41 anv_measure_snapshot(cmd_buffer,
42 params->snapshot_type,
43 NULL, 0);
44 }
45
46 static void *
blorp_emit_dwords(struct blorp_batch * batch,unsigned n)47 blorp_emit_dwords(struct blorp_batch *batch, unsigned n)
48 {
49 struct anv_cmd_buffer *cmd_buffer = batch->driver_batch;
50 return anv_batch_emit_dwords(&cmd_buffer->batch, n);
51 }
52
53 static uint64_t
blorp_emit_reloc(struct blorp_batch * batch,void * location,struct blorp_address address,uint32_t delta)54 blorp_emit_reloc(struct blorp_batch *batch,
55 void *location, struct blorp_address address, uint32_t delta)
56 {
57 struct anv_cmd_buffer *cmd_buffer = batch->driver_batch;
58 assert(cmd_buffer->batch.start <= location &&
59 location < cmd_buffer->batch.end);
60 return anv_batch_emit_reloc(&cmd_buffer->batch, location,
61 address.buffer, address.offset + delta);
62 }
63
64 static void
blorp_surface_reloc(struct blorp_batch * batch,uint32_t ss_offset,struct blorp_address address,uint32_t delta)65 blorp_surface_reloc(struct blorp_batch *batch, uint32_t ss_offset,
66 struct blorp_address address, uint32_t delta)
67 {
68 struct anv_cmd_buffer *cmd_buffer = batch->driver_batch;
69 VkResult result;
70
71 if (ANV_ALWAYS_SOFTPIN) {
72 result = anv_reloc_list_add_bo(&cmd_buffer->surface_relocs,
73 &cmd_buffer->pool->alloc,
74 address.buffer);
75 if (unlikely(result != VK_SUCCESS))
76 anv_batch_set_error(&cmd_buffer->batch, result);
77 return;
78 }
79
80 uint64_t address_u64 = 0;
81 result = anv_reloc_list_add(&cmd_buffer->surface_relocs,
82 &cmd_buffer->pool->alloc,
83 ss_offset, address.buffer,
84 address.offset + delta,
85 &address_u64);
86 if (result != VK_SUCCESS)
87 anv_batch_set_error(&cmd_buffer->batch, result);
88
89 void *dest = anv_block_pool_map(
90 &cmd_buffer->device->surface_state_pool.block_pool, ss_offset, 8);
91 write_reloc(cmd_buffer->device, dest, address_u64, false);
92 }
93
94 static uint64_t
blorp_get_surface_address(struct blorp_batch * blorp_batch,struct blorp_address address)95 blorp_get_surface_address(struct blorp_batch *blorp_batch,
96 struct blorp_address address)
97 {
98 if (ANV_ALWAYS_SOFTPIN) {
99 struct anv_address anv_addr = {
100 .bo = address.buffer,
101 .offset = address.offset,
102 };
103 return anv_address_physical(anv_addr);
104 } else {
105 /* We'll let blorp_surface_reloc write the address. */
106 return 0;
107 }
108 }
109
110 #if GFX_VER >= 7 && GFX_VER < 10
111 static struct blorp_address
blorp_get_surface_base_address(struct blorp_batch * batch)112 blorp_get_surface_base_address(struct blorp_batch *batch)
113 {
114 struct anv_cmd_buffer *cmd_buffer = batch->driver_batch;
115 return (struct blorp_address) {
116 .buffer = cmd_buffer->device->surface_state_pool.block_pool.bo,
117 .offset = 0,
118 };
119 }
120 #endif
121
122 static void *
blorp_alloc_dynamic_state(struct blorp_batch * batch,uint32_t size,uint32_t alignment,uint32_t * offset)123 blorp_alloc_dynamic_state(struct blorp_batch *batch,
124 uint32_t size,
125 uint32_t alignment,
126 uint32_t *offset)
127 {
128 struct anv_cmd_buffer *cmd_buffer = batch->driver_batch;
129
130 struct anv_state state =
131 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, size, alignment);
132
133 *offset = state.offset;
134 return state.map;
135 }
136
137 UNUSED static void *
blorp_alloc_general_state(struct blorp_batch * batch,uint32_t size,uint32_t alignment,uint32_t * offset)138 blorp_alloc_general_state(struct blorp_batch *batch,
139 uint32_t size,
140 uint32_t alignment,
141 uint32_t *offset)
142 {
143 struct anv_cmd_buffer *cmd_buffer = batch->driver_batch;
144
145 struct anv_state state =
146 anv_state_stream_alloc(&cmd_buffer->general_state_stream, size,
147 alignment);
148
149 *offset = state.offset;
150 return state.map;
151 }
152
153 static void
blorp_alloc_binding_table(struct blorp_batch * batch,unsigned num_entries,unsigned state_size,unsigned state_alignment,uint32_t * bt_offset,uint32_t * surface_offsets,void ** surface_maps)154 blorp_alloc_binding_table(struct blorp_batch *batch, unsigned num_entries,
155 unsigned state_size, unsigned state_alignment,
156 uint32_t *bt_offset,
157 uint32_t *surface_offsets, void **surface_maps)
158 {
159 struct anv_cmd_buffer *cmd_buffer = batch->driver_batch;
160
161 uint32_t state_offset;
162 struct anv_state bt_state;
163
164 VkResult result =
165 anv_cmd_buffer_alloc_blorp_binding_table(cmd_buffer, num_entries,
166 &state_offset, &bt_state);
167 if (result != VK_SUCCESS)
168 return;
169
170 uint32_t *bt_map = bt_state.map;
171 *bt_offset = bt_state.offset;
172
173 for (unsigned i = 0; i < num_entries; i++) {
174 struct anv_state surface_state =
175 anv_cmd_buffer_alloc_surface_state(cmd_buffer);
176 bt_map[i] = surface_state.offset + state_offset;
177 surface_offsets[i] = surface_state.offset;
178 surface_maps[i] = surface_state.map;
179 }
180 }
181
182 static void *
blorp_alloc_vertex_buffer(struct blorp_batch * batch,uint32_t size,struct blorp_address * addr)183 blorp_alloc_vertex_buffer(struct blorp_batch *batch, uint32_t size,
184 struct blorp_address *addr)
185 {
186 struct anv_cmd_buffer *cmd_buffer = batch->driver_batch;
187 struct anv_state vb_state =
188 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, size, 64);
189
190 *addr = (struct blorp_address) {
191 .buffer = cmd_buffer->device->dynamic_state_pool.block_pool.bo,
192 .offset = vb_state.offset,
193 .mocs = isl_mocs(&cmd_buffer->device->isl_dev,
194 ISL_SURF_USAGE_VERTEX_BUFFER_BIT, false),
195 };
196
197 return vb_state.map;
198 }
199
200 static void
blorp_vf_invalidate_for_vb_48b_transitions(struct blorp_batch * batch,const struct blorp_address * addrs,uint32_t * sizes,unsigned num_vbs)201 blorp_vf_invalidate_for_vb_48b_transitions(struct blorp_batch *batch,
202 const struct blorp_address *addrs,
203 uint32_t *sizes,
204 unsigned num_vbs)
205 {
206 struct anv_cmd_buffer *cmd_buffer = batch->driver_batch;
207
208 for (unsigned i = 0; i < num_vbs; i++) {
209 struct anv_address anv_addr = {
210 .bo = addrs[i].buffer,
211 .offset = addrs[i].offset,
212 };
213 genX(cmd_buffer_set_binding_for_gfx8_vb_flush)(cmd_buffer,
214 i, anv_addr, sizes[i]);
215 }
216
217 genX(cmd_buffer_apply_pipe_flushes)(cmd_buffer);
218
219 /* Technically, we should call this *after* 3DPRIMITIVE but it doesn't
220 * really matter for blorp because we never call apply_pipe_flushes after
221 * this point.
222 */
223 genX(cmd_buffer_update_dirty_vbs_for_gfx8_vb_flush)(cmd_buffer, SEQUENTIAL,
224 (1 << num_vbs) - 1);
225 }
226
227 UNUSED static struct blorp_address
blorp_get_workaround_address(struct blorp_batch * batch)228 blorp_get_workaround_address(struct blorp_batch *batch)
229 {
230 struct anv_cmd_buffer *cmd_buffer = batch->driver_batch;
231
232 return (struct blorp_address) {
233 .buffer = cmd_buffer->device->workaround_address.bo,
234 .offset = cmd_buffer->device->workaround_address.offset,
235 };
236 }
237
238 static void
blorp_flush_range(struct blorp_batch * batch,void * start,size_t size)239 blorp_flush_range(struct blorp_batch *batch, void *start, size_t size)
240 {
241 /* We don't need to flush states anymore, since everything will be snooped.
242 */
243 }
244
245 static const struct intel_l3_config *
blorp_get_l3_config(struct blorp_batch * batch)246 blorp_get_l3_config(struct blorp_batch *batch)
247 {
248 struct anv_cmd_buffer *cmd_buffer = batch->driver_batch;
249 return cmd_buffer->state.current_l3_config;
250 }
251
252 void
genX(blorp_exec)253 genX(blorp_exec)(struct blorp_batch *batch,
254 const struct blorp_params *params)
255 {
256 struct anv_cmd_buffer *cmd_buffer = batch->driver_batch;
257 if (batch->flags & BLORP_BATCH_USE_COMPUTE)
258 assert(cmd_buffer->pool->queue_family->queueFlags & VK_QUEUE_COMPUTE_BIT);
259 else
260 assert(cmd_buffer->pool->queue_family->queueFlags & VK_QUEUE_GRAPHICS_BIT);
261
262 if (!cmd_buffer->state.current_l3_config) {
263 const struct intel_l3_config *cfg =
264 intel_get_default_l3_config(&cmd_buffer->device->info);
265 genX(cmd_buffer_config_l3)(cmd_buffer, cfg);
266 }
267
268 const unsigned scale = params->fast_clear_op ? UINT_MAX : 1;
269 genX(cmd_buffer_emit_hashing_mode)(cmd_buffer, params->x1 - params->x0,
270 params->y1 - params->y0, scale);
271
272 #if GFX_VER >= 11
273 /* The PIPE_CONTROL command description says:
274 *
275 * "Whenever a Binding Table Index (BTI) used by a Render Taget Message
276 * points to a different RENDER_SURFACE_STATE, SW must issue a Render
277 * Target Cache Flush by enabling this bit. When render target flush
278 * is set due to new association of BTI, PS Scoreboard Stall bit must
279 * be set in this packet."
280 */
281 anv_add_pending_pipe_bits(cmd_buffer,
282 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT |
283 ANV_PIPE_STALL_AT_SCOREBOARD_BIT,
284 "before blorp BTI change");
285 #endif
286
287 if (params->depth.enabled &&
288 !(batch->flags & BLORP_BATCH_NO_EMIT_DEPTH_STENCIL))
289 genX(cmd_buffer_emit_gfx12_depth_wa)(cmd_buffer, ¶ms->depth.surf);
290
291 #if GFX_VER == 7
292 /* The MI_LOAD/STORE_REGISTER_MEM commands which BLORP uses to implement
293 * indirect fast-clear colors can cause GPU hangs if we don't stall first.
294 * See genX(cmd_buffer_mi_memcpy) for more details.
295 */
296 if (params->src.clear_color_addr.buffer ||
297 params->dst.clear_color_addr.buffer) {
298 anv_add_pending_pipe_bits(cmd_buffer,
299 ANV_PIPE_CS_STALL_BIT,
300 "before blorp prep fast clear");
301 }
302 #endif
303
304 genX(cmd_buffer_apply_pipe_flushes)(cmd_buffer);
305
306 if (batch->flags & BLORP_BATCH_USE_COMPUTE)
307 genX(flush_pipeline_select_gpgpu)(cmd_buffer);
308 else
309 genX(flush_pipeline_select_3d)(cmd_buffer);
310
311 genX(cmd_buffer_emit_gfx7_depth_flush)(cmd_buffer);
312
313 /* BLORP doesn't do anything fancy with depth such as discards, so we want
314 * the PMA fix off. Also, off is always the safe option.
315 */
316 genX(cmd_buffer_enable_pma_fix)(cmd_buffer, false);
317
318 blorp_exec(batch, params);
319
320 #if GFX_VER >= 11
321 /* The PIPE_CONTROL command description says:
322 *
323 * "Whenever a Binding Table Index (BTI) used by a Render Taget Message
324 * points to a different RENDER_SURFACE_STATE, SW must issue a Render
325 * Target Cache Flush by enabling this bit. When render target flush
326 * is set due to new association of BTI, PS Scoreboard Stall bit must
327 * be set in this packet."
328 */
329 anv_add_pending_pipe_bits(cmd_buffer,
330 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT |
331 ANV_PIPE_STALL_AT_SCOREBOARD_BIT,
332 "after blorp BTI change");
333 #endif
334
335 /* Calculate state that does not get touched by blorp.
336 * Flush everything else.
337 */
338 anv_cmd_dirty_mask_t skip_bits = ANV_CMD_DIRTY_DYNAMIC_SCISSOR |
339 ANV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS |
340 ANV_CMD_DIRTY_INDEX_BUFFER |
341 ANV_CMD_DIRTY_XFB_ENABLE |
342 ANV_CMD_DIRTY_DYNAMIC_LINE_STIPPLE |
343 ANV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS_TEST_ENABLE |
344 ANV_CMD_DIRTY_DYNAMIC_SAMPLE_LOCATIONS |
345 ANV_CMD_DIRTY_DYNAMIC_SHADING_RATE |
346 ANV_CMD_DIRTY_DYNAMIC_PRIMITIVE_RESTART_ENABLE;
347
348 if (!params->wm_prog_data) {
349 skip_bits |= ANV_CMD_DIRTY_DYNAMIC_COLOR_BLEND_STATE |
350 ANV_CMD_DIRTY_DYNAMIC_LOGIC_OP;
351 }
352
353 cmd_buffer->state.gfx.vb_dirty = ~0;
354 cmd_buffer->state.gfx.dirty |= ~skip_bits;
355 cmd_buffer->state.push_constants_dirty = ~0;
356 }
357