1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #ifndef BLORP_GENX_EXEC_H
25 #define BLORP_GENX_EXEC_H
26
27 #include "blorp_priv.h"
28 #include "dev/intel_device_info.h"
29 #include "common/intel_sample_positions.h"
30 #include "common/intel_l3_config.h"
31 #include "genxml/gen_macros.h"
32
33 /**
34 * This file provides the blorp pipeline setup and execution functionality.
35 * It defines the following function:
36 *
37 * static void
38 * blorp_exec(struct blorp_context *blorp, void *batch_data,
39 * const struct blorp_params *params);
40 *
41 * It is the job of whoever includes this header to wrap this in something
42 * to get an externally visible symbol.
43 *
44 * In order for the blorp_exec function to work, the driver must provide
45 * implementations of the following static helper functions.
46 */
47
48 static void *
49 blorp_emit_dwords(struct blorp_batch *batch, unsigned n);
50
51 static uint64_t
52 blorp_emit_reloc(struct blorp_batch *batch,
53 void *location, struct blorp_address address, uint32_t delta);
54
55 static void
56 blorp_measure_start(struct blorp_batch *batch,
57 const struct blorp_params *params);
58
59 static void *
60 blorp_alloc_dynamic_state(struct blorp_batch *batch,
61 uint32_t size,
62 uint32_t alignment,
63 uint32_t *offset);
64
65 UNUSED static void *
66 blorp_alloc_general_state(struct blorp_batch *batch,
67 uint32_t size,
68 uint32_t alignment,
69 uint32_t *offset);
70
71 static void *
72 blorp_alloc_vertex_buffer(struct blorp_batch *batch, uint32_t size,
73 struct blorp_address *addr);
74 static void
75 blorp_vf_invalidate_for_vb_48b_transitions(struct blorp_batch *batch,
76 const struct blorp_address *addrs,
77 uint32_t *sizes,
78 unsigned num_vbs);
79
80 UNUSED static struct blorp_address
81 blorp_get_workaround_address(struct blorp_batch *batch);
82
83 static void
84 blorp_alloc_binding_table(struct blorp_batch *batch, unsigned num_entries,
85 unsigned state_size, unsigned state_alignment,
86 uint32_t *bt_offset, uint32_t *surface_offsets,
87 void **surface_maps);
88
89 static void
90 blorp_flush_range(struct blorp_batch *batch, void *start, size_t size);
91
92 static void
93 blorp_surface_reloc(struct blorp_batch *batch, uint32_t ss_offset,
94 struct blorp_address address, uint32_t delta);
95
96 static uint64_t
97 blorp_get_surface_address(struct blorp_batch *batch,
98 struct blorp_address address);
99
100 #if GFX_VER >= 7 && GFX_VER < 10
101 static struct blorp_address
102 blorp_get_surface_base_address(struct blorp_batch *batch);
103 #endif
104
105 #if GFX_VER >= 7
106 static const struct intel_l3_config *
107 blorp_get_l3_config(struct blorp_batch *batch);
108 # else
109 static void
110 blorp_emit_urb_config(struct blorp_batch *batch,
111 unsigned vs_entry_size, unsigned sf_entry_size);
112 #endif
113
114 static void
115 blorp_emit_pipeline(struct blorp_batch *batch,
116 const struct blorp_params *params);
117
118 /***** BEGIN blorp_exec implementation ******/
119
120 static uint64_t
_blorp_combine_address(struct blorp_batch * batch,void * location,struct blorp_address address,uint32_t delta)121 _blorp_combine_address(struct blorp_batch *batch, void *location,
122 struct blorp_address address, uint32_t delta)
123 {
124 if (address.buffer == NULL) {
125 return address.offset + delta;
126 } else {
127 return blorp_emit_reloc(batch, location, address, delta);
128 }
129 }
130
131 #define __gen_address_type struct blorp_address
132 #define __gen_user_data struct blorp_batch
133 #define __gen_combine_address _blorp_combine_address
134
135 #include "genxml/genX_pack.h"
136
137 #define _blorp_cmd_length(cmd) cmd ## _length
138 #define _blorp_cmd_length_bias(cmd) cmd ## _length_bias
139 #define _blorp_cmd_header(cmd) cmd ## _header
140 #define _blorp_cmd_pack(cmd) cmd ## _pack
141
142 #define blorp_emit(batch, cmd, name) \
143 for (struct cmd name = { _blorp_cmd_header(cmd) }, \
144 *_dst = blorp_emit_dwords(batch, _blorp_cmd_length(cmd)); \
145 __builtin_expect(_dst != NULL, 1); \
146 _blorp_cmd_pack(cmd)(batch, (void *)_dst, &name), \
147 _dst = NULL)
148
149 #define blorp_emitn(batch, cmd, n, ...) ({ \
150 uint32_t *_dw = blorp_emit_dwords(batch, n); \
151 if (_dw) { \
152 struct cmd template = { \
153 _blorp_cmd_header(cmd), \
154 .DWordLength = n - _blorp_cmd_length_bias(cmd), \
155 __VA_ARGS__ \
156 }; \
157 _blorp_cmd_pack(cmd)(batch, _dw, &template); \
158 } \
159 _dw ? _dw + 1 : NULL; /* Array starts at dw[1] */ \
160 })
161
162 #define STRUCT_ZERO(S) ({ struct S t; memset(&t, 0, sizeof(t)); t; })
163
164 #define blorp_emit_dynamic(batch, state, name, align, offset) \
165 for (struct state name = STRUCT_ZERO(state), \
166 *_dst = blorp_alloc_dynamic_state(batch, \
167 _blorp_cmd_length(state) * 4, \
168 align, offset); \
169 __builtin_expect(_dst != NULL, 1); \
170 _blorp_cmd_pack(state)(batch, (void *)_dst, &name), \
171 blorp_flush_range(batch, _dst, _blorp_cmd_length(state) * 4), \
172 _dst = NULL)
173
174 /* 3DSTATE_URB
175 * 3DSTATE_URB_VS
176 * 3DSTATE_URB_HS
177 * 3DSTATE_URB_DS
178 * 3DSTATE_URB_GS
179 *
180 * Assign the entire URB to the VS. Even though the VS disabled, URB space
181 * is still needed because the clipper loads the VUE's from the URB. From
182 * the Sandybridge PRM, Volume 2, Part 1, Section 3DSTATE,
183 * Dword 1.15:0 "VS Number of URB Entries":
184 * This field is always used (even if VS Function Enable is DISABLED).
185 *
186 * The warning below appears in the PRM (Section 3DSTATE_URB), but we can
187 * safely ignore it because this batch contains only one draw call.
188 * Because of URB corruption caused by allocating a previous GS unit
189 * URB entry to the VS unit, software is required to send a “GS NULL
190 * Fence” (Send URB fence with VS URB size == 1 and GS URB size == 0)
191 * plus a dummy DRAW call before any case where VS will be taking over
192 * GS URB space.
193 *
194 * If the 3DSTATE_URB_VS is emitted, than the others must be also.
195 * From the Ivybridge PRM, Volume 2 Part 1, section 1.7.1 3DSTATE_URB_VS:
196 *
197 * 3DSTATE_URB_HS, 3DSTATE_URB_DS, and 3DSTATE_URB_GS must also be
198 * programmed in order for the programming of this state to be
199 * valid.
200 */
201 static void
emit_urb_config(struct blorp_batch * batch,const struct blorp_params * params,UNUSED enum intel_urb_deref_block_size * deref_block_size)202 emit_urb_config(struct blorp_batch *batch,
203 const struct blorp_params *params,
204 UNUSED enum intel_urb_deref_block_size *deref_block_size)
205 {
206 /* Once vertex fetcher has written full VUE entries with complete
207 * header the space requirement is as follows per vertex (in bytes):
208 *
209 * Header Position Program constants
210 * +--------+------------+-------------------+
211 * | 16 | 16 | n x 16 |
212 * +--------+------------+-------------------+
213 *
214 * where 'n' stands for number of varying inputs expressed as vec4s.
215 */
216 const unsigned num_varyings =
217 params->wm_prog_data ? params->wm_prog_data->num_varying_inputs : 0;
218 const unsigned total_needed = 16 + 16 + num_varyings * 16;
219
220 /* The URB size is expressed in units of 64 bytes (512 bits) */
221 const unsigned vs_entry_size = DIV_ROUND_UP(total_needed, 64);
222
223 ASSERTED const unsigned sf_entry_size =
224 params->sf_prog_data ? params->sf_prog_data->urb_entry_size : 0;
225
226 #if GFX_VER >= 7
227 assert(sf_entry_size == 0);
228 const unsigned entry_size[4] = { vs_entry_size, 1, 1, 1 };
229
230 unsigned entries[4], start[4];
231 bool constrained;
232 intel_get_urb_config(batch->blorp->compiler->devinfo,
233 blorp_get_l3_config(batch),
234 false, false, entry_size,
235 entries, start, deref_block_size, &constrained);
236
237 #if GFX_VERx10 == 70
238 /* From the IVB PRM Vol. 2, Part 1, Section 3.2.1:
239 *
240 * "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth stall
241 * needs to be sent just prior to any 3DSTATE_VS, 3DSTATE_URB_VS,
242 * 3DSTATE_CONSTANT_VS, 3DSTATE_BINDING_TABLE_POINTER_VS,
243 * 3DSTATE_SAMPLER_STATE_POINTER_VS command. Only one PIPE_CONTROL
244 * needs to be sent before any combination of VS associated 3DSTATE."
245 */
246 blorp_emit(batch, GENX(PIPE_CONTROL), pc) {
247 pc.DepthStallEnable = true;
248 pc.PostSyncOperation = WriteImmediateData;
249 pc.Address = blorp_get_workaround_address(batch);
250 }
251 #endif
252
253 for (int i = 0; i <= MESA_SHADER_GEOMETRY; i++) {
254 blorp_emit(batch, GENX(3DSTATE_URB_VS), urb) {
255 urb._3DCommandSubOpcode += i;
256 urb.VSURBStartingAddress = start[i];
257 urb.VSURBEntryAllocationSize = entry_size[i] - 1;
258 urb.VSNumberofURBEntries = entries[i];
259 }
260 }
261 #else /* GFX_VER < 7 */
262 blorp_emit_urb_config(batch, vs_entry_size, sf_entry_size);
263 #endif
264 }
265
266 #if GFX_VER >= 7
267 static void
268 blorp_emit_memcpy(struct blorp_batch *batch,
269 struct blorp_address dst,
270 struct blorp_address src,
271 uint32_t size);
272 #endif
273
274 static void
blorp_emit_vertex_data(struct blorp_batch * batch,const struct blorp_params * params,struct blorp_address * addr,uint32_t * size)275 blorp_emit_vertex_data(struct blorp_batch *batch,
276 const struct blorp_params *params,
277 struct blorp_address *addr,
278 uint32_t *size)
279 {
280 const float vertices[] = {
281 /* v0 */ (float)params->x1, (float)params->y1, params->z,
282 /* v1 */ (float)params->x0, (float)params->y1, params->z,
283 /* v2 */ (float)params->x0, (float)params->y0, params->z,
284 };
285
286 void *data = blorp_alloc_vertex_buffer(batch, sizeof(vertices), addr);
287 memcpy(data, vertices, sizeof(vertices));
288 *size = sizeof(vertices);
289 blorp_flush_range(batch, data, *size);
290 }
291
292 static void
blorp_emit_input_varying_data(struct blorp_batch * batch,const struct blorp_params * params,struct blorp_address * addr,uint32_t * size)293 blorp_emit_input_varying_data(struct blorp_batch *batch,
294 const struct blorp_params *params,
295 struct blorp_address *addr,
296 uint32_t *size)
297 {
298 const unsigned vec4_size_in_bytes = 4 * sizeof(float);
299 const unsigned max_num_varyings =
300 DIV_ROUND_UP(sizeof(params->wm_inputs), vec4_size_in_bytes);
301 const unsigned num_varyings =
302 params->wm_prog_data ? params->wm_prog_data->num_varying_inputs : 0;
303
304 *size = 16 + num_varyings * vec4_size_in_bytes;
305
306 const uint32_t *const inputs_src = (const uint32_t *)¶ms->wm_inputs;
307 void *data = blorp_alloc_vertex_buffer(batch, *size, addr);
308 uint32_t *inputs = data;
309
310 /* Copy in the VS inputs */
311 assert(sizeof(params->vs_inputs) == 16);
312 memcpy(inputs, ¶ms->vs_inputs, sizeof(params->vs_inputs));
313 inputs += 4;
314
315 if (params->wm_prog_data) {
316 /* Walk over the attribute slots, determine if the attribute is used by
317 * the program and when necessary copy the values from the input storage
318 * to the vertex data buffer.
319 */
320 for (unsigned i = 0; i < max_num_varyings; i++) {
321 const gl_varying_slot attr = VARYING_SLOT_VAR0 + i;
322
323 const int input_index = params->wm_prog_data->urb_setup[attr];
324 if (input_index < 0)
325 continue;
326
327 memcpy(inputs, inputs_src + i * 4, vec4_size_in_bytes);
328
329 inputs += 4;
330 }
331 }
332
333 blorp_flush_range(batch, data, *size);
334
335 if (params->dst_clear_color_as_input) {
336 #if GFX_VER >= 7
337 /* In this case, the clear color isn't known statically and instead
338 * comes in through an indirect which we have to copy into the vertex
339 * buffer before we execute the 3DPRIMITIVE. We already copied the
340 * value of params->wm_inputs.clear_color into the vertex buffer in the
341 * loop above. Now we emit code to stomp it from the GPU with the
342 * actual clear color value.
343 */
344 assert(num_varyings == 1);
345
346 /* The clear color is the first thing after the header */
347 struct blorp_address clear_color_input_addr = *addr;
348 clear_color_input_addr.offset += 16;
349
350 const unsigned clear_color_size =
351 GFX_VER < 10 ? batch->blorp->isl_dev->ss.clear_value_size : 4 * 4;
352 blorp_emit_memcpy(batch, clear_color_input_addr,
353 params->dst.clear_color_addr,
354 clear_color_size);
355 #else
356 unreachable("MCS partial resolve is not a thing on SNB and earlier");
357 #endif
358 }
359 }
360
361 static void
blorp_fill_vertex_buffer_state(struct GENX (VERTEX_BUFFER_STATE)* vb,unsigned idx,struct blorp_address addr,uint32_t size,uint32_t stride)362 blorp_fill_vertex_buffer_state(struct GENX(VERTEX_BUFFER_STATE) *vb,
363 unsigned idx,
364 struct blorp_address addr, uint32_t size,
365 uint32_t stride)
366 {
367 vb[idx].VertexBufferIndex = idx;
368 vb[idx].BufferStartingAddress = addr;
369 vb[idx].BufferPitch = stride;
370
371 #if GFX_VER >= 6
372 vb[idx].MOCS = addr.mocs;
373 #endif
374
375 #if GFX_VER >= 7
376 vb[idx].AddressModifyEnable = true;
377 #endif
378
379 #if GFX_VER >= 8
380 vb[idx].BufferSize = size;
381 #elif GFX_VER >= 5
382 vb[idx].BufferAccessType = stride > 0 ? VERTEXDATA : INSTANCEDATA;
383 vb[idx].EndAddress = vb[idx].BufferStartingAddress;
384 vb[idx].EndAddress.offset += size - 1;
385 #elif GFX_VER == 4
386 vb[idx].BufferAccessType = stride > 0 ? VERTEXDATA : INSTANCEDATA;
387 vb[idx].MaxIndex = stride > 0 ? size / stride : 0;
388 #endif
389
390 #if GFX_VER >= 12
391 vb[idx].L3BypassDisable = true;
392 #endif
393 }
394
395 static void
blorp_emit_vertex_buffers(struct blorp_batch * batch,const struct blorp_params * params)396 blorp_emit_vertex_buffers(struct blorp_batch *batch,
397 const struct blorp_params *params)
398 {
399 struct GENX(VERTEX_BUFFER_STATE) vb[3];
400 uint32_t num_vbs = 2;
401 memset(vb, 0, sizeof(vb));
402
403 struct blorp_address addrs[2] = {};
404 uint32_t sizes[2];
405 blorp_emit_vertex_data(batch, params, &addrs[0], &sizes[0]);
406 blorp_fill_vertex_buffer_state(vb, 0, addrs[0], sizes[0],
407 3 * sizeof(float));
408
409 blorp_emit_input_varying_data(batch, params, &addrs[1], &sizes[1]);
410 blorp_fill_vertex_buffer_state(vb, 1, addrs[1], sizes[1], 0);
411
412 blorp_vf_invalidate_for_vb_48b_transitions(batch, addrs, sizes, num_vbs);
413
414 const unsigned num_dwords = 1 + num_vbs * GENX(VERTEX_BUFFER_STATE_length);
415 uint32_t *dw = blorp_emitn(batch, GENX(3DSTATE_VERTEX_BUFFERS), num_dwords);
416 if (!dw)
417 return;
418
419 for (unsigned i = 0; i < num_vbs; i++) {
420 GENX(VERTEX_BUFFER_STATE_pack)(batch, dw, &vb[i]);
421 dw += GENX(VERTEX_BUFFER_STATE_length);
422 }
423 }
424
425 static void
blorp_emit_vertex_elements(struct blorp_batch * batch,const struct blorp_params * params)426 blorp_emit_vertex_elements(struct blorp_batch *batch,
427 const struct blorp_params *params)
428 {
429 const unsigned num_varyings =
430 params->wm_prog_data ? params->wm_prog_data->num_varying_inputs : 0;
431 bool need_ndc = batch->blorp->compiler->devinfo->ver <= 5;
432 const unsigned num_elements = 2 + need_ndc + num_varyings;
433
434 struct GENX(VERTEX_ELEMENT_STATE) ve[num_elements];
435 memset(ve, 0, num_elements * sizeof(*ve));
436
437 /* Setup VBO for the rectangle primitive..
438 *
439 * A rectangle primitive (3DPRIM_RECTLIST) consists of only three
440 * vertices. The vertices reside in screen space with DirectX
441 * coordinates (that is, (0, 0) is the upper left corner).
442 *
443 * v2 ------ implied
444 * | |
445 * | |
446 * v1 ----- v0
447 *
448 * Since the VS is disabled, the clipper loads each VUE directly from
449 * the URB. This is controlled by the 3DSTATE_VERTEX_BUFFERS and
450 * 3DSTATE_VERTEX_ELEMENTS packets below. The VUE contents are as follows:
451 * dw0: Reserved, MBZ.
452 * dw1: Render Target Array Index. Below vertex fetcher gets programmed
453 * to assign this with primitive instance identifier which will be
454 * used for layered clears. All other renders have only one instance
455 * and therefore the value will be effectively zero.
456 * dw2: Viewport Index. The HiZ op disables viewport mapping and
457 * scissoring, so set the dword to 0.
458 * dw3: Point Width: The HiZ op does not emit the POINTLIST primitive,
459 * so set the dword to 0.
460 * dw4: Vertex Position X.
461 * dw5: Vertex Position Y.
462 * dw6: Vertex Position Z.
463 * dw7: Vertex Position W.
464 *
465 * dw8: Flat vertex input 0
466 * dw9: Flat vertex input 1
467 * ...
468 * dwn: Flat vertex input n - 8
469 *
470 * For details, see the Sandybridge PRM, Volume 2, Part 1, Section 1.5.1
471 * "Vertex URB Entry (VUE) Formats".
472 *
473 * Only vertex position X and Y are going to be variable, Z is fixed to
474 * zero and W to one. Header words dw0,2,3 are zero. There is no need to
475 * include the fixed values in the vertex buffer. Vertex fetcher can be
476 * instructed to fill vertex elements with constant values of one and zero
477 * instead of reading them from the buffer.
478 * Flat inputs are program constants that are not interpolated. Moreover
479 * their values will be the same between vertices.
480 *
481 * See the vertex element setup below.
482 */
483 unsigned slot = 0;
484
485 ve[slot] = (struct GENX(VERTEX_ELEMENT_STATE)) {
486 .VertexBufferIndex = 1,
487 .Valid = true,
488 .SourceElementFormat = ISL_FORMAT_R32G32B32A32_FLOAT,
489 .SourceElementOffset = 0,
490 .Component0Control = VFCOMP_STORE_SRC,
491
492 /* From Gfx8 onwards hardware is no more instructed to overwrite
493 * components using an element specifier. Instead one has separate
494 * 3DSTATE_VF_SGVS (System Generated Value Setup) state packet for it.
495 */
496 #if GFX_VER >= 8
497 .Component1Control = VFCOMP_STORE_0,
498 #elif GFX_VER >= 5
499 .Component1Control = VFCOMP_STORE_IID,
500 #else
501 .Component1Control = VFCOMP_STORE_0,
502 #endif
503 .Component2Control = VFCOMP_STORE_0,
504 .Component3Control = VFCOMP_STORE_0,
505 #if GFX_VER <= 5
506 .DestinationElementOffset = slot * 4,
507 #endif
508 };
509 slot++;
510
511 #if GFX_VER <= 5
512 /* On Iron Lake and earlier, a native device coordinates version of the
513 * position goes right after the normal VUE header and before position.
514 * Since w == 1 for all of our coordinates, this is just a copy of the
515 * position.
516 */
517 ve[slot] = (struct GENX(VERTEX_ELEMENT_STATE)) {
518 .VertexBufferIndex = 0,
519 .Valid = true,
520 .SourceElementFormat = ISL_FORMAT_R32G32B32_FLOAT,
521 .SourceElementOffset = 0,
522 .Component0Control = VFCOMP_STORE_SRC,
523 .Component1Control = VFCOMP_STORE_SRC,
524 .Component2Control = VFCOMP_STORE_SRC,
525 .Component3Control = VFCOMP_STORE_1_FP,
526 .DestinationElementOffset = slot * 4,
527 };
528 slot++;
529 #endif
530
531 ve[slot] = (struct GENX(VERTEX_ELEMENT_STATE)) {
532 .VertexBufferIndex = 0,
533 .Valid = true,
534 .SourceElementFormat = ISL_FORMAT_R32G32B32_FLOAT,
535 .SourceElementOffset = 0,
536 .Component0Control = VFCOMP_STORE_SRC,
537 .Component1Control = VFCOMP_STORE_SRC,
538 .Component2Control = VFCOMP_STORE_SRC,
539 .Component3Control = VFCOMP_STORE_1_FP,
540 #if GFX_VER <= 5
541 .DestinationElementOffset = slot * 4,
542 #endif
543 };
544 slot++;
545
546 for (unsigned i = 0; i < num_varyings; ++i) {
547 ve[slot] = (struct GENX(VERTEX_ELEMENT_STATE)) {
548 .VertexBufferIndex = 1,
549 .Valid = true,
550 .SourceElementFormat = ISL_FORMAT_R32G32B32A32_FLOAT,
551 .SourceElementOffset = 16 + i * 4 * sizeof(float),
552 .Component0Control = VFCOMP_STORE_SRC,
553 .Component1Control = VFCOMP_STORE_SRC,
554 .Component2Control = VFCOMP_STORE_SRC,
555 .Component3Control = VFCOMP_STORE_SRC,
556 #if GFX_VER <= 5
557 .DestinationElementOffset = slot * 4,
558 #endif
559 };
560 slot++;
561 }
562
563 const unsigned num_dwords =
564 1 + GENX(VERTEX_ELEMENT_STATE_length) * num_elements;
565 uint32_t *dw = blorp_emitn(batch, GENX(3DSTATE_VERTEX_ELEMENTS), num_dwords);
566 if (!dw)
567 return;
568
569 for (unsigned i = 0; i < num_elements; i++) {
570 GENX(VERTEX_ELEMENT_STATE_pack)(batch, dw, &ve[i]);
571 dw += GENX(VERTEX_ELEMENT_STATE_length);
572 }
573
574 blorp_emit(batch, GENX(3DSTATE_VF_STATISTICS), vf) {
575 vf.StatisticsEnable = false;
576 }
577
578 #if GFX_VER >= 8
579 /* Overwrite Render Target Array Index (2nd dword) in the VUE header with
580 * primitive instance identifier. This is used for layered clears.
581 */
582 blorp_emit(batch, GENX(3DSTATE_VF_SGVS), sgvs) {
583 sgvs.InstanceIDEnable = true;
584 sgvs.InstanceIDComponentNumber = COMP_1;
585 sgvs.InstanceIDElementOffset = 0;
586 }
587
588 for (unsigned i = 0; i < num_elements; i++) {
589 blorp_emit(batch, GENX(3DSTATE_VF_INSTANCING), vf) {
590 vf.VertexElementIndex = i;
591 vf.InstancingEnable = false;
592 }
593 }
594
595 blorp_emit(batch, GENX(3DSTATE_VF_TOPOLOGY), topo) {
596 topo.PrimitiveTopologyType = _3DPRIM_RECTLIST;
597 }
598 #endif
599 }
600
601 /* 3DSTATE_VIEWPORT_STATE_POINTERS */
602 static uint32_t
blorp_emit_cc_viewport(struct blorp_batch * batch)603 blorp_emit_cc_viewport(struct blorp_batch *batch)
604 {
605 uint32_t cc_vp_offset;
606 blorp_emit_dynamic(batch, GENX(CC_VIEWPORT), vp, 32, &cc_vp_offset) {
607 vp.MinimumDepth = 0.0;
608 vp.MaximumDepth = 1.0;
609 }
610
611 #if GFX_VER >= 7
612 blorp_emit(batch, GENX(3DSTATE_VIEWPORT_STATE_POINTERS_CC), vsp) {
613 vsp.CCViewportPointer = cc_vp_offset;
614 }
615 #elif GFX_VER == 6
616 blorp_emit(batch, GENX(3DSTATE_VIEWPORT_STATE_POINTERS), vsp) {
617 vsp.CCViewportStateChange = true;
618 vsp.PointertoCC_VIEWPORT = cc_vp_offset;
619 }
620 #endif
621
622 return cc_vp_offset;
623 }
624
625 static uint32_t
blorp_emit_sampler_state(struct blorp_batch * batch)626 blorp_emit_sampler_state(struct blorp_batch *batch)
627 {
628 uint32_t offset;
629 blorp_emit_dynamic(batch, GENX(SAMPLER_STATE), sampler, 32, &offset) {
630 sampler.MipModeFilter = MIPFILTER_NONE;
631 sampler.MagModeFilter = MAPFILTER_LINEAR;
632 sampler.MinModeFilter = MAPFILTER_LINEAR;
633 sampler.MinLOD = 0;
634 sampler.MaxLOD = 0;
635 sampler.TCXAddressControlMode = TCM_CLAMP;
636 sampler.TCYAddressControlMode = TCM_CLAMP;
637 sampler.TCZAddressControlMode = TCM_CLAMP;
638 sampler.MaximumAnisotropy = RATIO21;
639 sampler.RAddressMinFilterRoundingEnable = true;
640 sampler.RAddressMagFilterRoundingEnable = true;
641 sampler.VAddressMinFilterRoundingEnable = true;
642 sampler.VAddressMagFilterRoundingEnable = true;
643 sampler.UAddressMinFilterRoundingEnable = true;
644 sampler.UAddressMagFilterRoundingEnable = true;
645 #if GFX_VER > 6
646 sampler.NonnormalizedCoordinateEnable = true;
647 #endif
648 }
649
650 return offset;
651 }
652
653 UNUSED static uint32_t
blorp_emit_sampler_state_ps(struct blorp_batch * batch)654 blorp_emit_sampler_state_ps(struct blorp_batch *batch)
655 {
656 uint32_t offset = blorp_emit_sampler_state(batch);
657
658 #if GFX_VER >= 7
659 blorp_emit(batch, GENX(3DSTATE_SAMPLER_STATE_POINTERS_PS), ssp) {
660 ssp.PointertoPSSamplerState = offset;
661 }
662 #elif GFX_VER == 6
663 blorp_emit(batch, GENX(3DSTATE_SAMPLER_STATE_POINTERS), ssp) {
664 ssp.VSSamplerStateChange = true;
665 ssp.GSSamplerStateChange = true;
666 ssp.PSSamplerStateChange = true;
667 ssp.PointertoPSSamplerState = offset;
668 }
669 #endif
670
671 return offset;
672 }
673
674 /* What follows is the code for setting up a "pipeline" on Sandy Bridge and
675 * later hardware. This file will be included by i965 for gfx4-5 as well, so
676 * this code is guarded by GFX_VER >= 6.
677 */
678 #if GFX_VER >= 6
679
680 static void
blorp_emit_vs_config(struct blorp_batch * batch,const struct blorp_params * params)681 blorp_emit_vs_config(struct blorp_batch *batch,
682 const struct blorp_params *params)
683 {
684 struct brw_vs_prog_data *vs_prog_data = params->vs_prog_data;
685 assert(!vs_prog_data || GFX_VER < 11 ||
686 vs_prog_data->base.dispatch_mode == DISPATCH_MODE_SIMD8);
687
688 blorp_emit(batch, GENX(3DSTATE_VS), vs) {
689 if (vs_prog_data) {
690 vs.Enable = true;
691
692 vs.KernelStartPointer = params->vs_prog_kernel;
693
694 vs.DispatchGRFStartRegisterForURBData =
695 vs_prog_data->base.base.dispatch_grf_start_reg;
696 vs.VertexURBEntryReadLength =
697 vs_prog_data->base.urb_read_length;
698 vs.VertexURBEntryReadOffset = 0;
699
700 vs.MaximumNumberofThreads =
701 batch->blorp->isl_dev->info->max_vs_threads - 1;
702
703 #if GFX_VER >= 8
704 vs.SIMD8DispatchEnable =
705 vs_prog_data->base.dispatch_mode == DISPATCH_MODE_SIMD8;
706 #endif
707 }
708 }
709 }
710
711 static void
blorp_emit_sf_config(struct blorp_batch * batch,const struct blorp_params * params,UNUSED enum intel_urb_deref_block_size urb_deref_block_size)712 blorp_emit_sf_config(struct blorp_batch *batch,
713 const struct blorp_params *params,
714 UNUSED enum intel_urb_deref_block_size urb_deref_block_size)
715 {
716 const struct brw_wm_prog_data *prog_data = params->wm_prog_data;
717
718 /* 3DSTATE_SF
719 *
720 * Disable ViewportTransformEnable (dw2.1)
721 *
722 * From the SandyBridge PRM, Volume 2, Part 1, Section 1.3, "3D
723 * Primitives Overview":
724 * RECTLIST: Viewport Mapping must be DISABLED (as is typical with the
725 * use of screen- space coordinates).
726 *
727 * A solid rectangle must be rendered, so set FrontFaceFillMode (dw2.4:3)
728 * and BackFaceFillMode (dw2.5:6) to SOLID(0).
729 *
730 * From the Sandy Bridge PRM, Volume 2, Part 1, Section
731 * 6.4.1.1 3DSTATE_SF, Field FrontFaceFillMode:
732 * SOLID: Any triangle or rectangle object found to be front-facing
733 * is rendered as a solid object. This setting is required when
734 * (rendering rectangle (RECTLIST) objects.
735 */
736
737 #if GFX_VER >= 8
738
739 blorp_emit(batch, GENX(3DSTATE_SF), sf) {
740 #if GFX_VER >= 12
741 sf.DerefBlockSize = urb_deref_block_size;
742 #endif
743 }
744
745 blorp_emit(batch, GENX(3DSTATE_RASTER), raster) {
746 raster.CullMode = CULLMODE_NONE;
747 }
748
749 blorp_emit(batch, GENX(3DSTATE_SBE), sbe) {
750 sbe.VertexURBEntryReadOffset = 1;
751 if (prog_data) {
752 sbe.NumberofSFOutputAttributes = prog_data->num_varying_inputs;
753 sbe.VertexURBEntryReadLength = brw_blorp_get_urb_length(prog_data);
754 sbe.ConstantInterpolationEnable = prog_data->flat_inputs;
755 } else {
756 sbe.NumberofSFOutputAttributes = 0;
757 sbe.VertexURBEntryReadLength = 1;
758 }
759 sbe.ForceVertexURBEntryReadLength = true;
760 sbe.ForceVertexURBEntryReadOffset = true;
761
762 #if GFX_VER >= 9
763 for (unsigned i = 0; i < 32; i++)
764 sbe.AttributeActiveComponentFormat[i] = ACF_XYZW;
765 #endif
766 }
767
768 #elif GFX_VER >= 7
769
770 blorp_emit(batch, GENX(3DSTATE_SF), sf) {
771 sf.FrontFaceFillMode = FILL_MODE_SOLID;
772 sf.BackFaceFillMode = FILL_MODE_SOLID;
773
774 sf.MultisampleRasterizationMode = params->num_samples > 1 ?
775 MSRASTMODE_ON_PATTERN : MSRASTMODE_OFF_PIXEL;
776
777 #if GFX_VER == 7
778 sf.DepthBufferSurfaceFormat = params->depth_format;
779 #endif
780 }
781
782 blorp_emit(batch, GENX(3DSTATE_SBE), sbe) {
783 sbe.VertexURBEntryReadOffset = 1;
784 if (prog_data) {
785 sbe.NumberofSFOutputAttributes = prog_data->num_varying_inputs;
786 sbe.VertexURBEntryReadLength = brw_blorp_get_urb_length(prog_data);
787 sbe.ConstantInterpolationEnable = prog_data->flat_inputs;
788 } else {
789 sbe.NumberofSFOutputAttributes = 0;
790 sbe.VertexURBEntryReadLength = 1;
791 }
792 }
793
794 #else /* GFX_VER <= 6 */
795
796 blorp_emit(batch, GENX(3DSTATE_SF), sf) {
797 sf.FrontFaceFillMode = FILL_MODE_SOLID;
798 sf.BackFaceFillMode = FILL_MODE_SOLID;
799
800 sf.MultisampleRasterizationMode = params->num_samples > 1 ?
801 MSRASTMODE_ON_PATTERN : MSRASTMODE_OFF_PIXEL;
802
803 sf.VertexURBEntryReadOffset = 1;
804 if (prog_data) {
805 sf.NumberofSFOutputAttributes = prog_data->num_varying_inputs;
806 sf.VertexURBEntryReadLength = brw_blorp_get_urb_length(prog_data);
807 sf.ConstantInterpolationEnable = prog_data->flat_inputs;
808 } else {
809 sf.NumberofSFOutputAttributes = 0;
810 sf.VertexURBEntryReadLength = 1;
811 }
812 }
813
814 #endif /* GFX_VER */
815 }
816
817 static void
blorp_emit_ps_config(struct blorp_batch * batch,const struct blorp_params * params)818 blorp_emit_ps_config(struct blorp_batch *batch,
819 const struct blorp_params *params)
820 {
821 const struct brw_wm_prog_data *prog_data = params->wm_prog_data;
822
823 /* Even when thread dispatch is disabled, max threads (dw5.25:31) must be
824 * nonzero to prevent the GPU from hanging. While the documentation doesn't
825 * mention this explicitly, it notes that the valid range for the field is
826 * [1,39] = [2,40] threads, which excludes zero.
827 *
828 * To be safe (and to minimize extraneous code) we go ahead and fully
829 * configure the WM state whether or not there is a WM program.
830 */
831
832 #if GFX_VER >= 8
833
834 blorp_emit(batch, GENX(3DSTATE_WM), wm);
835
836 blorp_emit(batch, GENX(3DSTATE_PS), ps) {
837 if (params->src.enabled) {
838 ps.SamplerCount = 1; /* Up to 4 samplers */
839 ps.BindingTableEntryCount = 2;
840 } else {
841 ps.BindingTableEntryCount = 1;
842 }
843
844 /* SAMPLER_STATE prefetching is broken on Gfx11 - Wa_1606682166 */
845 if (GFX_VER == 11)
846 ps.SamplerCount = 0;
847
848 if (prog_data) {
849 ps._8PixelDispatchEnable = prog_data->dispatch_8;
850 ps._16PixelDispatchEnable = prog_data->dispatch_16;
851 ps._32PixelDispatchEnable = prog_data->dispatch_32;
852
853 /* From the Sky Lake PRM 3DSTATE_PS::32 Pixel Dispatch Enable:
854 *
855 * "When NUM_MULTISAMPLES = 16 or FORCE_SAMPLE_COUNT = 16, SIMD32
856 * Dispatch must not be enabled for PER_PIXEL dispatch mode."
857 *
858 * Since 16x MSAA is first introduced on SKL, we don't need to apply
859 * the workaround on any older hardware.
860 */
861 if (GFX_VER >= 9 && !prog_data->persample_dispatch &&
862 params->num_samples == 16) {
863 assert(ps._8PixelDispatchEnable || ps._16PixelDispatchEnable);
864 ps._32PixelDispatchEnable = false;
865 }
866
867 ps.DispatchGRFStartRegisterForConstantSetupData0 =
868 brw_wm_prog_data_dispatch_grf_start_reg(prog_data, ps, 0);
869 ps.DispatchGRFStartRegisterForConstantSetupData1 =
870 brw_wm_prog_data_dispatch_grf_start_reg(prog_data, ps, 1);
871 ps.DispatchGRFStartRegisterForConstantSetupData2 =
872 brw_wm_prog_data_dispatch_grf_start_reg(prog_data, ps, 2);
873
874 ps.KernelStartPointer0 = params->wm_prog_kernel +
875 brw_wm_prog_data_prog_offset(prog_data, ps, 0);
876 ps.KernelStartPointer1 = params->wm_prog_kernel +
877 brw_wm_prog_data_prog_offset(prog_data, ps, 1);
878 ps.KernelStartPointer2 = params->wm_prog_kernel +
879 brw_wm_prog_data_prog_offset(prog_data, ps, 2);
880 }
881
882 /* 3DSTATE_PS expects the number of threads per PSD, which is always 64
883 * for pre Gfx11 and 128 for gfx11+; On gfx11+ If a programmed value is
884 * k, it implies 2(k+1) threads. It implicitly scales for different GT
885 * levels (which have some # of PSDs).
886 *
887 * In Gfx8 the format is U8-2 whereas in Gfx9+ it is U9-1.
888 */
889 if (GFX_VER >= 9)
890 ps.MaximumNumberofThreadsPerPSD = 64 - 1;
891 else
892 ps.MaximumNumberofThreadsPerPSD = 64 - 2;
893
894 switch (params->fast_clear_op) {
895 case ISL_AUX_OP_NONE:
896 break;
897 #if GFX_VER >= 10
898 case ISL_AUX_OP_AMBIGUATE:
899 ps.RenderTargetFastClearEnable = true;
900 ps.RenderTargetResolveType = FAST_CLEAR_0;
901 break;
902 #endif
903 #if GFX_VER >= 9
904 case ISL_AUX_OP_PARTIAL_RESOLVE:
905 ps.RenderTargetResolveType = RESOLVE_PARTIAL;
906 break;
907 case ISL_AUX_OP_FULL_RESOLVE:
908 ps.RenderTargetResolveType = RESOLVE_FULL;
909 break;
910 #else
911 case ISL_AUX_OP_FULL_RESOLVE:
912 ps.RenderTargetResolveEnable = true;
913 break;
914 #endif
915 case ISL_AUX_OP_FAST_CLEAR:
916 ps.RenderTargetFastClearEnable = true;
917 break;
918 default:
919 unreachable("Invalid fast clear op");
920 }
921 }
922
923 blorp_emit(batch, GENX(3DSTATE_PS_EXTRA), psx) {
924 if (prog_data) {
925 psx.PixelShaderValid = true;
926 psx.AttributeEnable = prog_data->num_varying_inputs > 0;
927 psx.PixelShaderIsPerSample = prog_data->persample_dispatch;
928 psx.PixelShaderComputedDepthMode = prog_data->computed_depth_mode;
929 #if GFX_VER >= 9
930 psx.PixelShaderComputesStencil = prog_data->computed_stencil;
931 #endif
932 }
933
934 if (params->src.enabled)
935 psx.PixelShaderKillsPixel = true;
936 }
937
938 #elif GFX_VER >= 7
939
940 blorp_emit(batch, GENX(3DSTATE_WM), wm) {
941 switch (params->hiz_op) {
942 case ISL_AUX_OP_FAST_CLEAR:
943 wm.DepthBufferClear = true;
944 break;
945 case ISL_AUX_OP_FULL_RESOLVE:
946 wm.DepthBufferResolveEnable = true;
947 break;
948 case ISL_AUX_OP_AMBIGUATE:
949 wm.HierarchicalDepthBufferResolveEnable = true;
950 break;
951 case ISL_AUX_OP_NONE:
952 break;
953 default:
954 unreachable("not reached");
955 }
956
957 if (prog_data) {
958 wm.ThreadDispatchEnable = true;
959 wm.PixelShaderComputedDepthMode = prog_data->computed_depth_mode;
960 }
961
962 if (params->src.enabled)
963 wm.PixelShaderKillsPixel = true;
964
965 if (params->num_samples > 1) {
966 wm.MultisampleRasterizationMode = MSRASTMODE_ON_PATTERN;
967 wm.MultisampleDispatchMode =
968 (prog_data && prog_data->persample_dispatch) ?
969 MSDISPMODE_PERSAMPLE : MSDISPMODE_PERPIXEL;
970 } else {
971 wm.MultisampleRasterizationMode = MSRASTMODE_OFF_PIXEL;
972 wm.MultisampleDispatchMode = MSDISPMODE_PERSAMPLE;
973 }
974 }
975
976 blorp_emit(batch, GENX(3DSTATE_PS), ps) {
977 ps.MaximumNumberofThreads =
978 batch->blorp->isl_dev->info->max_wm_threads - 1;
979
980 #if GFX_VERx10 == 75
981 ps.SampleMask = 1;
982 #endif
983
984 if (prog_data) {
985 ps._8PixelDispatchEnable = prog_data->dispatch_8;
986 ps._16PixelDispatchEnable = prog_data->dispatch_16;
987 ps._32PixelDispatchEnable = prog_data->dispatch_32;
988
989 ps.DispatchGRFStartRegisterForConstantSetupData0 =
990 brw_wm_prog_data_dispatch_grf_start_reg(prog_data, ps, 0);
991 ps.DispatchGRFStartRegisterForConstantSetupData1 =
992 brw_wm_prog_data_dispatch_grf_start_reg(prog_data, ps, 1);
993 ps.DispatchGRFStartRegisterForConstantSetupData2 =
994 brw_wm_prog_data_dispatch_grf_start_reg(prog_data, ps, 2);
995
996 ps.KernelStartPointer0 = params->wm_prog_kernel +
997 brw_wm_prog_data_prog_offset(prog_data, ps, 0);
998 ps.KernelStartPointer1 = params->wm_prog_kernel +
999 brw_wm_prog_data_prog_offset(prog_data, ps, 1);
1000 ps.KernelStartPointer2 = params->wm_prog_kernel +
1001 brw_wm_prog_data_prog_offset(prog_data, ps, 2);
1002
1003 ps.AttributeEnable = prog_data->num_varying_inputs > 0;
1004 } else {
1005 /* Gfx7 hardware gets angry if we don't enable at least one dispatch
1006 * mode, so just enable 16-pixel dispatch if we don't have a program.
1007 */
1008 ps._16PixelDispatchEnable = true;
1009 }
1010
1011 if (params->src.enabled)
1012 ps.SamplerCount = 1; /* Up to 4 samplers */
1013
1014 switch (params->fast_clear_op) {
1015 case ISL_AUX_OP_NONE:
1016 break;
1017 case ISL_AUX_OP_FULL_RESOLVE:
1018 ps.RenderTargetResolveEnable = true;
1019 break;
1020 case ISL_AUX_OP_FAST_CLEAR:
1021 ps.RenderTargetFastClearEnable = true;
1022 break;
1023 default:
1024 unreachable("Invalid fast clear op");
1025 }
1026 }
1027
1028 #else /* GFX_VER <= 6 */
1029
1030 blorp_emit(batch, GENX(3DSTATE_WM), wm) {
1031 wm.MaximumNumberofThreads =
1032 batch->blorp->isl_dev->info->max_wm_threads - 1;
1033
1034 switch (params->hiz_op) {
1035 case ISL_AUX_OP_FAST_CLEAR:
1036 wm.DepthBufferClear = true;
1037 break;
1038 case ISL_AUX_OP_FULL_RESOLVE:
1039 wm.DepthBufferResolveEnable = true;
1040 break;
1041 case ISL_AUX_OP_AMBIGUATE:
1042 wm.HierarchicalDepthBufferResolveEnable = true;
1043 break;
1044 case ISL_AUX_OP_NONE:
1045 break;
1046 default:
1047 unreachable("not reached");
1048 }
1049
1050 if (prog_data) {
1051 wm.ThreadDispatchEnable = true;
1052
1053 wm._8PixelDispatchEnable = prog_data->dispatch_8;
1054 wm._16PixelDispatchEnable = prog_data->dispatch_16;
1055 wm._32PixelDispatchEnable = prog_data->dispatch_32;
1056
1057 wm.DispatchGRFStartRegisterForConstantSetupData0 =
1058 brw_wm_prog_data_dispatch_grf_start_reg(prog_data, wm, 0);
1059 wm.DispatchGRFStartRegisterForConstantSetupData1 =
1060 brw_wm_prog_data_dispatch_grf_start_reg(prog_data, wm, 1);
1061 wm.DispatchGRFStartRegisterForConstantSetupData2 =
1062 brw_wm_prog_data_dispatch_grf_start_reg(prog_data, wm, 2);
1063
1064 wm.KernelStartPointer0 = params->wm_prog_kernel +
1065 brw_wm_prog_data_prog_offset(prog_data, wm, 0);
1066 wm.KernelStartPointer1 = params->wm_prog_kernel +
1067 brw_wm_prog_data_prog_offset(prog_data, wm, 1);
1068 wm.KernelStartPointer2 = params->wm_prog_kernel +
1069 brw_wm_prog_data_prog_offset(prog_data, wm, 2);
1070
1071 wm.NumberofSFOutputAttributes = prog_data->num_varying_inputs;
1072 }
1073
1074 if (params->src.enabled) {
1075 wm.SamplerCount = 1; /* Up to 4 samplers */
1076 wm.PixelShaderKillsPixel = true; /* TODO: temporarily smash on */
1077 }
1078
1079 if (params->num_samples > 1) {
1080 wm.MultisampleRasterizationMode = MSRASTMODE_ON_PATTERN;
1081 wm.MultisampleDispatchMode =
1082 (prog_data && prog_data->persample_dispatch) ?
1083 MSDISPMODE_PERSAMPLE : MSDISPMODE_PERPIXEL;
1084 } else {
1085 wm.MultisampleRasterizationMode = MSRASTMODE_OFF_PIXEL;
1086 wm.MultisampleDispatchMode = MSDISPMODE_PERSAMPLE;
1087 }
1088 }
1089
1090 #endif /* GFX_VER */
1091 }
1092
1093 static uint32_t
blorp_emit_blend_state(struct blorp_batch * batch,const struct blorp_params * params)1094 blorp_emit_blend_state(struct blorp_batch *batch,
1095 const struct blorp_params *params)
1096 {
1097 struct GENX(BLEND_STATE) blend = { };
1098
1099 uint32_t offset;
1100 int size = GENX(BLEND_STATE_length) * 4;
1101 size += GENX(BLEND_STATE_ENTRY_length) * 4 * params->num_draw_buffers;
1102 uint32_t *state = blorp_alloc_dynamic_state(batch, size, 64, &offset);
1103 uint32_t *pos = state;
1104
1105 GENX(BLEND_STATE_pack)(NULL, pos, &blend);
1106 pos += GENX(BLEND_STATE_length);
1107
1108 for (unsigned i = 0; i < params->num_draw_buffers; ++i) {
1109 struct GENX(BLEND_STATE_ENTRY) entry = {
1110 .PreBlendColorClampEnable = true,
1111 .PostBlendColorClampEnable = true,
1112 .ColorClampRange = COLORCLAMP_RTFORMAT,
1113
1114 .WriteDisableRed = params->color_write_disable & 1,
1115 .WriteDisableGreen = params->color_write_disable & 2,
1116 .WriteDisableBlue = params->color_write_disable & 4,
1117 .WriteDisableAlpha = params->color_write_disable & 8,
1118 };
1119 GENX(BLEND_STATE_ENTRY_pack)(NULL, pos, &entry);
1120 pos += GENX(BLEND_STATE_ENTRY_length);
1121 }
1122
1123 blorp_flush_range(batch, state, size);
1124
1125 #if GFX_VER >= 7
1126 blorp_emit(batch, GENX(3DSTATE_BLEND_STATE_POINTERS), sp) {
1127 sp.BlendStatePointer = offset;
1128 #if GFX_VER >= 8
1129 sp.BlendStatePointerValid = true;
1130 #endif
1131 }
1132 #endif
1133
1134 #if GFX_VER >= 8
1135 blorp_emit(batch, GENX(3DSTATE_PS_BLEND), ps_blend) {
1136 ps_blend.HasWriteableRT = true;
1137 }
1138 #endif
1139
1140 return offset;
1141 }
1142
1143 static uint32_t
blorp_emit_color_calc_state(struct blorp_batch * batch,UNUSED const struct blorp_params * params)1144 blorp_emit_color_calc_state(struct blorp_batch *batch,
1145 UNUSED const struct blorp_params *params)
1146 {
1147 uint32_t offset;
1148 blorp_emit_dynamic(batch, GENX(COLOR_CALC_STATE), cc, 64, &offset) {
1149 #if GFX_VER <= 8
1150 cc.StencilReferenceValue = params->stencil_ref;
1151 #endif
1152 }
1153
1154 #if GFX_VER >= 7
1155 blorp_emit(batch, GENX(3DSTATE_CC_STATE_POINTERS), sp) {
1156 sp.ColorCalcStatePointer = offset;
1157 #if GFX_VER >= 8
1158 sp.ColorCalcStatePointerValid = true;
1159 #endif
1160 }
1161 #endif
1162
1163 return offset;
1164 }
1165
1166 static uint32_t
blorp_emit_depth_stencil_state(struct blorp_batch * batch,const struct blorp_params * params)1167 blorp_emit_depth_stencil_state(struct blorp_batch *batch,
1168 const struct blorp_params *params)
1169 {
1170 #if GFX_VER >= 8
1171 struct GENX(3DSTATE_WM_DEPTH_STENCIL) ds = {
1172 GENX(3DSTATE_WM_DEPTH_STENCIL_header),
1173 };
1174 #else
1175 struct GENX(DEPTH_STENCIL_STATE) ds = { 0 };
1176 #endif
1177
1178 if (params->depth.enabled) {
1179 ds.DepthBufferWriteEnable = true;
1180
1181 switch (params->hiz_op) {
1182 /* See the following sections of the Sandy Bridge PRM, Volume 2, Part1:
1183 * - 7.5.3.1 Depth Buffer Clear
1184 * - 7.5.3.2 Depth Buffer Resolve
1185 * - 7.5.3.3 Hierarchical Depth Buffer Resolve
1186 */
1187 case ISL_AUX_OP_FULL_RESOLVE:
1188 ds.DepthTestEnable = true;
1189 ds.DepthTestFunction = COMPAREFUNCTION_NEVER;
1190 break;
1191
1192 case ISL_AUX_OP_NONE:
1193 case ISL_AUX_OP_FAST_CLEAR:
1194 case ISL_AUX_OP_AMBIGUATE:
1195 ds.DepthTestEnable = false;
1196 break;
1197 case ISL_AUX_OP_PARTIAL_RESOLVE:
1198 unreachable("Invalid HIZ op");
1199 }
1200 }
1201
1202 if (params->stencil.enabled) {
1203 ds.StencilBufferWriteEnable = true;
1204 ds.StencilTestEnable = true;
1205 ds.DoubleSidedStencilEnable = false;
1206
1207 ds.StencilTestFunction = COMPAREFUNCTION_ALWAYS;
1208 ds.StencilPassDepthPassOp = STENCILOP_REPLACE;
1209
1210 ds.StencilWriteMask = params->stencil_mask;
1211 #if GFX_VER >= 9
1212 ds.StencilReferenceValue = params->stencil_ref;
1213 #endif
1214 }
1215
1216 #if GFX_VER >= 8
1217 uint32_t offset = 0;
1218 uint32_t *dw = blorp_emit_dwords(batch,
1219 GENX(3DSTATE_WM_DEPTH_STENCIL_length));
1220 if (!dw)
1221 return 0;
1222
1223 GENX(3DSTATE_WM_DEPTH_STENCIL_pack)(NULL, dw, &ds);
1224 #else
1225 uint32_t offset;
1226 void *state = blorp_alloc_dynamic_state(batch,
1227 GENX(DEPTH_STENCIL_STATE_length) * 4,
1228 64, &offset);
1229 GENX(DEPTH_STENCIL_STATE_pack)(NULL, state, &ds);
1230 blorp_flush_range(batch, state, GENX(DEPTH_STENCIL_STATE_length) * 4);
1231 #endif
1232
1233 #if GFX_VER == 7
1234 blorp_emit(batch, GENX(3DSTATE_DEPTH_STENCIL_STATE_POINTERS), sp) {
1235 sp.PointertoDEPTH_STENCIL_STATE = offset;
1236 }
1237 #endif
1238
1239 return offset;
1240 }
1241
1242 static void
blorp_emit_3dstate_multisample(struct blorp_batch * batch,const struct blorp_params * params)1243 blorp_emit_3dstate_multisample(struct blorp_batch *batch,
1244 const struct blorp_params *params)
1245 {
1246 blorp_emit(batch, GENX(3DSTATE_MULTISAMPLE), ms) {
1247 ms.NumberofMultisamples = __builtin_ffs(params->num_samples) - 1;
1248
1249 #if GFX_VER >= 8
1250 /* The PRM says that this bit is valid only for DX9:
1251 *
1252 * SW can choose to set this bit only for DX9 API. DX10/OGL API's
1253 * should not have any effect by setting or not setting this bit.
1254 */
1255 ms.PixelPositionOffsetEnable = false;
1256 #elif GFX_VER >= 7
1257
1258 switch (params->num_samples) {
1259 case 1:
1260 INTEL_SAMPLE_POS_1X(ms.Sample);
1261 break;
1262 case 2:
1263 INTEL_SAMPLE_POS_2X(ms.Sample);
1264 break;
1265 case 4:
1266 INTEL_SAMPLE_POS_4X(ms.Sample);
1267 break;
1268 case 8:
1269 INTEL_SAMPLE_POS_8X(ms.Sample);
1270 break;
1271 default:
1272 break;
1273 }
1274 #else
1275 INTEL_SAMPLE_POS_4X(ms.Sample);
1276 #endif
1277 ms.PixelLocation = CENTER;
1278 }
1279 }
1280
1281 static void
blorp_emit_pipeline(struct blorp_batch * batch,const struct blorp_params * params)1282 blorp_emit_pipeline(struct blorp_batch *batch,
1283 const struct blorp_params *params)
1284 {
1285 uint32_t blend_state_offset = 0;
1286 uint32_t color_calc_state_offset;
1287 uint32_t depth_stencil_state_offset;
1288
1289 enum intel_urb_deref_block_size urb_deref_block_size;
1290 emit_urb_config(batch, params, &urb_deref_block_size);
1291
1292 if (params->wm_prog_data) {
1293 blend_state_offset = blorp_emit_blend_state(batch, params);
1294 }
1295 color_calc_state_offset = blorp_emit_color_calc_state(batch, params);
1296 depth_stencil_state_offset = blorp_emit_depth_stencil_state(batch, params);
1297
1298 #if GFX_VER == 6
1299 /* 3DSTATE_CC_STATE_POINTERS
1300 *
1301 * The pointer offsets are relative to
1302 * CMD_STATE_BASE_ADDRESS.DynamicStateBaseAddress.
1303 *
1304 * The HiZ op doesn't use BLEND_STATE or COLOR_CALC_STATE.
1305 *
1306 * The dynamic state emit helpers emit their own STATE_POINTERS packets on
1307 * gfx7+. However, on gfx6 and earlier, they're all lumpped together in
1308 * one CC_STATE_POINTERS packet so we have to emit that here.
1309 */
1310 blorp_emit(batch, GENX(3DSTATE_CC_STATE_POINTERS), cc) {
1311 cc.BLEND_STATEChange = true;
1312 cc.ColorCalcStatePointerValid = true;
1313 cc.DEPTH_STENCIL_STATEChange = true;
1314 cc.PointertoBLEND_STATE = blend_state_offset;
1315 cc.ColorCalcStatePointer = color_calc_state_offset;
1316 cc.PointertoDEPTH_STENCIL_STATE = depth_stencil_state_offset;
1317 }
1318 #else
1319 (void)blend_state_offset;
1320 (void)color_calc_state_offset;
1321 (void)depth_stencil_state_offset;
1322 #endif
1323
1324 #if GFX_VER >= 12
1325 blorp_emit(batch, GENX(3DSTATE_CONSTANT_ALL), pc) {
1326 /* Update empty push constants for all stages (bitmask = 11111b) */
1327 pc.ShaderUpdateEnable = 0x1f;
1328 }
1329 #else
1330 blorp_emit(batch, GENX(3DSTATE_CONSTANT_VS), vs);
1331 #if GFX_VER >= 7
1332 blorp_emit(batch, GENX(3DSTATE_CONSTANT_HS), hs);
1333 blorp_emit(batch, GENX(3DSTATE_CONSTANT_DS), DS);
1334 #endif
1335 blorp_emit(batch, GENX(3DSTATE_CONSTANT_GS), gs);
1336 blorp_emit(batch, GENX(3DSTATE_CONSTANT_PS), ps);
1337 #endif
1338
1339 if (params->src.enabled)
1340 blorp_emit_sampler_state_ps(batch);
1341
1342 blorp_emit_3dstate_multisample(batch, params);
1343
1344 blorp_emit(batch, GENX(3DSTATE_SAMPLE_MASK), mask) {
1345 mask.SampleMask = (1 << params->num_samples) - 1;
1346 }
1347
1348 /* From the BSpec, 3D Pipeline > Geometry > Vertex Shader > State,
1349 * 3DSTATE_VS, Dword 5.0 "VS Function Enable":
1350 *
1351 * [DevSNB] A pipeline flush must be programmed prior to a
1352 * 3DSTATE_VS command that causes the VS Function Enable to
1353 * toggle. Pipeline flush can be executed by sending a PIPE_CONTROL
1354 * command with CS stall bit set and a post sync operation.
1355 *
1356 * We've already done one at the start of the BLORP operation.
1357 */
1358 blorp_emit_vs_config(batch, params);
1359 #if GFX_VER >= 7
1360 blorp_emit(batch, GENX(3DSTATE_HS), hs);
1361 blorp_emit(batch, GENX(3DSTATE_TE), te);
1362 blorp_emit(batch, GENX(3DSTATE_DS), DS);
1363 blorp_emit(batch, GENX(3DSTATE_STREAMOUT), so);
1364 #endif
1365 blorp_emit(batch, GENX(3DSTATE_GS), gs);
1366
1367 blorp_emit(batch, GENX(3DSTATE_CLIP), clip) {
1368 clip.PerspectiveDivideDisable = true;
1369 }
1370
1371 blorp_emit_sf_config(batch, params, urb_deref_block_size);
1372 blorp_emit_ps_config(batch, params);
1373
1374 blorp_emit_cc_viewport(batch);
1375
1376 #if GFX_VER >= 12
1377 /* Disable Primitive Replication. */
1378 blorp_emit(batch, GENX(3DSTATE_PRIMITIVE_REPLICATION), pr);
1379 #endif
1380 }
1381
1382 /******** This is the end of the pipeline setup code ********/
1383
1384 #endif /* GFX_VER >= 6 */
1385
1386 #if GFX_VER >= 7
1387 static void
blorp_emit_memcpy(struct blorp_batch * batch,struct blorp_address dst,struct blorp_address src,uint32_t size)1388 blorp_emit_memcpy(struct blorp_batch *batch,
1389 struct blorp_address dst,
1390 struct blorp_address src,
1391 uint32_t size)
1392 {
1393 assert(size % 4 == 0);
1394
1395 for (unsigned dw = 0; dw < size; dw += 4) {
1396 #if GFX_VER >= 8
1397 blorp_emit(batch, GENX(MI_COPY_MEM_MEM), cp) {
1398 cp.DestinationMemoryAddress = dst;
1399 cp.SourceMemoryAddress = src;
1400 }
1401 #else
1402 /* IVB does not have a general purpose register for command streamer
1403 * commands. Therefore, we use an alternate temporary register.
1404 */
1405 #define BLORP_TEMP_REG 0x2440 /* GFX7_3DPRIM_BASE_VERTEX */
1406 blorp_emit(batch, GENX(MI_LOAD_REGISTER_MEM), load) {
1407 load.RegisterAddress = BLORP_TEMP_REG;
1408 load.MemoryAddress = src;
1409 }
1410 blorp_emit(batch, GENX(MI_STORE_REGISTER_MEM), store) {
1411 store.RegisterAddress = BLORP_TEMP_REG;
1412 store.MemoryAddress = dst;
1413 }
1414 #undef BLORP_TEMP_REG
1415 #endif
1416 dst.offset += 4;
1417 src.offset += 4;
1418 }
1419 }
1420 #endif
1421
1422 static void
blorp_emit_surface_state(struct blorp_batch * batch,const struct brw_blorp_surface_info * surface,UNUSED enum isl_aux_op aux_op,void * state,uint32_t state_offset,uint8_t color_write_disable,bool is_render_target)1423 blorp_emit_surface_state(struct blorp_batch *batch,
1424 const struct brw_blorp_surface_info *surface,
1425 UNUSED enum isl_aux_op aux_op,
1426 void *state, uint32_t state_offset,
1427 uint8_t color_write_disable,
1428 bool is_render_target)
1429 {
1430 const struct isl_device *isl_dev = batch->blorp->isl_dev;
1431 struct isl_surf surf = surface->surf;
1432
1433 if (surf.dim == ISL_SURF_DIM_1D &&
1434 surf.dim_layout == ISL_DIM_LAYOUT_GFX4_2D) {
1435 assert(surf.logical_level0_px.height == 1);
1436 surf.dim = ISL_SURF_DIM_2D;
1437 }
1438
1439 if (isl_aux_usage_has_hiz(surface->aux_usage)) {
1440 /* BLORP doesn't render with depth so we can't use HiZ */
1441 assert(!is_render_target);
1442 /* We can't reinterpret HiZ */
1443 assert(surface->surf.format == surface->view.format);
1444 }
1445
1446 enum isl_aux_usage aux_usage = surface->aux_usage;
1447
1448 /* On gfx12, implicit CCS has no aux buffer */
1449 bool use_aux_address = (aux_usage != ISL_AUX_USAGE_NONE) &&
1450 (surface->aux_addr.buffer != NULL);
1451
1452 isl_channel_mask_t write_disable_mask = 0;
1453 if (is_render_target && GFX_VER <= 5) {
1454 if (color_write_disable & BITFIELD_BIT(0))
1455 write_disable_mask |= ISL_CHANNEL_RED_BIT;
1456 if (color_write_disable & BITFIELD_BIT(1))
1457 write_disable_mask |= ISL_CHANNEL_GREEN_BIT;
1458 if (color_write_disable & BITFIELD_BIT(2))
1459 write_disable_mask |= ISL_CHANNEL_BLUE_BIT;
1460 if (color_write_disable & BITFIELD_BIT(3))
1461 write_disable_mask |= ISL_CHANNEL_ALPHA_BIT;
1462 }
1463
1464 const bool use_clear_address =
1465 GFX_VER >= 10 && (surface->clear_color_addr.buffer != NULL);
1466
1467 isl_surf_fill_state(batch->blorp->isl_dev, state,
1468 .surf = &surf, .view = &surface->view,
1469 .aux_surf = &surface->aux_surf, .aux_usage = aux_usage,
1470 .address =
1471 blorp_get_surface_address(batch, surface->addr),
1472 .aux_address = !use_aux_address ? 0 :
1473 blorp_get_surface_address(batch, surface->aux_addr),
1474 .clear_address = !use_clear_address ? 0 :
1475 blorp_get_surface_address(batch,
1476 surface->clear_color_addr),
1477 .mocs = surface->addr.mocs,
1478 .clear_color = surface->clear_color,
1479 .use_clear_address = use_clear_address,
1480 .write_disables = write_disable_mask);
1481
1482 blorp_surface_reloc(batch, state_offset + isl_dev->ss.addr_offset,
1483 surface->addr, 0);
1484
1485 if (use_aux_address) {
1486 /* On gfx7 and prior, the bottom 12 bits of the MCS base address are
1487 * used to store other information. This should be ok, however, because
1488 * surface buffer addresses are always 4K page alinged.
1489 */
1490 assert((surface->aux_addr.offset & 0xfff) == 0);
1491 uint32_t *aux_addr = state + isl_dev->ss.aux_addr_offset;
1492 blorp_surface_reloc(batch, state_offset + isl_dev->ss.aux_addr_offset,
1493 surface->aux_addr, *aux_addr);
1494 }
1495
1496 if (aux_usage != ISL_AUX_USAGE_NONE && surface->clear_color_addr.buffer) {
1497 #if GFX_VER >= 10
1498 assert((surface->clear_color_addr.offset & 0x3f) == 0);
1499 uint32_t *clear_addr = state + isl_dev->ss.clear_color_state_offset;
1500 blorp_surface_reloc(batch, state_offset +
1501 isl_dev->ss.clear_color_state_offset,
1502 surface->clear_color_addr, *clear_addr);
1503 #elif GFX_VER >= 7
1504 /* Fast clears just whack the AUX surface and don't actually use the
1505 * clear color for anything. We can avoid the MI memcpy on that case.
1506 */
1507 if (aux_op != ISL_AUX_OP_FAST_CLEAR) {
1508 struct blorp_address dst_addr = blorp_get_surface_base_address(batch);
1509 dst_addr.offset += state_offset + isl_dev->ss.clear_value_offset;
1510 blorp_emit_memcpy(batch, dst_addr, surface->clear_color_addr,
1511 isl_dev->ss.clear_value_size);
1512 }
1513 #else
1514 unreachable("Fast clears are only supported on gfx7+");
1515 #endif
1516 }
1517
1518 blorp_flush_range(batch, state, GENX(RENDER_SURFACE_STATE_length) * 4);
1519 }
1520
1521 static void
blorp_emit_null_surface_state(struct blorp_batch * batch,const struct brw_blorp_surface_info * surface,uint32_t * state)1522 blorp_emit_null_surface_state(struct blorp_batch *batch,
1523 const struct brw_blorp_surface_info *surface,
1524 uint32_t *state)
1525 {
1526 struct GENX(RENDER_SURFACE_STATE) ss = {
1527 .SurfaceType = SURFTYPE_NULL,
1528 .SurfaceFormat = ISL_FORMAT_R8G8B8A8_UNORM,
1529 .Width = surface->surf.logical_level0_px.width - 1,
1530 .Height = surface->surf.logical_level0_px.height - 1,
1531 .MIPCountLOD = surface->view.base_level,
1532 .MinimumArrayElement = surface->view.base_array_layer,
1533 .Depth = surface->view.array_len - 1,
1534 .RenderTargetViewExtent = surface->view.array_len - 1,
1535 #if GFX_VER >= 6
1536 .NumberofMultisamples = ffs(surface->surf.samples) - 1,
1537 #endif
1538
1539 #if GFX_VER >= 7
1540 .SurfaceArray = surface->surf.dim != ISL_SURF_DIM_3D,
1541 #endif
1542
1543 #if GFX_VERx10 >= 125
1544 .TileMode = TILE4,
1545 #elif GFX_VER >= 8
1546 .TileMode = YMAJOR,
1547 #else
1548 .TiledSurface = true,
1549 #endif
1550 };
1551
1552 GENX(RENDER_SURFACE_STATE_pack)(NULL, state, &ss);
1553
1554 blorp_flush_range(batch, state, GENX(RENDER_SURFACE_STATE_length) * 4);
1555 }
1556
1557 static uint32_t
blorp_setup_binding_table(struct blorp_batch * batch,const struct blorp_params * params)1558 blorp_setup_binding_table(struct blorp_batch *batch,
1559 const struct blorp_params *params)
1560 {
1561 const struct isl_device *isl_dev = batch->blorp->isl_dev;
1562 uint32_t surface_offsets[2], bind_offset = 0;
1563 void *surface_maps[2];
1564
1565 UNUSED bool has_indirect_clear_color = false;
1566 if (params->use_pre_baked_binding_table) {
1567 bind_offset = params->pre_baked_binding_table_offset;
1568 } else {
1569 unsigned num_surfaces = 1 + params->src.enabled;
1570 blorp_alloc_binding_table(batch, num_surfaces,
1571 isl_dev->ss.size, isl_dev->ss.align,
1572 &bind_offset, surface_offsets, surface_maps);
1573
1574 if (params->dst.enabled) {
1575 blorp_emit_surface_state(batch, ¶ms->dst,
1576 params->fast_clear_op,
1577 surface_maps[BLORP_RENDERBUFFER_BT_INDEX],
1578 surface_offsets[BLORP_RENDERBUFFER_BT_INDEX],
1579 params->color_write_disable, true);
1580 if (params->dst.clear_color_addr.buffer != NULL)
1581 has_indirect_clear_color = true;
1582 } else {
1583 assert(params->depth.enabled || params->stencil.enabled);
1584 const struct brw_blorp_surface_info *surface =
1585 params->depth.enabled ? ¶ms->depth : ¶ms->stencil;
1586 blorp_emit_null_surface_state(batch, surface,
1587 surface_maps[BLORP_RENDERBUFFER_BT_INDEX]);
1588 }
1589
1590 if (params->src.enabled) {
1591 blorp_emit_surface_state(batch, ¶ms->src,
1592 params->fast_clear_op,
1593 surface_maps[BLORP_TEXTURE_BT_INDEX],
1594 surface_offsets[BLORP_TEXTURE_BT_INDEX],
1595 0, false);
1596 if (params->src.clear_color_addr.buffer != NULL)
1597 has_indirect_clear_color = true;
1598 }
1599 }
1600
1601 #if GFX_VER >= 7
1602 if (has_indirect_clear_color) {
1603 /* Updating a surface state object may require that the state cache be
1604 * invalidated. From the SKL PRM, Shared Functions -> State -> State
1605 * Caching:
1606 *
1607 * Whenever the RENDER_SURFACE_STATE object in memory pointed to by
1608 * the Binding Table Pointer (BTP) and Binding Table Index (BTI) is
1609 * modified [...], the L1 state cache must be invalidated to ensure
1610 * the new surface or sampler state is fetched from system memory.
1611 */
1612 blorp_emit(batch, GENX(PIPE_CONTROL), pipe) {
1613 pipe.StateCacheInvalidationEnable = true;
1614 }
1615 }
1616 #endif
1617
1618 return bind_offset;
1619 }
1620
1621 static void
blorp_emit_btp(struct blorp_batch * batch,uint32_t bind_offset)1622 blorp_emit_btp(struct blorp_batch *batch, uint32_t bind_offset)
1623 {
1624 #if GFX_VER >= 7
1625 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS_VS), bt);
1626 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS_HS), bt);
1627 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS_DS), bt);
1628 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS_GS), bt);
1629
1630 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS_PS), bt) {
1631 bt.PointertoPSBindingTable = bind_offset;
1632 }
1633 #elif GFX_VER >= 6
1634 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS), bt) {
1635 bt.PSBindingTableChange = true;
1636 bt.PointertoPSBindingTable = bind_offset;
1637 }
1638 #else
1639 blorp_emit(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS), bt) {
1640 bt.PointertoPSBindingTable = bind_offset;
1641 }
1642 #endif
1643 }
1644
1645 static void
blorp_emit_depth_stencil_config(struct blorp_batch * batch,const struct blorp_params * params)1646 blorp_emit_depth_stencil_config(struct blorp_batch *batch,
1647 const struct blorp_params *params)
1648 {
1649 const struct isl_device *isl_dev = batch->blorp->isl_dev;
1650
1651 uint32_t *dw = blorp_emit_dwords(batch, isl_dev->ds.size / 4);
1652 if (dw == NULL)
1653 return;
1654
1655 struct isl_depth_stencil_hiz_emit_info info = { };
1656
1657 if (params->depth.enabled) {
1658 info.view = ¶ms->depth.view;
1659 info.mocs = params->depth.addr.mocs;
1660 } else if (params->stencil.enabled) {
1661 info.view = ¶ms->stencil.view;
1662 info.mocs = params->stencil.addr.mocs;
1663 }
1664
1665 if (params->depth.enabled) {
1666 info.depth_surf = ¶ms->depth.surf;
1667
1668 info.depth_address =
1669 blorp_emit_reloc(batch, dw + isl_dev->ds.depth_offset / 4,
1670 params->depth.addr, 0);
1671
1672 info.hiz_usage = params->depth.aux_usage;
1673 if (isl_aux_usage_has_hiz(info.hiz_usage)) {
1674 info.hiz_surf = ¶ms->depth.aux_surf;
1675
1676 struct blorp_address hiz_address = params->depth.aux_addr;
1677 #if GFX_VER == 6
1678 /* Sandy bridge hardware does not technically support mipmapped HiZ.
1679 * However, we have a special layout that allows us to make it work
1680 * anyway by manually offsetting to the specified miplevel.
1681 */
1682 assert(info.hiz_surf->dim_layout == ISL_DIM_LAYOUT_GFX6_STENCIL_HIZ);
1683 uint64_t offset_B;
1684 isl_surf_get_image_offset_B_tile_sa(info.hiz_surf,
1685 info.view->base_level, 0, 0,
1686 &offset_B, NULL, NULL);
1687 hiz_address.offset += offset_B;
1688 #endif
1689
1690 info.hiz_address =
1691 blorp_emit_reloc(batch, dw + isl_dev->ds.hiz_offset / 4,
1692 hiz_address, 0);
1693
1694 info.depth_clear_value = params->depth.clear_color.f32[0];
1695 }
1696 }
1697
1698 if (params->stencil.enabled) {
1699 info.stencil_surf = ¶ms->stencil.surf;
1700
1701 info.stencil_aux_usage = params->stencil.aux_usage;
1702 struct blorp_address stencil_address = params->stencil.addr;
1703 #if GFX_VER == 6
1704 /* Sandy bridge hardware does not technically support mipmapped stencil.
1705 * However, we have a special layout that allows us to make it work
1706 * anyway by manually offsetting to the specified miplevel.
1707 */
1708 assert(info.stencil_surf->dim_layout == ISL_DIM_LAYOUT_GFX6_STENCIL_HIZ);
1709 uint64_t offset_B;
1710 isl_surf_get_image_offset_B_tile_sa(info.stencil_surf,
1711 info.view->base_level, 0, 0,
1712 &offset_B, NULL, NULL);
1713 stencil_address.offset += offset_B;
1714 #endif
1715
1716 info.stencil_address =
1717 blorp_emit_reloc(batch, dw + isl_dev->ds.stencil_offset / 4,
1718 stencil_address, 0);
1719 }
1720
1721 isl_emit_depth_stencil_hiz_s(isl_dev, dw, &info);
1722
1723 #if GFX_VER >= 12
1724 /* Wa_1408224581
1725 *
1726 * Workaround: Gfx12LP Astep only An additional pipe control with
1727 * post-sync = store dword operation would be required.( w/a is to
1728 * have an additional pipe control after the stencil state whenever
1729 * the surface state bits of this state is changing).
1730 */
1731 blorp_emit(batch, GENX(PIPE_CONTROL), pc) {
1732 pc.PostSyncOperation = WriteImmediateData;
1733 pc.Address = blorp_get_workaround_address(batch);
1734 }
1735 #endif
1736 }
1737
1738 #if GFX_VER >= 8
1739 /* Emits the Optimized HiZ sequence specified in the BDW+ PRMs. The
1740 * depth/stencil buffer extents are ignored to handle APIs which perform
1741 * clearing operations without such information.
1742 * */
1743 static void
blorp_emit_gfx8_hiz_op(struct blorp_batch * batch,const struct blorp_params * params)1744 blorp_emit_gfx8_hiz_op(struct blorp_batch *batch,
1745 const struct blorp_params *params)
1746 {
1747 /* We should be performing an operation on a depth or stencil buffer.
1748 */
1749 assert(params->depth.enabled || params->stencil.enabled);
1750
1751 /* The stencil buffer should only be enabled if a fast clear operation is
1752 * requested.
1753 */
1754 if (params->stencil.enabled)
1755 assert(params->hiz_op == ISL_AUX_OP_FAST_CLEAR);
1756
1757 /* From the BDW PRM Volume 2, 3DSTATE_WM_HZ_OP:
1758 *
1759 * 3DSTATE_MULTISAMPLE packet must be used prior to this packet to change
1760 * the Number of Multisamples. This packet must not be used to change
1761 * Number of Multisamples in a rendering sequence.
1762 *
1763 * Since HIZ may be the first thing in a batch buffer, play safe and always
1764 * emit 3DSTATE_MULTISAMPLE.
1765 */
1766 blorp_emit_3dstate_multisample(batch, params);
1767
1768 /* From the BDW PRM Volume 7, Depth Buffer Clear:
1769 *
1770 * The clear value must be between the min and max depth values
1771 * (inclusive) defined in the CC_VIEWPORT. If the depth buffer format is
1772 * D32_FLOAT, then +/-DENORM values are also allowed.
1773 *
1774 * Set the bounds to match our hardware limits, [0.0, 1.0].
1775 */
1776 if (params->depth.enabled && params->hiz_op == ISL_AUX_OP_FAST_CLEAR) {
1777 assert(params->depth.clear_color.f32[0] >= 0.0f);
1778 assert(params->depth.clear_color.f32[0] <= 1.0f);
1779 blorp_emit_cc_viewport(batch);
1780 }
1781
1782 /* According to the SKL PRM formula for WM_INT::ThreadDispatchEnable, the
1783 * 3DSTATE_WM::ForceThreadDispatchEnable field can force WM thread dispatch
1784 * even when WM_HZ_OP is active. However, WM thread dispatch is normally
1785 * disabled for HiZ ops and it appears that force-enabling it can lead to
1786 * GPU hangs on at least Skylake. Since we don't know the current state of
1787 * the 3DSTATE_WM packet, just emit a dummy one prior to 3DSTATE_WM_HZ_OP.
1788 */
1789 blorp_emit(batch, GENX(3DSTATE_WM), wm);
1790
1791 /* If we can't alter the depth stencil config and multiple layers are
1792 * involved, the HiZ op will fail. This is because the op requires that a
1793 * new config is emitted for each additional layer.
1794 */
1795 if (batch->flags & BLORP_BATCH_NO_EMIT_DEPTH_STENCIL) {
1796 assert(params->num_layers <= 1);
1797 } else {
1798 blorp_emit_depth_stencil_config(batch, params);
1799 }
1800
1801 blorp_measure_start(batch, params);
1802
1803 blorp_emit(batch, GENX(3DSTATE_WM_HZ_OP), hzp) {
1804 switch (params->hiz_op) {
1805 case ISL_AUX_OP_FAST_CLEAR:
1806 hzp.StencilBufferClearEnable = params->stencil.enabled;
1807 hzp.DepthBufferClearEnable = params->depth.enabled;
1808 hzp.StencilClearValue = params->stencil_ref;
1809 hzp.FullSurfaceDepthandStencilClear = params->full_surface_hiz_op;
1810 break;
1811 case ISL_AUX_OP_FULL_RESOLVE:
1812 assert(params->full_surface_hiz_op);
1813 hzp.DepthBufferResolveEnable = true;
1814 break;
1815 case ISL_AUX_OP_AMBIGUATE:
1816 assert(params->full_surface_hiz_op);
1817 hzp.HierarchicalDepthBufferResolveEnable = true;
1818 break;
1819 case ISL_AUX_OP_PARTIAL_RESOLVE:
1820 case ISL_AUX_OP_NONE:
1821 unreachable("Invalid HIZ op");
1822 }
1823
1824 hzp.NumberofMultisamples = ffs(params->num_samples) - 1;
1825 hzp.SampleMask = 0xFFFF;
1826
1827 /* Due to a hardware issue, this bit MBZ */
1828 assert(hzp.ScissorRectangleEnable == false);
1829
1830 /* Contrary to the HW docs both fields are inclusive */
1831 hzp.ClearRectangleXMin = params->x0;
1832 hzp.ClearRectangleYMin = params->y0;
1833
1834 /* Contrary to the HW docs both fields are exclusive */
1835 hzp.ClearRectangleXMax = params->x1;
1836 hzp.ClearRectangleYMax = params->y1;
1837 }
1838
1839 /* PIPE_CONTROL w/ all bits clear except for “Post-Sync Operation” must set
1840 * to “Write Immediate Data” enabled.
1841 */
1842 blorp_emit(batch, GENX(PIPE_CONTROL), pc) {
1843 pc.PostSyncOperation = WriteImmediateData;
1844 pc.Address = blorp_get_workaround_address(batch);
1845 }
1846
1847 blorp_emit(batch, GENX(3DSTATE_WM_HZ_OP), hzp);
1848 }
1849 #endif
1850
1851 static void
blorp_update_clear_color(UNUSED struct blorp_batch * batch,const struct brw_blorp_surface_info * info,enum isl_aux_op op)1852 blorp_update_clear_color(UNUSED struct blorp_batch *batch,
1853 const struct brw_blorp_surface_info *info,
1854 enum isl_aux_op op)
1855 {
1856 if (info->clear_color_addr.buffer && op == ISL_AUX_OP_FAST_CLEAR) {
1857 #if GFX_VER == 11
1858 blorp_emit(batch, GENX(PIPE_CONTROL), pipe) {
1859 pipe.CommandStreamerStallEnable = true;
1860 }
1861
1862 /* 2 QWORDS */
1863 const unsigned inlinedata_dw = 2 * 2;
1864 const unsigned num_dwords = GENX(MI_ATOMIC_length) + inlinedata_dw;
1865
1866 struct blorp_address clear_addr = info->clear_color_addr;
1867 uint32_t *dw = blorp_emitn(batch, GENX(MI_ATOMIC), num_dwords,
1868 .DataSize = MI_ATOMIC_QWORD,
1869 .ATOMICOPCODE = MI_ATOMIC_OP_MOVE8B,
1870 .InlineData = true,
1871 .MemoryAddress = clear_addr);
1872 /* dw starts at dword 1, but we need to fill dwords 3 and 5 */
1873 dw[2] = info->clear_color.u32[0];
1874 dw[3] = 0;
1875 dw[4] = info->clear_color.u32[1];
1876 dw[5] = 0;
1877
1878 clear_addr.offset += 8;
1879 dw = blorp_emitn(batch, GENX(MI_ATOMIC), num_dwords,
1880 .DataSize = MI_ATOMIC_QWORD,
1881 .ATOMICOPCODE = MI_ATOMIC_OP_MOVE8B,
1882 .CSSTALL = true,
1883 .ReturnDataControl = true,
1884 .InlineData = true,
1885 .MemoryAddress = clear_addr);
1886 /* dw starts at dword 1, but we need to fill dwords 3 and 5 */
1887 dw[2] = info->clear_color.u32[2];
1888 dw[3] = 0;
1889 dw[4] = info->clear_color.u32[3];
1890 dw[5] = 0;
1891
1892 blorp_emit(batch, GENX(PIPE_CONTROL), pipe) {
1893 pipe.StateCacheInvalidationEnable = true;
1894 pipe.TextureCacheInvalidationEnable = true;
1895 }
1896 #elif GFX_VER >= 9
1897
1898 /* According to Wa_2201730850, in the Clear Color Programming Note
1899 * under the Red channel, "Software shall write the converted Depth
1900 * Clear to this dword." The only depth formats listed under the red
1901 * channel are IEEE_FP and UNORM24_X8. These two requirements are
1902 * incompatible with the UNORM16 depth format, so just ignore that case
1903 * and simply perform the conversion for all depth formats.
1904 */
1905 union isl_color_value fixed_color = info->clear_color;
1906 if (GFX_VER == 12 && isl_surf_usage_is_depth(info->surf.usage)) {
1907 isl_color_value_pack(&info->clear_color, info->surf.format,
1908 fixed_color.u32);
1909 }
1910
1911 for (int i = 0; i < 4; i++) {
1912 blorp_emit(batch, GENX(MI_STORE_DATA_IMM), sdi) {
1913 sdi.Address = info->clear_color_addr;
1914 sdi.Address.offset += i * 4;
1915 sdi.ImmediateData = fixed_color.u32[i];
1916 #if GFX_VER >= 12
1917 if (i == 3)
1918 sdi.ForceWriteCompletionCheck = true;
1919 #endif
1920 }
1921 }
1922
1923 /* The RENDER_SURFACE_STATE::ClearColor field states that software should
1924 * write the converted depth value 16B after the clear address:
1925 *
1926 * 3D Sampler will always fetch clear depth from the location 16-bytes
1927 * above this address, where the clear depth, converted to native
1928 * surface format by software, will be stored.
1929 *
1930 */
1931 #if GFX_VER >= 12
1932 if (isl_surf_usage_is_depth(info->surf.usage)) {
1933 blorp_emit(batch, GENX(MI_STORE_DATA_IMM), sdi) {
1934 sdi.Address = info->clear_color_addr;
1935 sdi.Address.offset += 4 * 4;
1936 sdi.ImmediateData = fixed_color.u32[0];
1937 sdi.ForceWriteCompletionCheck = true;
1938 }
1939 }
1940 #endif
1941
1942 #elif GFX_VER >= 7
1943 blorp_emit(batch, GENX(MI_STORE_DATA_IMM), sdi) {
1944 sdi.Address = info->clear_color_addr;
1945 sdi.ImmediateData = ISL_CHANNEL_SELECT_RED << 25 |
1946 ISL_CHANNEL_SELECT_GREEN << 22 |
1947 ISL_CHANNEL_SELECT_BLUE << 19 |
1948 ISL_CHANNEL_SELECT_ALPHA << 16;
1949 if (isl_format_has_int_channel(info->view.format)) {
1950 for (unsigned i = 0; i < 4; i++) {
1951 assert(info->clear_color.u32[i] == 0 ||
1952 info->clear_color.u32[i] == 1);
1953 }
1954 sdi.ImmediateData |= (info->clear_color.u32[0] != 0) << 31;
1955 sdi.ImmediateData |= (info->clear_color.u32[1] != 0) << 30;
1956 sdi.ImmediateData |= (info->clear_color.u32[2] != 0) << 29;
1957 sdi.ImmediateData |= (info->clear_color.u32[3] != 0) << 28;
1958 } else {
1959 for (unsigned i = 0; i < 4; i++) {
1960 assert(info->clear_color.f32[i] == 0.0f ||
1961 info->clear_color.f32[i] == 1.0f);
1962 }
1963 sdi.ImmediateData |= (info->clear_color.f32[0] != 0.0f) << 31;
1964 sdi.ImmediateData |= (info->clear_color.f32[1] != 0.0f) << 30;
1965 sdi.ImmediateData |= (info->clear_color.f32[2] != 0.0f) << 29;
1966 sdi.ImmediateData |= (info->clear_color.f32[3] != 0.0f) << 28;
1967 }
1968 }
1969 #endif
1970 }
1971 }
1972
1973 static void
blorp_exec_3d(struct blorp_batch * batch,const struct blorp_params * params)1974 blorp_exec_3d(struct blorp_batch *batch, const struct blorp_params *params)
1975 {
1976 if (!(batch->flags & BLORP_BATCH_NO_UPDATE_CLEAR_COLOR)) {
1977 blorp_update_clear_color(batch, ¶ms->dst, params->fast_clear_op);
1978 blorp_update_clear_color(batch, ¶ms->depth, params->hiz_op);
1979 }
1980
1981 #if GFX_VER >= 8
1982 if (params->hiz_op != ISL_AUX_OP_NONE) {
1983 blorp_emit_gfx8_hiz_op(batch, params);
1984 return;
1985 }
1986 #endif
1987
1988 blorp_emit_vertex_buffers(batch, params);
1989 blorp_emit_vertex_elements(batch, params);
1990
1991 blorp_emit_pipeline(batch, params);
1992
1993 blorp_emit_btp(batch, blorp_setup_binding_table(batch, params));
1994
1995 if (!(batch->flags & BLORP_BATCH_NO_EMIT_DEPTH_STENCIL))
1996 blorp_emit_depth_stencil_config(batch, params);
1997
1998 blorp_measure_start(batch, params);
1999
2000 blorp_emit(batch, GENX(3DPRIMITIVE), prim) {
2001 prim.VertexAccessType = SEQUENTIAL;
2002 prim.PrimitiveTopologyType = _3DPRIM_RECTLIST;
2003 #if GFX_VER >= 7
2004 prim.PredicateEnable = batch->flags & BLORP_BATCH_PREDICATE_ENABLE;
2005 #endif
2006 prim.VertexCountPerInstance = 3;
2007 prim.InstanceCount = params->num_layers;
2008 }
2009 }
2010
2011 #if GFX_VER >= 7
2012
2013 static void
blorp_get_compute_push_const(struct blorp_batch * batch,const struct blorp_params * params,uint32_t threads,uint32_t * state_offset,unsigned * state_size)2014 blorp_get_compute_push_const(struct blorp_batch *batch,
2015 const struct blorp_params *params,
2016 uint32_t threads,
2017 uint32_t *state_offset,
2018 unsigned *state_size)
2019 {
2020 const struct brw_cs_prog_data *cs_prog_data = params->cs_prog_data;
2021 const unsigned push_const_size =
2022 ALIGN(brw_cs_push_const_total_size(cs_prog_data, threads), 64);
2023 assert(cs_prog_data->push.cross_thread.size +
2024 cs_prog_data->push.per_thread.size == sizeof(params->wm_inputs));
2025
2026 if (push_const_size == 0) {
2027 *state_offset = 0;
2028 *state_size = 0;
2029 return;
2030 }
2031
2032 uint32_t push_const_offset;
2033 uint32_t *push_const =
2034 GFX_VERx10 >= 125 ?
2035 blorp_alloc_general_state(batch, push_const_size, 64,
2036 &push_const_offset) :
2037 blorp_alloc_dynamic_state(batch, push_const_size, 64,
2038 &push_const_offset);
2039 memset(push_const, 0x0, push_const_size);
2040
2041 void *dst = push_const;
2042 const void *src = (char *)¶ms->wm_inputs;
2043
2044 if (cs_prog_data->push.cross_thread.size > 0) {
2045 memcpy(dst, src, cs_prog_data->push.cross_thread.size);
2046 dst += cs_prog_data->push.cross_thread.size;
2047 src += cs_prog_data->push.cross_thread.size;
2048 }
2049
2050 assert(GFX_VERx10 < 125 || cs_prog_data->push.per_thread.size == 0);
2051 #if GFX_VERx10 < 125
2052 if (cs_prog_data->push.per_thread.size > 0) {
2053 for (unsigned t = 0; t < threads; t++) {
2054 memcpy(dst, src, (cs_prog_data->push.per_thread.dwords - 1) * 4);
2055
2056 uint32_t *subgroup_id = dst + cs_prog_data->push.per_thread.size - 4;
2057 *subgroup_id = t;
2058
2059 dst += cs_prog_data->push.per_thread.size;
2060 }
2061 }
2062 #endif
2063
2064 *state_offset = push_const_offset;
2065 *state_size = push_const_size;
2066 }
2067
2068 #endif /* GFX_VER >= 7 */
2069
2070 static void
blorp_exec_compute(struct blorp_batch * batch,const struct blorp_params * params)2071 blorp_exec_compute(struct blorp_batch *batch, const struct blorp_params *params)
2072 {
2073 assert(!(batch->flags & BLORP_BATCH_NO_UPDATE_CLEAR_COLOR));
2074 assert(params->hiz_op == ISL_AUX_OP_NONE);
2075
2076 #if GFX_VER >= 7
2077
2078 const struct brw_cs_prog_data *cs_prog_data = params->cs_prog_data;
2079 const struct brw_stage_prog_data *prog_data = &cs_prog_data->base;
2080 const struct brw_cs_dispatch_info dispatch =
2081 brw_cs_get_dispatch_info(batch->blorp->compiler->devinfo, cs_prog_data,
2082 NULL);
2083 const struct intel_device_info *devinfo = batch->blorp->compiler->devinfo;
2084
2085 uint32_t group_x0 = params->x0 / cs_prog_data->local_size[0];
2086 uint32_t group_y0 = params->y0 / cs_prog_data->local_size[1];
2087 uint32_t group_z0 = params->dst.z_offset;
2088 uint32_t group_x1 = DIV_ROUND_UP(params->x1, cs_prog_data->local_size[0]);
2089 uint32_t group_y1 = DIV_ROUND_UP(params->y1, cs_prog_data->local_size[1]);
2090 assert(params->num_layers >= 1);
2091 uint32_t group_z1 = params->dst.z_offset + params->num_layers;
2092 assert(cs_prog_data->local_size[2] == 1);
2093
2094 #endif /* GFX_VER >= 7 */
2095
2096 #if GFX_VERx10 >= 125
2097
2098 blorp_emit(batch, GENX(CFE_STATE), cfe) {
2099 cfe.MaximumNumberofThreads =
2100 devinfo->max_cs_threads * devinfo->subslice_total - 1;
2101 }
2102
2103 assert(cs_prog_data->push.per_thread.regs == 0);
2104 blorp_emit(batch, GENX(COMPUTE_WALKER), cw) {
2105 cw.SIMDSize = dispatch.simd_size / 16;
2106 cw.LocalXMaximum = cs_prog_data->local_size[0] - 1;
2107 cw.LocalYMaximum = cs_prog_data->local_size[1] - 1;
2108 cw.LocalZMaximum = cs_prog_data->local_size[2] - 1;
2109 cw.ThreadGroupIDStartingX = group_x0;
2110 cw.ThreadGroupIDStartingY = group_y0;
2111 cw.ThreadGroupIDStartingZ = group_z0;
2112 cw.ThreadGroupIDXDimension = group_x1;
2113 cw.ThreadGroupIDYDimension = group_y1;
2114 cw.ThreadGroupIDZDimension = group_z1;
2115 cw.ExecutionMask = 0xffffffff;
2116
2117 uint32_t surfaces_offset = blorp_setup_binding_table(batch, params);
2118
2119 uint32_t samplers_offset =
2120 params->src.enabled ? blorp_emit_sampler_state(batch) : 0;
2121
2122 uint32_t push_const_offset;
2123 unsigned push_const_size;
2124 blorp_get_compute_push_const(batch, params, dispatch.threads,
2125 &push_const_offset, &push_const_size);
2126 cw.IndirectDataStartAddress = push_const_offset;
2127 cw.IndirectDataLength = push_const_size;
2128
2129 cw.InterfaceDescriptor = (struct GENX(INTERFACE_DESCRIPTOR_DATA)) {
2130 .KernelStartPointer = params->cs_prog_kernel,
2131 .SamplerStatePointer = samplers_offset,
2132 .SamplerCount = params->src.enabled ? 1 : 0,
2133 .BindingTableEntryCount = params->src.enabled ? 2 : 1,
2134 .BindingTablePointer = surfaces_offset,
2135 .NumberofThreadsinGPGPUThreadGroup = dispatch.threads,
2136 .SharedLocalMemorySize =
2137 encode_slm_size(GFX_VER, prog_data->total_shared),
2138 .NumberOfBarriers = cs_prog_data->uses_barrier,
2139 };
2140 }
2141
2142 #elif GFX_VER >= 7
2143
2144 /* The MEDIA_VFE_STATE documentation for Gfx8+ says:
2145 *
2146 * "A stalling PIPE_CONTROL is required before MEDIA_VFE_STATE unless
2147 * the only bits that are changed are scoreboard related: Scoreboard
2148 * Enable, Scoreboard Type, Scoreboard Mask, Scoreboard * Delta. For
2149 * these scoreboard related states, a MEDIA_STATE_FLUSH is sufficient."
2150 *
2151 * Earlier generations say "MI_FLUSH" instead of "stalling PIPE_CONTROL",
2152 * but MI_FLUSH isn't really a thing, so we assume they meant PIPE_CONTROL.
2153 */
2154 blorp_emit(batch, GENX(PIPE_CONTROL), pc) {
2155 pc.CommandStreamerStallEnable = true;
2156 pc.StallAtPixelScoreboard = true;
2157 }
2158
2159 blorp_emit(batch, GENX(MEDIA_VFE_STATE), vfe) {
2160 assert(prog_data->total_scratch == 0);
2161 vfe.MaximumNumberofThreads =
2162 devinfo->max_cs_threads * devinfo->subslice_total - 1;
2163 vfe.NumberofURBEntries = GFX_VER >= 8 ? 2 : 0;
2164 #if GFX_VER < 11
2165 vfe.ResetGatewayTimer =
2166 Resettingrelativetimerandlatchingtheglobaltimestamp;
2167 #endif
2168 #if GFX_VER < 9
2169 vfe.BypassGatewayControl = BypassingOpenGatewayCloseGatewayprotocol;
2170 #endif
2171 #if GFX_VER == 7
2172 vfe.GPGPUMode = true;
2173 #endif
2174 vfe.URBEntryAllocationSize = GFX_VER >= 8 ? 2 : 0;
2175
2176 const uint32_t vfe_curbe_allocation =
2177 ALIGN(cs_prog_data->push.per_thread.regs * dispatch.threads +
2178 cs_prog_data->push.cross_thread.regs, 2);
2179 vfe.CURBEAllocationSize = vfe_curbe_allocation;
2180 }
2181
2182 uint32_t push_const_offset;
2183 unsigned push_const_size;
2184 blorp_get_compute_push_const(batch, params, dispatch.threads,
2185 &push_const_offset, &push_const_size);
2186
2187 blorp_emit(batch, GENX(MEDIA_CURBE_LOAD), curbe) {
2188 curbe.CURBETotalDataLength = push_const_size;
2189 curbe.CURBEDataStartAddress = push_const_offset;
2190 }
2191
2192 uint32_t surfaces_offset = blorp_setup_binding_table(batch, params);
2193
2194 uint32_t samplers_offset =
2195 params->src.enabled ? blorp_emit_sampler_state(batch) : 0;
2196
2197 struct GENX(INTERFACE_DESCRIPTOR_DATA) idd = {
2198 .KernelStartPointer = params->cs_prog_kernel,
2199 .SamplerStatePointer = samplers_offset,
2200 .SamplerCount = params->src.enabled ? 1 : 0,
2201 .BindingTableEntryCount = params->src.enabled ? 2 : 1,
2202 .BindingTablePointer = surfaces_offset,
2203 .ConstantURBEntryReadLength = cs_prog_data->push.per_thread.regs,
2204 .NumberofThreadsinGPGPUThreadGroup = dispatch.threads,
2205 .SharedLocalMemorySize = encode_slm_size(GFX_VER,
2206 prog_data->total_shared),
2207 .BarrierEnable = cs_prog_data->uses_barrier,
2208 #if GFX_VER >= 8 || GEN_IS_HASWELL
2209 .CrossThreadConstantDataReadLength =
2210 cs_prog_data->push.cross_thread.regs,
2211 #endif
2212 };
2213
2214 uint32_t idd_offset;
2215 uint32_t size = GENX(INTERFACE_DESCRIPTOR_DATA_length) * sizeof(uint32_t);
2216 void *state = blorp_alloc_dynamic_state(batch, size, 64, &idd_offset);
2217 GENX(INTERFACE_DESCRIPTOR_DATA_pack)(NULL, state, &idd);
2218
2219 blorp_emit(batch, GENX(MEDIA_INTERFACE_DESCRIPTOR_LOAD), mid) {
2220 mid.InterfaceDescriptorTotalLength = size;
2221 mid.InterfaceDescriptorDataStartAddress = idd_offset;
2222 }
2223
2224 blorp_emit(batch, GENX(GPGPU_WALKER), ggw) {
2225 ggw.SIMDSize = dispatch.simd_size / 16;
2226 ggw.ThreadDepthCounterMaximum = 0;
2227 ggw.ThreadHeightCounterMaximum = 0;
2228 ggw.ThreadWidthCounterMaximum = dispatch.threads - 1;
2229 ggw.ThreadGroupIDStartingX = group_x0;
2230 ggw.ThreadGroupIDStartingY = group_y0;
2231 #if GFX_VER >= 8
2232 ggw.ThreadGroupIDStartingResumeZ = group_z0;
2233 #else
2234 ggw.ThreadGroupIDStartingZ = group_z0;
2235 #endif
2236 ggw.ThreadGroupIDXDimension = group_x1;
2237 ggw.ThreadGroupIDYDimension = group_y1;
2238 ggw.ThreadGroupIDZDimension = group_z1;
2239 ggw.RightExecutionMask = dispatch.right_mask;
2240 ggw.BottomExecutionMask = 0xffffffff;
2241 }
2242
2243 #else /* GFX_VER >= 7 */
2244
2245 unreachable("Compute blorp is not supported on SNB and earlier");
2246
2247 #endif /* GFX_VER >= 7 */
2248 }
2249
2250
2251 /**
2252 * \brief Execute a blit or render pass operation.
2253 *
2254 * To execute the operation, this function manually constructs and emits a
2255 * batch to draw a rectangle primitive. The batchbuffer is flushed before
2256 * constructing and after emitting the batch.
2257 *
2258 * This function alters no GL state.
2259 */
2260 static void
blorp_exec(struct blorp_batch * batch,const struct blorp_params * params)2261 blorp_exec(struct blorp_batch *batch, const struct blorp_params *params)
2262 {
2263 if (batch->flags & BLORP_BATCH_USE_COMPUTE)
2264 blorp_exec_compute(batch, params);
2265 else
2266 blorp_exec_3d(batch, params);
2267 }
2268
2269 #endif /* BLORP_GENX_EXEC_H */
2270