1 /**************************************************************************
2 *
3 * Copyright 2009 VMware, Inc.
4 * Copyright 2007 VMware, Inc.
5 * All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
27 **************************************************************************/
28
29 /**
30 * @file
31 * Code generate the whole fragment pipeline.
32 *
33 * The fragment pipeline consists of the following stages:
34 * - early depth test
35 * - fragment shader
36 * - alpha test
37 * - depth/stencil test
38 * - blending
39 *
40 * This file has only the glue to assemble the fragment pipeline. The actual
41 * plumbing of converting Gallium state into LLVM IR is done elsewhere, in the
42 * lp_bld_*.[ch] files, and in a complete generic and reusable way. Here we
43 * muster the LLVM JIT execution engine to create a function that follows an
44 * established binary interface and that can be called from C directly.
45 *
46 * A big source of complexity here is that we often want to run different
47 * stages with different precisions and data types and precisions. For example,
48 * the fragment shader needs typically to be done in floats, but the
49 * depth/stencil test and blending is better done in the type that most closely
50 * matches the depth/stencil and color buffer respectively.
51 *
52 * Since the width of a SIMD vector register stays the same regardless of the
53 * element type, different types imply different number of elements, so we must
54 * code generate more instances of the stages with larger types to be able to
55 * feed/consume the stages with smaller types.
56 *
57 * @author Jose Fonseca <jfonseca@vmware.com>
58 */
59
60 #include <limits.h>
61 #include "pipe/p_defines.h"
62 #include "util/u_inlines.h"
63 #include "util/u_memory.h"
64 #include "util/u_pointer.h"
65 #include "util/format/u_format.h"
66 #include "util/u_dump.h"
67 #include "util/u_string.h"
68 #include "util/simple_list.h"
69 #include "util/u_dual_blend.h"
70 #include "util/u_upload_mgr.h"
71 #include "util/os_time.h"
72 #include "pipe/p_shader_tokens.h"
73 #include "draw/draw_context.h"
74 #include "tgsi/tgsi_dump.h"
75 #include "tgsi/tgsi_scan.h"
76 #include "tgsi/tgsi_parse.h"
77 #include "gallivm/lp_bld_type.h"
78 #include "gallivm/lp_bld_const.h"
79 #include "gallivm/lp_bld_conv.h"
80 #include "gallivm/lp_bld_init.h"
81 #include "gallivm/lp_bld_intr.h"
82 #include "gallivm/lp_bld_logic.h"
83 #include "gallivm/lp_bld_tgsi.h"
84 #include "gallivm/lp_bld_nir.h"
85 #include "gallivm/lp_bld_swizzle.h"
86 #include "gallivm/lp_bld_flow.h"
87 #include "gallivm/lp_bld_debug.h"
88 #include "gallivm/lp_bld_arit.h"
89 #include "gallivm/lp_bld_bitarit.h"
90 #include "gallivm/lp_bld_pack.h"
91 #include "gallivm/lp_bld_format.h"
92 #include "gallivm/lp_bld_quad.h"
93 #include "gallivm/lp_bld_gather.h"
94
95 #include "lp_bld_alpha.h"
96 #include "lp_bld_blend.h"
97 #include "lp_bld_depth.h"
98 #include "lp_bld_interp.h"
99 #include "lp_context.h"
100 #include "lp_debug.h"
101 #include "lp_perf.h"
102 #include "lp_setup.h"
103 #include "lp_state.h"
104 #include "lp_tex_sample.h"
105 #include "lp_flush.h"
106 #include "lp_state_fs.h"
107 #include "lp_rast.h"
108 #include "nir/nir_to_tgsi_info.h"
109
110 #include "lp_screen.h"
111 #include "compiler/nir/nir_serialize.h"
112 #include "util/mesa-sha1.h"
113 /** Fragment shader number (for debugging) */
114 static unsigned fs_no = 0;
115
116 static void
117 load_unswizzled_block(struct gallivm_state *gallivm,
118 LLVMValueRef base_ptr,
119 LLVMValueRef stride,
120 unsigned block_width,
121 unsigned block_height,
122 LLVMValueRef* dst,
123 struct lp_type dst_type,
124 unsigned dst_count,
125 unsigned dst_alignment);
126 /**
127 * Checks if a format description is an arithmetic format
128 *
129 * A format which has irregular channel sizes such as R3_G3_B2 or R5_G6_B5.
130 */
131 static inline boolean
is_arithmetic_format(const struct util_format_description * format_desc)132 is_arithmetic_format(const struct util_format_description *format_desc)
133 {
134 boolean arith = false;
135 unsigned i;
136
137 for (i = 0; i < format_desc->nr_channels; ++i) {
138 arith |= format_desc->channel[i].size != format_desc->channel[0].size;
139 arith |= (format_desc->channel[i].size % 8) != 0;
140 }
141
142 return arith;
143 }
144
145 /**
146 * Checks if this format requires special handling due to required expansion
147 * to floats for blending, and furthermore has "natural" packed AoS -> unpacked
148 * SoA conversion.
149 */
150 static inline boolean
format_expands_to_float_soa(const struct util_format_description * format_desc)151 format_expands_to_float_soa(const struct util_format_description *format_desc)
152 {
153 if (format_desc->format == PIPE_FORMAT_R11G11B10_FLOAT ||
154 format_desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB) {
155 return true;
156 }
157 return false;
158 }
159
160
161 /**
162 * Retrieves the type representing the memory layout for a format
163 *
164 * e.g. RGBA16F = 4x half-float and R3G3B2 = 1x byte
165 */
166 static inline void
lp_mem_type_from_format_desc(const struct util_format_description * format_desc,struct lp_type * type)167 lp_mem_type_from_format_desc(const struct util_format_description *format_desc,
168 struct lp_type* type)
169 {
170 unsigned i;
171 unsigned chan;
172
173 if (format_expands_to_float_soa(format_desc)) {
174 /* just make this a uint with width of block */
175 type->floating = false;
176 type->fixed = false;
177 type->sign = false;
178 type->norm = false;
179 type->width = format_desc->block.bits;
180 type->length = 1;
181 return;
182 }
183
184 for (i = 0; i < 4; i++)
185 if (format_desc->channel[i].type != UTIL_FORMAT_TYPE_VOID)
186 break;
187 chan = i;
188
189 memset(type, 0, sizeof(struct lp_type));
190 type->floating = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FLOAT;
191 type->fixed = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FIXED;
192 type->sign = format_desc->channel[chan].type != UTIL_FORMAT_TYPE_UNSIGNED;
193 type->norm = format_desc->channel[chan].normalized;
194
195 if (is_arithmetic_format(format_desc)) {
196 type->width = 0;
197 type->length = 1;
198
199 for (i = 0; i < format_desc->nr_channels; ++i) {
200 type->width += format_desc->channel[i].size;
201 }
202 } else {
203 type->width = format_desc->channel[chan].size;
204 type->length = format_desc->nr_channels;
205 }
206 }
207
208 /**
209 * Expand the relevant bits of mask_input to a n*4-dword mask for the
210 * n*four pixels in n 2x2 quads. This will set the n*four elements of the
211 * quad mask vector to 0 or ~0.
212 * Grouping is 01, 23 for 2 quad mode hence only 0 and 2 are valid
213 * quad arguments with fs length 8.
214 *
215 * \param first_quad which quad(s) of the quad group to test, in [0,3]
216 * \param mask_input bitwise mask for the whole 4x4 stamp
217 */
218 static LLVMValueRef
generate_quad_mask(struct gallivm_state * gallivm,struct lp_type fs_type,unsigned first_quad,unsigned sample,LLVMValueRef mask_input)219 generate_quad_mask(struct gallivm_state *gallivm,
220 struct lp_type fs_type,
221 unsigned first_quad,
222 unsigned sample,
223 LLVMValueRef mask_input) /* int64 */
224 {
225 LLVMBuilderRef builder = gallivm->builder;
226 struct lp_type mask_type;
227 LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context);
228 LLVMValueRef bits[16];
229 LLVMValueRef mask, bits_vec;
230 int shift, i;
231
232 /*
233 * XXX: We'll need a different path for 16 x u8
234 */
235 assert(fs_type.width == 32);
236 assert(fs_type.length <= ARRAY_SIZE(bits));
237 mask_type = lp_int_type(fs_type);
238
239 /*
240 * mask_input >>= (quad * 4)
241 */
242 switch (first_quad) {
243 case 0:
244 shift = 0;
245 break;
246 case 1:
247 assert(fs_type.length == 4);
248 shift = 2;
249 break;
250 case 2:
251 shift = 8;
252 break;
253 case 3:
254 assert(fs_type.length == 4);
255 shift = 10;
256 break;
257 default:
258 assert(0);
259 shift = 0;
260 }
261
262 mask_input = LLVMBuildLShr(builder, mask_input, lp_build_const_int64(gallivm, 16 * sample), "");
263 mask_input = LLVMBuildTrunc(builder, mask_input,
264 i32t, "");
265 mask_input = LLVMBuildAnd(builder, mask_input, lp_build_const_int32(gallivm, 0xffff), "");
266
267 mask_input = LLVMBuildLShr(builder,
268 mask_input,
269 LLVMConstInt(i32t, shift, 0),
270 "");
271
272 /*
273 * mask = { mask_input & (1 << i), for i in [0,3] }
274 */
275 mask = lp_build_broadcast(gallivm,
276 lp_build_vec_type(gallivm, mask_type),
277 mask_input);
278
279 for (i = 0; i < fs_type.length / 4; i++) {
280 unsigned j = 2 * (i % 2) + (i / 2) * 8;
281 bits[4*i + 0] = LLVMConstInt(i32t, 1ULL << (j + 0), 0);
282 bits[4*i + 1] = LLVMConstInt(i32t, 1ULL << (j + 1), 0);
283 bits[4*i + 2] = LLVMConstInt(i32t, 1ULL << (j + 4), 0);
284 bits[4*i + 3] = LLVMConstInt(i32t, 1ULL << (j + 5), 0);
285 }
286 bits_vec = LLVMConstVector(bits, fs_type.length);
287 mask = LLVMBuildAnd(builder, mask, bits_vec, "");
288
289 /*
290 * mask = mask == bits ? ~0 : 0
291 */
292 mask = lp_build_compare(gallivm,
293 mask_type, PIPE_FUNC_EQUAL,
294 mask, bits_vec);
295
296 return mask;
297 }
298
299
300 #define EARLY_DEPTH_TEST 0x1
301 #define LATE_DEPTH_TEST 0x2
302 #define EARLY_DEPTH_WRITE 0x4
303 #define LATE_DEPTH_WRITE 0x8
304 #define EARLY_DEPTH_TEST_INFERRED 0x10 //only with EARLY_DEPTH_TEST
305
306 static int
find_output_by_semantic(const struct tgsi_shader_info * info,unsigned semantic,unsigned index)307 find_output_by_semantic( const struct tgsi_shader_info *info,
308 unsigned semantic,
309 unsigned index )
310 {
311 int i;
312
313 for (i = 0; i < info->num_outputs; i++)
314 if (info->output_semantic_name[i] == semantic &&
315 info->output_semantic_index[i] == index)
316 return i;
317
318 return -1;
319 }
320
321
322 /**
323 * Fetch the specified lp_jit_viewport structure for a given viewport_index.
324 */
325 static LLVMValueRef
lp_llvm_viewport(LLVMValueRef context_ptr,struct gallivm_state * gallivm,LLVMValueRef viewport_index)326 lp_llvm_viewport(LLVMValueRef context_ptr,
327 struct gallivm_state *gallivm,
328 LLVMValueRef viewport_index)
329 {
330 LLVMBuilderRef builder = gallivm->builder;
331 LLVMValueRef ptr;
332 LLVMValueRef res;
333 struct lp_type viewport_type =
334 lp_type_float_vec(32, 32 * LP_JIT_VIEWPORT_NUM_FIELDS);
335
336 ptr = lp_jit_context_viewports(gallivm, context_ptr);
337 ptr = LLVMBuildPointerCast(builder, ptr,
338 LLVMPointerType(lp_build_vec_type(gallivm, viewport_type), 0), "");
339
340 res = lp_build_pointer_get(builder, ptr, viewport_index);
341
342 return res;
343 }
344
345
346 static LLVMValueRef
lp_build_depth_clamp(struct gallivm_state * gallivm,LLVMBuilderRef builder,struct lp_type type,LLVMValueRef context_ptr,LLVMValueRef thread_data_ptr,LLVMValueRef z)347 lp_build_depth_clamp(struct gallivm_state *gallivm,
348 LLVMBuilderRef builder,
349 struct lp_type type,
350 LLVMValueRef context_ptr,
351 LLVMValueRef thread_data_ptr,
352 LLVMValueRef z)
353 {
354 LLVMValueRef viewport, min_depth, max_depth;
355 LLVMValueRef viewport_index;
356 struct lp_build_context f32_bld;
357
358 assert(type.floating);
359 lp_build_context_init(&f32_bld, gallivm, type);
360
361 /*
362 * Assumes clamping of the viewport index will occur in setup/gs. Value
363 * is passed through the rasterization stage via lp_rast_shader_inputs.
364 *
365 * See: draw_clamp_viewport_idx and lp_clamp_viewport_idx for clamping
366 * semantics.
367 */
368 viewport_index = lp_jit_thread_data_raster_state_viewport_index(gallivm,
369 thread_data_ptr);
370
371 /*
372 * Load the min and max depth from the lp_jit_context.viewports
373 * array of lp_jit_viewport structures.
374 */
375 viewport = lp_llvm_viewport(context_ptr, gallivm, viewport_index);
376
377 /* viewports[viewport_index].min_depth */
378 min_depth = LLVMBuildExtractElement(builder, viewport,
379 lp_build_const_int32(gallivm, LP_JIT_VIEWPORT_MIN_DEPTH), "");
380 min_depth = lp_build_broadcast_scalar(&f32_bld, min_depth);
381
382 /* viewports[viewport_index].max_depth */
383 max_depth = LLVMBuildExtractElement(builder, viewport,
384 lp_build_const_int32(gallivm, LP_JIT_VIEWPORT_MAX_DEPTH), "");
385 max_depth = lp_build_broadcast_scalar(&f32_bld, max_depth);
386
387 /*
388 * Clamp to the min and max depth values for the given viewport.
389 */
390 return lp_build_clamp(&f32_bld, z, min_depth, max_depth);
391 }
392
393 static void
lp_build_sample_alpha_to_coverage(struct gallivm_state * gallivm,struct lp_type type,unsigned coverage_samples,LLVMValueRef num_loop,LLVMValueRef loop_counter,LLVMValueRef coverage_mask_store,LLVMValueRef alpha)394 lp_build_sample_alpha_to_coverage(struct gallivm_state *gallivm,
395 struct lp_type type,
396 unsigned coverage_samples,
397 LLVMValueRef num_loop,
398 LLVMValueRef loop_counter,
399 LLVMValueRef coverage_mask_store,
400 LLVMValueRef alpha)
401 {
402 struct lp_build_context bld;
403 LLVMBuilderRef builder = gallivm->builder;
404 float step = 1.0 / coverage_samples;
405
406 lp_build_context_init(&bld, gallivm, type);
407 for (unsigned s = 0; s < coverage_samples; s++) {
408 LLVMValueRef alpha_ref_value = lp_build_const_vec(gallivm, type, step * s);
409 LLVMValueRef test = lp_build_cmp(&bld, PIPE_FUNC_GREATER, alpha, alpha_ref_value);
410
411 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, lp_build_const_int32(gallivm, s), num_loop, "");
412 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_counter, "");
413 LLVMValueRef s_mask_ptr = LLVMBuildGEP(builder, coverage_mask_store, &s_mask_idx, 1, "");
414 LLVMValueRef s_mask = LLVMBuildLoad(builder, s_mask_ptr, "");
415 s_mask = LLVMBuildAnd(builder, s_mask, test, "");
416 LLVMBuildStore(builder, s_mask, s_mask_ptr);
417 }
418 };
419
420 struct lp_build_fs_llvm_iface {
421 struct lp_build_fs_iface base;
422 struct lp_build_interp_soa_context *interp;
423 struct lp_build_for_loop_state *loop_state;
424 LLVMValueRef mask_store;
425 LLVMValueRef sample_id;
426 LLVMValueRef color_ptr_ptr;
427 LLVMValueRef color_stride_ptr;
428 LLVMValueRef color_sample_stride_ptr;
429 const struct lp_fragment_shader_variant_key *key;
430 };
431
fs_interp(const struct lp_build_fs_iface * iface,struct lp_build_context * bld,unsigned attrib,unsigned chan,bool centroid,bool sample,LLVMValueRef attrib_indir,LLVMValueRef offsets[2])432 static LLVMValueRef fs_interp(const struct lp_build_fs_iface *iface,
433 struct lp_build_context *bld,
434 unsigned attrib, unsigned chan,
435 bool centroid, bool sample,
436 LLVMValueRef attrib_indir,
437 LLVMValueRef offsets[2])
438 {
439 struct lp_build_fs_llvm_iface *fs_iface = (struct lp_build_fs_llvm_iface *)iface;
440 struct lp_build_interp_soa_context *interp = fs_iface->interp;
441 unsigned loc = TGSI_INTERPOLATE_LOC_CENTER;
442 if (centroid)
443 loc = TGSI_INTERPOLATE_LOC_CENTROID;
444 if (sample)
445 loc = TGSI_INTERPOLATE_LOC_SAMPLE;
446
447 return lp_build_interp_soa(interp, bld->gallivm, fs_iface->loop_state->counter,
448 fs_iface->mask_store,
449 attrib, chan, loc, attrib_indir, offsets);
450 }
451
fs_fb_fetch(const struct lp_build_fs_iface * iface,struct lp_build_context * bld,int location,LLVMValueRef result[4])452 static void fs_fb_fetch(const struct lp_build_fs_iface *iface,
453 struct lp_build_context *bld,
454 int location,
455 LLVMValueRef result[4])
456 {
457 assert(location >= FRAG_RESULT_DATA0 && location <= FRAG_RESULT_DATA7);
458 const int cbuf = location - FRAG_RESULT_DATA0;
459
460 struct lp_build_fs_llvm_iface *fs_iface = (struct lp_build_fs_llvm_iface *)iface;
461 struct gallivm_state *gallivm = bld->gallivm;
462 LLVMBuilderRef builder = gallivm->builder;
463 const struct lp_fragment_shader_variant_key *key = fs_iface->key;
464 LLVMValueRef index = lp_build_const_int32(gallivm, cbuf);
465 LLVMValueRef color_ptr = LLVMBuildLoad(builder, LLVMBuildGEP(builder, fs_iface->color_ptr_ptr, &index, 1, ""), "");
466 LLVMValueRef stride = LLVMBuildLoad(builder, LLVMBuildGEP(builder, fs_iface->color_stride_ptr, &index, 1, ""), "");
467
468 enum pipe_format cbuf_format = key->cbuf_format[cbuf];
469 const struct util_format_description* out_format_desc = util_format_description(cbuf_format);
470 if (out_format_desc->format == PIPE_FORMAT_NONE) {
471 result[0] = result[1] = result[2] = result[3] = bld->undef;
472 return;
473 }
474
475 unsigned block_size = bld->type.length;
476 unsigned block_height = key->resource_1d ? 1 : 2;
477 unsigned block_width = block_size / block_height;
478
479 if (key->multisample) {
480 LLVMValueRef sample_stride = LLVMBuildLoad(builder,
481 LLVMBuildGEP(builder, fs_iface->color_sample_stride_ptr,
482 &index, 1, ""), "");
483 LLVMValueRef sample_offset = LLVMBuildMul(builder, sample_stride, fs_iface->sample_id, "");
484 color_ptr = LLVMBuildGEP(builder, color_ptr, &sample_offset, 1, "");
485 }
486 /* fragment shader executes on 4x4 blocks. depending on vector width it can execute 2 or 4 iterations.
487 * only move to the next row once the top row has completed 8 wide 1 iteration, 4 wide 2 iterations */
488 LLVMValueRef x_offset = NULL, y_offset = NULL;
489 if (!key->resource_1d) {
490 LLVMValueRef counter = fs_iface->loop_state->counter;
491
492 if (block_size == 4) {
493 x_offset = LLVMBuildShl(builder,
494 LLVMBuildAnd(builder, fs_iface->loop_state->counter, lp_build_const_int32(gallivm, 1), ""),
495 lp_build_const_int32(gallivm, 1), "");
496 counter = LLVMBuildLShr(builder, fs_iface->loop_state->counter, lp_build_const_int32(gallivm, 1), "");
497 }
498 y_offset = LLVMBuildMul(builder, counter, lp_build_const_int32(gallivm, 2), "");
499 }
500
501 LLVMValueRef offsets[4 * 4];
502 for (unsigned i = 0; i < block_size; i++) {
503 unsigned x = i % block_width;
504 unsigned y = i / block_width;
505
506 if (block_size == 8) {
507 /* remap the raw slots into the fragment shader execution mode. */
508 /* this math took me way too long to work out, I'm sure it's overkill. */
509 x = (i & 1) + ((i >> 2) << 1);
510 if (!key->resource_1d)
511 y = (i & 2) >> 1;
512 }
513
514 LLVMValueRef x_val;
515 if (x_offset) {
516 x_val = LLVMBuildAdd(builder, lp_build_const_int32(gallivm, x), x_offset, "");
517 x_val = LLVMBuildMul(builder, x_val, lp_build_const_int32(gallivm, out_format_desc->block.bits / 8), "");
518 } else {
519 x_val = lp_build_const_int32(gallivm, x * (out_format_desc->block.bits / 8));
520 }
521
522 LLVMValueRef y_val = lp_build_const_int32(gallivm, y);
523 if (y_offset)
524 y_val = LLVMBuildAdd(builder, y_val, y_offset, "");
525 y_val = LLVMBuildMul(builder, y_val, stride, "");
526
527 offsets[i] = LLVMBuildAdd(builder, x_val, y_val, "");
528 }
529 LLVMValueRef offset = lp_build_gather_values(gallivm, offsets, block_size);
530
531 struct lp_type texel_type = bld->type;
532 if (out_format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB &&
533 out_format_desc->channel[0].pure_integer) {
534 if (out_format_desc->channel[0].type == UTIL_FORMAT_TYPE_SIGNED) {
535 texel_type = lp_type_int_vec(bld->type.width, bld->type.width * bld->type.length);
536 }
537 else if (out_format_desc->channel[0].type == UTIL_FORMAT_TYPE_UNSIGNED) {
538 texel_type = lp_type_uint_vec(bld->type.width, bld->type.width * bld->type.length);
539 }
540 }
541
542 lp_build_fetch_rgba_soa(gallivm, out_format_desc, texel_type,
543 true, color_ptr, offset,
544 NULL, NULL, NULL, result);
545 }
546
547 /**
548 * Generate the fragment shader, depth/stencil test, and alpha tests.
549 */
550 static void
generate_fs_loop(struct gallivm_state * gallivm,struct lp_fragment_shader * shader,const struct lp_fragment_shader_variant_key * key,LLVMBuilderRef builder,struct lp_type type,LLVMValueRef context_ptr,LLVMValueRef sample_pos_array,LLVMValueRef num_loop,struct lp_build_interp_soa_context * interp,const struct lp_build_sampler_soa * sampler,const struct lp_build_image_soa * image,LLVMValueRef mask_store,LLVMValueRef (* out_color)[4],LLVMValueRef depth_base_ptr,LLVMValueRef depth_stride,LLVMValueRef depth_sample_stride,LLVMValueRef color_ptr_ptr,LLVMValueRef color_stride_ptr,LLVMValueRef color_sample_stride_ptr,LLVMValueRef facing,LLVMValueRef thread_data_ptr)551 generate_fs_loop(struct gallivm_state *gallivm,
552 struct lp_fragment_shader *shader,
553 const struct lp_fragment_shader_variant_key *key,
554 LLVMBuilderRef builder,
555 struct lp_type type,
556 LLVMValueRef context_ptr,
557 LLVMValueRef sample_pos_array,
558 LLVMValueRef num_loop,
559 struct lp_build_interp_soa_context *interp,
560 const struct lp_build_sampler_soa *sampler,
561 const struct lp_build_image_soa *image,
562 LLVMValueRef mask_store,
563 LLVMValueRef (*out_color)[4],
564 LLVMValueRef depth_base_ptr,
565 LLVMValueRef depth_stride,
566 LLVMValueRef depth_sample_stride,
567 LLVMValueRef color_ptr_ptr,
568 LLVMValueRef color_stride_ptr,
569 LLVMValueRef color_sample_stride_ptr,
570 LLVMValueRef facing,
571 LLVMValueRef thread_data_ptr)
572 {
573 const struct util_format_description *zs_format_desc = NULL;
574 const struct tgsi_token *tokens = shader->base.tokens;
575 struct lp_type int_type = lp_int_type(type);
576 LLVMTypeRef vec_type, int_vec_type;
577 LLVMValueRef mask_ptr = NULL, mask_val = NULL;
578 LLVMValueRef consts_ptr, num_consts_ptr;
579 LLVMValueRef ssbo_ptr, num_ssbo_ptr;
580 LLVMValueRef z;
581 LLVMValueRef z_value, s_value;
582 LLVMValueRef z_fb, s_fb;
583 LLVMValueRef depth_ptr;
584 LLVMValueRef stencil_refs[2];
585 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
586 LLVMValueRef zs_samples = lp_build_const_int32(gallivm, key->zsbuf_nr_samples);
587 LLVMValueRef z_out = NULL, s_out = NULL;
588 struct lp_build_for_loop_state loop_state, sample_loop_state = {0};
589 struct lp_build_mask_context mask;
590 /*
591 * TODO: figure out if simple_shader optimization is really worthwile to
592 * keep. Disabled because it may hide some real bugs in the (depth/stencil)
593 * code since tests tend to take another codepath than real shaders.
594 */
595 boolean simple_shader = (shader->info.base.file_count[TGSI_FILE_SAMPLER] == 0 &&
596 shader->info.base.num_inputs < 3 &&
597 shader->info.base.num_instructions < 8) && 0;
598 const boolean dual_source_blend = key->blend.rt[0].blend_enable &&
599 util_blend_state_is_dual(&key->blend, 0);
600 const bool post_depth_coverage = shader->info.base.properties[TGSI_PROPERTY_FS_POST_DEPTH_COVERAGE];
601 unsigned attrib;
602 unsigned chan;
603 unsigned cbuf;
604 unsigned depth_mode;
605
606 struct lp_bld_tgsi_system_values system_values;
607
608 memset(&system_values, 0, sizeof(system_values));
609
610 /* truncate then sign extend. */
611 system_values.front_facing = LLVMBuildTrunc(gallivm->builder, facing, LLVMInt1TypeInContext(gallivm->context), "");
612 system_values.front_facing = LLVMBuildSExt(gallivm->builder, system_values.front_facing, LLVMInt32TypeInContext(gallivm->context), "");
613 system_values.view_index = lp_jit_thread_data_raster_state_view_index(gallivm,
614 thread_data_ptr);
615 if (key->depth.enabled ||
616 key->stencil[0].enabled) {
617
618 zs_format_desc = util_format_description(key->zsbuf_format);
619 assert(zs_format_desc);
620
621 if (shader->info.base.properties[TGSI_PROPERTY_FS_EARLY_DEPTH_STENCIL])
622 depth_mode = EARLY_DEPTH_TEST | EARLY_DEPTH_WRITE;
623 else if (!shader->info.base.writes_z && !shader->info.base.writes_stencil) {
624 if (shader->info.base.writes_memory)
625 depth_mode = LATE_DEPTH_TEST | LATE_DEPTH_WRITE;
626 else if (key->alpha.enabled ||
627 key->blend.alpha_to_coverage ||
628 shader->info.base.uses_kill ||
629 shader->info.base.writes_samplemask) {
630 /* With alpha test and kill, can do the depth test early
631 * and hopefully eliminate some quads. But need to do a
632 * special deferred depth write once the final mask value
633 * is known. This only works though if there's either no
634 * stencil test or the stencil value isn't written.
635 */
636 if (key->stencil[0].enabled && (key->stencil[0].writemask ||
637 (key->stencil[1].enabled &&
638 key->stencil[1].writemask)))
639 depth_mode = LATE_DEPTH_TEST | LATE_DEPTH_WRITE;
640 else
641 depth_mode = EARLY_DEPTH_TEST | LATE_DEPTH_WRITE | EARLY_DEPTH_TEST_INFERRED;
642 }
643 else
644 depth_mode = EARLY_DEPTH_TEST | EARLY_DEPTH_WRITE | EARLY_DEPTH_TEST_INFERRED;
645 }
646 else {
647 depth_mode = LATE_DEPTH_TEST | LATE_DEPTH_WRITE;
648 }
649
650 if (!(key->depth.enabled && key->depth.writemask) &&
651 !(key->stencil[0].enabled && (key->stencil[0].writemask ||
652 (key->stencil[1].enabled &&
653 key->stencil[1].writemask))))
654 depth_mode &= ~(LATE_DEPTH_WRITE | EARLY_DEPTH_WRITE);
655 }
656 else {
657 depth_mode = 0;
658 }
659
660 vec_type = lp_build_vec_type(gallivm, type);
661 int_vec_type = lp_build_vec_type(gallivm, int_type);
662
663 stencil_refs[0] = lp_jit_context_stencil_ref_front_value(gallivm, context_ptr);
664 stencil_refs[1] = lp_jit_context_stencil_ref_back_value(gallivm, context_ptr);
665 /* convert scalar stencil refs into vectors */
666 stencil_refs[0] = lp_build_broadcast(gallivm, int_vec_type, stencil_refs[0]);
667 stencil_refs[1] = lp_build_broadcast(gallivm, int_vec_type, stencil_refs[1]);
668
669 consts_ptr = lp_jit_context_constants(gallivm, context_ptr);
670 num_consts_ptr = lp_jit_context_num_constants(gallivm, context_ptr);
671
672 ssbo_ptr = lp_jit_context_ssbos(gallivm, context_ptr);
673 num_ssbo_ptr = lp_jit_context_num_ssbos(gallivm, context_ptr);
674
675 memset(outputs, 0, sizeof outputs);
676
677 /* Allocate color storage for each fragment sample */
678 LLVMValueRef color_store_size = num_loop;
679 if (key->min_samples > 1)
680 color_store_size = LLVMBuildMul(builder, num_loop, lp_build_const_int32(gallivm, key->min_samples), "");
681
682 for(cbuf = 0; cbuf < key->nr_cbufs; cbuf++) {
683 for(chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
684 out_color[cbuf][chan] = lp_build_array_alloca(gallivm,
685 lp_build_vec_type(gallivm,
686 type),
687 color_store_size, "color");
688 }
689 }
690 if (dual_source_blend) {
691 assert(key->nr_cbufs <= 1);
692 for(chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
693 out_color[1][chan] = lp_build_array_alloca(gallivm,
694 lp_build_vec_type(gallivm,
695 type),
696 color_store_size, "color1");
697 }
698 }
699 if (shader->info.base.writes_z) {
700 z_out = lp_build_array_alloca(gallivm,
701 lp_build_vec_type(gallivm, type),
702 color_store_size, "depth");
703 }
704
705 if (shader->info.base.writes_stencil) {
706 s_out = lp_build_array_alloca(gallivm,
707 lp_build_vec_type(gallivm, type),
708 color_store_size, "depth");
709 }
710
711 lp_build_for_loop_begin(&loop_state, gallivm,
712 lp_build_const_int32(gallivm, 0),
713 LLVMIntULT,
714 num_loop,
715 lp_build_const_int32(gallivm, 1));
716
717 LLVMValueRef sample_mask_in;
718 if (key->multisample) {
719 sample_mask_in = lp_build_const_int_vec(gallivm, type, 0);
720 /* create shader execution mask by combining all sample masks. */
721 for (unsigned s = 0; s < key->coverage_samples; s++) {
722 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, num_loop, lp_build_const_int32(gallivm, s), "");
723 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, "");
724 LLVMValueRef s_mask = lp_build_pointer_get(builder, mask_store, s_mask_idx);
725 if (s == 0)
726 mask_val = s_mask;
727 else
728 mask_val = LLVMBuildOr(builder, s_mask, mask_val, "");
729
730 LLVMValueRef mask_in = LLVMBuildAnd(builder, s_mask, lp_build_const_int_vec(gallivm, type, (1ll << s)), "");
731 sample_mask_in = LLVMBuildOr(builder, sample_mask_in, mask_in, "");
732 }
733 } else {
734 sample_mask_in = lp_build_const_int_vec(gallivm, type, 1);
735 mask_ptr = LLVMBuildGEP(builder, mask_store,
736 &loop_state.counter, 1, "mask_ptr");
737 mask_val = LLVMBuildLoad(builder, mask_ptr, "");
738
739 LLVMValueRef mask_in = LLVMBuildAnd(builder, mask_val, lp_build_const_int_vec(gallivm, type, 1), "");
740 sample_mask_in = LLVMBuildOr(builder, sample_mask_in, mask_in, "");
741 }
742
743 /* 'mask' will control execution based on quad's pixel alive/killed state */
744 lp_build_mask_begin(&mask, gallivm, type, mask_val);
745
746 if (!(depth_mode & EARLY_DEPTH_TEST) && !simple_shader)
747 lp_build_mask_check(&mask);
748
749 /* Create storage for recombining sample masks after early Z pass. */
750 LLVMValueRef s_mask_or = lp_build_alloca(gallivm, lp_build_int_vec_type(gallivm, type), "cov_mask_early_depth");
751 LLVMBuildStore(builder, LLVMConstNull(lp_build_int_vec_type(gallivm, type)), s_mask_or);
752
753 /* Create storage for post depth sample mask */
754 LLVMValueRef post_depth_sample_mask_in = NULL;
755 if (post_depth_coverage)
756 post_depth_sample_mask_in = lp_build_alloca(gallivm, int_vec_type, "post_depth_sample_mask_in");
757
758 LLVMValueRef s_mask = NULL, s_mask_ptr = NULL;
759 LLVMValueRef z_sample_value_store = NULL, s_sample_value_store = NULL;
760 LLVMValueRef z_fb_store = NULL, s_fb_store = NULL;
761 LLVMTypeRef z_type = NULL, z_fb_type = NULL;
762
763 /* Run early depth once per sample */
764 if (key->multisample) {
765
766 if (zs_format_desc) {
767 struct lp_type zs_type = lp_depth_type(zs_format_desc, type.length);
768 struct lp_type z_type = zs_type;
769 struct lp_type s_type = zs_type;
770 if (zs_format_desc->block.bits < type.width)
771 z_type.width = type.width;
772 if (zs_format_desc->block.bits == 8)
773 s_type.width = type.width;
774
775 else if (zs_format_desc->block.bits > 32) {
776 z_type.width = z_type.width / 2;
777 s_type.width = s_type.width / 2;
778 s_type.floating = 0;
779 }
780 z_sample_value_store = lp_build_array_alloca(gallivm, lp_build_int_vec_type(gallivm, type),
781 zs_samples, "z_sample_store");
782 s_sample_value_store = lp_build_array_alloca(gallivm, lp_build_int_vec_type(gallivm, type),
783 zs_samples, "s_sample_store");
784 z_fb_store = lp_build_array_alloca(gallivm, lp_build_vec_type(gallivm, z_type),
785 zs_samples, "z_fb_store");
786 s_fb_store = lp_build_array_alloca(gallivm, lp_build_vec_type(gallivm, s_type),
787 zs_samples, "s_fb_store");
788 }
789 lp_build_for_loop_begin(&sample_loop_state, gallivm,
790 lp_build_const_int32(gallivm, 0),
791 LLVMIntULT, lp_build_const_int32(gallivm, key->coverage_samples),
792 lp_build_const_int32(gallivm, 1));
793
794 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, sample_loop_state.counter, num_loop, "");
795 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, "");
796 s_mask_ptr = LLVMBuildGEP(builder, mask_store, &s_mask_idx, 1, "");
797
798 s_mask = LLVMBuildLoad(builder, s_mask_ptr, "");
799 s_mask = LLVMBuildAnd(builder, s_mask, mask_val, "");
800 }
801
802
803 /* for multisample Z needs to be interpolated at sample points for testing. */
804 lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, key->multisample ? sample_loop_state.counter : NULL);
805 z = interp->pos[2];
806
807 depth_ptr = depth_base_ptr;
808 if (key->multisample) {
809 LLVMValueRef sample_offset = LLVMBuildMul(builder, sample_loop_state.counter, depth_sample_stride, "");
810 depth_ptr = LLVMBuildGEP(builder, depth_ptr, &sample_offset, 1, "");
811 }
812
813 if (depth_mode & EARLY_DEPTH_TEST) {
814 /*
815 * Clamp according to ARB_depth_clamp semantics.
816 */
817 if (key->depth_clamp) {
818 z = lp_build_depth_clamp(gallivm, builder, type, context_ptr,
819 thread_data_ptr, z);
820 }
821 lp_build_depth_stencil_load_swizzled(gallivm, type,
822 zs_format_desc, key->resource_1d,
823 depth_ptr, depth_stride,
824 &z_fb, &s_fb, loop_state.counter);
825 lp_build_depth_stencil_test(gallivm,
826 &key->depth,
827 key->stencil,
828 type,
829 zs_format_desc,
830 key->multisample ? NULL : &mask,
831 &s_mask,
832 stencil_refs,
833 z, z_fb, s_fb,
834 facing,
835 &z_value, &s_value,
836 !simple_shader && !key->multisample);
837
838 if (depth_mode & EARLY_DEPTH_WRITE) {
839 lp_build_depth_stencil_write_swizzled(gallivm, type,
840 zs_format_desc, key->resource_1d,
841 NULL, NULL, NULL, loop_state.counter,
842 depth_ptr, depth_stride,
843 z_value, s_value);
844 }
845 /*
846 * Note mask check if stencil is enabled must be after ds write not after
847 * stencil test otherwise new stencil values may not get written if all
848 * fragments got killed by depth/stencil test.
849 */
850 if (!simple_shader && key->stencil[0].enabled && !key->multisample)
851 lp_build_mask_check(&mask);
852
853 if (key->multisample) {
854 z_fb_type = LLVMTypeOf(z_fb);
855 z_type = LLVMTypeOf(z_value);
856 lp_build_pointer_set(builder, z_sample_value_store, sample_loop_state.counter, LLVMBuildBitCast(builder, z_value, lp_build_int_vec_type(gallivm, type), ""));
857 lp_build_pointer_set(builder, s_sample_value_store, sample_loop_state.counter, LLVMBuildBitCast(builder, s_value, lp_build_int_vec_type(gallivm, type), ""));
858 lp_build_pointer_set(builder, z_fb_store, sample_loop_state.counter, z_fb);
859 lp_build_pointer_set(builder, s_fb_store, sample_loop_state.counter, s_fb);
860 }
861 }
862
863 if (key->multisample) {
864 /*
865 * Store the post-early Z coverage mask.
866 * Recombine the resulting coverage masks post early Z into the fragment
867 * shader execution mask.
868 */
869 LLVMValueRef tmp_s_mask_or = LLVMBuildLoad(builder, s_mask_or, "");
870 tmp_s_mask_or = LLVMBuildOr(builder, tmp_s_mask_or, s_mask, "");
871 LLVMBuildStore(builder, tmp_s_mask_or, s_mask_or);
872
873 if (post_depth_coverage) {
874 LLVMValueRef mask_bit_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, "");
875 LLVMValueRef post_depth_mask_in = LLVMBuildLoad(builder, post_depth_sample_mask_in, "");
876 mask_bit_idx = LLVMBuildAnd(builder, s_mask, lp_build_broadcast(gallivm, int_vec_type, mask_bit_idx), "");
877 post_depth_mask_in = LLVMBuildOr(builder, post_depth_mask_in, mask_bit_idx, "");
878 LLVMBuildStore(builder, post_depth_mask_in, post_depth_sample_mask_in);
879 }
880
881 LLVMBuildStore(builder, s_mask, s_mask_ptr);
882
883 lp_build_for_loop_end(&sample_loop_state);
884
885 /* recombined all the coverage masks in the shader exec mask. */
886 tmp_s_mask_or = LLVMBuildLoad(builder, s_mask_or, "");
887 lp_build_mask_update(&mask, tmp_s_mask_or);
888
889 if (key->min_samples == 1) {
890 /* for multisample Z needs to be re interpolated at pixel center */
891 lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, NULL);
892 z = interp->pos[2];
893 lp_build_mask_update(&mask, tmp_s_mask_or);
894 }
895 } else {
896 if (post_depth_coverage) {
897 LLVMValueRef post_depth_mask_in = LLVMBuildAnd(builder, lp_build_mask_value(&mask), lp_build_const_int_vec(gallivm, type, 1), "");
898 LLVMBuildStore(builder, post_depth_mask_in, post_depth_sample_mask_in);
899 }
900 }
901
902 LLVMValueRef out_sample_mask_storage = NULL;
903 if (shader->info.base.writes_samplemask) {
904 out_sample_mask_storage = lp_build_alloca(gallivm, int_vec_type, "write_mask");
905 if (key->min_samples > 1)
906 LLVMBuildStore(builder, LLVMConstNull(int_vec_type), out_sample_mask_storage);
907 }
908
909 if (post_depth_coverage) {
910 system_values.sample_mask_in = LLVMBuildLoad(builder, post_depth_sample_mask_in, "");
911 }
912 else
913 system_values.sample_mask_in = sample_mask_in;
914 if (key->multisample && key->min_samples > 1) {
915 lp_build_for_loop_begin(&sample_loop_state, gallivm,
916 lp_build_const_int32(gallivm, 0),
917 LLVMIntULT,
918 lp_build_const_int32(gallivm, key->min_samples),
919 lp_build_const_int32(gallivm, 1));
920
921 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, sample_loop_state.counter, num_loop, "");
922 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, "");
923 s_mask_ptr = LLVMBuildGEP(builder, mask_store, &s_mask_idx, 1, "");
924 s_mask = LLVMBuildLoad(builder, s_mask_ptr, "");
925 lp_build_mask_force(&mask, s_mask);
926 lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, sample_loop_state.counter);
927 system_values.sample_id = sample_loop_state.counter;
928 system_values.sample_mask_in = LLVMBuildAnd(builder, system_values.sample_mask_in,
929 lp_build_broadcast(gallivm, int_vec_type,
930 LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, "")), "");
931 } else {
932 system_values.sample_id = lp_build_const_int32(gallivm, 0);
933
934 }
935 system_values.sample_pos = sample_pos_array;
936
937 lp_build_interp_soa_update_inputs_dyn(interp, gallivm, loop_state.counter, mask_store, sample_loop_state.counter);
938
939 struct lp_build_fs_llvm_iface fs_iface = {
940 .base.interp_fn = fs_interp,
941 .base.fb_fetch = fs_fb_fetch,
942 .interp = interp,
943 .loop_state = &loop_state,
944 .sample_id = system_values.sample_id,
945 .mask_store = mask_store,
946 .color_ptr_ptr = color_ptr_ptr,
947 .color_stride_ptr = color_stride_ptr,
948 .color_sample_stride_ptr = color_sample_stride_ptr,
949 .key = key,
950 };
951
952 struct lp_build_tgsi_params params;
953 memset(¶ms, 0, sizeof(params));
954
955 params.type = type;
956 params.mask = &mask;
957 params.fs_iface = &fs_iface.base;
958 params.consts_ptr = consts_ptr;
959 params.const_sizes_ptr = num_consts_ptr;
960 params.system_values = &system_values;
961 params.inputs = interp->inputs;
962 params.context_ptr = context_ptr;
963 params.thread_data_ptr = thread_data_ptr;
964 params.sampler = sampler;
965 params.info = &shader->info.base;
966 params.ssbo_ptr = ssbo_ptr;
967 params.ssbo_sizes_ptr = num_ssbo_ptr;
968 params.image = image;
969 params.aniso_filter_table = lp_jit_context_aniso_filter_table(gallivm, context_ptr);
970
971 /* Build the actual shader */
972 if (shader->base.type == PIPE_SHADER_IR_TGSI)
973 lp_build_tgsi_soa(gallivm, tokens, ¶ms,
974 outputs);
975 else
976 lp_build_nir_soa(gallivm, shader->base.ir.nir, ¶ms,
977 outputs);
978
979 /* Alpha test */
980 if (key->alpha.enabled) {
981 int color0 = find_output_by_semantic(&shader->info.base,
982 TGSI_SEMANTIC_COLOR,
983 0);
984
985 if (color0 != -1 && outputs[color0][3]) {
986 const struct util_format_description *cbuf_format_desc;
987 LLVMValueRef alpha = LLVMBuildLoad(builder, outputs[color0][3], "alpha");
988 LLVMValueRef alpha_ref_value;
989
990 alpha_ref_value = lp_jit_context_alpha_ref_value(gallivm, context_ptr);
991 alpha_ref_value = lp_build_broadcast(gallivm, vec_type, alpha_ref_value);
992
993 cbuf_format_desc = util_format_description(key->cbuf_format[0]);
994
995 lp_build_alpha_test(gallivm, key->alpha.func, type, cbuf_format_desc,
996 &mask, alpha, alpha_ref_value,
997 (depth_mode & LATE_DEPTH_TEST) != 0);
998 }
999 }
1000
1001 /* Emulate Alpha to Coverage with Alpha test */
1002 if (key->blend.alpha_to_coverage) {
1003 int color0 = find_output_by_semantic(&shader->info.base,
1004 TGSI_SEMANTIC_COLOR,
1005 0);
1006
1007 if (color0 != -1 && outputs[color0][3]) {
1008 LLVMValueRef alpha = LLVMBuildLoad(builder, outputs[color0][3], "alpha");
1009
1010 if (!key->multisample) {
1011 lp_build_alpha_to_coverage(gallivm, type,
1012 &mask, alpha,
1013 (depth_mode & LATE_DEPTH_TEST) != 0);
1014 } else {
1015 lp_build_sample_alpha_to_coverage(gallivm, type, key->coverage_samples, num_loop,
1016 loop_state.counter,
1017 mask_store, alpha);
1018 }
1019 }
1020 }
1021 if (key->blend.alpha_to_one && key->multisample) {
1022 for (attrib = 0; attrib < shader->info.base.num_outputs; ++attrib) {
1023 unsigned cbuf = shader->info.base.output_semantic_index[attrib];
1024 if ((shader->info.base.output_semantic_name[attrib] == TGSI_SEMANTIC_COLOR) &&
1025 ((cbuf < key->nr_cbufs) || (cbuf == 1 && dual_source_blend)))
1026 if (outputs[cbuf][3]) {
1027 LLVMBuildStore(builder, lp_build_const_vec(gallivm, type, 1.0), outputs[cbuf][3]);
1028 }
1029 }
1030 }
1031 if (shader->info.base.writes_samplemask) {
1032 LLVMValueRef output_smask = NULL;
1033 int smaski = find_output_by_semantic(&shader->info.base,
1034 TGSI_SEMANTIC_SAMPLEMASK,
1035 0);
1036 struct lp_build_context smask_bld;
1037 lp_build_context_init(&smask_bld, gallivm, int_type);
1038
1039 assert(smaski >= 0);
1040 output_smask = LLVMBuildLoad(builder, outputs[smaski][0], "smask");
1041 output_smask = LLVMBuildBitCast(builder, output_smask, smask_bld.vec_type, "");
1042 if (!key->multisample && key->no_ms_sample_mask_out) {
1043 output_smask = lp_build_and(&smask_bld, output_smask, smask_bld.one);
1044 output_smask = lp_build_cmp(&smask_bld, PIPE_FUNC_NOTEQUAL, output_smask, smask_bld.zero);
1045 lp_build_mask_update(&mask, output_smask);
1046 }
1047
1048 if (key->min_samples > 1) {
1049 /* only the bit corresponding to this sample is to be used. */
1050 LLVMValueRef tmp_mask = LLVMBuildLoad(builder, out_sample_mask_storage, "tmp_mask");
1051 LLVMValueRef out_smask_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, "");
1052 LLVMValueRef smask_bit = LLVMBuildAnd(builder, output_smask, lp_build_broadcast(gallivm, int_vec_type, out_smask_idx), "");
1053 output_smask = LLVMBuildOr(builder, tmp_mask, smask_bit, "");
1054 }
1055
1056 LLVMBuildStore(builder, output_smask, out_sample_mask_storage);
1057 }
1058
1059 if (shader->info.base.writes_z) {
1060 int pos0 = find_output_by_semantic(&shader->info.base,
1061 TGSI_SEMANTIC_POSITION,
1062 0);
1063 LLVMValueRef out = LLVMBuildLoad(builder, outputs[pos0][2], "");
1064 LLVMValueRef idx = loop_state.counter;
1065 if (key->min_samples > 1)
1066 idx = LLVMBuildAdd(builder, idx,
1067 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), "");
1068 LLVMValueRef ptr = LLVMBuildGEP(builder, z_out, &idx, 1, "");
1069 LLVMBuildStore(builder, out, ptr);
1070 }
1071
1072 if (shader->info.base.writes_stencil) {
1073 int sten_out = find_output_by_semantic(&shader->info.base,
1074 TGSI_SEMANTIC_STENCIL,
1075 0);
1076 LLVMValueRef out = LLVMBuildLoad(builder, outputs[sten_out][1], "output.s");
1077 LLVMValueRef idx = loop_state.counter;
1078 if (key->min_samples > 1)
1079 idx = LLVMBuildAdd(builder, idx,
1080 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), "");
1081 LLVMValueRef ptr = LLVMBuildGEP(builder, s_out, &idx, 1, "");
1082 LLVMBuildStore(builder, out, ptr);
1083 }
1084
1085 bool has_cbuf0_write = false;
1086 /* Color write - per fragment sample */
1087 for (attrib = 0; attrib < shader->info.base.num_outputs; ++attrib)
1088 {
1089 unsigned cbuf = shader->info.base.output_semantic_index[attrib];
1090 if ((shader->info.base.output_semantic_name[attrib] == TGSI_SEMANTIC_COLOR) &&
1091 ((cbuf < key->nr_cbufs) || (cbuf == 1 && dual_source_blend)))
1092 {
1093 if (cbuf == 0 && shader->info.base.properties[TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS]) {
1094 /* XXX: there is an edge case with FB fetch where gl_FragColor and gl_LastFragData[0]
1095 * are used together. This creates both FRAG_RESULT_COLOR and FRAG_RESULT_DATA* output
1096 * variables. This loop then writes to cbuf 0 twice, owerwriting the correct value
1097 * from gl_FragColor with some garbage. This case is excercised in one of deqp tests.
1098 * A similar bug can happen if gl_SecondaryFragColorEXT and gl_LastFragData[1]
1099 * are mixed in the same fashion...
1100 * This workaround will break if gl_LastFragData[0] goes in outputs list before
1101 * gl_FragColor. This doesn't seem to happen though.
1102 */
1103 if (has_cbuf0_write)
1104 continue;
1105 has_cbuf0_write = true;
1106 }
1107
1108 for(chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
1109 if(outputs[attrib][chan]) {
1110 /* XXX: just initialize outputs to point at colors[] and
1111 * skip this.
1112 */
1113 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
1114 LLVMValueRef color_ptr;
1115 LLVMValueRef color_idx = loop_state.counter;
1116 if (key->min_samples > 1)
1117 color_idx = LLVMBuildAdd(builder, color_idx,
1118 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), "");
1119 color_ptr = LLVMBuildGEP(builder, out_color[cbuf][chan],
1120 &color_idx, 1, "");
1121 lp_build_name(out, "color%u.%c", attrib, "rgba"[chan]);
1122 LLVMBuildStore(builder, out, color_ptr);
1123 }
1124 }
1125 }
1126 }
1127
1128 if (key->multisample && key->min_samples > 1) {
1129 LLVMBuildStore(builder, lp_build_mask_value(&mask), s_mask_ptr);
1130 lp_build_for_loop_end(&sample_loop_state);
1131 }
1132
1133 if (key->multisample) {
1134 /* execute depth test for each sample */
1135 lp_build_for_loop_begin(&sample_loop_state, gallivm,
1136 lp_build_const_int32(gallivm, 0),
1137 LLVMIntULT, lp_build_const_int32(gallivm, key->coverage_samples),
1138 lp_build_const_int32(gallivm, 1));
1139
1140 /* load the per-sample coverage mask */
1141 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, sample_loop_state.counter, num_loop, "");
1142 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, "");
1143 s_mask_ptr = LLVMBuildGEP(builder, mask_store, &s_mask_idx, 1, "");
1144
1145 /* combine the execution mask post fragment shader with the coverage mask. */
1146 s_mask = LLVMBuildLoad(builder, s_mask_ptr, "");
1147 if (key->min_samples == 1)
1148 s_mask = LLVMBuildAnd(builder, s_mask, lp_build_mask_value(&mask), "");
1149
1150 /* if the shader writes sample mask use that,
1151 * but only if this isn't genuine early-depth to avoid breaking occlusion query */
1152 if (shader->info.base.writes_samplemask &&
1153 (!(depth_mode & EARLY_DEPTH_TEST) || (depth_mode & (EARLY_DEPTH_TEST_INFERRED)))) {
1154 LLVMValueRef out_smask_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, "");
1155 out_smask_idx = lp_build_broadcast(gallivm, int_vec_type, out_smask_idx);
1156 LLVMValueRef output_smask = LLVMBuildLoad(builder, out_sample_mask_storage, "");
1157 LLVMValueRef smask_bit = LLVMBuildAnd(builder, output_smask, out_smask_idx, "");
1158 LLVMValueRef cmp = LLVMBuildICmp(builder, LLVMIntNE, smask_bit, lp_build_const_int_vec(gallivm, int_type, 0), "");
1159 smask_bit = LLVMBuildSExt(builder, cmp, int_vec_type, "");
1160
1161 s_mask = LLVMBuildAnd(builder, s_mask, smask_bit, "");
1162 }
1163 }
1164
1165 depth_ptr = depth_base_ptr;
1166 if (key->multisample) {
1167 LLVMValueRef sample_offset = LLVMBuildMul(builder, sample_loop_state.counter, depth_sample_stride, "");
1168 depth_ptr = LLVMBuildGEP(builder, depth_ptr, &sample_offset, 1, "");
1169 }
1170
1171 /* Late Z test */
1172 if (depth_mode & LATE_DEPTH_TEST) {
1173 if (shader->info.base.writes_z) {
1174 LLVMValueRef idx = loop_state.counter;
1175 if (key->min_samples > 1)
1176 idx = LLVMBuildAdd(builder, idx,
1177 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), "");
1178 LLVMValueRef ptr = LLVMBuildGEP(builder, z_out, &idx, 1, "");
1179 z = LLVMBuildLoad(builder, ptr, "output.z");
1180 } else {
1181 if (key->multisample) {
1182 lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, key->multisample ? sample_loop_state.counter : NULL);
1183 z = interp->pos[2];
1184 }
1185 }
1186
1187 /*
1188 * Clamp according to ARB_depth_clamp semantics.
1189 */
1190 if (key->depth_clamp) {
1191 z = lp_build_depth_clamp(gallivm, builder, type, context_ptr,
1192 thread_data_ptr, z);
1193 } else {
1194 struct lp_build_context f32_bld;
1195 lp_build_context_init(&f32_bld, gallivm, type);
1196 z = lp_build_clamp(&f32_bld, z,
1197 lp_build_const_vec(gallivm, type, 0.0),
1198 lp_build_const_vec(gallivm, type, 1.0));
1199 }
1200
1201 if (shader->info.base.writes_stencil) {
1202 LLVMValueRef idx = loop_state.counter;
1203 if (key->min_samples > 1)
1204 idx = LLVMBuildAdd(builder, idx,
1205 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), "");
1206 LLVMValueRef ptr = LLVMBuildGEP(builder, s_out, &idx, 1, "");
1207 stencil_refs[0] = LLVMBuildLoad(builder, ptr, "output.s");
1208 /* there's only one value, and spec says to discard additional bits */
1209 LLVMValueRef s_max_mask = lp_build_const_int_vec(gallivm, int_type, 255);
1210 stencil_refs[0] = LLVMBuildBitCast(builder, stencil_refs[0], int_vec_type, "");
1211 stencil_refs[0] = LLVMBuildAnd(builder, stencil_refs[0], s_max_mask, "");
1212 stencil_refs[1] = stencil_refs[0];
1213 }
1214
1215 lp_build_depth_stencil_load_swizzled(gallivm, type,
1216 zs_format_desc, key->resource_1d,
1217 depth_ptr, depth_stride,
1218 &z_fb, &s_fb, loop_state.counter);
1219
1220 lp_build_depth_stencil_test(gallivm,
1221 &key->depth,
1222 key->stencil,
1223 type,
1224 zs_format_desc,
1225 key->multisample ? NULL : &mask,
1226 &s_mask,
1227 stencil_refs,
1228 z, z_fb, s_fb,
1229 facing,
1230 &z_value, &s_value,
1231 !simple_shader);
1232 /* Late Z write */
1233 if (depth_mode & LATE_DEPTH_WRITE) {
1234 lp_build_depth_stencil_write_swizzled(gallivm, type,
1235 zs_format_desc, key->resource_1d,
1236 NULL, NULL, NULL, loop_state.counter,
1237 depth_ptr, depth_stride,
1238 z_value, s_value);
1239 }
1240 }
1241 else if ((depth_mode & EARLY_DEPTH_TEST) &&
1242 (depth_mode & LATE_DEPTH_WRITE))
1243 {
1244 /* Need to apply a reduced mask to the depth write. Reload the
1245 * depth value, update from zs_value with the new mask value and
1246 * write that out.
1247 */
1248 if (key->multisample) {
1249 z_value = LLVMBuildBitCast(builder, lp_build_pointer_get(builder, z_sample_value_store, sample_loop_state.counter), z_type, "");;
1250 s_value = lp_build_pointer_get(builder, s_sample_value_store, sample_loop_state.counter);
1251 z_fb = LLVMBuildBitCast(builder, lp_build_pointer_get(builder, z_fb_store, sample_loop_state.counter), z_fb_type, "");
1252 s_fb = lp_build_pointer_get(builder, s_fb_store, sample_loop_state.counter);
1253 }
1254 lp_build_depth_stencil_write_swizzled(gallivm, type,
1255 zs_format_desc, key->resource_1d,
1256 key->multisample ? s_mask : lp_build_mask_value(&mask), z_fb, s_fb, loop_state.counter,
1257 depth_ptr, depth_stride,
1258 z_value, s_value);
1259 }
1260
1261 if (key->occlusion_count) {
1262 LLVMValueRef counter = lp_jit_thread_data_counter(gallivm, thread_data_ptr);
1263 lp_build_name(counter, "counter");
1264
1265 lp_build_occlusion_count(gallivm, type,
1266 key->multisample ? s_mask : lp_build_mask_value(&mask), counter);
1267 }
1268
1269 /* if this is genuine early-depth in the shader, write samplemask now
1270 * after occlusion count has been updated
1271 */
1272 if (key->multisample && shader->info.base.writes_samplemask &&
1273 (depth_mode & (EARLY_DEPTH_TEST_INFERRED | EARLY_DEPTH_TEST)) == EARLY_DEPTH_TEST) {
1274 /* if the shader writes sample mask use that */
1275 LLVMValueRef out_smask_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, "");
1276 out_smask_idx = lp_build_broadcast(gallivm, int_vec_type, out_smask_idx);
1277 LLVMValueRef output_smask = LLVMBuildLoad(builder, out_sample_mask_storage, "");
1278 LLVMValueRef smask_bit = LLVMBuildAnd(builder, output_smask, out_smask_idx, "");
1279 LLVMValueRef cmp = LLVMBuildICmp(builder, LLVMIntNE, smask_bit, lp_build_const_int_vec(gallivm, int_type, 0), "");
1280 smask_bit = LLVMBuildSExt(builder, cmp, int_vec_type, "");
1281
1282 s_mask = LLVMBuildAnd(builder, s_mask, smask_bit, "");
1283 }
1284
1285
1286 if (key->multisample) {
1287 /* store the sample mask for this loop */
1288 LLVMBuildStore(builder, s_mask, s_mask_ptr);
1289 lp_build_for_loop_end(&sample_loop_state);
1290 }
1291
1292 mask_val = lp_build_mask_end(&mask);
1293 if (!key->multisample)
1294 LLVMBuildStore(builder, mask_val, mask_ptr);
1295 lp_build_for_loop_end(&loop_state);
1296 }
1297
1298
1299 /**
1300 * This function will reorder pixels from the fragment shader SoA to memory layout AoS
1301 *
1302 * Fragment Shader outputs pixels in small 2x2 blocks
1303 * e.g. (0, 0), (1, 0), (0, 1), (1, 1) ; (2, 0) ...
1304 *
1305 * However in memory pixels are stored in rows
1306 * e.g. (0, 0), (1, 0), (2, 0), (3, 0) ; (0, 1) ...
1307 *
1308 * @param type fragment shader type (4x or 8x float)
1309 * @param num_fs number of fs_src
1310 * @param is_1d whether we're outputting to a 1d resource
1311 * @param dst_channels number of output channels
1312 * @param fs_src output from fragment shader
1313 * @param dst pointer to store result
1314 * @param pad_inline is channel padding inline or at end of row
1315 * @return the number of dsts
1316 */
1317 static int
generate_fs_twiddle(struct gallivm_state * gallivm,struct lp_type type,unsigned num_fs,unsigned dst_channels,LLVMValueRef fs_src[][4],LLVMValueRef * dst,bool pad_inline)1318 generate_fs_twiddle(struct gallivm_state *gallivm,
1319 struct lp_type type,
1320 unsigned num_fs,
1321 unsigned dst_channels,
1322 LLVMValueRef fs_src[][4],
1323 LLVMValueRef* dst,
1324 bool pad_inline)
1325 {
1326 LLVMValueRef src[16];
1327
1328 bool swizzle_pad;
1329 bool twiddle;
1330 bool split;
1331
1332 unsigned pixels = type.length / 4;
1333 unsigned reorder_group;
1334 unsigned src_channels;
1335 unsigned src_count;
1336 unsigned i;
1337
1338 src_channels = dst_channels < 3 ? dst_channels : 4;
1339 src_count = num_fs * src_channels;
1340
1341 assert(pixels == 2 || pixels == 1);
1342 assert(num_fs * src_channels <= ARRAY_SIZE(src));
1343
1344 /*
1345 * Transpose from SoA -> AoS
1346 */
1347 for (i = 0; i < num_fs; ++i) {
1348 lp_build_transpose_aos_n(gallivm, type, &fs_src[i][0], src_channels, &src[i * src_channels]);
1349 }
1350
1351 /*
1352 * Pick transformation options
1353 */
1354 swizzle_pad = false;
1355 twiddle = false;
1356 split = false;
1357 reorder_group = 0;
1358
1359 if (dst_channels == 1) {
1360 twiddle = true;
1361
1362 if (pixels == 2) {
1363 split = true;
1364 }
1365 } else if (dst_channels == 2) {
1366 if (pixels == 1) {
1367 reorder_group = 1;
1368 }
1369 } else if (dst_channels > 2) {
1370 if (pixels == 1) {
1371 reorder_group = 2;
1372 } else {
1373 twiddle = true;
1374 }
1375
1376 if (!pad_inline && dst_channels == 3 && pixels > 1) {
1377 swizzle_pad = true;
1378 }
1379 }
1380
1381 /*
1382 * Split the src in half
1383 */
1384 if (split) {
1385 for (i = num_fs; i > 0; --i) {
1386 src[(i - 1)*2 + 1] = lp_build_extract_range(gallivm, src[i - 1], 4, 4);
1387 src[(i - 1)*2 + 0] = lp_build_extract_range(gallivm, src[i - 1], 0, 4);
1388 }
1389
1390 src_count *= 2;
1391 type.length = 4;
1392 }
1393
1394 /*
1395 * Ensure pixels are in memory order
1396 */
1397 if (reorder_group) {
1398 /* Twiddle pixels by reordering the array, e.g.:
1399 *
1400 * src_count = 8 -> 0 2 1 3 4 6 5 7
1401 * src_count = 16 -> 0 1 4 5 2 3 6 7 8 9 12 13 10 11 14 15
1402 */
1403 const unsigned reorder_sw[] = { 0, 2, 1, 3 };
1404
1405 for (i = 0; i < src_count; ++i) {
1406 unsigned group = i / reorder_group;
1407 unsigned block = (group / 4) * 4 * reorder_group;
1408 unsigned j = block + (reorder_sw[group % 4] * reorder_group) + (i % reorder_group);
1409 dst[i] = src[j];
1410 }
1411 } else if (twiddle) {
1412 /* Twiddle pixels across elements of array */
1413 /*
1414 * XXX: we should avoid this in some cases, but would need to tell
1415 * lp_build_conv to reorder (or deal with it ourselves).
1416 */
1417 lp_bld_quad_twiddle(gallivm, type, src, src_count, dst);
1418 } else {
1419 /* Do nothing */
1420 memcpy(dst, src, sizeof(LLVMValueRef) * src_count);
1421 }
1422
1423 /*
1424 * Moves any padding between pixels to the end
1425 * e.g. RGBXRGBX -> RGBRGBXX
1426 */
1427 if (swizzle_pad) {
1428 unsigned char swizzles[16];
1429 unsigned elems = pixels * dst_channels;
1430
1431 for (i = 0; i < type.length; ++i) {
1432 if (i < elems)
1433 swizzles[i] = i % dst_channels + (i / dst_channels) * 4;
1434 else
1435 swizzles[i] = LP_BLD_SWIZZLE_DONTCARE;
1436 }
1437
1438 for (i = 0; i < src_count; ++i) {
1439 dst[i] = lp_build_swizzle_aos_n(gallivm, dst[i], swizzles, type.length, type.length);
1440 }
1441 }
1442
1443 return src_count;
1444 }
1445
1446
1447 /*
1448 * Untwiddle and transpose, much like the above.
1449 * However, this is after conversion, so we get packed vectors.
1450 * At this time only handle 4x16i8 rgba / 2x16i8 rg / 1x16i8 r data,
1451 * the vectors will look like:
1452 * r0r1r4r5r2r3r6r7r8r9r12... (albeit color channels may
1453 * be swizzled here). Extending to 16bit should be trivial.
1454 * Should also be extended to handle twice wide vectors with AVX2...
1455 */
1456 static void
fs_twiddle_transpose(struct gallivm_state * gallivm,struct lp_type type,LLVMValueRef * src,unsigned src_count,LLVMValueRef * dst)1457 fs_twiddle_transpose(struct gallivm_state *gallivm,
1458 struct lp_type type,
1459 LLVMValueRef *src,
1460 unsigned src_count,
1461 LLVMValueRef *dst)
1462 {
1463 unsigned i, j;
1464 struct lp_type type64, type16, type32;
1465 LLVMTypeRef type64_t, type8_t, type16_t, type32_t;
1466 LLVMBuilderRef builder = gallivm->builder;
1467 LLVMValueRef tmp[4], shuf[8];
1468 for (j = 0; j < 2; j++) {
1469 shuf[j*4 + 0] = lp_build_const_int32(gallivm, j*4 + 0);
1470 shuf[j*4 + 1] = lp_build_const_int32(gallivm, j*4 + 2);
1471 shuf[j*4 + 2] = lp_build_const_int32(gallivm, j*4 + 1);
1472 shuf[j*4 + 3] = lp_build_const_int32(gallivm, j*4 + 3);
1473 }
1474
1475 assert(src_count == 4 || src_count == 2 || src_count == 1);
1476 assert(type.width == 8);
1477 assert(type.length == 16);
1478
1479 type8_t = lp_build_vec_type(gallivm, type);
1480
1481 type64 = type;
1482 type64.length /= 8;
1483 type64.width *= 8;
1484 type64_t = lp_build_vec_type(gallivm, type64);
1485
1486 type16 = type;
1487 type16.length /= 2;
1488 type16.width *= 2;
1489 type16_t = lp_build_vec_type(gallivm, type16);
1490
1491 type32 = type;
1492 type32.length /= 4;
1493 type32.width *= 4;
1494 type32_t = lp_build_vec_type(gallivm, type32);
1495
1496 lp_build_transpose_aos_n(gallivm, type, src, src_count, tmp);
1497
1498 if (src_count == 1) {
1499 /* transpose was no-op, just untwiddle */
1500 LLVMValueRef shuf_vec;
1501 shuf_vec = LLVMConstVector(shuf, 8);
1502 tmp[0] = LLVMBuildBitCast(builder, src[0], type16_t, "");
1503 tmp[0] = LLVMBuildShuffleVector(builder, tmp[0], tmp[0], shuf_vec, "");
1504 dst[0] = LLVMBuildBitCast(builder, tmp[0], type8_t, "");
1505 } else if (src_count == 2) {
1506 LLVMValueRef shuf_vec;
1507 shuf_vec = LLVMConstVector(shuf, 4);
1508
1509 for (i = 0; i < 2; i++) {
1510 tmp[i] = LLVMBuildBitCast(builder, tmp[i], type32_t, "");
1511 tmp[i] = LLVMBuildShuffleVector(builder, tmp[i], tmp[i], shuf_vec, "");
1512 dst[i] = LLVMBuildBitCast(builder, tmp[i], type8_t, "");
1513 }
1514 } else {
1515 for (j = 0; j < 2; j++) {
1516 LLVMValueRef lo, hi, lo2, hi2;
1517 /*
1518 * Note that if we only really have 3 valid channels (rgb)
1519 * and we don't need alpha we could substitute a undef here
1520 * for the respective channel (causing llvm to drop conversion
1521 * for alpha).
1522 */
1523 /* we now have rgba0rgba1rgba4rgba5 etc, untwiddle */
1524 lo2 = LLVMBuildBitCast(builder, tmp[j*2], type64_t, "");
1525 hi2 = LLVMBuildBitCast(builder, tmp[j*2 + 1], type64_t, "");
1526 lo = lp_build_interleave2(gallivm, type64, lo2, hi2, 0);
1527 hi = lp_build_interleave2(gallivm, type64, lo2, hi2, 1);
1528 dst[j*2] = LLVMBuildBitCast(builder, lo, type8_t, "");
1529 dst[j*2 + 1] = LLVMBuildBitCast(builder, hi, type8_t, "");
1530 }
1531 }
1532 }
1533
1534
1535 /**
1536 * Load an unswizzled block of pixels from memory
1537 */
1538 static void
load_unswizzled_block(struct gallivm_state * gallivm,LLVMValueRef base_ptr,LLVMValueRef stride,unsigned block_width,unsigned block_height,LLVMValueRef * dst,struct lp_type dst_type,unsigned dst_count,unsigned dst_alignment)1539 load_unswizzled_block(struct gallivm_state *gallivm,
1540 LLVMValueRef base_ptr,
1541 LLVMValueRef stride,
1542 unsigned block_width,
1543 unsigned block_height,
1544 LLVMValueRef* dst,
1545 struct lp_type dst_type,
1546 unsigned dst_count,
1547 unsigned dst_alignment)
1548 {
1549 LLVMBuilderRef builder = gallivm->builder;
1550 unsigned row_size = dst_count / block_height;
1551 unsigned i;
1552
1553 /* Ensure block exactly fits into dst */
1554 assert((block_width * block_height) % dst_count == 0);
1555
1556 for (i = 0; i < dst_count; ++i) {
1557 unsigned x = i % row_size;
1558 unsigned y = i / row_size;
1559
1560 LLVMValueRef bx = lp_build_const_int32(gallivm, x * (dst_type.width / 8) * dst_type.length);
1561 LLVMValueRef by = LLVMBuildMul(builder, lp_build_const_int32(gallivm, y), stride, "");
1562
1563 LLVMValueRef gep[2];
1564 LLVMValueRef dst_ptr;
1565
1566 gep[0] = lp_build_const_int32(gallivm, 0);
1567 gep[1] = LLVMBuildAdd(builder, bx, by, "");
1568
1569 dst_ptr = LLVMBuildGEP(builder, base_ptr, gep, 2, "");
1570 dst_ptr = LLVMBuildBitCast(builder, dst_ptr,
1571 LLVMPointerType(lp_build_vec_type(gallivm, dst_type), 0), "");
1572
1573 dst[i] = LLVMBuildLoad(builder, dst_ptr, "");
1574
1575 LLVMSetAlignment(dst[i], dst_alignment);
1576 }
1577 }
1578
1579
1580 /**
1581 * Store an unswizzled block of pixels to memory
1582 */
1583 static void
store_unswizzled_block(struct gallivm_state * gallivm,LLVMValueRef base_ptr,LLVMValueRef stride,unsigned block_width,unsigned block_height,LLVMValueRef * src,struct lp_type src_type,unsigned src_count,unsigned src_alignment)1584 store_unswizzled_block(struct gallivm_state *gallivm,
1585 LLVMValueRef base_ptr,
1586 LLVMValueRef stride,
1587 unsigned block_width,
1588 unsigned block_height,
1589 LLVMValueRef* src,
1590 struct lp_type src_type,
1591 unsigned src_count,
1592 unsigned src_alignment)
1593 {
1594 LLVMBuilderRef builder = gallivm->builder;
1595 unsigned row_size = src_count / block_height;
1596 unsigned i;
1597
1598 /* Ensure src exactly fits into block */
1599 assert((block_width * block_height) % src_count == 0);
1600
1601 for (i = 0; i < src_count; ++i) {
1602 unsigned x = i % row_size;
1603 unsigned y = i / row_size;
1604
1605 LLVMValueRef bx = lp_build_const_int32(gallivm, x * (src_type.width / 8) * src_type.length);
1606 LLVMValueRef by = LLVMBuildMul(builder, lp_build_const_int32(gallivm, y), stride, "");
1607
1608 LLVMValueRef gep[2];
1609 LLVMValueRef src_ptr;
1610
1611 gep[0] = lp_build_const_int32(gallivm, 0);
1612 gep[1] = LLVMBuildAdd(builder, bx, by, "");
1613
1614 src_ptr = LLVMBuildGEP(builder, base_ptr, gep, 2, "");
1615 src_ptr = LLVMBuildBitCast(builder, src_ptr,
1616 LLVMPointerType(lp_build_vec_type(gallivm, src_type), 0), "");
1617
1618 src_ptr = LLVMBuildStore(builder, src[i], src_ptr);
1619
1620 LLVMSetAlignment(src_ptr, src_alignment);
1621 }
1622 }
1623
1624
1625
1626 /**
1627 * Retrieves the type for a format which is usable in the blending code.
1628 *
1629 * e.g. RGBA16F = 4x float, R3G3B2 = 3x byte
1630 */
1631 static inline void
lp_blend_type_from_format_desc(const struct util_format_description * format_desc,struct lp_type * type)1632 lp_blend_type_from_format_desc(const struct util_format_description *format_desc,
1633 struct lp_type* type)
1634 {
1635 unsigned i;
1636 unsigned chan;
1637
1638 if (format_expands_to_float_soa(format_desc)) {
1639 /* always use ordinary floats for blending */
1640 type->floating = true;
1641 type->fixed = false;
1642 type->sign = true;
1643 type->norm = false;
1644 type->width = 32;
1645 type->length = 4;
1646 return;
1647 }
1648
1649 for (i = 0; i < 4; i++)
1650 if (format_desc->channel[i].type != UTIL_FORMAT_TYPE_VOID)
1651 break;
1652 chan = i;
1653
1654 memset(type, 0, sizeof(struct lp_type));
1655 type->floating = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FLOAT;
1656 type->fixed = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FIXED;
1657 type->sign = format_desc->channel[chan].type != UTIL_FORMAT_TYPE_UNSIGNED;
1658 type->norm = format_desc->channel[chan].normalized;
1659 type->width = format_desc->channel[chan].size;
1660 type->length = format_desc->nr_channels;
1661
1662 for (i = 1; i < format_desc->nr_channels; ++i) {
1663 if (format_desc->channel[i].size > type->width)
1664 type->width = format_desc->channel[i].size;
1665 }
1666
1667 if (type->floating) {
1668 type->width = 32;
1669 } else {
1670 if (type->width <= 8) {
1671 type->width = 8;
1672 } else if (type->width <= 16) {
1673 type->width = 16;
1674 } else {
1675 type->width = 32;
1676 }
1677 }
1678
1679 if (is_arithmetic_format(format_desc) && type->length == 3) {
1680 type->length = 4;
1681 }
1682 }
1683
1684
1685 /**
1686 * Scale a normalized value from src_bits to dst_bits.
1687 *
1688 * The exact calculation is
1689 *
1690 * dst = iround(src * dst_mask / src_mask)
1691 *
1692 * or with integer rounding
1693 *
1694 * dst = src * (2*dst_mask + sign(src)*src_mask) / (2*src_mask)
1695 *
1696 * where
1697 *
1698 * src_mask = (1 << src_bits) - 1
1699 * dst_mask = (1 << dst_bits) - 1
1700 *
1701 * but we try to avoid division and multiplication through shifts.
1702 */
1703 static inline LLVMValueRef
scale_bits(struct gallivm_state * gallivm,int src_bits,int dst_bits,LLVMValueRef src,struct lp_type src_type)1704 scale_bits(struct gallivm_state *gallivm,
1705 int src_bits,
1706 int dst_bits,
1707 LLVMValueRef src,
1708 struct lp_type src_type)
1709 {
1710 LLVMBuilderRef builder = gallivm->builder;
1711 LLVMValueRef result = src;
1712
1713 if (dst_bits < src_bits) {
1714 int delta_bits = src_bits - dst_bits;
1715
1716 if (delta_bits <= dst_bits) {
1717
1718 if (dst_bits == 4) {
1719 struct lp_type flt_type = lp_type_float_vec(32, src_type.length * 32);
1720
1721 result = lp_build_unsigned_norm_to_float(gallivm, src_bits, flt_type, src);
1722 result = lp_build_clamped_float_to_unsigned_norm(gallivm, flt_type, dst_bits, result);
1723 result = LLVMBuildTrunc(gallivm->builder, result, lp_build_int_vec_type(gallivm, src_type), "");
1724 return result;
1725 }
1726
1727 /*
1728 * Approximate the rescaling with a single shift.
1729 *
1730 * This gives the wrong rounding.
1731 */
1732
1733 result = LLVMBuildLShr(builder,
1734 src,
1735 lp_build_const_int_vec(gallivm, src_type, delta_bits),
1736 "");
1737
1738 } else {
1739 /*
1740 * Try more accurate rescaling.
1741 */
1742
1743 /*
1744 * Drop the least significant bits to make space for the multiplication.
1745 *
1746 * XXX: A better approach would be to use a wider integer type as intermediate. But
1747 * this is enough to convert alpha from 16bits -> 2 when rendering to
1748 * PIPE_FORMAT_R10G10B10A2_UNORM.
1749 */
1750 result = LLVMBuildLShr(builder,
1751 src,
1752 lp_build_const_int_vec(gallivm, src_type, dst_bits),
1753 "");
1754
1755
1756 result = LLVMBuildMul(builder,
1757 result,
1758 lp_build_const_int_vec(gallivm, src_type, (1LL << dst_bits) - 1),
1759 "");
1760
1761 /*
1762 * Add a rounding term before the division.
1763 *
1764 * TODO: Handle signed integers too.
1765 */
1766 if (!src_type.sign) {
1767 result = LLVMBuildAdd(builder,
1768 result,
1769 lp_build_const_int_vec(gallivm, src_type, (1LL << (delta_bits - 1))),
1770 "");
1771 }
1772
1773 /*
1774 * Approximate the division by src_mask with a src_bits shift.
1775 *
1776 * Given the src has already been shifted by dst_bits, all we need
1777 * to do is to shift by the difference.
1778 */
1779
1780 result = LLVMBuildLShr(builder,
1781 result,
1782 lp_build_const_int_vec(gallivm, src_type, delta_bits),
1783 "");
1784 }
1785
1786 } else if (dst_bits > src_bits) {
1787 /* Scale up bits */
1788 int db = dst_bits - src_bits;
1789
1790 /* Shift left by difference in bits */
1791 result = LLVMBuildShl(builder,
1792 src,
1793 lp_build_const_int_vec(gallivm, src_type, db),
1794 "");
1795
1796 if (db <= src_bits) {
1797 /* Enough bits in src to fill the remainder */
1798 LLVMValueRef lower = LLVMBuildLShr(builder,
1799 src,
1800 lp_build_const_int_vec(gallivm, src_type, src_bits - db),
1801 "");
1802
1803 result = LLVMBuildOr(builder, result, lower, "");
1804 } else if (db > src_bits) {
1805 /* Need to repeatedly copy src bits to fill remainder in dst */
1806 unsigned n;
1807
1808 for (n = src_bits; n < dst_bits; n *= 2) {
1809 LLVMValueRef shuv = lp_build_const_int_vec(gallivm, src_type, n);
1810
1811 result = LLVMBuildOr(builder,
1812 result,
1813 LLVMBuildLShr(builder, result, shuv, ""),
1814 "");
1815 }
1816 }
1817 }
1818
1819 return result;
1820 }
1821
1822 /**
1823 * If RT is a smallfloat (needing denorms) format
1824 */
1825 static inline int
have_smallfloat_format(struct lp_type dst_type,enum pipe_format format)1826 have_smallfloat_format(struct lp_type dst_type,
1827 enum pipe_format format)
1828 {
1829 return ((dst_type.floating && dst_type.width != 32) ||
1830 /* due to format handling hacks this format doesn't have floating set
1831 * here (and actually has width set to 32 too) so special case this. */
1832 (format == PIPE_FORMAT_R11G11B10_FLOAT));
1833 }
1834
1835
1836 /**
1837 * Convert from memory format to blending format
1838 *
1839 * e.g. GL_R3G3B2 is 1 byte in memory but 3 bytes for blending
1840 */
1841 static void
convert_to_blend_type(struct gallivm_state * gallivm,unsigned block_size,const struct util_format_description * src_fmt,struct lp_type src_type,struct lp_type dst_type,LLVMValueRef * src,unsigned num_srcs)1842 convert_to_blend_type(struct gallivm_state *gallivm,
1843 unsigned block_size,
1844 const struct util_format_description *src_fmt,
1845 struct lp_type src_type,
1846 struct lp_type dst_type,
1847 LLVMValueRef* src, // and dst
1848 unsigned num_srcs)
1849 {
1850 LLVMValueRef *dst = src;
1851 LLVMBuilderRef builder = gallivm->builder;
1852 struct lp_type blend_type;
1853 struct lp_type mem_type;
1854 unsigned i, j;
1855 unsigned pixels = block_size / num_srcs;
1856 bool is_arith;
1857
1858 /*
1859 * full custom path for packed floats and srgb formats - none of the later
1860 * functions would do anything useful, and given the lp_type representation they
1861 * can't be fixed. Should really have some SoA blend path for these kind of
1862 * formats rather than hacking them in here.
1863 */
1864 if (format_expands_to_float_soa(src_fmt)) {
1865 LLVMValueRef tmpsrc[4];
1866 /*
1867 * This is pretty suboptimal for this case blending in SoA would be much
1868 * better, since conversion gets us SoA values so need to convert back.
1869 */
1870 assert(src_type.width == 32 || src_type.width == 16);
1871 assert(dst_type.floating);
1872 assert(dst_type.width == 32);
1873 assert(dst_type.length % 4 == 0);
1874 assert(num_srcs % 4 == 0);
1875
1876 if (src_type.width == 16) {
1877 /* expand 4x16bit values to 4x32bit */
1878 struct lp_type type32x4 = src_type;
1879 LLVMTypeRef ltype32x4;
1880 unsigned num_fetch = dst_type.length == 8 ? num_srcs / 2 : num_srcs / 4;
1881 type32x4.width = 32;
1882 ltype32x4 = lp_build_vec_type(gallivm, type32x4);
1883 for (i = 0; i < num_fetch; i++) {
1884 src[i] = LLVMBuildZExt(builder, src[i], ltype32x4, "");
1885 }
1886 src_type.width = 32;
1887 }
1888 for (i = 0; i < 4; i++) {
1889 tmpsrc[i] = src[i];
1890 }
1891 for (i = 0; i < num_srcs / 4; i++) {
1892 LLVMValueRef tmpsoa[4];
1893 LLVMValueRef tmps = tmpsrc[i];
1894 if (dst_type.length == 8) {
1895 LLVMValueRef shuffles[8];
1896 unsigned j;
1897 /* fetch was 4 values but need 8-wide output values */
1898 tmps = lp_build_concat(gallivm, &tmpsrc[i * 2], src_type, 2);
1899 /*
1900 * for 8-wide aos transpose would give us wrong order not matching
1901 * incoming converted fs values and mask. ARGH.
1902 */
1903 for (j = 0; j < 4; j++) {
1904 shuffles[j] = lp_build_const_int32(gallivm, j * 2);
1905 shuffles[j + 4] = lp_build_const_int32(gallivm, j * 2 + 1);
1906 }
1907 tmps = LLVMBuildShuffleVector(builder, tmps, tmps,
1908 LLVMConstVector(shuffles, 8), "");
1909 }
1910 if (src_fmt->format == PIPE_FORMAT_R11G11B10_FLOAT) {
1911 lp_build_r11g11b10_to_float(gallivm, tmps, tmpsoa);
1912 }
1913 else {
1914 lp_build_unpack_rgba_soa(gallivm, src_fmt, dst_type, tmps, tmpsoa);
1915 }
1916 lp_build_transpose_aos(gallivm, dst_type, tmpsoa, &src[i * 4]);
1917 }
1918 return;
1919 }
1920
1921 lp_mem_type_from_format_desc(src_fmt, &mem_type);
1922 lp_blend_type_from_format_desc(src_fmt, &blend_type);
1923
1924 /* Is the format arithmetic */
1925 is_arith = blend_type.length * blend_type.width != mem_type.width * mem_type.length;
1926 is_arith &= !(mem_type.width == 16 && mem_type.floating);
1927
1928 /* Pad if necessary */
1929 if (!is_arith && src_type.length < dst_type.length) {
1930 for (i = 0; i < num_srcs; ++i) {
1931 dst[i] = lp_build_pad_vector(gallivm, src[i], dst_type.length);
1932 }
1933
1934 src_type.length = dst_type.length;
1935 }
1936
1937 /* Special case for half-floats */
1938 if (mem_type.width == 16 && mem_type.floating) {
1939 assert(blend_type.width == 32 && blend_type.floating);
1940 lp_build_conv_auto(gallivm, src_type, &dst_type, dst, num_srcs, dst);
1941 is_arith = false;
1942 }
1943
1944 if (!is_arith) {
1945 return;
1946 }
1947
1948 src_type.width = blend_type.width * blend_type.length;
1949 blend_type.length *= pixels;
1950 src_type.length *= pixels / (src_type.length / mem_type.length);
1951
1952 for (i = 0; i < num_srcs; ++i) {
1953 LLVMValueRef chans[4];
1954 LLVMValueRef res = NULL;
1955
1956 dst[i] = LLVMBuildZExt(builder, src[i], lp_build_vec_type(gallivm, src_type), "");
1957
1958 for (j = 0; j < src_fmt->nr_channels; ++j) {
1959 unsigned mask = 0;
1960 unsigned sa = src_fmt->channel[j].shift;
1961 #if UTIL_ARCH_LITTLE_ENDIAN
1962 unsigned from_lsb = j;
1963 #else
1964 unsigned from_lsb = src_fmt->nr_channels - j - 1;
1965 #endif
1966
1967 mask = (1 << src_fmt->channel[j].size) - 1;
1968
1969 /* Extract bits from source */
1970 chans[j] = LLVMBuildLShr(builder,
1971 dst[i],
1972 lp_build_const_int_vec(gallivm, src_type, sa),
1973 "");
1974
1975 chans[j] = LLVMBuildAnd(builder,
1976 chans[j],
1977 lp_build_const_int_vec(gallivm, src_type, mask),
1978 "");
1979
1980 /* Scale bits */
1981 if (src_type.norm) {
1982 chans[j] = scale_bits(gallivm, src_fmt->channel[j].size,
1983 blend_type.width, chans[j], src_type);
1984 }
1985
1986 /* Insert bits into correct position */
1987 chans[j] = LLVMBuildShl(builder,
1988 chans[j],
1989 lp_build_const_int_vec(gallivm, src_type, from_lsb * blend_type.width),
1990 "");
1991
1992 if (j == 0) {
1993 res = chans[j];
1994 } else {
1995 res = LLVMBuildOr(builder, res, chans[j], "");
1996 }
1997 }
1998
1999 dst[i] = LLVMBuildBitCast(builder, res, lp_build_vec_type(gallivm, blend_type), "");
2000 }
2001 }
2002
2003
2004 /**
2005 * Convert from blending format to memory format
2006 *
2007 * e.g. GL_R3G3B2 is 3 bytes for blending but 1 byte in memory
2008 */
2009 static void
convert_from_blend_type(struct gallivm_state * gallivm,unsigned block_size,const struct util_format_description * src_fmt,struct lp_type src_type,struct lp_type dst_type,LLVMValueRef * src,unsigned num_srcs)2010 convert_from_blend_type(struct gallivm_state *gallivm,
2011 unsigned block_size,
2012 const struct util_format_description *src_fmt,
2013 struct lp_type src_type,
2014 struct lp_type dst_type,
2015 LLVMValueRef* src, // and dst
2016 unsigned num_srcs)
2017 {
2018 LLVMValueRef* dst = src;
2019 unsigned i, j, k;
2020 struct lp_type mem_type;
2021 struct lp_type blend_type;
2022 LLVMBuilderRef builder = gallivm->builder;
2023 unsigned pixels = block_size / num_srcs;
2024 bool is_arith;
2025
2026 /*
2027 * full custom path for packed floats and srgb formats - none of the later
2028 * functions would do anything useful, and given the lp_type representation they
2029 * can't be fixed. Should really have some SoA blend path for these kind of
2030 * formats rather than hacking them in here.
2031 */
2032 if (format_expands_to_float_soa(src_fmt)) {
2033 /*
2034 * This is pretty suboptimal for this case blending in SoA would be much
2035 * better - we need to transpose the AoS values back to SoA values for
2036 * conversion/packing.
2037 */
2038 assert(src_type.floating);
2039 assert(src_type.width == 32);
2040 assert(src_type.length % 4 == 0);
2041 assert(dst_type.width == 32 || dst_type.width == 16);
2042
2043 for (i = 0; i < num_srcs / 4; i++) {
2044 LLVMValueRef tmpsoa[4], tmpdst;
2045 lp_build_transpose_aos(gallivm, src_type, &src[i * 4], tmpsoa);
2046 /* really really need SoA here */
2047
2048 if (src_fmt->format == PIPE_FORMAT_R11G11B10_FLOAT) {
2049 tmpdst = lp_build_float_to_r11g11b10(gallivm, tmpsoa);
2050 }
2051 else {
2052 tmpdst = lp_build_float_to_srgb_packed(gallivm, src_fmt,
2053 src_type, tmpsoa);
2054 }
2055
2056 if (src_type.length == 8) {
2057 LLVMValueRef tmpaos, shuffles[8];
2058 unsigned j;
2059 /*
2060 * for 8-wide aos transpose has given us wrong order not matching
2061 * output order. HMPF. Also need to split the output values manually.
2062 */
2063 for (j = 0; j < 4; j++) {
2064 shuffles[j * 2] = lp_build_const_int32(gallivm, j);
2065 shuffles[j * 2 + 1] = lp_build_const_int32(gallivm, j + 4);
2066 }
2067 tmpaos = LLVMBuildShuffleVector(builder, tmpdst, tmpdst,
2068 LLVMConstVector(shuffles, 8), "");
2069 src[i * 2] = lp_build_extract_range(gallivm, tmpaos, 0, 4);
2070 src[i * 2 + 1] = lp_build_extract_range(gallivm, tmpaos, 4, 4);
2071 }
2072 else {
2073 src[i] = tmpdst;
2074 }
2075 }
2076 if (dst_type.width == 16) {
2077 struct lp_type type16x8 = dst_type;
2078 struct lp_type type32x4 = dst_type;
2079 LLVMTypeRef ltype16x4, ltypei64, ltypei128;
2080 unsigned num_fetch = src_type.length == 8 ? num_srcs / 2 : num_srcs / 4;
2081 type16x8.length = 8;
2082 type32x4.width = 32;
2083 ltypei128 = LLVMIntTypeInContext(gallivm->context, 128);
2084 ltypei64 = LLVMIntTypeInContext(gallivm->context, 64);
2085 ltype16x4 = lp_build_vec_type(gallivm, dst_type);
2086 /* We could do vector truncation but it doesn't generate very good code */
2087 for (i = 0; i < num_fetch; i++) {
2088 src[i] = lp_build_pack2(gallivm, type32x4, type16x8,
2089 src[i], lp_build_zero(gallivm, type32x4));
2090 src[i] = LLVMBuildBitCast(builder, src[i], ltypei128, "");
2091 src[i] = LLVMBuildTrunc(builder, src[i], ltypei64, "");
2092 src[i] = LLVMBuildBitCast(builder, src[i], ltype16x4, "");
2093 }
2094 }
2095 return;
2096 }
2097
2098 lp_mem_type_from_format_desc(src_fmt, &mem_type);
2099 lp_blend_type_from_format_desc(src_fmt, &blend_type);
2100
2101 is_arith = (blend_type.length * blend_type.width != mem_type.width * mem_type.length);
2102
2103 /* Special case for half-floats */
2104 if (mem_type.width == 16 && mem_type.floating) {
2105 int length = dst_type.length;
2106 assert(blend_type.width == 32 && blend_type.floating);
2107
2108 dst_type.length = src_type.length;
2109
2110 lp_build_conv_auto(gallivm, src_type, &dst_type, dst, num_srcs, dst);
2111
2112 dst_type.length = length;
2113 is_arith = false;
2114 }
2115
2116 /* Remove any padding */
2117 if (!is_arith && (src_type.length % mem_type.length)) {
2118 src_type.length -= (src_type.length % mem_type.length);
2119
2120 for (i = 0; i < num_srcs; ++i) {
2121 dst[i] = lp_build_extract_range(gallivm, dst[i], 0, src_type.length);
2122 }
2123 }
2124
2125 /* No bit arithmetic to do */
2126 if (!is_arith) {
2127 return;
2128 }
2129
2130 src_type.length = pixels;
2131 src_type.width = blend_type.length * blend_type.width;
2132 dst_type.length = pixels;
2133
2134 for (i = 0; i < num_srcs; ++i) {
2135 LLVMValueRef chans[4];
2136 LLVMValueRef res = NULL;
2137
2138 dst[i] = LLVMBuildBitCast(builder, src[i], lp_build_vec_type(gallivm, src_type), "");
2139
2140 for (j = 0; j < src_fmt->nr_channels; ++j) {
2141 unsigned mask = 0;
2142 unsigned sa = src_fmt->channel[j].shift;
2143 unsigned sz_a = src_fmt->channel[j].size;
2144 #if UTIL_ARCH_LITTLE_ENDIAN
2145 unsigned from_lsb = j;
2146 #else
2147 unsigned from_lsb = src_fmt->nr_channels - j - 1;
2148 #endif
2149
2150 assert(blend_type.width > src_fmt->channel[j].size);
2151
2152 for (k = 0; k < blend_type.width; ++k) {
2153 mask |= 1 << k;
2154 }
2155
2156 /* Extract bits */
2157 chans[j] = LLVMBuildLShr(builder,
2158 dst[i],
2159 lp_build_const_int_vec(gallivm, src_type,
2160 from_lsb * blend_type.width),
2161 "");
2162
2163 chans[j] = LLVMBuildAnd(builder,
2164 chans[j],
2165 lp_build_const_int_vec(gallivm, src_type, mask),
2166 "");
2167
2168 /* Scale down bits */
2169 if (src_type.norm) {
2170 chans[j] = scale_bits(gallivm, blend_type.width,
2171 src_fmt->channel[j].size, chans[j], src_type);
2172 } else if (!src_type.floating && sz_a < blend_type.width) {
2173 LLVMValueRef mask_val = lp_build_const_int_vec(gallivm, src_type, (1UL << sz_a) - 1);
2174 LLVMValueRef mask = LLVMBuildICmp(builder, LLVMIntUGT, chans[j], mask_val, "");
2175 chans[j] = LLVMBuildSelect(builder, mask, mask_val, chans[j], "");
2176 }
2177
2178 /* Insert bits */
2179 chans[j] = LLVMBuildShl(builder,
2180 chans[j],
2181 lp_build_const_int_vec(gallivm, src_type, sa),
2182 "");
2183
2184 sa += src_fmt->channel[j].size;
2185
2186 if (j == 0) {
2187 res = chans[j];
2188 } else {
2189 res = LLVMBuildOr(builder, res, chans[j], "");
2190 }
2191 }
2192
2193 assert (dst_type.width != 24);
2194
2195 dst[i] = LLVMBuildTrunc(builder, res, lp_build_vec_type(gallivm, dst_type), "");
2196 }
2197 }
2198
2199
2200 /**
2201 * Convert alpha to same blend type as src
2202 */
2203 static void
convert_alpha(struct gallivm_state * gallivm,struct lp_type row_type,struct lp_type alpha_type,const unsigned block_size,const unsigned block_height,const unsigned src_count,const unsigned dst_channels,const bool pad_inline,LLVMValueRef * src_alpha)2204 convert_alpha(struct gallivm_state *gallivm,
2205 struct lp_type row_type,
2206 struct lp_type alpha_type,
2207 const unsigned block_size,
2208 const unsigned block_height,
2209 const unsigned src_count,
2210 const unsigned dst_channels,
2211 const bool pad_inline,
2212 LLVMValueRef* src_alpha)
2213 {
2214 LLVMBuilderRef builder = gallivm->builder;
2215 unsigned i, j;
2216 unsigned length = row_type.length;
2217 row_type.length = alpha_type.length;
2218
2219 /* Twiddle the alpha to match pixels */
2220 lp_bld_quad_twiddle(gallivm, alpha_type, src_alpha, block_height, src_alpha);
2221
2222 /*
2223 * TODO this should use single lp_build_conv call for
2224 * src_count == 1 && dst_channels == 1 case (dropping the concat below)
2225 */
2226 for (i = 0; i < block_height; ++i) {
2227 lp_build_conv(gallivm, alpha_type, row_type, &src_alpha[i], 1, &src_alpha[i], 1);
2228 }
2229
2230 alpha_type = row_type;
2231 row_type.length = length;
2232
2233 /* If only one channel we can only need the single alpha value per pixel */
2234 if (src_count == 1 && dst_channels == 1) {
2235
2236 lp_build_concat_n(gallivm, alpha_type, src_alpha, block_height, src_alpha, src_count);
2237 } else {
2238 /* If there are more srcs than rows then we need to split alpha up */
2239 if (src_count > block_height) {
2240 for (i = src_count; i > 0; --i) {
2241 unsigned pixels = block_size / src_count;
2242 unsigned idx = i - 1;
2243
2244 src_alpha[idx] = lp_build_extract_range(gallivm, src_alpha[(idx * pixels) / 4],
2245 (idx * pixels) % 4, pixels);
2246 }
2247 }
2248
2249 /* If there is a src for each pixel broadcast the alpha across whole row */
2250 if (src_count == block_size) {
2251 for (i = 0; i < src_count; ++i) {
2252 src_alpha[i] = lp_build_broadcast(gallivm,
2253 lp_build_vec_type(gallivm, row_type), src_alpha[i]);
2254 }
2255 } else {
2256 unsigned pixels = block_size / src_count;
2257 unsigned channels = pad_inline ? TGSI_NUM_CHANNELS : dst_channels;
2258 unsigned alpha_span = 1;
2259 LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH];
2260
2261 /* Check if we need 2 src_alphas for our shuffles */
2262 if (pixels > alpha_type.length) {
2263 alpha_span = 2;
2264 }
2265
2266 /* Broadcast alpha across all channels, e.g. a1a2 to a1a1a1a1a2a2a2a2 */
2267 for (j = 0; j < row_type.length; ++j) {
2268 if (j < pixels * channels) {
2269 shuffles[j] = lp_build_const_int32(gallivm, j / channels);
2270 } else {
2271 shuffles[j] = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context));
2272 }
2273 }
2274
2275 for (i = 0; i < src_count; ++i) {
2276 unsigned idx1 = i, idx2 = i;
2277
2278 if (alpha_span > 1){
2279 idx1 *= alpha_span;
2280 idx2 = idx1 + 1;
2281 }
2282
2283 src_alpha[i] = LLVMBuildShuffleVector(builder,
2284 src_alpha[idx1],
2285 src_alpha[idx2],
2286 LLVMConstVector(shuffles, row_type.length),
2287 "");
2288 }
2289 }
2290 }
2291 }
2292
2293
2294 /**
2295 * Generates the blend function for unswizzled colour buffers
2296 * Also generates the read & write from colour buffer
2297 */
2298 static void
generate_unswizzled_blend(struct gallivm_state * gallivm,unsigned rt,struct lp_fragment_shader_variant * variant,enum pipe_format out_format,unsigned int num_fs,struct lp_type fs_type,LLVMValueRef * fs_mask,LLVMValueRef fs_out_color[PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS][4],LLVMValueRef context_ptr,LLVMValueRef color_ptr,LLVMValueRef stride,unsigned partial_mask,boolean do_branch)2299 generate_unswizzled_blend(struct gallivm_state *gallivm,
2300 unsigned rt,
2301 struct lp_fragment_shader_variant *variant,
2302 enum pipe_format out_format,
2303 unsigned int num_fs,
2304 struct lp_type fs_type,
2305 LLVMValueRef* fs_mask,
2306 LLVMValueRef fs_out_color[PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS][4],
2307 LLVMValueRef context_ptr,
2308 LLVMValueRef color_ptr,
2309 LLVMValueRef stride,
2310 unsigned partial_mask,
2311 boolean do_branch)
2312 {
2313 const unsigned alpha_channel = 3;
2314 const unsigned block_width = LP_RASTER_BLOCK_SIZE;
2315 const unsigned block_height = LP_RASTER_BLOCK_SIZE;
2316 const unsigned block_size = block_width * block_height;
2317 const unsigned lp_integer_vector_width = 128;
2318
2319 LLVMBuilderRef builder = gallivm->builder;
2320 LLVMValueRef fs_src[4][TGSI_NUM_CHANNELS];
2321 LLVMValueRef fs_src1[4][TGSI_NUM_CHANNELS];
2322 LLVMValueRef src_alpha[4 * 4];
2323 LLVMValueRef src1_alpha[4 * 4] = { NULL };
2324 LLVMValueRef src_mask[4 * 4];
2325 LLVMValueRef src[4 * 4];
2326 LLVMValueRef src1[4 * 4];
2327 LLVMValueRef dst[4 * 4];
2328 LLVMValueRef blend_color;
2329 LLVMValueRef blend_alpha;
2330 LLVMValueRef i32_zero;
2331 LLVMValueRef check_mask;
2332 LLVMValueRef undef_src_val;
2333
2334 struct lp_build_mask_context mask_ctx;
2335 struct lp_type mask_type;
2336 struct lp_type blend_type;
2337 struct lp_type row_type;
2338 struct lp_type dst_type;
2339 struct lp_type ls_type;
2340
2341 unsigned char swizzle[TGSI_NUM_CHANNELS];
2342 unsigned vector_width;
2343 unsigned src_channels = TGSI_NUM_CHANNELS;
2344 unsigned dst_channels;
2345 unsigned dst_count;
2346 unsigned src_count;
2347 unsigned i, j;
2348
2349 const struct util_format_description* out_format_desc = util_format_description(out_format);
2350
2351 unsigned dst_alignment;
2352
2353 bool pad_inline = is_arithmetic_format(out_format_desc);
2354 bool has_alpha = false;
2355 const boolean dual_source_blend = variant->key.blend.rt[0].blend_enable &&
2356 util_blend_state_is_dual(&variant->key.blend, 0);
2357
2358 const boolean is_1d = variant->key.resource_1d;
2359 boolean twiddle_after_convert = FALSE;
2360 unsigned num_fullblock_fs = is_1d ? 2 * num_fs : num_fs;
2361 LLVMValueRef fpstate = 0;
2362
2363 /* Get type from output format */
2364 lp_blend_type_from_format_desc(out_format_desc, &row_type);
2365 lp_mem_type_from_format_desc(out_format_desc, &dst_type);
2366
2367 /*
2368 * Technically this code should go into lp_build_smallfloat_to_float
2369 * and lp_build_float_to_smallfloat but due to the
2370 * http://llvm.org/bugs/show_bug.cgi?id=6393
2371 * llvm reorders the mxcsr intrinsics in a way that breaks the code.
2372 * So the ordering is important here and there shouldn't be any
2373 * llvm ir instrunctions in this function before
2374 * this, otherwise half-float format conversions won't work
2375 * (again due to llvm bug #6393).
2376 */
2377 if (have_smallfloat_format(dst_type, out_format)) {
2378 /* We need to make sure that denorms are ok for half float
2379 conversions */
2380 fpstate = lp_build_fpstate_get(gallivm);
2381 lp_build_fpstate_set_denorms_zero(gallivm, FALSE);
2382 }
2383
2384 mask_type = lp_int32_vec4_type();
2385 mask_type.length = fs_type.length;
2386
2387 for (i = num_fs; i < num_fullblock_fs; i++) {
2388 fs_mask[i] = lp_build_zero(gallivm, mask_type);
2389 }
2390
2391 /* Do not bother executing code when mask is empty.. */
2392 if (do_branch) {
2393 check_mask = LLVMConstNull(lp_build_int_vec_type(gallivm, mask_type));
2394
2395 for (i = 0; i < num_fullblock_fs; ++i) {
2396 check_mask = LLVMBuildOr(builder, check_mask, fs_mask[i], "");
2397 }
2398
2399 lp_build_mask_begin(&mask_ctx, gallivm, mask_type, check_mask);
2400 lp_build_mask_check(&mask_ctx);
2401 }
2402
2403 partial_mask |= !variant->opaque;
2404 i32_zero = lp_build_const_int32(gallivm, 0);
2405
2406 undef_src_val = lp_build_undef(gallivm, fs_type);
2407
2408 row_type.length = fs_type.length;
2409 vector_width = dst_type.floating ? lp_native_vector_width : lp_integer_vector_width;
2410
2411 /* Compute correct swizzle and count channels */
2412 memset(swizzle, LP_BLD_SWIZZLE_DONTCARE, TGSI_NUM_CHANNELS);
2413 dst_channels = 0;
2414
2415 for (i = 0; i < TGSI_NUM_CHANNELS; ++i) {
2416 /* Ensure channel is used */
2417 if (out_format_desc->swizzle[i] >= TGSI_NUM_CHANNELS) {
2418 continue;
2419 }
2420
2421 /* Ensure not already written to (happens in case with GL_ALPHA) */
2422 if (swizzle[out_format_desc->swizzle[i]] < TGSI_NUM_CHANNELS) {
2423 continue;
2424 }
2425
2426 /* Ensure we haven't already found all channels */
2427 if (dst_channels >= out_format_desc->nr_channels) {
2428 continue;
2429 }
2430
2431 swizzle[out_format_desc->swizzle[i]] = i;
2432 ++dst_channels;
2433
2434 if (i == alpha_channel) {
2435 has_alpha = true;
2436 }
2437 }
2438
2439 if (format_expands_to_float_soa(out_format_desc)) {
2440 /*
2441 * the code above can't work for layout_other
2442 * for srgb it would sort of work but we short-circuit swizzles, etc.
2443 * as that is done as part of unpack / pack.
2444 */
2445 dst_channels = 4; /* HACK: this is fake 4 really but need it due to transpose stuff later */
2446 has_alpha = true;
2447 swizzle[0] = 0;
2448 swizzle[1] = 1;
2449 swizzle[2] = 2;
2450 swizzle[3] = 3;
2451 pad_inline = true; /* HACK: prevent rgbxrgbx->rgbrgbxx conversion later */
2452 }
2453
2454 /* If 3 channels then pad to include alpha for 4 element transpose */
2455 if (dst_channels == 3) {
2456 assert (!has_alpha);
2457 for (i = 0; i < TGSI_NUM_CHANNELS; i++) {
2458 if (swizzle[i] > TGSI_NUM_CHANNELS)
2459 swizzle[i] = 3;
2460 }
2461 if (out_format_desc->nr_channels == 4) {
2462 dst_channels = 4;
2463 /*
2464 * We use alpha from the color conversion, not separate one.
2465 * We had to include it for transpose, hence it will get converted
2466 * too (albeit when doing transpose after conversion, that would
2467 * no longer be the case necessarily).
2468 * (It works only with 4 channel dsts, e.g. rgbx formats, because
2469 * otherwise we really have padding, not alpha, included.)
2470 */
2471 has_alpha = true;
2472 }
2473 }
2474
2475 /*
2476 * Load shader output
2477 */
2478 for (i = 0; i < num_fullblock_fs; ++i) {
2479 /* Always load alpha for use in blending */
2480 LLVMValueRef alpha;
2481 if (i < num_fs) {
2482 alpha = LLVMBuildLoad(builder, fs_out_color[rt][alpha_channel][i], "");
2483 }
2484 else {
2485 alpha = undef_src_val;
2486 }
2487
2488 /* Load each channel */
2489 for (j = 0; j < dst_channels; ++j) {
2490 assert(swizzle[j] < 4);
2491 if (i < num_fs) {
2492 fs_src[i][j] = LLVMBuildLoad(builder, fs_out_color[rt][swizzle[j]][i], "");
2493 }
2494 else {
2495 fs_src[i][j] = undef_src_val;
2496 }
2497 }
2498
2499 /* If 3 channels then pad to include alpha for 4 element transpose */
2500 /*
2501 * XXX If we include that here maybe could actually use it instead of
2502 * separate alpha for blending?
2503 * (Difficult though we actually convert pad channels, not alpha.)
2504 */
2505 if (dst_channels == 3 && !has_alpha) {
2506 fs_src[i][3] = alpha;
2507 }
2508
2509 /* We split the row_mask and row_alpha as we want 128bit interleave */
2510 if (fs_type.length == 8) {
2511 src_mask[i*2 + 0] = lp_build_extract_range(gallivm, fs_mask[i],
2512 0, src_channels);
2513 src_mask[i*2 + 1] = lp_build_extract_range(gallivm, fs_mask[i],
2514 src_channels, src_channels);
2515
2516 src_alpha[i*2 + 0] = lp_build_extract_range(gallivm, alpha, 0, src_channels);
2517 src_alpha[i*2 + 1] = lp_build_extract_range(gallivm, alpha,
2518 src_channels, src_channels);
2519 } else {
2520 src_mask[i] = fs_mask[i];
2521 src_alpha[i] = alpha;
2522 }
2523 }
2524 if (dual_source_blend) {
2525 /* same as above except different src/dst, skip masks and comments... */
2526 for (i = 0; i < num_fullblock_fs; ++i) {
2527 LLVMValueRef alpha;
2528 if (i < num_fs) {
2529 alpha = LLVMBuildLoad(builder, fs_out_color[1][alpha_channel][i], "");
2530 }
2531 else {
2532 alpha = undef_src_val;
2533 }
2534
2535 for (j = 0; j < dst_channels; ++j) {
2536 assert(swizzle[j] < 4);
2537 if (i < num_fs) {
2538 fs_src1[i][j] = LLVMBuildLoad(builder, fs_out_color[1][swizzle[j]][i], "");
2539 }
2540 else {
2541 fs_src1[i][j] = undef_src_val;
2542 }
2543 }
2544 if (dst_channels == 3 && !has_alpha) {
2545 fs_src1[i][3] = alpha;
2546 }
2547 if (fs_type.length == 8) {
2548 src1_alpha[i*2 + 0] = lp_build_extract_range(gallivm, alpha, 0, src_channels);
2549 src1_alpha[i*2 + 1] = lp_build_extract_range(gallivm, alpha,
2550 src_channels, src_channels);
2551 } else {
2552 src1_alpha[i] = alpha;
2553 }
2554 }
2555 }
2556
2557 if (util_format_is_pure_integer(out_format)) {
2558 /*
2559 * In this case fs_type was really ints or uints disguised as floats,
2560 * fix that up now.
2561 */
2562 fs_type.floating = 0;
2563 fs_type.sign = dst_type.sign;
2564 for (i = 0; i < num_fullblock_fs; ++i) {
2565 for (j = 0; j < dst_channels; ++j) {
2566 fs_src[i][j] = LLVMBuildBitCast(builder, fs_src[i][j],
2567 lp_build_vec_type(gallivm, fs_type), "");
2568 }
2569 if (dst_channels == 3 && !has_alpha) {
2570 fs_src[i][3] = LLVMBuildBitCast(builder, fs_src[i][3],
2571 lp_build_vec_type(gallivm, fs_type), "");
2572 }
2573 }
2574 }
2575
2576 /*
2577 * We actually should generally do conversion first (for non-1d cases)
2578 * when the blend format is 8 or 16 bits. The reason is obvious,
2579 * there's 2 or 4 times less vectors to deal with for the interleave...
2580 * Albeit for the AVX (not AVX2) case there's no benefit with 16 bit
2581 * vectors (as it can do 32bit unpack with 256bit vectors, but 8/16bit
2582 * unpack only with 128bit vectors).
2583 * Note: for 16bit sizes really need matching pack conversion code
2584 */
2585 if (!is_1d && dst_channels != 3 && dst_type.width == 8) {
2586 twiddle_after_convert = TRUE;
2587 }
2588
2589 /*
2590 * Pixel twiddle from fragment shader order to memory order
2591 */
2592 if (!twiddle_after_convert) {
2593 src_count = generate_fs_twiddle(gallivm, fs_type, num_fullblock_fs,
2594 dst_channels, fs_src, src, pad_inline);
2595 if (dual_source_blend) {
2596 generate_fs_twiddle(gallivm, fs_type, num_fullblock_fs, dst_channels,
2597 fs_src1, src1, pad_inline);
2598 }
2599 } else {
2600 src_count = num_fullblock_fs * dst_channels;
2601 /*
2602 * We reorder things a bit here, so the cases for 4-wide and 8-wide
2603 * (AVX) turn out the same later when untwiddling/transpose (albeit
2604 * for true AVX2 path untwiddle needs to be different).
2605 * For now just order by colors first (so we can use unpack later).
2606 */
2607 for (j = 0; j < num_fullblock_fs; j++) {
2608 for (i = 0; i < dst_channels; i++) {
2609 src[i*num_fullblock_fs + j] = fs_src[j][i];
2610 if (dual_source_blend) {
2611 src1[i*num_fullblock_fs + j] = fs_src1[j][i];
2612 }
2613 }
2614 }
2615 }
2616
2617 src_channels = dst_channels < 3 ? dst_channels : 4;
2618 if (src_count != num_fullblock_fs * src_channels) {
2619 unsigned ds = src_count / (num_fullblock_fs * src_channels);
2620 row_type.length /= ds;
2621 fs_type.length = row_type.length;
2622 }
2623
2624 blend_type = row_type;
2625 mask_type.length = 4;
2626
2627 /* Convert src to row_type */
2628 if (dual_source_blend) {
2629 struct lp_type old_row_type = row_type;
2630 lp_build_conv_auto(gallivm, fs_type, &row_type, src, src_count, src);
2631 src_count = lp_build_conv_auto(gallivm, fs_type, &old_row_type, src1, src_count, src1);
2632 }
2633 else {
2634 src_count = lp_build_conv_auto(gallivm, fs_type, &row_type, src, src_count, src);
2635 }
2636
2637 /* If the rows are not an SSE vector, combine them to become SSE size! */
2638 if ((row_type.width * row_type.length) % 128) {
2639 unsigned bits = row_type.width * row_type.length;
2640 unsigned combined;
2641
2642 assert(src_count >= (vector_width / bits));
2643
2644 dst_count = src_count / (vector_width / bits);
2645
2646 combined = lp_build_concat_n(gallivm, row_type, src, src_count, src, dst_count);
2647 if (dual_source_blend) {
2648 lp_build_concat_n(gallivm, row_type, src1, src_count, src1, dst_count);
2649 }
2650
2651 row_type.length *= combined;
2652 src_count /= combined;
2653
2654 bits = row_type.width * row_type.length;
2655 assert(bits == 128 || bits == 256);
2656 }
2657
2658 if (twiddle_after_convert) {
2659 fs_twiddle_transpose(gallivm, row_type, src, src_count, src);
2660 if (dual_source_blend) {
2661 fs_twiddle_transpose(gallivm, row_type, src1, src_count, src1);
2662 }
2663 }
2664
2665 /*
2666 * Blend Colour conversion
2667 */
2668 blend_color = lp_jit_context_f_blend_color(gallivm, context_ptr);
2669 blend_color = LLVMBuildPointerCast(builder, blend_color,
2670 LLVMPointerType(lp_build_vec_type(gallivm, fs_type), 0), "");
2671 blend_color = LLVMBuildLoad(builder, LLVMBuildGEP(builder, blend_color,
2672 &i32_zero, 1, ""), "");
2673
2674 /* Convert */
2675 lp_build_conv(gallivm, fs_type, blend_type, &blend_color, 1, &blend_color, 1);
2676
2677 if (out_format_desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB) {
2678 /*
2679 * since blending is done with floats, there was no conversion.
2680 * However, the rules according to fixed point renderbuffers still
2681 * apply, that is we must clamp inputs to 0.0/1.0.
2682 * (This would apply to separate alpha conversion too but we currently
2683 * force has_alpha to be true.)
2684 * TODO: should skip this with "fake" blend, since post-blend conversion
2685 * will clamp anyway.
2686 * TODO: could also skip this if fragment color clamping is enabled. We
2687 * don't support it natively so it gets baked into the shader however, so
2688 * can't really tell here.
2689 */
2690 struct lp_build_context f32_bld;
2691 assert(row_type.floating);
2692 lp_build_context_init(&f32_bld, gallivm, row_type);
2693 for (i = 0; i < src_count; i++) {
2694 src[i] = lp_build_clamp_zero_one_nanzero(&f32_bld, src[i]);
2695 }
2696 if (dual_source_blend) {
2697 for (i = 0; i < src_count; i++) {
2698 src1[i] = lp_build_clamp_zero_one_nanzero(&f32_bld, src1[i]);
2699 }
2700 }
2701 /* probably can't be different than row_type but better safe than sorry... */
2702 lp_build_context_init(&f32_bld, gallivm, blend_type);
2703 blend_color = lp_build_clamp(&f32_bld, blend_color, f32_bld.zero, f32_bld.one);
2704 }
2705
2706 /* Extract alpha */
2707 blend_alpha = lp_build_extract_broadcast(gallivm, blend_type, row_type, blend_color, lp_build_const_int32(gallivm, 3));
2708
2709 /* Swizzle to appropriate channels, e.g. from RGBA to BGRA BGRA */
2710 pad_inline &= (dst_channels * (block_size / src_count) * row_type.width) != vector_width;
2711 if (pad_inline) {
2712 /* Use all 4 channels e.g. from RGBA RGBA to RGxx RGxx */
2713 blend_color = lp_build_swizzle_aos_n(gallivm, blend_color, swizzle, TGSI_NUM_CHANNELS, row_type.length);
2714 } else {
2715 /* Only use dst_channels e.g. RGBA RGBA to RG RG xxxx */
2716 blend_color = lp_build_swizzle_aos_n(gallivm, blend_color, swizzle, dst_channels, row_type.length);
2717 }
2718
2719 /*
2720 * Mask conversion
2721 */
2722 lp_bld_quad_twiddle(gallivm, mask_type, &src_mask[0], block_height, &src_mask[0]);
2723
2724 if (src_count < block_height) {
2725 lp_build_concat_n(gallivm, mask_type, src_mask, 4, src_mask, src_count);
2726 } else if (src_count > block_height) {
2727 for (i = src_count; i > 0; --i) {
2728 unsigned pixels = block_size / src_count;
2729 unsigned idx = i - 1;
2730
2731 src_mask[idx] = lp_build_extract_range(gallivm, src_mask[(idx * pixels) / 4],
2732 (idx * pixels) % 4, pixels);
2733 }
2734 }
2735
2736 assert(mask_type.width == 32);
2737
2738 for (i = 0; i < src_count; ++i) {
2739 unsigned pixels = block_size / src_count;
2740 unsigned pixel_width = row_type.width * dst_channels;
2741
2742 if (pixel_width == 24) {
2743 mask_type.width = 8;
2744 mask_type.length = vector_width / mask_type.width;
2745 } else {
2746 mask_type.length = pixels;
2747 mask_type.width = row_type.width * dst_channels;
2748
2749 /*
2750 * If mask_type width is smaller than 32bit, this doesn't quite
2751 * generate the most efficient code (could use some pack).
2752 */
2753 src_mask[i] = LLVMBuildIntCast(builder, src_mask[i],
2754 lp_build_int_vec_type(gallivm, mask_type), "");
2755
2756 mask_type.length *= dst_channels;
2757 mask_type.width /= dst_channels;
2758 }
2759
2760 src_mask[i] = LLVMBuildBitCast(builder, src_mask[i],
2761 lp_build_int_vec_type(gallivm, mask_type), "");
2762 src_mask[i] = lp_build_pad_vector(gallivm, src_mask[i], row_type.length);
2763 }
2764
2765 /*
2766 * Alpha conversion
2767 */
2768 if (!has_alpha) {
2769 struct lp_type alpha_type = fs_type;
2770 alpha_type.length = 4;
2771 convert_alpha(gallivm, row_type, alpha_type,
2772 block_size, block_height,
2773 src_count, dst_channels,
2774 pad_inline, src_alpha);
2775 if (dual_source_blend) {
2776 convert_alpha(gallivm, row_type, alpha_type,
2777 block_size, block_height,
2778 src_count, dst_channels,
2779 pad_inline, src1_alpha);
2780 }
2781 }
2782
2783
2784 /*
2785 * Load dst from memory
2786 */
2787 if (src_count < block_height) {
2788 dst_count = block_height;
2789 } else {
2790 dst_count = src_count;
2791 }
2792
2793 dst_type.length *= block_size / dst_count;
2794
2795 if (format_expands_to_float_soa(out_format_desc)) {
2796 /*
2797 * we need multiple values at once for the conversion, so can as well
2798 * load them vectorized here too instead of concatenating later.
2799 * (Still need concatenation later for 8-wide vectors).
2800 */
2801 dst_count = block_height;
2802 dst_type.length = block_width;
2803 }
2804
2805 /*
2806 * Compute the alignment of the destination pointer in bytes
2807 * We fetch 1-4 pixels, if the format has pot alignment then those fetches
2808 * are always aligned by MIN2(16, fetch_width) except for buffers (not
2809 * 1d tex but can't distinguish here) so need to stick with per-pixel
2810 * alignment in this case.
2811 */
2812 if (is_1d) {
2813 dst_alignment = (out_format_desc->block.bits + 7)/(out_format_desc->block.width * 8);
2814 }
2815 else {
2816 dst_alignment = dst_type.length * dst_type.width / 8;
2817 }
2818 /* Force power-of-two alignment by extracting only the least-significant-bit */
2819 dst_alignment = 1 << (ffs(dst_alignment) - 1);
2820 /*
2821 * Resource base and stride pointers are aligned to 16 bytes, so that's
2822 * the maximum alignment we can guarantee
2823 */
2824 dst_alignment = MIN2(16, dst_alignment);
2825
2826 ls_type = dst_type;
2827
2828 if (dst_count > src_count) {
2829 if ((dst_type.width == 8 || dst_type.width == 16) &&
2830 util_is_power_of_two_or_zero(dst_type.length) &&
2831 dst_type.length * dst_type.width < 128) {
2832 /*
2833 * Never try to load values as 4xi8 which we will then
2834 * concatenate to larger vectors. This gives llvm a real
2835 * headache (the problem is the type legalizer (?) will
2836 * try to load that as 4xi8 zext to 4xi32 to fill the vector,
2837 * then the shuffles to concatenate are more or less impossible
2838 * - llvm is easily capable of generating a sequence of 32
2839 * pextrb/pinsrb instructions for that. Albeit it appears to
2840 * be fixed in llvm 4.0. So, load and concatenate with 32bit
2841 * width to avoid the trouble (16bit seems not as bad, llvm
2842 * probably recognizes the load+shuffle as only one shuffle
2843 * is necessary, but we can do just the same anyway).
2844 */
2845 ls_type.length = dst_type.length * dst_type.width / 32;
2846 ls_type.width = 32;
2847 }
2848 }
2849
2850 if (is_1d) {
2851 load_unswizzled_block(gallivm, color_ptr, stride, block_width, 1,
2852 dst, ls_type, dst_count / 4, dst_alignment);
2853 for (i = dst_count / 4; i < dst_count; i++) {
2854 dst[i] = lp_build_undef(gallivm, ls_type);
2855 }
2856
2857 }
2858 else {
2859 load_unswizzled_block(gallivm, color_ptr, stride, block_width, block_height,
2860 dst, ls_type, dst_count, dst_alignment);
2861 }
2862
2863
2864 /*
2865 * Convert from dst/output format to src/blending format.
2866 *
2867 * This is necessary as we can only read 1 row from memory at a time,
2868 * so the minimum dst_count will ever be at this point is 4.
2869 *
2870 * With, for example, R8 format you can have all 16 pixels in a 128 bit vector,
2871 * this will take the 4 dsts and combine them into 1 src so we can perform blending
2872 * on all 16 pixels in that single vector at once.
2873 */
2874 if (dst_count > src_count) {
2875 if (ls_type.length != dst_type.length && ls_type.length == 1) {
2876 LLVMTypeRef elem_type = lp_build_elem_type(gallivm, ls_type);
2877 LLVMTypeRef ls_vec_type = LLVMVectorType(elem_type, 1);
2878 for (i = 0; i < dst_count; i++) {
2879 dst[i] = LLVMBuildBitCast(builder, dst[i], ls_vec_type, "");
2880 }
2881 }
2882
2883 lp_build_concat_n(gallivm, ls_type, dst, 4, dst, src_count);
2884
2885 if (ls_type.length != dst_type.length) {
2886 struct lp_type tmp_type = dst_type;
2887 tmp_type.length = dst_type.length * 4 / src_count;
2888 for (i = 0; i < src_count; i++) {
2889 dst[i] = LLVMBuildBitCast(builder, dst[i],
2890 lp_build_vec_type(gallivm, tmp_type), "");
2891 }
2892 }
2893 }
2894
2895 /*
2896 * Blending
2897 */
2898 /* XXX this is broken for RGB8 formats -
2899 * they get expanded from 12 to 16 elements (to include alpha)
2900 * by convert_to_blend_type then reduced to 15 instead of 12
2901 * by convert_from_blend_type (a simple fix though breaks A8...).
2902 * R16G16B16 also crashes differently however something going wrong
2903 * inside llvm handling npot vector sizes seemingly.
2904 * It seems some cleanup could be done here (like skipping conversion/blend
2905 * when not needed).
2906 */
2907 convert_to_blend_type(gallivm, block_size, out_format_desc, dst_type,
2908 row_type, dst, src_count);
2909
2910 /*
2911 * FIXME: Really should get logic ops / masks out of generic blend / row
2912 * format. Logic ops will definitely not work on the blend float format
2913 * used for SRGB here and I think OpenGL expects this to work as expected
2914 * (that is incoming values converted to srgb then logic op applied).
2915 */
2916 for (i = 0; i < src_count; ++i) {
2917 dst[i] = lp_build_blend_aos(gallivm,
2918 &variant->key.blend,
2919 out_format,
2920 row_type,
2921 rt,
2922 src[i],
2923 has_alpha ? NULL : src_alpha[i],
2924 src1[i],
2925 has_alpha ? NULL : src1_alpha[i],
2926 dst[i],
2927 partial_mask ? src_mask[i] : NULL,
2928 blend_color,
2929 has_alpha ? NULL : blend_alpha,
2930 swizzle,
2931 pad_inline ? 4 : dst_channels);
2932 }
2933
2934 convert_from_blend_type(gallivm, block_size, out_format_desc,
2935 row_type, dst_type, dst, src_count);
2936
2937 /* Split the blend rows back to memory rows */
2938 if (dst_count > src_count) {
2939 row_type.length = dst_type.length * (dst_count / src_count);
2940
2941 if (src_count == 1) {
2942 dst[1] = lp_build_extract_range(gallivm, dst[0], row_type.length / 2, row_type.length / 2);
2943 dst[0] = lp_build_extract_range(gallivm, dst[0], 0, row_type.length / 2);
2944
2945 row_type.length /= 2;
2946 src_count *= 2;
2947 }
2948
2949 dst[3] = lp_build_extract_range(gallivm, dst[1], row_type.length / 2, row_type.length / 2);
2950 dst[2] = lp_build_extract_range(gallivm, dst[1], 0, row_type.length / 2);
2951 dst[1] = lp_build_extract_range(gallivm, dst[0], row_type.length / 2, row_type.length / 2);
2952 dst[0] = lp_build_extract_range(gallivm, dst[0], 0, row_type.length / 2);
2953
2954 row_type.length /= 2;
2955 src_count *= 2;
2956 }
2957
2958 /*
2959 * Store blend result to memory
2960 */
2961 if (is_1d) {
2962 store_unswizzled_block(gallivm, color_ptr, stride, block_width, 1,
2963 dst, dst_type, dst_count / 4, dst_alignment);
2964 }
2965 else {
2966 store_unswizzled_block(gallivm, color_ptr, stride, block_width, block_height,
2967 dst, dst_type, dst_count, dst_alignment);
2968 }
2969
2970 if (have_smallfloat_format(dst_type, out_format)) {
2971 lp_build_fpstate_set(gallivm, fpstate);
2972 }
2973
2974 if (do_branch) {
2975 lp_build_mask_end(&mask_ctx);
2976 }
2977 }
2978
2979
2980 /**
2981 * Generate the runtime callable function for the whole fragment pipeline.
2982 * Note that the function which we generate operates on a block of 16
2983 * pixels at at time. The block contains 2x2 quads. Each quad contains
2984 * 2x2 pixels.
2985 */
2986 static void
generate_fragment(struct llvmpipe_context * lp,struct lp_fragment_shader * shader,struct lp_fragment_shader_variant * variant,unsigned partial_mask)2987 generate_fragment(struct llvmpipe_context *lp,
2988 struct lp_fragment_shader *shader,
2989 struct lp_fragment_shader_variant *variant,
2990 unsigned partial_mask)
2991 {
2992 struct gallivm_state *gallivm = variant->gallivm;
2993 struct lp_fragment_shader_variant_key *key = &variant->key;
2994 struct lp_shader_input inputs[PIPE_MAX_SHADER_INPUTS];
2995 char func_name[64];
2996 struct lp_type fs_type;
2997 struct lp_type blend_type;
2998 LLVMTypeRef fs_elem_type;
2999 LLVMTypeRef blend_vec_type;
3000 LLVMTypeRef arg_types[15];
3001 LLVMTypeRef func_type;
3002 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
3003 LLVMTypeRef int8_type = LLVMInt8TypeInContext(gallivm->context);
3004 LLVMValueRef context_ptr;
3005 LLVMValueRef x;
3006 LLVMValueRef y;
3007 LLVMValueRef a0_ptr;
3008 LLVMValueRef dadx_ptr;
3009 LLVMValueRef dady_ptr;
3010 LLVMValueRef color_ptr_ptr;
3011 LLVMValueRef stride_ptr;
3012 LLVMValueRef color_sample_stride_ptr;
3013 LLVMValueRef depth_ptr;
3014 LLVMValueRef depth_stride;
3015 LLVMValueRef depth_sample_stride;
3016 LLVMValueRef mask_input;
3017 LLVMValueRef thread_data_ptr;
3018 LLVMBasicBlockRef block;
3019 LLVMBuilderRef builder;
3020 struct lp_build_sampler_soa *sampler;
3021 struct lp_build_image_soa *image;
3022 struct lp_build_interp_soa_context interp;
3023 LLVMValueRef fs_mask[(16 / 4) * LP_MAX_SAMPLES];
3024 LLVMValueRef fs_out_color[LP_MAX_SAMPLES][PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS][16 / 4];
3025 LLVMValueRef function;
3026 LLVMValueRef facing;
3027 unsigned num_fs;
3028 unsigned i;
3029 unsigned chan;
3030 unsigned cbuf;
3031 boolean cbuf0_write_all;
3032 const boolean dual_source_blend = key->blend.rt[0].blend_enable &&
3033 util_blend_state_is_dual(&key->blend, 0);
3034
3035 assert(lp_native_vector_width / 32 >= 4);
3036
3037 /* Adjust color input interpolation according to flatshade state:
3038 */
3039 memcpy(inputs, shader->inputs, shader->info.base.num_inputs * sizeof inputs[0]);
3040 for (i = 0; i < shader->info.base.num_inputs; i++) {
3041 if (inputs[i].interp == LP_INTERP_COLOR) {
3042 if (key->flatshade)
3043 inputs[i].interp = LP_INTERP_CONSTANT;
3044 else
3045 inputs[i].interp = LP_INTERP_PERSPECTIVE;
3046 }
3047 }
3048
3049 /* check if writes to cbuf[0] are to be copied to all cbufs */
3050 cbuf0_write_all =
3051 shader->info.base.properties[TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS];
3052
3053 /* TODO: actually pick these based on the fs and color buffer
3054 * characteristics. */
3055
3056 memset(&fs_type, 0, sizeof fs_type);
3057 fs_type.floating = TRUE; /* floating point values */
3058 fs_type.sign = TRUE; /* values are signed */
3059 fs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */
3060 fs_type.width = 32; /* 32-bit float */
3061 fs_type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
3062
3063 memset(&blend_type, 0, sizeof blend_type);
3064 blend_type.floating = FALSE; /* values are integers */
3065 blend_type.sign = FALSE; /* values are unsigned */
3066 blend_type.norm = TRUE; /* values are in [0,1] or [-1,1] */
3067 blend_type.width = 8; /* 8-bit ubyte values */
3068 blend_type.length = 16; /* 16 elements per vector */
3069
3070 /*
3071 * Generate the function prototype. Any change here must be reflected in
3072 * lp_jit.h's lp_jit_frag_func function pointer type, and vice-versa.
3073 */
3074
3075 fs_elem_type = lp_build_elem_type(gallivm, fs_type);
3076
3077 blend_vec_type = lp_build_vec_type(gallivm, blend_type);
3078
3079 snprintf(func_name, sizeof(func_name), "fs_variant_%s",
3080 partial_mask ? "partial" : "whole");
3081
3082 arg_types[0] = variant->jit_context_ptr_type; /* context */
3083 arg_types[1] = int32_type; /* x */
3084 arg_types[2] = int32_type; /* y */
3085 arg_types[3] = int32_type; /* facing */
3086 arg_types[4] = LLVMPointerType(fs_elem_type, 0); /* a0 */
3087 arg_types[5] = LLVMPointerType(fs_elem_type, 0); /* dadx */
3088 arg_types[6] = LLVMPointerType(fs_elem_type, 0); /* dady */
3089 arg_types[7] = LLVMPointerType(LLVMPointerType(int8_type, 0), 0); /* color */
3090 arg_types[8] = LLVMPointerType(int8_type, 0); /* depth */
3091 arg_types[9] = LLVMInt64TypeInContext(gallivm->context); /* mask_input */
3092 arg_types[10] = variant->jit_thread_data_ptr_type; /* per thread data */
3093 arg_types[11] = LLVMPointerType(int32_type, 0); /* stride */
3094 arg_types[12] = int32_type; /* depth_stride */
3095 arg_types[13] = LLVMPointerType(int32_type, 0); /* color sample strides */
3096 arg_types[14] = int32_type; /* depth sample stride */
3097
3098 func_type = LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context),
3099 arg_types, ARRAY_SIZE(arg_types), 0);
3100
3101 function = LLVMAddFunction(gallivm->module, func_name, func_type);
3102 LLVMSetFunctionCallConv(function, LLVMCCallConv);
3103
3104 variant->function[partial_mask] = function;
3105
3106 /* XXX: need to propagate noalias down into color param now we are
3107 * passing a pointer-to-pointer?
3108 */
3109 for(i = 0; i < ARRAY_SIZE(arg_types); ++i)
3110 if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
3111 lp_add_function_attr(function, i + 1, LP_FUNC_ATTR_NOALIAS);
3112
3113 if (variant->gallivm->cache->data_size)
3114 return;
3115
3116 context_ptr = LLVMGetParam(function, 0);
3117 x = LLVMGetParam(function, 1);
3118 y = LLVMGetParam(function, 2);
3119 facing = LLVMGetParam(function, 3);
3120 a0_ptr = LLVMGetParam(function, 4);
3121 dadx_ptr = LLVMGetParam(function, 5);
3122 dady_ptr = LLVMGetParam(function, 6);
3123 color_ptr_ptr = LLVMGetParam(function, 7);
3124 depth_ptr = LLVMGetParam(function, 8);
3125 mask_input = LLVMGetParam(function, 9);
3126 thread_data_ptr = LLVMGetParam(function, 10);
3127 stride_ptr = LLVMGetParam(function, 11);
3128 depth_stride = LLVMGetParam(function, 12);
3129 color_sample_stride_ptr = LLVMGetParam(function, 13);
3130 depth_sample_stride = LLVMGetParam(function, 14);
3131
3132 lp_build_name(context_ptr, "context");
3133 lp_build_name(x, "x");
3134 lp_build_name(y, "y");
3135 lp_build_name(a0_ptr, "a0");
3136 lp_build_name(dadx_ptr, "dadx");
3137 lp_build_name(dady_ptr, "dady");
3138 lp_build_name(color_ptr_ptr, "color_ptr_ptr");
3139 lp_build_name(depth_ptr, "depth");
3140 lp_build_name(mask_input, "mask_input");
3141 lp_build_name(thread_data_ptr, "thread_data");
3142 lp_build_name(stride_ptr, "stride_ptr");
3143 lp_build_name(depth_stride, "depth_stride");
3144 lp_build_name(color_sample_stride_ptr, "color_sample_stride_ptr");
3145 lp_build_name(depth_sample_stride, "depth_sample_stride");
3146
3147 /*
3148 * Function body
3149 */
3150
3151 block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
3152 builder = gallivm->builder;
3153 assert(builder);
3154 LLVMPositionBuilderAtEnd(builder, block);
3155
3156 /*
3157 * Must not count ps invocations if there's a null shader.
3158 * (It would be ok to count with null shader if there's d/s tests,
3159 * but only if there's d/s buffers too, which is different
3160 * to implicit rasterization disable which must not depend
3161 * on the d/s buffers.)
3162 * Could use popcount on mask, but pixel accuracy is not required.
3163 * Could disable if there's no stats query, but maybe not worth it.
3164 */
3165 if (shader->info.base.num_instructions > 1) {
3166 LLVMValueRef invocs, val;
3167 invocs = lp_jit_thread_data_invocations(gallivm, thread_data_ptr);
3168 val = LLVMBuildLoad(builder, invocs, "");
3169 val = LLVMBuildAdd(builder, val,
3170 LLVMConstInt(LLVMInt64TypeInContext(gallivm->context), 1, 0),
3171 "invoc_count");
3172 LLVMBuildStore(builder, val, invocs);
3173 }
3174
3175 /* code generated texture sampling */
3176 sampler = lp_llvm_sampler_soa_create(lp_fs_variant_key_samplers(key), key->nr_samplers);
3177 image = lp_llvm_image_soa_create(lp_fs_variant_key_images(key), key->nr_images);
3178
3179 num_fs = 16 / fs_type.length; /* number of loops per 4x4 stamp */
3180 /* for 1d resources only run "upper half" of stamp */
3181 if (key->resource_1d)
3182 num_fs /= 2;
3183
3184 {
3185 LLVMValueRef num_loop = lp_build_const_int32(gallivm, num_fs);
3186 LLVMTypeRef mask_type = lp_build_int_vec_type(gallivm, fs_type);
3187 LLVMValueRef num_loop_samp = lp_build_const_int32(gallivm, num_fs * key->coverage_samples);
3188 LLVMValueRef mask_store = lp_build_array_alloca(gallivm, mask_type,
3189 num_loop_samp, "mask_store");
3190
3191 LLVMTypeRef flt_type = LLVMFloatTypeInContext(gallivm->context);
3192 LLVMValueRef glob_sample_pos = LLVMAddGlobal(gallivm->module, LLVMArrayType(flt_type, key->coverage_samples * 2), "");
3193 LLVMValueRef sample_pos_array;
3194
3195 if (key->multisample && key->coverage_samples == 4) {
3196 LLVMValueRef sample_pos_arr[8];
3197 for (unsigned i = 0; i < 4; i++) {
3198 sample_pos_arr[i * 2] = LLVMConstReal(flt_type, lp_sample_pos_4x[i][0]);
3199 sample_pos_arr[i * 2 + 1] = LLVMConstReal(flt_type, lp_sample_pos_4x[i][1]);
3200 }
3201 sample_pos_array = LLVMConstArray(LLVMFloatTypeInContext(gallivm->context), sample_pos_arr, 8);
3202 } else {
3203 LLVMValueRef sample_pos_arr[2];
3204 sample_pos_arr[0] = LLVMConstReal(flt_type, 0.5);
3205 sample_pos_arr[1] = LLVMConstReal(flt_type, 0.5);
3206 sample_pos_array = LLVMConstArray(LLVMFloatTypeInContext(gallivm->context), sample_pos_arr, 2);
3207 }
3208 LLVMSetInitializer(glob_sample_pos, sample_pos_array);
3209
3210 LLVMValueRef color_store[PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS];
3211 boolean pixel_center_integer =
3212 shader->info.base.properties[TGSI_PROPERTY_FS_COORD_PIXEL_CENTER];
3213
3214 /*
3215 * The shader input interpolation info is not explicitely baked in the
3216 * shader key, but everything it derives from (TGSI, and flatshade) is
3217 * already included in the shader key.
3218 */
3219 lp_build_interp_soa_init(&interp,
3220 gallivm,
3221 shader->info.base.num_inputs,
3222 inputs,
3223 pixel_center_integer,
3224 key->coverage_samples, glob_sample_pos,
3225 num_loop,
3226 key->depth_clamp,
3227 builder, fs_type,
3228 a0_ptr, dadx_ptr, dady_ptr,
3229 x, y);
3230
3231 for (i = 0; i < num_fs; i++) {
3232 if (key->multisample) {
3233 LLVMValueRef smask_val = LLVMBuildLoad(builder, lp_jit_context_sample_mask(gallivm, context_ptr), "");
3234
3235 /*
3236 * For multisampling, extract the per-sample mask from the incoming 64-bit mask,
3237 * store to the per sample mask storage. Or all of them together to generate
3238 * the fragment shader mask. (sample shading TODO).
3239 * Take the incoming state coverage mask into account.
3240 */
3241 for (unsigned s = 0; s < key->coverage_samples; s++) {
3242 LLVMValueRef sindexi = lp_build_const_int32(gallivm, i + (s * num_fs));
3243 LLVMValueRef sample_mask_ptr = LLVMBuildGEP(builder, mask_store,
3244 &sindexi, 1, "sample_mask_ptr");
3245 LLVMValueRef s_mask = generate_quad_mask(gallivm, fs_type,
3246 i*fs_type.length/4, s, mask_input);
3247
3248 LLVMValueRef smask_bit = LLVMBuildAnd(builder, smask_val, lp_build_const_int32(gallivm, (1 << s)), "");
3249 LLVMValueRef cmp = LLVMBuildICmp(builder, LLVMIntNE, smask_bit, lp_build_const_int32(gallivm, 0), "");
3250 smask_bit = LLVMBuildSExt(builder, cmp, int32_type, "");
3251 smask_bit = lp_build_broadcast(gallivm, mask_type, smask_bit);
3252
3253 s_mask = LLVMBuildAnd(builder, s_mask, smask_bit, "");
3254 LLVMBuildStore(builder, s_mask, sample_mask_ptr);
3255 }
3256 } else {
3257 LLVMValueRef mask;
3258 LLVMValueRef indexi = lp_build_const_int32(gallivm, i);
3259 LLVMValueRef mask_ptr = LLVMBuildGEP(builder, mask_store,
3260 &indexi, 1, "mask_ptr");
3261
3262 if (partial_mask) {
3263 mask = generate_quad_mask(gallivm, fs_type,
3264 i*fs_type.length/4, 0, mask_input);
3265 }
3266 else {
3267 mask = lp_build_const_int_vec(gallivm, fs_type, ~0);
3268 }
3269 LLVMBuildStore(builder, mask, mask_ptr);
3270 }
3271 }
3272
3273 generate_fs_loop(gallivm,
3274 shader, key,
3275 builder,
3276 fs_type,
3277 context_ptr,
3278 glob_sample_pos,
3279 num_loop,
3280 &interp,
3281 sampler,
3282 image,
3283 mask_store, /* output */
3284 color_store,
3285 depth_ptr,
3286 depth_stride,
3287 depth_sample_stride,
3288 color_ptr_ptr,
3289 stride_ptr,
3290 color_sample_stride_ptr,
3291 facing,
3292 thread_data_ptr);
3293
3294 for (i = 0; i < num_fs; i++) {
3295 LLVMValueRef ptr;
3296 for (unsigned s = 0; s < key->coverage_samples; s++) {
3297 int idx = (i + (s * num_fs));
3298 LLVMValueRef sindexi = lp_build_const_int32(gallivm, idx);
3299 ptr = LLVMBuildGEP(builder, mask_store, &sindexi, 1, "");
3300
3301 fs_mask[idx] = LLVMBuildLoad(builder, ptr, "smask");
3302 }
3303
3304 for (unsigned s = 0; s < key->min_samples; s++) {
3305 /* This is fucked up need to reorganize things */
3306 int idx = s * num_fs + i;
3307 LLVMValueRef sindexi = lp_build_const_int32(gallivm, idx);
3308 for (cbuf = 0; cbuf < key->nr_cbufs; cbuf++) {
3309 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
3310 ptr = LLVMBuildGEP(builder,
3311 color_store[cbuf * !cbuf0_write_all][chan],
3312 &sindexi, 1, "");
3313 fs_out_color[s][cbuf][chan][i] = ptr;
3314 }
3315 }
3316 if (dual_source_blend) {
3317 /* only support one dual source blend target hence always use output 1 */
3318 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
3319 ptr = LLVMBuildGEP(builder,
3320 color_store[1][chan],
3321 &sindexi, 1, "");
3322 fs_out_color[s][1][chan][i] = ptr;
3323 }
3324 }
3325 }
3326 }
3327 }
3328
3329 sampler->destroy(sampler);
3330 image->destroy(image);
3331 /* Loop over color outputs / color buffers to do blending.
3332 */
3333 for(cbuf = 0; cbuf < key->nr_cbufs; cbuf++) {
3334 if (key->cbuf_format[cbuf] != PIPE_FORMAT_NONE) {
3335 LLVMValueRef color_ptr;
3336 LLVMValueRef stride;
3337 LLVMValueRef sample_stride = NULL;
3338 LLVMValueRef index = lp_build_const_int32(gallivm, cbuf);
3339
3340 boolean do_branch = ((key->depth.enabled
3341 || key->stencil[0].enabled
3342 || key->alpha.enabled)
3343 && !shader->info.base.uses_kill);
3344
3345 color_ptr = LLVMBuildLoad(builder,
3346 LLVMBuildGEP(builder, color_ptr_ptr,
3347 &index, 1, ""),
3348 "");
3349
3350 stride = LLVMBuildLoad(builder,
3351 LLVMBuildGEP(builder, stride_ptr, &index, 1, ""),
3352 "");
3353
3354 if (key->cbuf_nr_samples[cbuf] > 1)
3355 sample_stride = LLVMBuildLoad(builder,
3356 LLVMBuildGEP(builder, color_sample_stride_ptr,
3357 &index, 1, ""), "");
3358
3359 for (unsigned s = 0; s < key->cbuf_nr_samples[cbuf]; s++) {
3360 unsigned mask_idx = num_fs * (key->multisample ? s : 0);
3361 unsigned out_idx = key->min_samples == 1 ? 0 : s;
3362 LLVMValueRef out_ptr = color_ptr;;
3363
3364 if (sample_stride) {
3365 LLVMValueRef sample_offset = LLVMBuildMul(builder, sample_stride, lp_build_const_int32(gallivm, s), "");
3366 out_ptr = LLVMBuildGEP(builder, out_ptr, &sample_offset, 1, "");
3367 }
3368 out_ptr = LLVMBuildBitCast(builder, out_ptr, LLVMPointerType(blend_vec_type, 0), "");
3369
3370 lp_build_name(out_ptr, "color_ptr%d", cbuf);
3371
3372 generate_unswizzled_blend(gallivm, cbuf, variant,
3373 key->cbuf_format[cbuf],
3374 num_fs, fs_type, &fs_mask[mask_idx], fs_out_color[out_idx],
3375 context_ptr, out_ptr, stride,
3376 partial_mask, do_branch);
3377 }
3378 }
3379 }
3380
3381 LLVMBuildRetVoid(builder);
3382
3383 gallivm_verify_function(gallivm, function);
3384 }
3385
3386
3387 static void
dump_fs_variant_key(struct lp_fragment_shader_variant_key * key)3388 dump_fs_variant_key(struct lp_fragment_shader_variant_key *key)
3389 {
3390 unsigned i;
3391
3392 debug_printf("fs variant %p:\n", (void *) key);
3393
3394 if (key->flatshade) {
3395 debug_printf("flatshade = 1\n");
3396 }
3397 if (key->depth_clamp)
3398 debug_printf("depth_clamp = 1\n");
3399
3400 if (key->multisample) {
3401 debug_printf("multisample = 1\n");
3402 debug_printf("coverage samples = %d\n", key->coverage_samples);
3403 debug_printf("min samples = %d\n", key->min_samples);
3404 }
3405 for (i = 0; i < key->nr_cbufs; ++i) {
3406 debug_printf("cbuf_format[%u] = %s\n", i, util_format_name(key->cbuf_format[i]));
3407 debug_printf("cbuf nr_samples[%u] = %d\n", i, key->cbuf_nr_samples[i]);
3408 }
3409 if (key->depth.enabled || key->stencil[0].enabled) {
3410 debug_printf("depth.format = %s\n", util_format_name(key->zsbuf_format));
3411 debug_printf("depth nr_samples = %d\n", key->zsbuf_nr_samples);
3412 }
3413 if (key->depth.enabled) {
3414 debug_printf("depth.func = %s\n", util_str_func(key->depth.func, TRUE));
3415 debug_printf("depth.writemask = %u\n", key->depth.writemask);
3416 }
3417
3418 for (i = 0; i < 2; ++i) {
3419 if (key->stencil[i].enabled) {
3420 debug_printf("stencil[%u].func = %s\n", i, util_str_func(key->stencil[i].func, TRUE));
3421 debug_printf("stencil[%u].fail_op = %s\n", i, util_str_stencil_op(key->stencil[i].fail_op, TRUE));
3422 debug_printf("stencil[%u].zpass_op = %s\n", i, util_str_stencil_op(key->stencil[i].zpass_op, TRUE));
3423 debug_printf("stencil[%u].zfail_op = %s\n", i, util_str_stencil_op(key->stencil[i].zfail_op, TRUE));
3424 debug_printf("stencil[%u].valuemask = 0x%x\n", i, key->stencil[i].valuemask);
3425 debug_printf("stencil[%u].writemask = 0x%x\n", i, key->stencil[i].writemask);
3426 }
3427 }
3428
3429 if (key->alpha.enabled) {
3430 debug_printf("alpha.func = %s\n", util_str_func(key->alpha.func, TRUE));
3431 }
3432
3433 if (key->occlusion_count) {
3434 debug_printf("occlusion_count = 1\n");
3435 }
3436
3437 if (key->blend.logicop_enable) {
3438 debug_printf("blend.logicop_func = %s\n", util_str_logicop(key->blend.logicop_func, TRUE));
3439 }
3440 else if (key->blend.rt[0].blend_enable) {
3441 debug_printf("blend.rgb_func = %s\n", util_str_blend_func (key->blend.rt[0].rgb_func, TRUE));
3442 debug_printf("blend.rgb_src_factor = %s\n", util_str_blend_factor(key->blend.rt[0].rgb_src_factor, TRUE));
3443 debug_printf("blend.rgb_dst_factor = %s\n", util_str_blend_factor(key->blend.rt[0].rgb_dst_factor, TRUE));
3444 debug_printf("blend.alpha_func = %s\n", util_str_blend_func (key->blend.rt[0].alpha_func, TRUE));
3445 debug_printf("blend.alpha_src_factor = %s\n", util_str_blend_factor(key->blend.rt[0].alpha_src_factor, TRUE));
3446 debug_printf("blend.alpha_dst_factor = %s\n", util_str_blend_factor(key->blend.rt[0].alpha_dst_factor, TRUE));
3447 }
3448 debug_printf("blend.colormask = 0x%x\n", key->blend.rt[0].colormask);
3449 if (key->blend.alpha_to_coverage) {
3450 debug_printf("blend.alpha_to_coverage is enabled\n");
3451 }
3452 for (i = 0; i < key->nr_samplers; ++i) {
3453 const struct lp_sampler_static_state *samplers = lp_fs_variant_key_samplers(key);
3454 const struct lp_static_sampler_state *sampler = &samplers[i].sampler_state;
3455 debug_printf("sampler[%u] = \n", i);
3456 debug_printf(" .wrap = %s %s %s\n",
3457 util_str_tex_wrap(sampler->wrap_s, TRUE),
3458 util_str_tex_wrap(sampler->wrap_t, TRUE),
3459 util_str_tex_wrap(sampler->wrap_r, TRUE));
3460 debug_printf(" .min_img_filter = %s\n",
3461 util_str_tex_filter(sampler->min_img_filter, TRUE));
3462 debug_printf(" .min_mip_filter = %s\n",
3463 util_str_tex_mipfilter(sampler->min_mip_filter, TRUE));
3464 debug_printf(" .mag_img_filter = %s\n",
3465 util_str_tex_filter(sampler->mag_img_filter, TRUE));
3466 if (sampler->compare_mode != PIPE_TEX_COMPARE_NONE)
3467 debug_printf(" .compare_func = %s\n", util_str_func(sampler->compare_func, TRUE));
3468 debug_printf(" .normalized_coords = %u\n", sampler->normalized_coords);
3469 debug_printf(" .min_max_lod_equal = %u\n", sampler->min_max_lod_equal);
3470 debug_printf(" .lod_bias_non_zero = %u\n", sampler->lod_bias_non_zero);
3471 debug_printf(" .apply_min_lod = %u\n", sampler->apply_min_lod);
3472 debug_printf(" .apply_max_lod = %u\n", sampler->apply_max_lod);
3473 debug_printf(" .reduction_mode = %u\n", sampler->reduction_mode);
3474 debug_printf(" .aniso = %u\n", sampler->aniso);
3475 }
3476 for (i = 0; i < key->nr_sampler_views; ++i) {
3477 const struct lp_sampler_static_state *samplers = lp_fs_variant_key_samplers(key);
3478 const struct lp_static_texture_state *texture = &samplers[i].texture_state;
3479 debug_printf("texture[%u] = \n", i);
3480 debug_printf(" .format = %s\n",
3481 util_format_name(texture->format));
3482 debug_printf(" .target = %s\n",
3483 util_str_tex_target(texture->target, TRUE));
3484 debug_printf(" .level_zero_only = %u\n",
3485 texture->level_zero_only);
3486 debug_printf(" .pot = %u %u %u\n",
3487 texture->pot_width,
3488 texture->pot_height,
3489 texture->pot_depth);
3490 }
3491 struct lp_image_static_state *images = lp_fs_variant_key_images(key);
3492 for (i = 0; i < key->nr_images; ++i) {
3493 const struct lp_static_texture_state *image = &images[i].image_state;
3494 debug_printf("image[%u] = \n", i);
3495 debug_printf(" .format = %s\n",
3496 util_format_name(image->format));
3497 debug_printf(" .target = %s\n",
3498 util_str_tex_target(image->target, TRUE));
3499 debug_printf(" .level_zero_only = %u\n",
3500 image->level_zero_only);
3501 debug_printf(" .pot = %u %u %u\n",
3502 image->pot_width,
3503 image->pot_height,
3504 image->pot_depth);
3505 }
3506 }
3507
3508 const char *
lp_debug_fs_kind(enum lp_fs_kind kind)3509 lp_debug_fs_kind(enum lp_fs_kind kind)
3510 {
3511 switch(kind) {
3512 case LP_FS_KIND_GENERAL:
3513 return "GENERAL";
3514 case LP_FS_KIND_BLIT_RGBA:
3515 return "BLIT_RGBA";
3516 case LP_FS_KIND_BLIT_RGB1:
3517 return "BLIT_RGB1";
3518 case LP_FS_KIND_AERO_MINIFICATION:
3519 return "AERO_MINIFICATION";
3520 case LP_FS_KIND_LLVM_LINEAR:
3521 return "LLVM_LINEAR";
3522 default:
3523 return "unknown";
3524 }
3525 }
3526
3527 void
lp_debug_fs_variant(struct lp_fragment_shader_variant * variant)3528 lp_debug_fs_variant(struct lp_fragment_shader_variant *variant)
3529 {
3530 debug_printf("llvmpipe: Fragment shader #%u variant #%u:\n",
3531 variant->shader->no, variant->no);
3532 if (variant->shader->base.type == PIPE_SHADER_IR_TGSI)
3533 tgsi_dump(variant->shader->base.tokens, 0);
3534 else
3535 nir_print_shader(variant->shader->base.ir.nir, stderr);
3536 dump_fs_variant_key(&variant->key);
3537 debug_printf("variant->opaque = %u\n", variant->opaque);
3538 debug_printf("variant->potentially_opaque = %u\n", variant->potentially_opaque);
3539 debug_printf("variant->blit = %u\n", variant->blit);
3540 debug_printf("shader->kind = %s\n", lp_debug_fs_kind(variant->shader->kind));
3541 debug_printf("\n");
3542 }
3543
3544 static void
lp_fs_get_ir_cache_key(struct lp_fragment_shader_variant * variant,unsigned char ir_sha1_cache_key[20])3545 lp_fs_get_ir_cache_key(struct lp_fragment_shader_variant *variant,
3546 unsigned char ir_sha1_cache_key[20])
3547 {
3548 struct blob blob = { 0 };
3549 unsigned ir_size;
3550 void *ir_binary;
3551
3552 blob_init(&blob);
3553 nir_serialize(&blob, variant->shader->base.ir.nir, true);
3554 ir_binary = blob.data;
3555 ir_size = blob.size;
3556
3557 struct mesa_sha1 ctx;
3558 _mesa_sha1_init(&ctx);
3559 _mesa_sha1_update(&ctx, &variant->key, variant->shader->variant_key_size);
3560 _mesa_sha1_update(&ctx, ir_binary, ir_size);
3561 _mesa_sha1_final(&ctx, ir_sha1_cache_key);
3562
3563 blob_finish(&blob);
3564 }
3565
3566 /**
3567 * Generate a new fragment shader variant from the shader code and
3568 * other state indicated by the key.
3569 */
3570 static struct lp_fragment_shader_variant *
generate_variant(struct llvmpipe_context * lp,struct lp_fragment_shader * shader,const struct lp_fragment_shader_variant_key * key)3571 generate_variant(struct llvmpipe_context *lp,
3572 struct lp_fragment_shader *shader,
3573 const struct lp_fragment_shader_variant_key *key)
3574 {
3575 struct llvmpipe_screen *screen = llvmpipe_screen(lp->pipe.screen);
3576 struct lp_fragment_shader_variant *variant;
3577 const struct util_format_description *cbuf0_format_desc = NULL;
3578 boolean fullcolormask;
3579 boolean no_kill;
3580 boolean linear;
3581 char module_name[64];
3582 unsigned char ir_sha1_cache_key[20];
3583 struct lp_cached_code cached = { 0 };
3584 bool needs_caching = false;
3585 variant = MALLOC(sizeof *variant + shader->variant_key_size - sizeof variant->key);
3586 if (!variant)
3587 return NULL;
3588
3589 memset(variant, 0, sizeof(*variant));
3590 snprintf(module_name, sizeof(module_name), "fs%u_variant%u",
3591 shader->no, shader->variants_created);
3592
3593 pipe_reference_init(&variant->reference, 1);
3594 lp_fs_reference(lp, &variant->shader, shader);
3595
3596 memcpy(&variant->key, key, shader->variant_key_size);
3597
3598 if (shader->base.ir.nir) {
3599 lp_fs_get_ir_cache_key(variant, ir_sha1_cache_key);
3600
3601 lp_disk_cache_find_shader(screen, &cached, ir_sha1_cache_key);
3602 if (!cached.data_size)
3603 needs_caching = true;
3604 }
3605 variant->gallivm = gallivm_create(module_name, lp->context, &cached);
3606 if (!variant->gallivm) {
3607 FREE(variant);
3608 return NULL;
3609 }
3610
3611 variant->list_item_global.base = variant;
3612 variant->list_item_local.base = variant;
3613 variant->no = shader->variants_created++;
3614
3615
3616
3617 /*
3618 * Determine whether we are touching all channels in the color buffer.
3619 */
3620 fullcolormask = FALSE;
3621 if (key->nr_cbufs == 1) {
3622 cbuf0_format_desc = util_format_description(key->cbuf_format[0]);
3623 fullcolormask = util_format_colormask_full(cbuf0_format_desc, key->blend.rt[0].colormask);
3624 }
3625
3626 /* The scissor is ignored here as only tiles inside the scissoring
3627 * rectangle will refer to this */
3628 no_kill =
3629 fullcolormask &&
3630 !key->stencil[0].enabled &&
3631 !key->alpha.enabled &&
3632 !key->multisample &&
3633 !key->blend.alpha_to_coverage &&
3634 !key->depth.enabled &&
3635 !shader->info.base.uses_kill &&
3636 !shader->info.base.writes_samplemask &&
3637 !shader->info.base.uses_fbfetch;
3638
3639 variant->opaque =
3640 no_kill &&
3641 !key->blend.logicop_enable &&
3642 !key->blend.rt[0].blend_enable
3643 ? TRUE : FALSE;
3644
3645 variant->potentially_opaque =
3646 no_kill &&
3647 !key->blend.logicop_enable &&
3648 key->blend.rt[0].blend_enable &&
3649 key->blend.rt[0].rgb_func == PIPE_BLEND_ADD &&
3650 key->blend.rt[0].rgb_dst_factor == PIPE_BLENDFACTOR_INV_SRC_ALPHA &&
3651 key->blend.rt[0].alpha_func == key->blend.rt[0].rgb_func &&
3652 key->blend.rt[0].alpha_dst_factor == key->blend.rt[0].rgb_dst_factor &&
3653 shader->base.type == PIPE_SHADER_IR_TGSI &&
3654 /*
3655 * FIXME: for NIR, all of the fields of info.xxx (except info.base)
3656 * are zeros, hence shader analysis (here and elsewhere) using these
3657 * bits cannot work and will silently fail (cbuf is the only pointer
3658 * field, hence causing a crash).
3659 */
3660 shader->info.cbuf[0][3].file != TGSI_FILE_NULL
3661 ? TRUE : FALSE;
3662
3663 /* We only care about opaque blits for now */
3664 if (variant->opaque &&
3665 (shader->kind == LP_FS_KIND_BLIT_RGBA ||
3666 shader->kind == LP_FS_KIND_BLIT_RGB1)) {
3667 unsigned target, min_img_filter, mag_img_filter, min_mip_filter;
3668 enum pipe_format texture_format;
3669 struct lp_sampler_static_state *samp0 = lp_fs_variant_key_sampler_idx(key, 0);
3670 assert(samp0);
3671 texture_format = samp0->texture_state.format;
3672 target = samp0->texture_state.target;
3673 min_img_filter = samp0->sampler_state.min_img_filter;
3674 mag_img_filter = samp0->sampler_state.mag_img_filter;
3675 if (samp0->texture_state.level_zero_only) {
3676 min_mip_filter = PIPE_TEX_MIPFILTER_NONE;
3677 } else {
3678 min_mip_filter = samp0->sampler_state.min_mip_filter;
3679 }
3680
3681 if (target == PIPE_TEXTURE_2D &&
3682 min_img_filter == PIPE_TEX_FILTER_NEAREST &&
3683 mag_img_filter == PIPE_TEX_FILTER_NEAREST &&
3684 min_mip_filter == PIPE_TEX_MIPFILTER_NONE &&
3685 ((texture_format &&
3686 util_is_format_compatible(util_format_description(texture_format),
3687 cbuf0_format_desc)) ||
3688 (shader->kind == LP_FS_KIND_BLIT_RGB1 &&
3689 (texture_format == PIPE_FORMAT_B8G8R8A8_UNORM ||
3690 texture_format == PIPE_FORMAT_B8G8R8X8_UNORM) &&
3691 (key->cbuf_format[0] == PIPE_FORMAT_B8G8R8A8_UNORM ||
3692 key->cbuf_format[0] == PIPE_FORMAT_B8G8R8X8_UNORM))))
3693 variant->blit = 1;
3694 }
3695
3696
3697 /* Whether this is a candidate for the linear path */
3698 linear =
3699 !key->stencil[0].enabled &&
3700 !key->depth.enabled &&
3701 !shader->info.base.uses_kill &&
3702 !key->blend.logicop_enable &&
3703 (key->cbuf_format[0] == PIPE_FORMAT_B8G8R8A8_UNORM ||
3704 key->cbuf_format[0] == PIPE_FORMAT_B8G8R8X8_UNORM);
3705
3706 memcpy(&variant->key, key, sizeof *key);
3707
3708 if ((LP_DEBUG & DEBUG_FS) || (gallivm_debug & GALLIVM_DEBUG_IR)) {
3709 lp_debug_fs_variant(variant);
3710 }
3711
3712 llvmpipe_fs_variant_fastpath(variant);
3713
3714 lp_jit_init_types(variant);
3715
3716 if (variant->jit_function[RAST_EDGE_TEST] == NULL)
3717 generate_fragment(lp, shader, variant, RAST_EDGE_TEST);
3718
3719 if (variant->jit_function[RAST_WHOLE] == NULL) {
3720 if (variant->opaque) {
3721 /* Specialized shader, which doesn't need to read the color buffer. */
3722 generate_fragment(lp, shader, variant, RAST_WHOLE);
3723 }
3724 }
3725
3726 if (linear) {
3727 /* Currently keeping both the old fastpaths and new linear path
3728 * active. The older code is still somewhat faster for the cases
3729 * it covers.
3730 *
3731 * XXX: consider restricting this to aero-mode only.
3732 */
3733 if (fullcolormask &&
3734 !key->alpha.enabled &&
3735 !key->blend.alpha_to_coverage) {
3736 llvmpipe_fs_variant_linear_fastpath(variant);
3737 }
3738
3739 /* If the original fastpath doesn't cover this variant, try the new
3740 * code:
3741 */
3742 if (variant->jit_linear == NULL) {
3743 if (shader->kind == LP_FS_KIND_BLIT_RGBA ||
3744 shader->kind == LP_FS_KIND_BLIT_RGB1 ||
3745 shader->kind == LP_FS_KIND_LLVM_LINEAR) {
3746 llvmpipe_fs_variant_linear_llvm(lp, shader, variant);
3747 }
3748 }
3749 } else {
3750 if (LP_DEBUG & DEBUG_LINEAR) {
3751 lp_debug_fs_variant(variant);
3752 debug_printf(" ----> no linear path for this variant\n");
3753 }
3754 }
3755
3756 /*
3757 * Compile everything
3758 */
3759
3760 gallivm_compile_module(variant->gallivm);
3761
3762 variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module);
3763
3764 if (variant->function[RAST_EDGE_TEST]) {
3765 variant->jit_function[RAST_EDGE_TEST] = (lp_jit_frag_func)
3766 gallivm_jit_function(variant->gallivm,
3767 variant->function[RAST_EDGE_TEST]);
3768 }
3769
3770 if (variant->function[RAST_WHOLE]) {
3771 variant->jit_function[RAST_WHOLE] = (lp_jit_frag_func)
3772 gallivm_jit_function(variant->gallivm,
3773 variant->function[RAST_WHOLE]);
3774 } else if (!variant->jit_function[RAST_WHOLE]) {
3775 variant->jit_function[RAST_WHOLE] = variant->jit_function[RAST_EDGE_TEST];
3776 }
3777
3778 if (linear) {
3779 if (variant->linear_function) {
3780 variant->jit_linear_llvm = (lp_jit_linear_llvm_func)
3781 gallivm_jit_function(variant->gallivm, variant->linear_function);
3782 }
3783
3784 /*
3785 * This must be done after LLVM compilation, as it will call the JIT'ed
3786 * code to determine active inputs.
3787 */
3788 lp_linear_check_variant(variant);
3789 }
3790
3791 if (needs_caching) {
3792 lp_disk_cache_insert_shader(screen, &cached, ir_sha1_cache_key);
3793 }
3794
3795 gallivm_free_ir(variant->gallivm);
3796
3797 return variant;
3798 }
3799
3800
3801 static void *
llvmpipe_create_fs_state(struct pipe_context * pipe,const struct pipe_shader_state * templ)3802 llvmpipe_create_fs_state(struct pipe_context *pipe,
3803 const struct pipe_shader_state *templ)
3804 {
3805 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
3806 struct lp_fragment_shader *shader;
3807 int nr_samplers;
3808 int nr_sampler_views;
3809 int nr_images;
3810 int i;
3811
3812 shader = CALLOC_STRUCT(lp_fragment_shader);
3813 if (!shader)
3814 return NULL;
3815
3816 pipe_reference_init(&shader->reference, 1);
3817 shader->no = fs_no++;
3818 make_empty_list(&shader->variants);
3819
3820 shader->base.type = templ->type;
3821 if (templ->type == PIPE_SHADER_IR_TGSI) {
3822 /* get/save the summary info for this shader */
3823 lp_build_tgsi_info(templ->tokens, &shader->info);
3824
3825 /* we need to keep a local copy of the tokens */
3826 shader->base.tokens = tgsi_dup_tokens(templ->tokens);
3827 } else {
3828 shader->base.ir.nir = templ->ir.nir;
3829 nir_tgsi_scan_shader(templ->ir.nir, &shader->info.base, true);
3830 }
3831
3832 shader->draw_data = draw_create_fragment_shader(llvmpipe->draw, templ);
3833 if (shader->draw_data == NULL) {
3834 FREE((void *) shader->base.tokens);
3835 FREE(shader);
3836 return NULL;
3837 }
3838
3839 nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1;
3840 nr_sampler_views = shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
3841 nr_images = shader->info.base.file_max[TGSI_FILE_IMAGE] + 1;
3842 shader->variant_key_size = lp_fs_variant_key_size(MAX2(nr_samplers, nr_sampler_views), nr_images);
3843
3844 for (i = 0; i < shader->info.base.num_inputs; i++) {
3845 shader->inputs[i].usage_mask = shader->info.base.input_usage_mask[i];
3846 shader->inputs[i].location = shader->info.base.input_interpolate_loc[i];
3847
3848 switch (shader->info.base.input_interpolate[i]) {
3849 case TGSI_INTERPOLATE_CONSTANT:
3850 shader->inputs[i].interp = LP_INTERP_CONSTANT;
3851 break;
3852 case TGSI_INTERPOLATE_LINEAR:
3853 shader->inputs[i].interp = LP_INTERP_LINEAR;
3854 break;
3855 case TGSI_INTERPOLATE_PERSPECTIVE:
3856 shader->inputs[i].interp = LP_INTERP_PERSPECTIVE;
3857 break;
3858 case TGSI_INTERPOLATE_COLOR:
3859 shader->inputs[i].interp = LP_INTERP_COLOR;
3860 break;
3861 default:
3862 assert(0);
3863 break;
3864 }
3865
3866 switch (shader->info.base.input_semantic_name[i]) {
3867 case TGSI_SEMANTIC_FACE:
3868 shader->inputs[i].interp = LP_INTERP_FACING;
3869 break;
3870 case TGSI_SEMANTIC_POSITION:
3871 /* Position was already emitted above
3872 */
3873 shader->inputs[i].interp = LP_INTERP_POSITION;
3874 shader->inputs[i].src_index = 0;
3875 continue;
3876 }
3877
3878 /* XXX this is a completely pointless index map... */
3879 shader->inputs[i].src_index = i+1;
3880 }
3881
3882 if (LP_DEBUG & DEBUG_TGSI && templ->type == PIPE_SHADER_IR_TGSI) {
3883 unsigned attrib;
3884 debug_printf("llvmpipe: Create fragment shader #%u %p:\n",
3885 shader->no, (void *) shader);
3886 tgsi_dump(templ->tokens, 0);
3887 debug_printf("usage masks:\n");
3888 for (attrib = 0; attrib < shader->info.base.num_inputs; ++attrib) {
3889 unsigned usage_mask = shader->info.base.input_usage_mask[attrib];
3890 debug_printf(" IN[%u].%s%s%s%s\n",
3891 attrib,
3892 usage_mask & TGSI_WRITEMASK_X ? "x" : "",
3893 usage_mask & TGSI_WRITEMASK_Y ? "y" : "",
3894 usage_mask & TGSI_WRITEMASK_Z ? "z" : "",
3895 usage_mask & TGSI_WRITEMASK_W ? "w" : "");
3896 }
3897 debug_printf("\n");
3898 }
3899
3900 /* This will put a derived copy of the tokens into shader->base.tokens */
3901 if (templ->type == PIPE_SHADER_IR_TGSI)
3902 llvmpipe_fs_analyse(shader, templ->tokens);
3903 else
3904 llvmpipe_fs_analyse_nir(shader);
3905
3906 return shader;
3907 }
3908
3909
3910 static void
llvmpipe_bind_fs_state(struct pipe_context * pipe,void * fs)3911 llvmpipe_bind_fs_state(struct pipe_context *pipe, void *fs)
3912 {
3913 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
3914 struct lp_fragment_shader *lp_fs = (struct lp_fragment_shader *)fs;
3915 if (llvmpipe->fs == lp_fs)
3916 return;
3917
3918 draw_bind_fragment_shader(llvmpipe->draw,
3919 (lp_fs ? lp_fs->draw_data : NULL));
3920
3921 lp_fs_reference(llvmpipe, &llvmpipe->fs, lp_fs);
3922
3923 /* invalidate the setup link, NEW_FS will make it update */
3924 lp_setup_set_fs_variant(llvmpipe->setup, NULL);
3925 llvmpipe->dirty |= LP_NEW_FS;
3926 }
3927
3928
3929 /**
3930 * Remove shader variant from two lists: the shader's variant list
3931 * and the context's variant list.
3932 */
3933
3934 static
llvmpipe_remove_shader_variant(struct llvmpipe_context * lp,struct lp_fragment_shader_variant * variant)3935 void llvmpipe_remove_shader_variant(struct llvmpipe_context *lp,
3936 struct lp_fragment_shader_variant *variant)
3937 {
3938 if ((LP_DEBUG & DEBUG_FS) || (gallivm_debug & GALLIVM_DEBUG_IR)) {
3939 debug_printf("llvmpipe: del fs #%u var %u v created %u v cached %u "
3940 "v total cached %u inst %u total inst %u\n",
3941 variant->shader->no, variant->no,
3942 variant->shader->variants_created,
3943 variant->shader->variants_cached,
3944 lp->nr_fs_variants, variant->nr_instrs, lp->nr_fs_instrs);
3945 }
3946
3947 /* remove from shader's list */
3948 remove_from_list(&variant->list_item_local);
3949 variant->shader->variants_cached--;
3950
3951 /* remove from context's list */
3952 remove_from_list(&variant->list_item_global);
3953 lp->nr_fs_variants--;
3954 lp->nr_fs_instrs -= variant->nr_instrs;
3955 }
3956
3957 void
llvmpipe_destroy_shader_variant(struct llvmpipe_context * lp,struct lp_fragment_shader_variant * variant)3958 llvmpipe_destroy_shader_variant(struct llvmpipe_context *lp,
3959 struct lp_fragment_shader_variant *variant)
3960 {
3961 gallivm_destroy(variant->gallivm);
3962
3963 lp_fs_reference(lp, &variant->shader, NULL);
3964
3965 FREE(variant);
3966 }
3967
3968 void
llvmpipe_destroy_fs(struct llvmpipe_context * llvmpipe,struct lp_fragment_shader * shader)3969 llvmpipe_destroy_fs(struct llvmpipe_context *llvmpipe,
3970 struct lp_fragment_shader *shader)
3971 {
3972 /* Delete draw module's data */
3973 draw_delete_fragment_shader(llvmpipe->draw, shader->draw_data);
3974
3975 if (shader->base.ir.nir)
3976 ralloc_free(shader->base.ir.nir);
3977 assert(shader->variants_cached == 0);
3978 FREE((void *) shader->base.tokens);
3979 FREE(shader);
3980 }
3981
3982 static void
llvmpipe_delete_fs_state(struct pipe_context * pipe,void * fs)3983 llvmpipe_delete_fs_state(struct pipe_context *pipe, void *fs)
3984 {
3985 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
3986 struct lp_fragment_shader *shader = fs;
3987 struct lp_fs_variant_list_item *li;
3988
3989 /* Delete all the variants */
3990 li = first_elem(&shader->variants);
3991 while(!at_end(&shader->variants, li)) {
3992 struct lp_fs_variant_list_item *next = next_elem(li);
3993 struct lp_fragment_shader_variant *variant;
3994 variant = li->base;
3995 llvmpipe_remove_shader_variant(llvmpipe, li->base);
3996 lp_fs_variant_reference(llvmpipe, &variant, NULL);
3997 li = next;
3998 }
3999
4000 lp_fs_reference(llvmpipe, &shader, NULL);
4001 }
4002
4003 static void
llvmpipe_set_constant_buffer(struct pipe_context * pipe,enum pipe_shader_type shader,uint index,bool take_ownership,const struct pipe_constant_buffer * cb)4004 llvmpipe_set_constant_buffer(struct pipe_context *pipe,
4005 enum pipe_shader_type shader, uint index,
4006 bool take_ownership,
4007 const struct pipe_constant_buffer *cb)
4008 {
4009 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
4010 struct pipe_constant_buffer *constants = &llvmpipe->constants[shader][index];
4011
4012 assert(shader < PIPE_SHADER_TYPES);
4013 assert(index < ARRAY_SIZE(llvmpipe->constants[shader]));
4014
4015 /* note: reference counting */
4016 util_copy_constant_buffer(&llvmpipe->constants[shader][index], cb,
4017 take_ownership);
4018
4019 /* user_buffer is only valid until the next set_constant_buffer (at most,
4020 * possibly until shader deletion), so we need to upload it now to make sure
4021 * it doesn't get updated/freed out from under us.
4022 */
4023 if (constants->user_buffer) {
4024 u_upload_data(llvmpipe->pipe.const_uploader, 0, constants->buffer_size, 16,
4025 constants->user_buffer, &constants->buffer_offset,
4026 &constants->buffer);
4027 }
4028 if (constants->buffer) {
4029 if (!(constants->buffer->bind & PIPE_BIND_CONSTANT_BUFFER)) {
4030 debug_printf("Illegal set constant without bind flag\n");
4031 constants->buffer->bind |= PIPE_BIND_CONSTANT_BUFFER;
4032 }
4033 }
4034
4035 if (shader == PIPE_SHADER_VERTEX ||
4036 shader == PIPE_SHADER_GEOMETRY ||
4037 shader == PIPE_SHADER_TESS_CTRL ||
4038 shader == PIPE_SHADER_TESS_EVAL) {
4039 /* Pass the constants to the 'draw' module */
4040 const unsigned size = cb ? cb->buffer_size : 0;
4041
4042 const ubyte *data = NULL;
4043 if (constants->buffer)
4044 data = (ubyte *) llvmpipe_resource_data(constants->buffer) + constants->buffer_offset;
4045
4046 draw_set_mapped_constant_buffer(llvmpipe->draw, shader,
4047 index, data, size);
4048 }
4049 else if (shader == PIPE_SHADER_COMPUTE)
4050 llvmpipe->cs_dirty |= LP_CSNEW_CONSTANTS;
4051 else
4052 llvmpipe->dirty |= LP_NEW_FS_CONSTANTS;
4053 }
4054
4055 static void
llvmpipe_set_shader_buffers(struct pipe_context * pipe,enum pipe_shader_type shader,unsigned start_slot,unsigned count,const struct pipe_shader_buffer * buffers,unsigned writable_bitmask)4056 llvmpipe_set_shader_buffers(struct pipe_context *pipe,
4057 enum pipe_shader_type shader, unsigned start_slot,
4058 unsigned count, const struct pipe_shader_buffer *buffers,
4059 unsigned writable_bitmask)
4060 {
4061 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
4062 unsigned i, idx;
4063 for (i = start_slot, idx = 0; i < start_slot + count; i++, idx++) {
4064 const struct pipe_shader_buffer *buffer = buffers ? &buffers[idx] : NULL;
4065
4066 util_copy_shader_buffer(&llvmpipe->ssbos[shader][i], buffer);
4067
4068 if (buffer && buffer->buffer) {
4069 boolean read_only = !(writable_bitmask & (1 << idx));
4070 llvmpipe_flush_resource(pipe, buffer->buffer, 0, read_only, false,
4071 false, "buffer");
4072 }
4073
4074 if (shader == PIPE_SHADER_VERTEX ||
4075 shader == PIPE_SHADER_GEOMETRY ||
4076 shader == PIPE_SHADER_TESS_CTRL ||
4077 shader == PIPE_SHADER_TESS_EVAL) {
4078 const unsigned size = buffer ? buffer->buffer_size : 0;
4079 const ubyte *data = NULL;
4080 if (buffer && buffer->buffer)
4081 data = (ubyte *) llvmpipe_resource_data(buffer->buffer);
4082 if (data)
4083 data += buffer->buffer_offset;
4084 draw_set_mapped_shader_buffer(llvmpipe->draw, shader,
4085 i, data, size);
4086 } else if (shader == PIPE_SHADER_COMPUTE) {
4087 llvmpipe->cs_dirty |= LP_CSNEW_SSBOS;
4088 } else if (shader == PIPE_SHADER_FRAGMENT) {
4089 llvmpipe->fs_ssbo_write_mask &= ~(((1 << count) - 1) << start_slot);
4090 llvmpipe->fs_ssbo_write_mask |= writable_bitmask << start_slot;
4091 llvmpipe->dirty |= LP_NEW_FS_SSBOS;
4092 }
4093 }
4094 }
4095
4096 static void
llvmpipe_set_shader_images(struct pipe_context * pipe,enum pipe_shader_type shader,unsigned start_slot,unsigned count,unsigned unbind_num_trailing_slots,const struct pipe_image_view * images)4097 llvmpipe_set_shader_images(struct pipe_context *pipe,
4098 enum pipe_shader_type shader, unsigned start_slot,
4099 unsigned count, unsigned unbind_num_trailing_slots,
4100 const struct pipe_image_view *images)
4101 {
4102 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
4103 unsigned i, idx;
4104
4105 draw_flush(llvmpipe->draw);
4106 for (i = start_slot, idx = 0; i < start_slot + count; i++, idx++) {
4107 const struct pipe_image_view *image = images ? &images[idx] : NULL;
4108
4109 util_copy_image_view(&llvmpipe->images[shader][i], image);
4110
4111 if (image && image->resource) {
4112 bool read_only = !(image->access & PIPE_IMAGE_ACCESS_WRITE);
4113 llvmpipe_flush_resource(pipe, image->resource, 0, read_only, false,
4114 false, "image");
4115 }
4116 }
4117
4118 llvmpipe->num_images[shader] = start_slot + count;
4119 if (shader == PIPE_SHADER_VERTEX ||
4120 shader == PIPE_SHADER_GEOMETRY ||
4121 shader == PIPE_SHADER_TESS_CTRL ||
4122 shader == PIPE_SHADER_TESS_EVAL) {
4123 draw_set_images(llvmpipe->draw,
4124 shader,
4125 llvmpipe->images[shader],
4126 start_slot + count);
4127 } else if (shader == PIPE_SHADER_COMPUTE)
4128 llvmpipe->cs_dirty |= LP_CSNEW_IMAGES;
4129 else
4130 llvmpipe->dirty |= LP_NEW_FS_IMAGES;
4131
4132 if (unbind_num_trailing_slots) {
4133 llvmpipe_set_shader_images(pipe, shader, start_slot + count,
4134 unbind_num_trailing_slots, 0, NULL);
4135 }
4136 }
4137
4138 /**
4139 * Return the blend factor equivalent to a destination alpha of one.
4140 */
4141 static inline unsigned
force_dst_alpha_one(unsigned factor,boolean clamped_zero)4142 force_dst_alpha_one(unsigned factor, boolean clamped_zero)
4143 {
4144 switch(factor) {
4145 case PIPE_BLENDFACTOR_DST_ALPHA:
4146 return PIPE_BLENDFACTOR_ONE;
4147 case PIPE_BLENDFACTOR_INV_DST_ALPHA:
4148 return PIPE_BLENDFACTOR_ZERO;
4149 case PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE:
4150 if (clamped_zero)
4151 return PIPE_BLENDFACTOR_ZERO;
4152 else
4153 return PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE;
4154 }
4155
4156 return factor;
4157 }
4158
4159
4160 /**
4161 * We need to generate several variants of the fragment pipeline to match
4162 * all the combinations of the contributing state atoms.
4163 *
4164 * TODO: there is actually no reason to tie this to context state -- the
4165 * generated code could be cached globally in the screen.
4166 */
4167 static struct lp_fragment_shader_variant_key *
make_variant_key(struct llvmpipe_context * lp,struct lp_fragment_shader * shader,char * store)4168 make_variant_key(struct llvmpipe_context *lp,
4169 struct lp_fragment_shader *shader,
4170 char *store)
4171 {
4172 unsigned i;
4173 struct lp_fragment_shader_variant_key *key;
4174
4175 key = (struct lp_fragment_shader_variant_key *)store;
4176
4177 memset(key, 0, sizeof(*key));
4178
4179 if (lp->framebuffer.zsbuf) {
4180 enum pipe_format zsbuf_format = lp->framebuffer.zsbuf->format;
4181 const struct util_format_description *zsbuf_desc =
4182 util_format_description(zsbuf_format);
4183
4184 if (lp->depth_stencil->depth_enabled &&
4185 util_format_has_depth(zsbuf_desc)) {
4186 key->zsbuf_format = zsbuf_format;
4187 key->depth.enabled = lp->depth_stencil->depth_enabled;
4188 key->depth.writemask = lp->depth_stencil->depth_writemask;
4189 key->depth.func = lp->depth_stencil->depth_func;
4190 }
4191 if (lp->depth_stencil->stencil[0].enabled &&
4192 util_format_has_stencil(zsbuf_desc)) {
4193 key->zsbuf_format = zsbuf_format;
4194 memcpy(&key->stencil, &lp->depth_stencil->stencil, sizeof key->stencil);
4195 }
4196 if (llvmpipe_resource_is_1d(lp->framebuffer.zsbuf->texture)) {
4197 key->resource_1d = TRUE;
4198 }
4199 key->zsbuf_nr_samples = util_res_sample_count(lp->framebuffer.zsbuf->texture);
4200 }
4201
4202 /*
4203 * Propagate the depth clamp setting from the rasterizer state.
4204 */
4205 key->depth_clamp = lp->rasterizer->depth_clamp;
4206
4207 /* alpha test only applies if render buffer 0 is non-integer (or does not exist) */
4208 if (!lp->framebuffer.nr_cbufs ||
4209 !lp->framebuffer.cbufs[0] ||
4210 !util_format_is_pure_integer(lp->framebuffer.cbufs[0]->format)) {
4211 key->alpha.enabled = lp->depth_stencil->alpha_enabled;
4212 }
4213 if(key->alpha.enabled)
4214 key->alpha.func = lp->depth_stencil->alpha_func;
4215 /* alpha.ref_value is passed in jit_context */
4216
4217 key->flatshade = lp->rasterizer->flatshade;
4218 key->multisample = lp->rasterizer->multisample;
4219 key->no_ms_sample_mask_out = lp->rasterizer->no_ms_sample_mask_out;
4220 if (lp->active_occlusion_queries && !lp->queries_disabled) {
4221 key->occlusion_count = TRUE;
4222 }
4223
4224 memcpy(&key->blend, lp->blend, sizeof key->blend);
4225
4226 key->coverage_samples = 1;
4227 key->min_samples = 1;
4228 if (key->multisample) {
4229 key->coverage_samples = util_framebuffer_get_num_samples(&lp->framebuffer);
4230 key->min_samples = lp->min_samples == 1 ? 1 : key->coverage_samples;
4231 }
4232 key->nr_cbufs = lp->framebuffer.nr_cbufs;
4233
4234 if (!key->blend.independent_blend_enable) {
4235 /* we always need independent blend otherwise the fixups below won't work */
4236 for (i = 1; i < key->nr_cbufs; i++) {
4237 memcpy(&key->blend.rt[i], &key->blend.rt[0], sizeof(key->blend.rt[0]));
4238 }
4239 key->blend.independent_blend_enable = 1;
4240 }
4241
4242 for (i = 0; i < lp->framebuffer.nr_cbufs; i++) {
4243 struct pipe_rt_blend_state *blend_rt = &key->blend.rt[i];
4244
4245 if (lp->framebuffer.cbufs[i]) {
4246 enum pipe_format format = lp->framebuffer.cbufs[i]->format;
4247 const struct util_format_description *format_desc;
4248
4249 key->cbuf_format[i] = format;
4250 key->cbuf_nr_samples[i] = util_res_sample_count(lp->framebuffer.cbufs[i]->texture);
4251
4252 /*
4253 * Figure out if this is a 1d resource. Note that OpenGL allows crazy
4254 * mixing of 2d textures with height 1 and 1d textures, so make sure
4255 * we pick 1d if any cbuf or zsbuf is 1d.
4256 */
4257 if (llvmpipe_resource_is_1d(lp->framebuffer.cbufs[i]->texture)) {
4258 key->resource_1d = TRUE;
4259 }
4260
4261 format_desc = util_format_description(format);
4262 assert(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB ||
4263 format_desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB);
4264
4265 /*
4266 * Mask out color channels not present in the color buffer.
4267 */
4268 blend_rt->colormask &= util_format_colormask(format_desc);
4269
4270 /*
4271 * Disable blend for integer formats.
4272 */
4273 if (util_format_is_pure_integer(format)) {
4274 blend_rt->blend_enable = 0;
4275 }
4276
4277 /*
4278 * Our swizzled render tiles always have an alpha channel, but the
4279 * linear render target format often does not, so force here the dst
4280 * alpha to be one.
4281 *
4282 * This is not a mere optimization. Wrong results will be produced if
4283 * the dst alpha is used, the dst format does not have alpha, and the
4284 * previous rendering was not flushed from the swizzled to linear
4285 * buffer. For example, NonPowTwo DCT.
4286 *
4287 * TODO: This should be generalized to all channels for better
4288 * performance, but only alpha causes correctness issues.
4289 *
4290 * Also, force rgb/alpha func/factors match, to make AoS blending
4291 * easier.
4292 */
4293 if (format_desc->swizzle[3] > PIPE_SWIZZLE_W ||
4294 format_desc->swizzle[3] == format_desc->swizzle[0]) {
4295 /* Doesn't cover mixed snorm/unorm but can't render to them anyway */
4296 boolean clamped_zero = !util_format_is_float(format) &&
4297 !util_format_is_snorm(format);
4298 blend_rt->rgb_src_factor =
4299 force_dst_alpha_one(blend_rt->rgb_src_factor, clamped_zero);
4300 blend_rt->rgb_dst_factor =
4301 force_dst_alpha_one(blend_rt->rgb_dst_factor, clamped_zero);
4302 blend_rt->alpha_func = blend_rt->rgb_func;
4303 blend_rt->alpha_src_factor = blend_rt->rgb_src_factor;
4304 blend_rt->alpha_dst_factor = blend_rt->rgb_dst_factor;
4305 }
4306 }
4307 else {
4308 /* no color buffer for this fragment output */
4309 key->cbuf_format[i] = PIPE_FORMAT_NONE;
4310 key->cbuf_nr_samples[i] = 0;
4311 blend_rt->colormask = 0x0;
4312 blend_rt->blend_enable = 0;
4313 }
4314 }
4315
4316 /* This value will be the same for all the variants of a given shader:
4317 */
4318 key->nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1;
4319
4320 struct lp_sampler_static_state *fs_sampler;
4321
4322 fs_sampler = lp_fs_variant_key_samplers(key);
4323
4324 memset(fs_sampler, 0, MAX2(key->nr_samplers, key->nr_sampler_views) * sizeof *fs_sampler);
4325
4326 for(i = 0; i < key->nr_samplers; ++i) {
4327 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
4328 lp_sampler_static_sampler_state(&fs_sampler[i].sampler_state,
4329 lp->samplers[PIPE_SHADER_FRAGMENT][i]);
4330 }
4331 }
4332
4333 /*
4334 * XXX If TGSI_FILE_SAMPLER_VIEW exists assume all texture opcodes
4335 * are dx10-style? Can't really have mixed opcodes, at least not
4336 * if we want to skip the holes here (without rescanning tgsi).
4337 */
4338 if (shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) {
4339 key->nr_sampler_views = shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
4340 for(i = 0; i < key->nr_sampler_views; ++i) {
4341 /*
4342 * Note sview may exceed what's representable by file_mask.
4343 * This will still work, the only downside is that not actually
4344 * used views may be included in the shader key.
4345 */
4346 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER_VIEW] & (1u << (i & 31))) {
4347 lp_sampler_static_texture_state(&fs_sampler[i].texture_state,
4348 lp->sampler_views[PIPE_SHADER_FRAGMENT][i]);
4349 }
4350 }
4351 }
4352 else {
4353 key->nr_sampler_views = key->nr_samplers;
4354 for(i = 0; i < key->nr_sampler_views; ++i) {
4355 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
4356 lp_sampler_static_texture_state(&fs_sampler[i].texture_state,
4357 lp->sampler_views[PIPE_SHADER_FRAGMENT][i]);
4358 }
4359 }
4360 }
4361
4362 struct lp_image_static_state *lp_image;
4363 lp_image = lp_fs_variant_key_images(key);
4364 key->nr_images = shader->info.base.file_max[TGSI_FILE_IMAGE] + 1;
4365 for (i = 0; i < key->nr_images; ++i) {
4366 if (shader->info.base.file_mask[TGSI_FILE_IMAGE] & (1 << i)) {
4367 lp_sampler_static_texture_state_image(&lp_image[i].image_state,
4368 &lp->images[PIPE_SHADER_FRAGMENT][i]);
4369 }
4370 }
4371
4372 if (shader->kind == LP_FS_KIND_AERO_MINIFICATION) {
4373 struct lp_sampler_static_state *samp0 = lp_fs_variant_key_sampler_idx(key, 0);
4374 assert(samp0);
4375 samp0->sampler_state.min_img_filter = PIPE_TEX_FILTER_NEAREST;
4376 samp0->sampler_state.mag_img_filter = PIPE_TEX_FILTER_NEAREST;
4377 }
4378
4379 return key;
4380 }
4381
4382
4383 /**
4384 * Update fragment shader state. This is called just prior to drawing
4385 * something when some fragment-related state has changed.
4386 */
4387 void
llvmpipe_update_fs(struct llvmpipe_context * lp)4388 llvmpipe_update_fs(struct llvmpipe_context *lp)
4389 {
4390 struct lp_fragment_shader *shader = lp->fs;
4391 struct lp_fragment_shader_variant_key *key;
4392 struct lp_fragment_shader_variant *variant = NULL;
4393 struct lp_fs_variant_list_item *li;
4394 char store[LP_FS_MAX_VARIANT_KEY_SIZE];
4395
4396 key = make_variant_key(lp, shader, store);
4397
4398 /* Search the variants for one which matches the key */
4399 li = first_elem(&shader->variants);
4400 while(!at_end(&shader->variants, li)) {
4401 if(memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
4402 variant = li->base;
4403 break;
4404 }
4405 li = next_elem(li);
4406 }
4407
4408 if (variant) {
4409 /* Move this variant to the head of the list to implement LRU
4410 * deletion of shader's when we have too many.
4411 */
4412 move_to_head(&lp->fs_variants_list, &variant->list_item_global);
4413 }
4414 else {
4415 /* variant not found, create it now */
4416 int64_t t0, t1, dt;
4417 unsigned i;
4418 unsigned variants_to_cull;
4419
4420 if (LP_DEBUG & DEBUG_FS) {
4421 debug_printf("%u variants,\t%u instrs,\t%u instrs/variant\n",
4422 lp->nr_fs_variants,
4423 lp->nr_fs_instrs,
4424 lp->nr_fs_variants ? lp->nr_fs_instrs / lp->nr_fs_variants : 0);
4425 }
4426
4427 /* First, check if we've exceeded the max number of shader variants.
4428 * If so, free 6.25% of them (the least recently used ones).
4429 */
4430 variants_to_cull = lp->nr_fs_variants >= LP_MAX_SHADER_VARIANTS ? LP_MAX_SHADER_VARIANTS / 16 : 0;
4431
4432 if (variants_to_cull ||
4433 lp->nr_fs_instrs >= LP_MAX_SHADER_INSTRUCTIONS) {
4434 if (gallivm_debug & GALLIVM_DEBUG_PERF) {
4435 debug_printf("Evicting FS: %u fs variants,\t%u total variants,"
4436 "\t%u instrs,\t%u instrs/variant\n",
4437 shader->variants_cached,
4438 lp->nr_fs_variants, lp->nr_fs_instrs,
4439 lp->nr_fs_instrs / lp->nr_fs_variants);
4440 }
4441
4442 /*
4443 * We need to re-check lp->nr_fs_variants because an arbitrarliy large
4444 * number of shader variants (potentially all of them) could be
4445 * pending for destruction on flush.
4446 */
4447
4448 for (i = 0; i < variants_to_cull || lp->nr_fs_instrs >= LP_MAX_SHADER_INSTRUCTIONS; i++) {
4449 struct lp_fs_variant_list_item *item;
4450 if (is_empty_list(&lp->fs_variants_list)) {
4451 break;
4452 }
4453 item = last_elem(&lp->fs_variants_list);
4454 assert(item);
4455 assert(item->base);
4456 llvmpipe_remove_shader_variant(lp, item->base);
4457 struct lp_fragment_shader_variant *variant = item->base;
4458 lp_fs_variant_reference(lp, &variant, NULL);
4459 }
4460 }
4461
4462 /*
4463 * Generate the new variant.
4464 */
4465 t0 = os_time_get();
4466 variant = generate_variant(lp, shader, key);
4467 t1 = os_time_get();
4468 dt = t1 - t0;
4469 LP_COUNT_ADD(llvm_compile_time, dt);
4470 LP_COUNT_ADD(nr_llvm_compiles, 2); /* emit vs. omit in/out test */
4471
4472 /* Put the new variant into the list */
4473 if (variant) {
4474 insert_at_head(&shader->variants, &variant->list_item_local);
4475 insert_at_head(&lp->fs_variants_list, &variant->list_item_global);
4476 lp->nr_fs_variants++;
4477 lp->nr_fs_instrs += variant->nr_instrs;
4478 shader->variants_cached++;
4479 }
4480 }
4481
4482 /* Bind this variant */
4483 lp_setup_set_fs_variant(lp->setup, variant);
4484 }
4485
4486
4487
4488
4489
4490 void
llvmpipe_init_fs_funcs(struct llvmpipe_context * llvmpipe)4491 llvmpipe_init_fs_funcs(struct llvmpipe_context *llvmpipe)
4492 {
4493 llvmpipe->pipe.create_fs_state = llvmpipe_create_fs_state;
4494 llvmpipe->pipe.bind_fs_state = llvmpipe_bind_fs_state;
4495 llvmpipe->pipe.delete_fs_state = llvmpipe_delete_fs_state;
4496
4497 llvmpipe->pipe.set_constant_buffer = llvmpipe_set_constant_buffer;
4498
4499 llvmpipe->pipe.set_shader_buffers = llvmpipe_set_shader_buffers;
4500 llvmpipe->pipe.set_shader_images = llvmpipe_set_shader_images;
4501 }
4502
4503
4504