1 /**************************************************************************
2 *
3 * Copyright 2010 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 #include "util/u_math.h"
29 #include "util/u_memory.h"
30 #include "util/u_prim.h"
31 #include "draw/draw_context.h"
32 #include "draw/draw_gs.h"
33 #include "draw/draw_tess.h"
34 #include "draw/draw_vbuf.h"
35 #include "draw/draw_vertex.h"
36 #include "draw/draw_pt.h"
37 #include "draw/draw_prim_assembler.h"
38 #include "draw/draw_vs.h"
39 #include "draw/draw_llvm.h"
40 #include "gallivm/lp_bld_init.h"
41 #include "gallivm/lp_bld_debug.h"
42
43
44 struct llvm_middle_end {
45 struct draw_pt_middle_end base;
46 struct draw_context *draw;
47
48 struct pt_emit *emit;
49 struct pt_so_emit *so_emit;
50 struct pt_fetch *fetch;
51 struct pt_post_vs *post_vs;
52
53
54 unsigned vertex_data_offset;
55 unsigned vertex_size;
56 unsigned input_prim;
57 unsigned opt;
58
59 struct draw_llvm *llvm;
60 struct draw_llvm_variant *current_variant;
61 };
62
63
64 /** cast wrapper */
65 static inline struct llvm_middle_end *
llvm_middle_end(struct draw_pt_middle_end * middle)66 llvm_middle_end(struct draw_pt_middle_end *middle)
67 {
68 return (struct llvm_middle_end *) middle;
69 }
70
71
72 static void
llvm_middle_end_prepare_gs(struct llvm_middle_end * fpme)73 llvm_middle_end_prepare_gs(struct llvm_middle_end *fpme)
74 {
75 struct draw_context *draw = fpme->draw;
76 struct draw_llvm *llvm = fpme->llvm;
77 struct draw_geometry_shader *gs = draw->gs.geometry_shader;
78 struct draw_gs_llvm_variant_key *key;
79 struct draw_gs_llvm_variant *variant = NULL;
80 struct draw_gs_llvm_variant_list_item *li;
81 struct llvm_geometry_shader *shader = llvm_geometry_shader(gs);
82 char store[DRAW_GS_LLVM_MAX_VARIANT_KEY_SIZE];
83 unsigned i;
84
85 key = draw_gs_llvm_make_variant_key(llvm, store);
86
87 /* Search shader's list of variants for the key */
88 li = first_elem(&shader->variants);
89 while (!at_end(&shader->variants, li)) {
90 if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
91 variant = li->base;
92 break;
93 }
94 li = next_elem(li);
95 }
96
97 if (variant) {
98 /* found the variant, move to head of global list (for LRU) */
99 move_to_head(&llvm->gs_variants_list, &variant->list_item_global);
100 }
101 else {
102 /* Need to create new variant */
103
104 /* First check if we've created too many variants. If so, free
105 * 3.125% of the LRU to avoid using too much memory.
106 */
107 if (llvm->nr_gs_variants >= DRAW_MAX_SHADER_VARIANTS) {
108 if (gallivm_debug & GALLIVM_DEBUG_PERF) {
109 debug_printf("Evicting GS: %u gs variants,\t%u total variants\n",
110 shader->variants_cached, llvm->nr_gs_variants);
111 }
112
113 /*
114 * XXX: should we flush here ?
115 */
116 for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
117 struct draw_gs_llvm_variant_list_item *item;
118 if (is_empty_list(&llvm->gs_variants_list)) {
119 break;
120 }
121 item = last_elem(&llvm->gs_variants_list);
122 assert(item);
123 assert(item->base);
124 draw_gs_llvm_destroy_variant(item->base);
125 }
126 }
127
128 variant = draw_gs_llvm_create_variant(llvm, gs->info.num_outputs, key);
129
130 if (variant) {
131 insert_at_head(&shader->variants, &variant->list_item_local);
132 insert_at_head(&llvm->gs_variants_list,
133 &variant->list_item_global);
134 llvm->nr_gs_variants++;
135 shader->variants_cached++;
136 }
137 }
138
139 gs->current_variant = variant;
140 }
141
142 static void
llvm_middle_end_prepare_tcs(struct llvm_middle_end * fpme)143 llvm_middle_end_prepare_tcs(struct llvm_middle_end *fpme)
144 {
145 struct draw_context *draw = fpme->draw;
146 struct draw_llvm *llvm = fpme->llvm;
147 struct draw_tess_ctrl_shader *tcs = draw->tcs.tess_ctrl_shader;
148 struct draw_tcs_llvm_variant_key *key;
149 struct draw_tcs_llvm_variant *variant = NULL;
150 struct draw_tcs_llvm_variant_list_item *li;
151 struct llvm_tess_ctrl_shader *shader = llvm_tess_ctrl_shader(tcs);
152 char store[DRAW_TCS_LLVM_MAX_VARIANT_KEY_SIZE];
153 unsigned i;
154
155 key = draw_tcs_llvm_make_variant_key(llvm, store);
156
157 /* Search shader's list of variants for the key */
158 li = first_elem(&shader->variants);
159 while (!at_end(&shader->variants, li)) {
160 if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
161 variant = li->base;
162 break;
163 }
164 li = next_elem(li);
165 }
166
167 if (variant) {
168 /* found the variant, move to head of global list (for LRU) */
169 move_to_head(&llvm->tcs_variants_list, &variant->list_item_global);
170 }
171 else {
172 /* Need to create new variant */
173
174 /* First check if we've created too many variants. If so, free
175 * 3.125% of the LRU to avoid using too much memory.
176 */
177 if (llvm->nr_tcs_variants >= DRAW_MAX_SHADER_VARIANTS) {
178 if (gallivm_debug & GALLIVM_DEBUG_PERF) {
179 debug_printf("Evicting TCS: %u tcs variants,\t%u total variants\n",
180 shader->variants_cached, llvm->nr_tcs_variants);
181 }
182
183 /*
184 * XXX: should we flush here ?
185 */
186 for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
187 struct draw_tcs_llvm_variant_list_item *item;
188 if (is_empty_list(&llvm->tcs_variants_list)) {
189 break;
190 }
191 item = last_elem(&llvm->tcs_variants_list);
192 assert(item);
193 assert(item->base);
194 draw_tcs_llvm_destroy_variant(item->base);
195 }
196 }
197
198 variant = draw_tcs_llvm_create_variant(llvm, 0, key);
199
200 if (variant) {
201 insert_at_head(&shader->variants, &variant->list_item_local);
202 insert_at_head(&llvm->tcs_variants_list,
203 &variant->list_item_global);
204 llvm->nr_tcs_variants++;
205 shader->variants_cached++;
206 }
207 }
208
209 tcs->current_variant = variant;
210 }
211
212 static void
llvm_middle_end_prepare_tes(struct llvm_middle_end * fpme)213 llvm_middle_end_prepare_tes(struct llvm_middle_end *fpme)
214 {
215 struct draw_context *draw = fpme->draw;
216 struct draw_llvm *llvm = fpme->llvm;
217 struct draw_tess_eval_shader *tes = draw->tes.tess_eval_shader;
218 struct draw_tes_llvm_variant_key *key;
219 struct draw_tes_llvm_variant *variant = NULL;
220 struct draw_tes_llvm_variant_list_item *li;
221 struct llvm_tess_eval_shader *shader = llvm_tess_eval_shader(tes);
222 char store[DRAW_TES_LLVM_MAX_VARIANT_KEY_SIZE];
223 unsigned i;
224
225 key = draw_tes_llvm_make_variant_key(llvm, store);
226
227 /* Search shader's list of variants for the key */
228 li = first_elem(&shader->variants);
229 while (!at_end(&shader->variants, li)) {
230 if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
231 variant = li->base;
232 break;
233 }
234 li = next_elem(li);
235 }
236
237 if (variant) {
238 /* found the variant, move to head of global list (for LRU) */
239 move_to_head(&llvm->tes_variants_list, &variant->list_item_global);
240 }
241 else {
242 /* Need to create new variant */
243
244 /* First check if we've created too many variants. If so, free
245 * 3.125% of the LRU to avoid using too much memory.
246 */
247 if (llvm->nr_tes_variants >= DRAW_MAX_SHADER_VARIANTS) {
248 if (gallivm_debug & GALLIVM_DEBUG_PERF) {
249 debug_printf("Evicting TES: %u tes variants,\t%u total variants\n",
250 shader->variants_cached, llvm->nr_tes_variants);
251 }
252
253 /*
254 * XXX: should we flush here ?
255 */
256 for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
257 struct draw_tes_llvm_variant_list_item *item;
258 if (is_empty_list(&llvm->tes_variants_list)) {
259 break;
260 }
261 item = last_elem(&llvm->tes_variants_list);
262 assert(item);
263 assert(item->base);
264 draw_tes_llvm_destroy_variant(item->base);
265 }
266 }
267
268 variant = draw_tes_llvm_create_variant(llvm, draw_total_tes_outputs(draw), key);
269
270 if (variant) {
271 insert_at_head(&shader->variants, &variant->list_item_local);
272 insert_at_head(&llvm->tes_variants_list,
273 &variant->list_item_global);
274 llvm->nr_tes_variants++;
275 shader->variants_cached++;
276 }
277 }
278
279 tes->current_variant = variant;
280 }
281
282 /**
283 * Prepare/validate middle part of the vertex pipeline.
284 * NOTE: if you change this function, also look at the non-LLVM
285 * function fetch_pipeline_prepare() for similar changes.
286 */
287 static void
llvm_middle_end_prepare(struct draw_pt_middle_end * middle,unsigned in_prim,unsigned opt,unsigned * max_vertices)288 llvm_middle_end_prepare( struct draw_pt_middle_end *middle,
289 unsigned in_prim,
290 unsigned opt,
291 unsigned *max_vertices )
292 {
293 struct llvm_middle_end *fpme = llvm_middle_end(middle);
294 struct draw_context *draw = fpme->draw;
295 struct draw_llvm *llvm = fpme->llvm;
296 struct draw_vertex_shader *vs = draw->vs.vertex_shader;
297 struct draw_geometry_shader *gs = draw->gs.geometry_shader;
298 struct draw_tess_ctrl_shader *tcs = draw->tcs.tess_ctrl_shader;
299 struct draw_tess_eval_shader *tes = draw->tes.tess_eval_shader;
300 const unsigned out_prim = gs ? gs->output_primitive : tes ? get_tes_output_prim(tes) :
301 u_assembled_prim(in_prim);
302 unsigned point_clip = draw->rasterizer->fill_front == PIPE_POLYGON_MODE_POINT ||
303 out_prim == PIPE_PRIM_POINTS;
304 unsigned nr;
305
306 fpme->input_prim = in_prim;
307 fpme->opt = opt;
308
309 draw_pt_post_vs_prepare( fpme->post_vs,
310 draw->clip_xy,
311 draw->clip_z,
312 draw->clip_user,
313 point_clip ? draw->guard_band_points_xy :
314 draw->guard_band_xy,
315 draw->bypass_viewport,
316 draw->rasterizer->clip_halfz,
317 (draw->vs.edgeflag_output ? TRUE : FALSE) );
318
319 draw_pt_so_emit_prepare( fpme->so_emit, gs == NULL );
320
321 if (!(opt & PT_PIPELINE)) {
322 draw_pt_emit_prepare( fpme->emit, out_prim,
323 max_vertices );
324
325 *max_vertices = MAX2( *max_vertices, 4096 );
326 }
327 else {
328 /* limit max fetches by limiting max_vertices */
329 *max_vertices = 4096;
330 }
331
332 /* Get the number of float[4] attributes per vertex.
333 * Note: this must be done after draw_pt_emit_prepare() since that
334 * can effect the vertex size.
335 */
336 nr = MAX2(vs->info.num_inputs, draw_total_vs_outputs(draw));
337
338 /* Always leave room for the vertex header whether we need it or
339 * not. It's hard to get rid of it in particular because of the
340 * viewport code in draw_pt_post_vs.c.
341 */
342 fpme->vertex_size = sizeof(struct vertex_header) + nr * 4 * sizeof(float);
343
344 /* return even number */
345 *max_vertices = *max_vertices & ~1;
346
347 /* Find/create the vertex shader variant */
348 {
349 struct draw_llvm_variant_key *key;
350 struct draw_llvm_variant *variant = NULL;
351 struct draw_llvm_variant_list_item *li;
352 struct llvm_vertex_shader *shader = llvm_vertex_shader(vs);
353 char store[DRAW_LLVM_MAX_VARIANT_KEY_SIZE];
354 unsigned i;
355
356 key = draw_llvm_make_variant_key(llvm, store);
357
358 /* Search shader's list of variants for the key */
359 li = first_elem(&shader->variants);
360 while (!at_end(&shader->variants, li)) {
361 if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
362 variant = li->base;
363 break;
364 }
365 li = next_elem(li);
366 }
367
368 if (variant) {
369 /* found the variant, move to head of global list (for LRU) */
370 move_to_head(&llvm->vs_variants_list, &variant->list_item_global);
371 }
372 else {
373 /* Need to create new variant */
374
375 /* First check if we've created too many variants. If so, free
376 * 3.125% of the LRU to avoid using too much memory.
377 */
378 if (llvm->nr_variants >= DRAW_MAX_SHADER_VARIANTS) {
379 if (gallivm_debug & GALLIVM_DEBUG_PERF) {
380 debug_printf("Evicting VS: %u vs variants,\t%u total variants\n",
381 shader->variants_cached, llvm->nr_variants);
382 }
383
384 /*
385 * XXX: should we flush here ?
386 */
387 for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
388 struct draw_llvm_variant_list_item *item;
389 if (is_empty_list(&llvm->vs_variants_list)) {
390 break;
391 }
392 item = last_elem(&llvm->vs_variants_list);
393 assert(item);
394 assert(item->base);
395 draw_llvm_destroy_variant(item->base);
396 }
397 }
398
399 variant = draw_llvm_create_variant(llvm, nr, key);
400
401 if (variant) {
402 insert_at_head(&shader->variants, &variant->list_item_local);
403 insert_at_head(&llvm->vs_variants_list,
404 &variant->list_item_global);
405 llvm->nr_variants++;
406 shader->variants_cached++;
407 }
408 }
409
410 fpme->current_variant = variant;
411 }
412
413 if (gs) {
414 llvm_middle_end_prepare_gs(fpme);
415 }
416 if (tcs) {
417 llvm_middle_end_prepare_tcs(fpme);
418 }
419 if (tes) {
420 llvm_middle_end_prepare_tes(fpme);
421 }
422 }
423
424 static unsigned
get_num_consts_robust(struct draw_context * draw,unsigned * sizes,unsigned idx)425 get_num_consts_robust(struct draw_context *draw, unsigned *sizes, unsigned idx)
426 {
427 unsigned const_bytes = sizes[idx];
428
429 if (const_bytes < sizeof(float))
430 return 0;
431
432 return DIV_ROUND_UP(const_bytes, draw->constant_buffer_stride);
433 }
434
435 /**
436 * Bind/update constant buffer pointers, clip planes and viewport dims.
437 * These are "light weight" parameters which aren't baked into the
438 * generated code. Updating these items is much cheaper than revalidating
439 * and rebuilding the generated pipeline code.
440 */
441 static void
llvm_middle_end_bind_parameters(struct draw_pt_middle_end * middle)442 llvm_middle_end_bind_parameters(struct draw_pt_middle_end *middle)
443 {
444 static const float fake_const_buf[4];
445 struct llvm_middle_end *fpme = llvm_middle_end(middle);
446 struct draw_context *draw = fpme->draw;
447 struct draw_llvm *llvm = fpme->llvm;
448 unsigned i;
449
450 for (i = 0; i < ARRAY_SIZE(llvm->jit_context.vs_constants); ++i) {
451 /*
452 * There could be a potential issue with rounding this up, as the
453 * shader expects 16-byte allocations, the fix is likely to move
454 * to LOAD intrinsic in the future and remove the vec4 constraint.
455 */
456 int num_consts = get_num_consts_robust(draw, draw->pt.user.vs_constants_size, i);
457 llvm->jit_context.vs_constants[i] = draw->pt.user.vs_constants[i];
458 llvm->jit_context.num_vs_constants[i] = num_consts;
459 if (num_consts == 0) {
460 llvm->jit_context.vs_constants[i] = fake_const_buf;
461 }
462 }
463 for (i = 0; i < ARRAY_SIZE(llvm->jit_context.vs_ssbos); ++i) {
464 int num_ssbos = draw->pt.user.vs_ssbos_size[i];
465 llvm->jit_context.vs_ssbos[i] = draw->pt.user.vs_ssbos[i];
466 llvm->jit_context.num_vs_ssbos[i] = num_ssbos;
467 if (num_ssbos == 0) {
468 llvm->jit_context.vs_ssbos[i] = (const uint32_t *)fake_const_buf;
469 }
470 }
471
472 for (i = 0; i < ARRAY_SIZE(llvm->gs_jit_context.constants); ++i) {
473 int num_consts = get_num_consts_robust(draw, draw->pt.user.gs_constants_size, i);
474 llvm->gs_jit_context.constants[i] = draw->pt.user.gs_constants[i];
475 llvm->gs_jit_context.num_constants[i] = num_consts;
476 if (num_consts == 0) {
477 llvm->gs_jit_context.constants[i] = fake_const_buf;
478 }
479 }
480 for (i = 0; i < ARRAY_SIZE(llvm->gs_jit_context.ssbos); ++i) {
481 int num_ssbos = draw->pt.user.gs_ssbos_size[i];
482 llvm->gs_jit_context.ssbos[i] = draw->pt.user.gs_ssbos[i];
483 llvm->gs_jit_context.num_ssbos[i] = num_ssbos;
484 if (num_ssbos == 0) {
485 llvm->gs_jit_context.ssbos[i] = (const uint32_t *)fake_const_buf;
486 }
487 }
488
489 for (i = 0; i < ARRAY_SIZE(llvm->tcs_jit_context.constants); ++i) {
490 int num_consts = get_num_consts_robust(draw, draw->pt.user.tcs_constants_size, i);
491 llvm->tcs_jit_context.constants[i] = draw->pt.user.tcs_constants[i];
492 llvm->tcs_jit_context.num_constants[i] = num_consts;
493 if (num_consts == 0) {
494 llvm->tcs_jit_context.constants[i] = fake_const_buf;
495 }
496 }
497 for (i = 0; i < ARRAY_SIZE(llvm->tcs_jit_context.ssbos); ++i) {
498 int num_ssbos = draw->pt.user.tcs_ssbos_size[i];
499 llvm->tcs_jit_context.ssbos[i] = draw->pt.user.tcs_ssbos[i];
500 llvm->tcs_jit_context.num_ssbos[i] = num_ssbos;
501 if (num_ssbos == 0) {
502 llvm->tcs_jit_context.ssbos[i] = (const uint32_t *)fake_const_buf;
503 }
504 }
505
506 for (i = 0; i < ARRAY_SIZE(llvm->tes_jit_context.constants); ++i) {
507 int num_consts = get_num_consts_robust(draw, draw->pt.user.tes_constants_size, i);
508 llvm->tes_jit_context.constants[i] = draw->pt.user.tes_constants[i];
509 llvm->tes_jit_context.num_constants[i] = num_consts;
510 if (num_consts == 0) {
511 llvm->tes_jit_context.constants[i] = fake_const_buf;
512 }
513 }
514 for (i = 0; i < ARRAY_SIZE(llvm->tes_jit_context.ssbos); ++i) {
515 int num_ssbos = draw->pt.user.tes_ssbos_size[i];
516 llvm->tes_jit_context.ssbos[i] = draw->pt.user.tes_ssbos[i];
517 llvm->tes_jit_context.num_ssbos[i] = num_ssbos;
518 if (num_ssbos == 0) {
519 llvm->tes_jit_context.ssbos[i] = (const uint32_t *)fake_const_buf;
520 }
521 }
522
523 llvm->jit_context.planes =
524 (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
525 llvm->gs_jit_context.planes =
526 (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
527
528 llvm->jit_context.viewports = draw->viewports;
529 llvm->gs_jit_context.viewports = draw->viewports;
530
531 llvm->jit_context.aniso_filter_table = lp_build_sample_aniso_filter_table();
532 llvm->gs_jit_context.aniso_filter_table = lp_build_sample_aniso_filter_table();
533 llvm->tcs_jit_context.aniso_filter_table = lp_build_sample_aniso_filter_table();
534 llvm->tes_jit_context.aniso_filter_table = lp_build_sample_aniso_filter_table();
535 }
536
537
538 static void
pipeline(struct llvm_middle_end * llvm,const struct draw_vertex_info * vert_info,const struct draw_prim_info * prim_info)539 pipeline(struct llvm_middle_end *llvm,
540 const struct draw_vertex_info *vert_info,
541 const struct draw_prim_info *prim_info)
542 {
543 if (prim_info->linear)
544 draw_pipeline_run_linear( llvm->draw,
545 vert_info,
546 prim_info);
547 else
548 draw_pipeline_run( llvm->draw,
549 vert_info,
550 prim_info );
551 }
552
553
554 static void
emit(struct pt_emit * emit,const struct draw_vertex_info * vert_info,const struct draw_prim_info * prim_info)555 emit(struct pt_emit *emit,
556 const struct draw_vertex_info *vert_info,
557 const struct draw_prim_info *prim_info)
558 {
559 if (prim_info->linear) {
560 draw_pt_emit_linear(emit, vert_info, prim_info);
561 }
562 else {
563 draw_pt_emit(emit, vert_info, prim_info);
564 }
565 }
566
567
568 static void
llvm_pipeline_generic(struct draw_pt_middle_end * middle,const struct draw_fetch_info * fetch_info,const struct draw_prim_info * in_prim_info)569 llvm_pipeline_generic(struct draw_pt_middle_end *middle,
570 const struct draw_fetch_info *fetch_info,
571 const struct draw_prim_info *in_prim_info)
572 {
573 struct llvm_middle_end *fpme = llvm_middle_end(middle);
574 struct draw_context *draw = fpme->draw;
575 struct draw_geometry_shader *gshader = draw->gs.geometry_shader;
576 struct draw_tess_ctrl_shader *tcs_shader = draw->tcs.tess_ctrl_shader;
577 struct draw_tess_eval_shader *tes_shader = draw->tes.tess_eval_shader;
578 struct draw_prim_info tcs_prim_info;
579 struct draw_prim_info tes_prim_info;
580 struct draw_prim_info gs_prim_info[TGSI_MAX_VERTEX_STREAMS];
581 struct draw_vertex_info llvm_vert_info;
582 struct draw_vertex_info tcs_vert_info;
583 struct draw_vertex_info tes_vert_info;
584 struct draw_vertex_info gs_vert_info[TGSI_MAX_VERTEX_STREAMS];
585 struct draw_vertex_info *vert_info;
586 struct draw_prim_info ia_prim_info;
587 struct draw_vertex_info ia_vert_info;
588 const struct draw_prim_info *prim_info = in_prim_info;
589 boolean free_prim_info = FALSE;
590 unsigned opt = fpme->opt;
591 boolean clipped = 0;
592 unsigned start_or_maxelt, vid_base;
593 const unsigned *elts;
594 ushort *tes_elts_out = NULL;
595
596 memset(&gs_vert_info, 0, sizeof(struct draw_vertex_info) * TGSI_MAX_VERTEX_STREAMS);
597 assert(fetch_info->count > 0);
598 llvm_vert_info.count = fetch_info->count;
599 llvm_vert_info.vertex_size = fpme->vertex_size;
600 llvm_vert_info.stride = fpme->vertex_size;
601 llvm_vert_info.verts = (struct vertex_header *)
602 MALLOC(fpme->vertex_size *
603 align(fetch_info->count, lp_native_vector_width / 32) +
604 DRAW_EXTRA_VERTICES_PADDING);
605 if (!llvm_vert_info.verts) {
606 assert(0);
607 return;
608 }
609
610 if (draw->collect_statistics) {
611 draw->statistics.ia_vertices += prim_info->count;
612 if (prim_info->prim == PIPE_PRIM_PATCHES)
613 draw->statistics.ia_primitives += prim_info->count / draw->pt.vertices_per_patch;
614 else
615 draw->statistics.ia_primitives +=
616 u_decomposed_prims_for_vertices(prim_info->prim, prim_info->count);
617 draw->statistics.vs_invocations += fetch_info->count;
618 }
619
620 if (fetch_info->linear) {
621 start_or_maxelt = fetch_info->start;
622 vid_base = draw->start_index;
623 elts = NULL;
624 }
625 else {
626 start_or_maxelt = draw->pt.user.eltMax;
627 vid_base = draw->pt.user.eltBias;
628 elts = fetch_info->elts;
629 }
630 clipped = fpme->current_variant->jit_func(&fpme->llvm->jit_context,
631 llvm_vert_info.verts,
632 draw->pt.user.vbuffer,
633 fetch_info->count,
634 start_or_maxelt,
635 fpme->vertex_size,
636 draw->pt.vertex_buffer,
637 draw->instance_id,
638 vid_base,
639 draw->start_instance,
640 elts, draw->pt.user.drawid,
641 draw->pt.user.viewid);
642
643 /* Finished with fetch and vs:
644 */
645 fetch_info = NULL;
646 vert_info = &llvm_vert_info;
647
648 if (opt & PT_SHADE) {
649 struct draw_vertex_shader *vshader = draw->vs.vertex_shader;
650 if (tcs_shader) {
651 draw_tess_ctrl_shader_run(tcs_shader,
652 draw->pt.user.tcs_constants,
653 draw->pt.user.tcs_constants_size,
654 vert_info,
655 prim_info,
656 &vshader->info,
657 &tcs_vert_info,
658 &tcs_prim_info);
659 FREE(vert_info->verts);
660 vert_info = &tcs_vert_info;
661 prim_info = &tcs_prim_info;
662
663 } else if (tes_shader) {
664 unsigned num_prims = prim_info->count / draw->pt.vertices_per_patch;
665 tcs_prim_info = *prim_info;
666 tcs_prim_info.primitive_count = num_prims;
667 prim_info = &tcs_prim_info;
668 }
669
670 if (tes_shader) {
671 draw_tess_eval_shader_run(tes_shader,
672 draw->pt.user.tes_constants,
673 draw->pt.user.tes_constants_size,
674 tcs_shader ? tcs_shader->vertices_out : draw->pt.vertices_per_patch,
675 vert_info,
676 prim_info,
677 tcs_shader ? &tcs_shader->info : &vshader->info,
678 &tes_vert_info,
679 &tes_prim_info, &tes_elts_out);
680
681 FREE(vert_info->verts);
682 vert_info = &tes_vert_info;
683 prim_info = &tes_prim_info;
684 free_prim_info = TRUE;
685
686 /*
687 * pt emit can only handle ushort number of vertices (see
688 * render->allocate_vertices).
689 * vsplit guarantees there's never more than 4096, however GS can
690 * easily blow this up (by a factor of 256 (or even 1024) max).
691 */
692 if (vert_info->count > 65535) {
693 opt |= PT_PIPELINE;
694 }
695 }
696 }
697
698 if ((opt & PT_SHADE) && gshader) {
699 struct draw_vertex_shader *vshader = draw->vs.vertex_shader;
700 draw_geometry_shader_run(gshader,
701 draw->pt.user.gs_constants,
702 draw->pt.user.gs_constants_size,
703 vert_info,
704 prim_info,
705 tes_shader ? &tes_shader->info : &vshader->info,
706 gs_vert_info,
707 gs_prim_info);
708
709 FREE(vert_info->verts);
710 if (free_prim_info) {
711 FREE(prim_info->primitive_lengths);
712 FREE(tes_elts_out);
713 }
714 vert_info = &gs_vert_info[0];
715 prim_info = &gs_prim_info[0];
716 free_prim_info = FALSE;
717 /*
718 * pt emit can only handle ushort number of vertices (see
719 * render->allocate_vertices).
720 * vsplit guarantees there's never more than 4096, however GS can
721 * easily blow this up (by a factor of 256 (or even 1024) max).
722 */
723 if (vert_info->count > 65535) {
724 opt |= PT_PIPELINE;
725 }
726 } else {
727 if (!tes_shader && draw_prim_assembler_is_required(draw, prim_info, vert_info)) {
728 draw_prim_assembler_run(draw, prim_info, vert_info,
729 &ia_prim_info, &ia_vert_info);
730
731 if (ia_vert_info.count) {
732 FREE(vert_info->verts);
733 if (free_prim_info) {
734 FREE(prim_info->primitive_lengths);
735 FREE(tes_elts_out);
736 tes_elts_out = NULL;
737 }
738 vert_info = &ia_vert_info;
739 prim_info = &ia_prim_info;
740 free_prim_info = TRUE;
741 }
742 }
743 }
744
745 /* stream output needs to be done before clipping */
746 draw_pt_so_emit( fpme->so_emit, gshader ? gshader->num_vertex_streams : 1, vert_info, prim_info );
747
748 if (prim_info->count == 0) {
749 debug_printf("GS/IA didn't emit any vertices!\n");
750 goto out;
751 }
752
753 draw_stats_clipper_primitives(draw, prim_info);
754
755 /*
756 * if there's no position, need to stop now, or the latter stages
757 * will try to access non-existent position output.
758 */
759 if (draw_current_shader_position_output(draw) != -1) {
760 if ((opt & PT_SHADE) && (gshader || tes_shader ||
761 draw->vs.vertex_shader->info.writes_viewport_index)) {
762 clipped = draw_pt_post_vs_run( fpme->post_vs, vert_info, prim_info );
763 }
764 /* "clipped" also includes non-one edgeflag */
765 if (clipped) {
766 opt |= PT_PIPELINE;
767 }
768
769 /* Do we need to run the pipeline? Now will come here if clipped
770 */
771 if (opt & PT_PIPELINE) {
772 pipeline( fpme, vert_info, prim_info );
773 }
774 else {
775 emit( fpme->emit, vert_info, prim_info );
776 }
777 }
778 out:
779 FREE(vert_info->verts);
780 if (gshader && gshader->num_vertex_streams > 1)
781 for (unsigned i = 1; i < gshader->num_vertex_streams; i++)
782 FREE(gs_vert_info[i].verts);
783
784 if (free_prim_info) {
785 FREE(tes_elts_out);
786 FREE(prim_info->primitive_lengths);
787 }
788 }
789
790
791 static inline unsigned
prim_type(unsigned prim,unsigned flags)792 prim_type(unsigned prim, unsigned flags)
793 {
794 if (flags & DRAW_LINE_LOOP_AS_STRIP)
795 return PIPE_PRIM_LINE_STRIP;
796 else
797 return prim;
798 }
799
800
801 static void
llvm_middle_end_run(struct draw_pt_middle_end * middle,const unsigned * fetch_elts,unsigned fetch_count,const ushort * draw_elts,unsigned draw_count,unsigned prim_flags)802 llvm_middle_end_run(struct draw_pt_middle_end *middle,
803 const unsigned *fetch_elts,
804 unsigned fetch_count,
805 const ushort *draw_elts,
806 unsigned draw_count,
807 unsigned prim_flags)
808 {
809 struct llvm_middle_end *fpme = llvm_middle_end(middle);
810 struct draw_fetch_info fetch_info;
811 struct draw_prim_info prim_info;
812
813 fetch_info.linear = FALSE;
814 fetch_info.start = 0;
815 fetch_info.elts = fetch_elts;
816 fetch_info.count = fetch_count;
817
818 prim_info.linear = FALSE;
819 prim_info.start = 0;
820 prim_info.count = draw_count;
821 prim_info.elts = draw_elts;
822 prim_info.prim = prim_type(fpme->input_prim, prim_flags);
823 prim_info.flags = prim_flags;
824 prim_info.primitive_count = 1;
825 prim_info.primitive_lengths = &draw_count;
826
827 llvm_pipeline_generic( middle, &fetch_info, &prim_info );
828 }
829
830
831 static void
llvm_middle_end_linear_run(struct draw_pt_middle_end * middle,unsigned start,unsigned count,unsigned prim_flags)832 llvm_middle_end_linear_run(struct draw_pt_middle_end *middle,
833 unsigned start,
834 unsigned count,
835 unsigned prim_flags)
836 {
837 struct llvm_middle_end *fpme = llvm_middle_end(middle);
838 struct draw_fetch_info fetch_info;
839 struct draw_prim_info prim_info;
840
841 fetch_info.linear = TRUE;
842 fetch_info.start = start;
843 fetch_info.count = count;
844 fetch_info.elts = NULL;
845
846 prim_info.linear = TRUE;
847 prim_info.start = 0;
848 prim_info.count = count;
849 prim_info.elts = NULL;
850 prim_info.prim = prim_type(fpme->input_prim, prim_flags);
851 prim_info.flags = prim_flags;
852 prim_info.primitive_count = 1;
853 prim_info.primitive_lengths = &count;
854
855 llvm_pipeline_generic( middle, &fetch_info, &prim_info );
856 }
857
858
859 static boolean
llvm_middle_end_linear_run_elts(struct draw_pt_middle_end * middle,unsigned start,unsigned count,const ushort * draw_elts,unsigned draw_count,unsigned prim_flags)860 llvm_middle_end_linear_run_elts(struct draw_pt_middle_end *middle,
861 unsigned start,
862 unsigned count,
863 const ushort *draw_elts,
864 unsigned draw_count,
865 unsigned prim_flags)
866 {
867 struct llvm_middle_end *fpme = llvm_middle_end(middle);
868 struct draw_fetch_info fetch_info;
869 struct draw_prim_info prim_info;
870
871 fetch_info.linear = TRUE;
872 fetch_info.start = start;
873 fetch_info.count = count;
874 fetch_info.elts = NULL;
875
876 prim_info.linear = FALSE;
877 prim_info.start = 0;
878 prim_info.count = draw_count;
879 prim_info.elts = draw_elts;
880 prim_info.prim = prim_type(fpme->input_prim, prim_flags);
881 prim_info.flags = prim_flags;
882 prim_info.primitive_count = 1;
883 prim_info.primitive_lengths = &draw_count;
884
885 llvm_pipeline_generic( middle, &fetch_info, &prim_info );
886
887 return TRUE;
888 }
889
890
891 static void
llvm_middle_end_finish(struct draw_pt_middle_end * middle)892 llvm_middle_end_finish(struct draw_pt_middle_end *middle)
893 {
894 /* nothing to do */
895 }
896
897
898 static void
llvm_middle_end_destroy(struct draw_pt_middle_end * middle)899 llvm_middle_end_destroy(struct draw_pt_middle_end *middle)
900 {
901 struct llvm_middle_end *fpme = llvm_middle_end(middle);
902
903 if (fpme->fetch)
904 draw_pt_fetch_destroy( fpme->fetch );
905
906 if (fpme->emit)
907 draw_pt_emit_destroy( fpme->emit );
908
909 if (fpme->so_emit)
910 draw_pt_so_emit_destroy( fpme->so_emit );
911
912 if (fpme->post_vs)
913 draw_pt_post_vs_destroy( fpme->post_vs );
914
915 FREE(middle);
916 }
917
918
919 struct draw_pt_middle_end *
draw_pt_fetch_pipeline_or_emit_llvm(struct draw_context * draw)920 draw_pt_fetch_pipeline_or_emit_llvm(struct draw_context *draw)
921 {
922 struct llvm_middle_end *fpme = 0;
923
924 if (!draw->llvm)
925 return NULL;
926
927 fpme = CALLOC_STRUCT( llvm_middle_end );
928 if (!fpme)
929 goto fail;
930
931 fpme->base.prepare = llvm_middle_end_prepare;
932 fpme->base.bind_parameters = llvm_middle_end_bind_parameters;
933 fpme->base.run = llvm_middle_end_run;
934 fpme->base.run_linear = llvm_middle_end_linear_run;
935 fpme->base.run_linear_elts = llvm_middle_end_linear_run_elts;
936 fpme->base.finish = llvm_middle_end_finish;
937 fpme->base.destroy = llvm_middle_end_destroy;
938
939 fpme->draw = draw;
940
941 fpme->fetch = draw_pt_fetch_create( draw );
942 if (!fpme->fetch)
943 goto fail;
944
945 fpme->post_vs = draw_pt_post_vs_create( draw );
946 if (!fpme->post_vs)
947 goto fail;
948
949 fpme->emit = draw_pt_emit_create( draw );
950 if (!fpme->emit)
951 goto fail;
952
953 fpme->so_emit = draw_pt_so_emit_create( draw );
954 if (!fpme->so_emit)
955 goto fail;
956
957 fpme->llvm = draw->llvm;
958 if (!fpme->llvm)
959 goto fail;
960
961 fpme->current_variant = NULL;
962
963 return &fpme->base;
964
965 fail:
966 if (fpme)
967 llvm_middle_end_destroy( &fpme->base );
968
969 return NULL;
970 }
971