/**************************************************************************
 *
 * Copyright 2010 VMware, Inc.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 **************************************************************************/

#include "util/u_math.h"
#include "util/u_memory.h"
#include "util/u_prim.h"
#include "draw/draw_context.h"
#include "draw/draw_gs.h"
#include "draw/draw_tess.h"
#include "draw/draw_vbuf.h"
#include "draw/draw_vertex.h"
#include "draw/draw_pt.h"
#include "draw/draw_prim_assembler.h"
#include "draw/draw_vs.h"
#include "draw/draw_llvm.h"
#include "gallivm/lp_bld_init.h"
#include "gallivm/lp_bld_debug.h"


struct llvm_middle_end {
   struct draw_pt_middle_end base;
   struct draw_context *draw;

   struct pt_emit *emit;
   struct pt_so_emit *so_emit;
   struct pt_fetch *fetch;
   struct pt_post_vs *post_vs;


   unsigned vertex_data_offset;
   unsigned vertex_size;
   unsigned input_prim;
   unsigned opt;

   struct draw_llvm *llvm;
   struct draw_llvm_variant *current_variant;
};


/** cast wrapper */
static inline struct llvm_middle_end *
llvm_middle_end(struct draw_pt_middle_end *middle)
{
   return (struct llvm_middle_end *) middle;
}


static void
llvm_middle_end_prepare_gs(struct llvm_middle_end *fpme)
{
   struct draw_context *draw = fpme->draw;
   struct draw_llvm *llvm = fpme->llvm;
   struct draw_geometry_shader *gs = draw->gs.geometry_shader;
   struct draw_gs_llvm_variant_key *key;
   struct draw_gs_llvm_variant *variant = NULL;
   struct draw_gs_llvm_variant_list_item *li;
   struct llvm_geometry_shader *shader = llvm_geometry_shader(gs);
   char store[DRAW_GS_LLVM_MAX_VARIANT_KEY_SIZE];
   unsigned i;

   key = draw_gs_llvm_make_variant_key(llvm, store);

   /* Search shader's list of variants for the key */
   li = first_elem(&shader->variants);
   while (!at_end(&shader->variants, li)) {
      if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
         variant = li->base;
         break;
      }
      li = next_elem(li);
   }

   if (variant) {
      /* found the variant, move to head of global list (for LRU) */
      move_to_head(&llvm->gs_variants_list, &variant->list_item_global);
   }
   else {
      /* Need to create new variant */

      /* First check if we've created too many variants.  If so, free
       * 3.125% of the LRU to avoid using too much memory.
       */
      if (llvm->nr_gs_variants >= DRAW_MAX_SHADER_VARIANTS) {
         if (gallivm_debug & GALLIVM_DEBUG_PERF) {
            debug_printf("Evicting GS: %u gs variants,\t%u total variants\n",
                      shader->variants_cached, llvm->nr_gs_variants);
         }

         /*
          * XXX: should we flush here ?
          */
         for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
            struct draw_gs_llvm_variant_list_item *item;
            if (is_empty_list(&llvm->gs_variants_list)) {
               break;
            }
            item = last_elem(&llvm->gs_variants_list);
            assert(item);
            assert(item->base);
            draw_gs_llvm_destroy_variant(item->base);
         }
      }

      variant = draw_gs_llvm_create_variant(llvm, gs->info.num_outputs, key);

      if (variant) {
         insert_at_head(&shader->variants, &variant->list_item_local);
         insert_at_head(&llvm->gs_variants_list,
                        &variant->list_item_global);
         llvm->nr_gs_variants++;
         shader->variants_cached++;
      }
   }

   gs->current_variant = variant;
}

static void
llvm_middle_end_prepare_tcs(struct llvm_middle_end *fpme)
{
   struct draw_context *draw = fpme->draw;
   struct draw_llvm *llvm = fpme->llvm;
   struct draw_tess_ctrl_shader *tcs = draw->tcs.tess_ctrl_shader;
   struct draw_tcs_llvm_variant_key *key;
   struct draw_tcs_llvm_variant *variant = NULL;
   struct draw_tcs_llvm_variant_list_item *li;
   struct llvm_tess_ctrl_shader *shader = llvm_tess_ctrl_shader(tcs);
   char store[DRAW_TCS_LLVM_MAX_VARIANT_KEY_SIZE];
   unsigned i;

   key = draw_tcs_llvm_make_variant_key(llvm, store);

   /* Search shader's list of variants for the key */
   li = first_elem(&shader->variants);
   while (!at_end(&shader->variants, li)) {
      if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
         variant = li->base;
         break;
      }
      li = next_elem(li);
   }

   if (variant) {
      /* found the variant, move to head of global list (for LRU) */
      move_to_head(&llvm->tcs_variants_list, &variant->list_item_global);
   }
   else {
      /* Need to create new variant */

      /* First check if we've created too many variants.  If so, free
       * 3.125% of the LRU to avoid using too much memory.
       */
      if (llvm->nr_tcs_variants >= DRAW_MAX_SHADER_VARIANTS) {
         if (gallivm_debug & GALLIVM_DEBUG_PERF) {
            debug_printf("Evicting TCS: %u tcs variants,\t%u total variants\n",
                      shader->variants_cached, llvm->nr_tcs_variants);
         }

         /*
          * XXX: should we flush here ?
          */
         for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
            struct draw_tcs_llvm_variant_list_item *item;
            if (is_empty_list(&llvm->tcs_variants_list)) {
               break;
            }
            item = last_elem(&llvm->tcs_variants_list);
            assert(item);
            assert(item->base);
            draw_tcs_llvm_destroy_variant(item->base);
         }
      }

      variant = draw_tcs_llvm_create_variant(llvm, 0, key);

      if (variant) {
         insert_at_head(&shader->variants, &variant->list_item_local);
         insert_at_head(&llvm->tcs_variants_list,
                        &variant->list_item_global);
         llvm->nr_tcs_variants++;
         shader->variants_cached++;
      }
   }

   tcs->current_variant = variant;
}

static void
llvm_middle_end_prepare_tes(struct llvm_middle_end *fpme)
{
   struct draw_context *draw = fpme->draw;
   struct draw_llvm *llvm = fpme->llvm;
   struct draw_tess_eval_shader *tes = draw->tes.tess_eval_shader;
   struct draw_tes_llvm_variant_key *key;
   struct draw_tes_llvm_variant *variant = NULL;
   struct draw_tes_llvm_variant_list_item *li;
   struct llvm_tess_eval_shader *shader = llvm_tess_eval_shader(tes);
   char store[DRAW_TES_LLVM_MAX_VARIANT_KEY_SIZE];
   unsigned i;

   key = draw_tes_llvm_make_variant_key(llvm, store);

   /* Search shader's list of variants for the key */
   li = first_elem(&shader->variants);
   while (!at_end(&shader->variants, li)) {
      if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
         variant = li->base;
         break;
      }
      li = next_elem(li);
   }

   if (variant) {
      /* found the variant, move to head of global list (for LRU) */
      move_to_head(&llvm->tes_variants_list, &variant->list_item_global);
   }
   else {
      /* Need to create new variant */

      /* First check if we've created too many variants.  If so, free
       * 3.125% of the LRU to avoid using too much memory.
       */
      if (llvm->nr_tes_variants >= DRAW_MAX_SHADER_VARIANTS) {
         if (gallivm_debug & GALLIVM_DEBUG_PERF) {
            debug_printf("Evicting TES: %u tes variants,\t%u total variants\n",
                      shader->variants_cached, llvm->nr_tes_variants);
         }

         /*
          * XXX: should we flush here ?
          */
         for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
            struct draw_tes_llvm_variant_list_item *item;
            if (is_empty_list(&llvm->tes_variants_list)) {
               break;
            }
            item = last_elem(&llvm->tes_variants_list);
            assert(item);
            assert(item->base);
            draw_tes_llvm_destroy_variant(item->base);
         }
      }

      variant = draw_tes_llvm_create_variant(llvm, draw_total_tes_outputs(draw), key);

      if (variant) {
         insert_at_head(&shader->variants, &variant->list_item_local);
         insert_at_head(&llvm->tes_variants_list,
                        &variant->list_item_global);
         llvm->nr_tes_variants++;
         shader->variants_cached++;
      }
   }

   tes->current_variant = variant;
}

/**
 * Prepare/validate middle part of the vertex pipeline.
 * NOTE: if you change this function, also look at the non-LLVM
 * function fetch_pipeline_prepare() for similar changes.
 */
static void
llvm_middle_end_prepare( struct draw_pt_middle_end *middle,
                         unsigned in_prim,
                         unsigned opt,
                         unsigned *max_vertices )
{
   struct llvm_middle_end *fpme = llvm_middle_end(middle);
   struct draw_context *draw = fpme->draw;
   struct draw_llvm *llvm = fpme->llvm;
   struct draw_vertex_shader *vs = draw->vs.vertex_shader;
   struct draw_geometry_shader *gs = draw->gs.geometry_shader;
   struct draw_tess_ctrl_shader *tcs = draw->tcs.tess_ctrl_shader;
   struct draw_tess_eval_shader *tes = draw->tes.tess_eval_shader;
   const unsigned out_prim = gs ? gs->output_primitive : tes ? get_tes_output_prim(tes) :
      u_assembled_prim(in_prim);
   unsigned point_clip = draw->rasterizer->fill_front == PIPE_POLYGON_MODE_POINT ||
                         out_prim == PIPE_PRIM_POINTS;
   unsigned nr;

   fpme->input_prim = in_prim;
   fpme->opt = opt;

   draw_pt_post_vs_prepare( fpme->post_vs,
                            draw->clip_xy,
                            draw->clip_z,
                            draw->clip_user,
                            point_clip ? draw->guard_band_points_xy :
                                         draw->guard_band_xy,
                            draw->bypass_viewport,
                            draw->rasterizer->clip_halfz,
                            (draw->vs.edgeflag_output ? TRUE : FALSE) );

   draw_pt_so_emit_prepare( fpme->so_emit, gs == NULL );

   if (!(opt & PT_PIPELINE)) {
      draw_pt_emit_prepare( fpme->emit, out_prim,
                            max_vertices );

      *max_vertices = MAX2( *max_vertices, 4096 );
   }
   else {
      /* limit max fetches by limiting max_vertices */
      *max_vertices = 4096;
   }

   /* Get the number of float[4] attributes per vertex.
    * Note: this must be done after draw_pt_emit_prepare() since that
    * can effect the vertex size.
    */
   nr = MAX2(vs->info.num_inputs, draw_total_vs_outputs(draw));

   /* Always leave room for the vertex header whether we need it or
    * not.  It's hard to get rid of it in particular because of the
    * viewport code in draw_pt_post_vs.c.
    */
   fpme->vertex_size = sizeof(struct vertex_header) + nr * 4 * sizeof(float);

   /* return even number */
   *max_vertices = *max_vertices & ~1;

   /* Find/create the vertex shader variant */
   {
      struct draw_llvm_variant_key *key;
      struct draw_llvm_variant *variant = NULL;
      struct draw_llvm_variant_list_item *li;
      struct llvm_vertex_shader *shader = llvm_vertex_shader(vs);
      char store[DRAW_LLVM_MAX_VARIANT_KEY_SIZE];
      unsigned i;

      key = draw_llvm_make_variant_key(llvm, store);

      /* Search shader's list of variants for the key */
      li = first_elem(&shader->variants);
      while (!at_end(&shader->variants, li)) {
         if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
            variant = li->base;
            break;
         }
         li = next_elem(li);
      }

      if (variant) {
         /* found the variant, move to head of global list (for LRU) */
         move_to_head(&llvm->vs_variants_list, &variant->list_item_global);
      }
      else {
         /* Need to create new variant */

         /* First check if we've created too many variants.  If so, free
          * 3.125% of the LRU to avoid using too much memory.
          */
         if (llvm->nr_variants >= DRAW_MAX_SHADER_VARIANTS) {
            if (gallivm_debug & GALLIVM_DEBUG_PERF) {
               debug_printf("Evicting VS: %u vs variants,\t%u total variants\n",
                         shader->variants_cached, llvm->nr_variants);
            }

            /*
             * XXX: should we flush here ?
             */
            for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
               struct draw_llvm_variant_list_item *item;
               if (is_empty_list(&llvm->vs_variants_list)) {
                  break;
               }
               item = last_elem(&llvm->vs_variants_list);
               assert(item);
               assert(item->base);
               draw_llvm_destroy_variant(item->base);
            }
         }

         variant = draw_llvm_create_variant(llvm, nr, key);

         if (variant) {
            insert_at_head(&shader->variants, &variant->list_item_local);
            insert_at_head(&llvm->vs_variants_list,
                           &variant->list_item_global);
            llvm->nr_variants++;
            shader->variants_cached++;
         }
      }

      fpme->current_variant = variant;
   }

   if (gs) {
      llvm_middle_end_prepare_gs(fpme);
   }
   if (tcs) {
      llvm_middle_end_prepare_tcs(fpme);
   }
   if (tes) {
      llvm_middle_end_prepare_tes(fpme);
   }
}

static unsigned
get_num_consts_robust(struct draw_context *draw, unsigned *sizes, unsigned idx)
{
   unsigned const_bytes = sizes[idx];

   if (const_bytes < sizeof(float))
      return 0;

   return DIV_ROUND_UP(const_bytes, draw->constant_buffer_stride);
}

/**
 * Bind/update constant buffer pointers, clip planes and viewport dims.
 * These are "light weight" parameters which aren't baked into the
 * generated code.  Updating these items is much cheaper than revalidating
 * and rebuilding the generated pipeline code.
 */
static void
llvm_middle_end_bind_parameters(struct draw_pt_middle_end *middle)
{
   static const float fake_const_buf[4];
   struct llvm_middle_end *fpme = llvm_middle_end(middle);
   struct draw_context *draw = fpme->draw;
   struct draw_llvm *llvm = fpme->llvm;
   unsigned i;

   for (i = 0; i < ARRAY_SIZE(llvm->jit_context.vs_constants); ++i) {
      /*
       * There could be a potential issue with rounding this up, as the
       * shader expects 16-byte allocations, the fix is likely to move
       * to LOAD intrinsic in the future and remove the vec4 constraint.
       */
      int num_consts = get_num_consts_robust(draw, draw->pt.user.vs_constants_size, i);
      llvm->jit_context.vs_constants[i] = draw->pt.user.vs_constants[i];
      llvm->jit_context.num_vs_constants[i] = num_consts;
      if (num_consts == 0) {
         llvm->jit_context.vs_constants[i] = fake_const_buf;
      }
   }
   for (i = 0; i < ARRAY_SIZE(llvm->jit_context.vs_ssbos); ++i) {
      int num_ssbos = draw->pt.user.vs_ssbos_size[i];
      llvm->jit_context.vs_ssbos[i] = draw->pt.user.vs_ssbos[i];
      llvm->jit_context.num_vs_ssbos[i] = num_ssbos;
      if (num_ssbos == 0) {
         llvm->jit_context.vs_ssbos[i] = (const uint32_t *)fake_const_buf;
      }
   }

   for (i = 0; i < ARRAY_SIZE(llvm->gs_jit_context.constants); ++i) {
      int num_consts = get_num_consts_robust(draw, draw->pt.user.gs_constants_size, i);
      llvm->gs_jit_context.constants[i] = draw->pt.user.gs_constants[i];
      llvm->gs_jit_context.num_constants[i] = num_consts;
      if (num_consts == 0) {
         llvm->gs_jit_context.constants[i] = fake_const_buf;
      }
   }
   for (i = 0; i < ARRAY_SIZE(llvm->gs_jit_context.ssbos); ++i) {
      int num_ssbos = draw->pt.user.gs_ssbos_size[i];
      llvm->gs_jit_context.ssbos[i] = draw->pt.user.gs_ssbos[i];
      llvm->gs_jit_context.num_ssbos[i] = num_ssbos;
      if (num_ssbos == 0) {
         llvm->gs_jit_context.ssbos[i] = (const uint32_t *)fake_const_buf;
      }
   }

   for (i = 0; i < ARRAY_SIZE(llvm->tcs_jit_context.constants); ++i) {
      int num_consts = get_num_consts_robust(draw, draw->pt.user.tcs_constants_size, i);
      llvm->tcs_jit_context.constants[i] = draw->pt.user.tcs_constants[i];
      llvm->tcs_jit_context.num_constants[i] = num_consts;
      if (num_consts == 0) {
         llvm->tcs_jit_context.constants[i] = fake_const_buf;
      }
   }
   for (i = 0; i < ARRAY_SIZE(llvm->tcs_jit_context.ssbos); ++i) {
      int num_ssbos = draw->pt.user.tcs_ssbos_size[i];
      llvm->tcs_jit_context.ssbos[i] = draw->pt.user.tcs_ssbos[i];
      llvm->tcs_jit_context.num_ssbos[i] = num_ssbos;
      if (num_ssbos == 0) {
         llvm->tcs_jit_context.ssbos[i] = (const uint32_t *)fake_const_buf;
      }
   }

   for (i = 0; i < ARRAY_SIZE(llvm->tes_jit_context.constants); ++i) {
      int num_consts = get_num_consts_robust(draw, draw->pt.user.tes_constants_size, i);
      llvm->tes_jit_context.constants[i] = draw->pt.user.tes_constants[i];
      llvm->tes_jit_context.num_constants[i] = num_consts;
      if (num_consts == 0) {
         llvm->tes_jit_context.constants[i] = fake_const_buf;
      }
   }
   for (i = 0; i < ARRAY_SIZE(llvm->tes_jit_context.ssbos); ++i) {
      int num_ssbos = draw->pt.user.tes_ssbos_size[i];
      llvm->tes_jit_context.ssbos[i] = draw->pt.user.tes_ssbos[i];
      llvm->tes_jit_context.num_ssbos[i] = num_ssbos;
      if (num_ssbos == 0) {
         llvm->tes_jit_context.ssbos[i] = (const uint32_t *)fake_const_buf;
      }
   }

   llvm->jit_context.planes =
      (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
   llvm->gs_jit_context.planes =
      (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];

   llvm->jit_context.viewports = draw->viewports;
   llvm->gs_jit_context.viewports = draw->viewports;

   llvm->jit_context.aniso_filter_table = lp_build_sample_aniso_filter_table();
   llvm->gs_jit_context.aniso_filter_table = lp_build_sample_aniso_filter_table();
   llvm->tcs_jit_context.aniso_filter_table = lp_build_sample_aniso_filter_table();
   llvm->tes_jit_context.aniso_filter_table = lp_build_sample_aniso_filter_table();
}


static void
pipeline(struct llvm_middle_end *llvm,
         const struct draw_vertex_info *vert_info,
         const struct draw_prim_info *prim_info)
{
   if (prim_info->linear)
      draw_pipeline_run_linear( llvm->draw,
                                vert_info,
                                prim_info);
   else
      draw_pipeline_run( llvm->draw,
                         vert_info,
                         prim_info );
}


static void
emit(struct pt_emit *emit,
     const struct draw_vertex_info *vert_info,
     const struct draw_prim_info *prim_info)
{
   if (prim_info->linear) {
      draw_pt_emit_linear(emit, vert_info, prim_info);
   }
   else {
      draw_pt_emit(emit, vert_info, prim_info);
   }
}


static void
llvm_pipeline_generic(struct draw_pt_middle_end *middle,
                      const struct draw_fetch_info *fetch_info,
                      const struct draw_prim_info *in_prim_info)
{
   struct llvm_middle_end *fpme = llvm_middle_end(middle);
   struct draw_context *draw = fpme->draw;
   struct draw_geometry_shader *gshader = draw->gs.geometry_shader;
   struct draw_tess_ctrl_shader *tcs_shader = draw->tcs.tess_ctrl_shader;
   struct draw_tess_eval_shader *tes_shader = draw->tes.tess_eval_shader;
   struct draw_prim_info tcs_prim_info;
   struct draw_prim_info tes_prim_info;
   struct draw_prim_info gs_prim_info[TGSI_MAX_VERTEX_STREAMS];
   struct draw_vertex_info llvm_vert_info;
   struct draw_vertex_info tcs_vert_info;
   struct draw_vertex_info tes_vert_info;
   struct draw_vertex_info gs_vert_info[TGSI_MAX_VERTEX_STREAMS];
   struct draw_vertex_info *vert_info;
   struct draw_prim_info ia_prim_info;
   struct draw_vertex_info ia_vert_info;
   const struct draw_prim_info *prim_info = in_prim_info;
   boolean free_prim_info = FALSE;
   unsigned opt = fpme->opt;
   boolean clipped = 0;
   unsigned start_or_maxelt, vid_base;
   const unsigned *elts;
   ushort *tes_elts_out = NULL;

   memset(&gs_vert_info, 0, sizeof(struct draw_vertex_info) * TGSI_MAX_VERTEX_STREAMS);
   assert(fetch_info->count > 0);
   llvm_vert_info.count = fetch_info->count;
   llvm_vert_info.vertex_size = fpme->vertex_size;
   llvm_vert_info.stride = fpme->vertex_size;
   llvm_vert_info.verts = (struct vertex_header *)
      MALLOC(fpme->vertex_size *
             align(fetch_info->count, lp_native_vector_width / 32) +
             DRAW_EXTRA_VERTICES_PADDING);
   if (!llvm_vert_info.verts) {
      assert(0);
      return;
   }

   if (draw->collect_statistics) {
      draw->statistics.ia_vertices += prim_info->count;
      if (prim_info->prim == PIPE_PRIM_PATCHES)
         draw->statistics.ia_primitives += prim_info->count / draw->pt.vertices_per_patch;
      else
         draw->statistics.ia_primitives +=
            u_decomposed_prims_for_vertices(prim_info->prim, prim_info->count);
      draw->statistics.vs_invocations += fetch_info->count;
   }

   if (fetch_info->linear) {
      start_or_maxelt = fetch_info->start;
      vid_base = draw->start_index;
      elts = NULL;
   }
   else {
      start_or_maxelt = draw->pt.user.eltMax;
      vid_base = draw->pt.user.eltBias;
      elts = fetch_info->elts;
   }
   clipped = fpme->current_variant->jit_func(&fpme->llvm->jit_context,
                                             llvm_vert_info.verts,
                                             draw->pt.user.vbuffer,
                                             fetch_info->count,
                                             start_or_maxelt,
                                             fpme->vertex_size,
                                             draw->pt.vertex_buffer,
                                             draw->instance_id,
                                             vid_base,
                                             draw->start_instance,
                                             elts, draw->pt.user.drawid,
                                             draw->pt.user.viewid);

   /* Finished with fetch and vs:
    */
   fetch_info = NULL;
   vert_info = &llvm_vert_info;

   if (opt & PT_SHADE) {
      struct draw_vertex_shader *vshader = draw->vs.vertex_shader;
      if (tcs_shader) {
         draw_tess_ctrl_shader_run(tcs_shader,
                                   draw->pt.user.tcs_constants,
                                   draw->pt.user.tcs_constants_size,
                                   vert_info,
                                   prim_info,
                                   &vshader->info,
                                   &tcs_vert_info,
                                   &tcs_prim_info);
         FREE(vert_info->verts);
         vert_info = &tcs_vert_info;
         prim_info = &tcs_prim_info;

      } else if (tes_shader) {
         unsigned num_prims = prim_info->count / draw->pt.vertices_per_patch;
         tcs_prim_info = *prim_info;
         tcs_prim_info.primitive_count = num_prims;
         prim_info = &tcs_prim_info;
      }

      if (tes_shader) {
         draw_tess_eval_shader_run(tes_shader,
                                   draw->pt.user.tes_constants,
                                   draw->pt.user.tes_constants_size,
                                   tcs_shader ? tcs_shader->vertices_out : draw->pt.vertices_per_patch,
                                   vert_info,
                                   prim_info,
                                   tcs_shader ? &tcs_shader->info : &vshader->info,
                                   &tes_vert_info,
                                   &tes_prim_info, &tes_elts_out);

         FREE(vert_info->verts);
         vert_info = &tes_vert_info;
         prim_info = &tes_prim_info;
         free_prim_info = TRUE;

         /*
          * pt emit can only handle ushort number of vertices (see
          * render->allocate_vertices).
          * vsplit guarantees there's never more than 4096, however GS can
          * easily blow this up (by a factor of 256 (or even 1024) max).
          */
         if (vert_info->count > 65535) {
            opt |= PT_PIPELINE;
         }
      }
   }

   if ((opt & PT_SHADE) && gshader) {
      struct draw_vertex_shader *vshader = draw->vs.vertex_shader;
      draw_geometry_shader_run(gshader,
                               draw->pt.user.gs_constants,
                               draw->pt.user.gs_constants_size,
                               vert_info,
                               prim_info,
                               tes_shader ? &tes_shader->info : &vshader->info,
                               gs_vert_info,
                               gs_prim_info);

      FREE(vert_info->verts);
      if (free_prim_info) {
         FREE(prim_info->primitive_lengths);
         FREE(tes_elts_out);
      }
      vert_info = &gs_vert_info[0];
      prim_info = &gs_prim_info[0];
      free_prim_info = FALSE;
      /*
       * pt emit can only handle ushort number of vertices (see
       * render->allocate_vertices).
       * vsplit guarantees there's never more than 4096, however GS can
       * easily blow this up (by a factor of 256 (or even 1024) max).
       */
      if (vert_info->count > 65535) {
         opt |= PT_PIPELINE;
      }
   } else {
      if (!tes_shader && draw_prim_assembler_is_required(draw, prim_info, vert_info)) {
         draw_prim_assembler_run(draw, prim_info, vert_info,
                                 &ia_prim_info, &ia_vert_info);

         if (ia_vert_info.count) {
            FREE(vert_info->verts);
            if (free_prim_info) {
               FREE(prim_info->primitive_lengths);
               FREE(tes_elts_out);
               tes_elts_out = NULL;
            }
            vert_info = &ia_vert_info;
            prim_info = &ia_prim_info;
            free_prim_info = TRUE;
         }
      }
   }

   /* stream output needs to be done before clipping */
   draw_pt_so_emit( fpme->so_emit, gshader ? gshader->num_vertex_streams : 1, vert_info, prim_info );

   if (prim_info->count == 0) {
      debug_printf("GS/IA didn't emit any vertices!\n");
      goto out;
   }

   draw_stats_clipper_primitives(draw, prim_info);

   /*
    * if there's no position, need to stop now, or the latter stages
    * will try to access non-existent position output.
    */
   if (draw_current_shader_position_output(draw) != -1) {
      if ((opt & PT_SHADE) && (gshader || tes_shader ||
                               draw->vs.vertex_shader->info.writes_viewport_index)) {
         clipped = draw_pt_post_vs_run( fpme->post_vs, vert_info, prim_info );
      }
      /* "clipped" also includes non-one edgeflag */
      if (clipped) {
         opt |= PT_PIPELINE;
      }

      /* Do we need to run the pipeline? Now will come here if clipped
       */
      if (opt & PT_PIPELINE) {
         pipeline( fpme, vert_info, prim_info );
      }
      else {
         emit( fpme->emit, vert_info, prim_info );
      }
   }
out:
   FREE(vert_info->verts);
   if (gshader && gshader->num_vertex_streams > 1)
     for (unsigned i = 1; i < gshader->num_vertex_streams; i++)
       FREE(gs_vert_info[i].verts);

   if (free_prim_info) {
      FREE(tes_elts_out);
      FREE(prim_info->primitive_lengths);
   }
}


static inline unsigned
prim_type(unsigned prim, unsigned flags)
{
   if (flags & DRAW_LINE_LOOP_AS_STRIP)
      return PIPE_PRIM_LINE_STRIP;
   else
      return prim;
}


static void
llvm_middle_end_run(struct draw_pt_middle_end *middle,
                    const unsigned *fetch_elts,
                    unsigned fetch_count,
                    const ushort *draw_elts,
                    unsigned draw_count,
                    unsigned prim_flags)
{
   struct llvm_middle_end *fpme = llvm_middle_end(middle);
   struct draw_fetch_info fetch_info;
   struct draw_prim_info prim_info;

   fetch_info.linear = FALSE;
   fetch_info.start = 0;
   fetch_info.elts = fetch_elts;
   fetch_info.count = fetch_count;

   prim_info.linear = FALSE;
   prim_info.start = 0;
   prim_info.count = draw_count;
   prim_info.elts = draw_elts;
   prim_info.prim = prim_type(fpme->input_prim, prim_flags);
   prim_info.flags = prim_flags;
   prim_info.primitive_count = 1;
   prim_info.primitive_lengths = &draw_count;

   llvm_pipeline_generic( middle, &fetch_info, &prim_info );
}


static void
llvm_middle_end_linear_run(struct draw_pt_middle_end *middle,
                           unsigned start,
                           unsigned count,
                           unsigned prim_flags)
{
   struct llvm_middle_end *fpme = llvm_middle_end(middle);
   struct draw_fetch_info fetch_info;
   struct draw_prim_info prim_info;

   fetch_info.linear = TRUE;
   fetch_info.start = start;
   fetch_info.count = count;
   fetch_info.elts = NULL;

   prim_info.linear = TRUE;
   prim_info.start = 0;
   prim_info.count = count;
   prim_info.elts = NULL;
   prim_info.prim = prim_type(fpme->input_prim, prim_flags);
   prim_info.flags = prim_flags;
   prim_info.primitive_count = 1;
   prim_info.primitive_lengths = &count;

   llvm_pipeline_generic( middle, &fetch_info, &prim_info );
}


static boolean
llvm_middle_end_linear_run_elts(struct draw_pt_middle_end *middle,
                                unsigned start,
                                unsigned count,
                                const ushort *draw_elts,
                                unsigned draw_count,
                                unsigned prim_flags)
{
   struct llvm_middle_end *fpme = llvm_middle_end(middle);
   struct draw_fetch_info fetch_info;
   struct draw_prim_info prim_info;

   fetch_info.linear = TRUE;
   fetch_info.start = start;
   fetch_info.count = count;
   fetch_info.elts = NULL;

   prim_info.linear = FALSE;
   prim_info.start = 0;
   prim_info.count = draw_count;
   prim_info.elts = draw_elts;
   prim_info.prim = prim_type(fpme->input_prim, prim_flags);
   prim_info.flags = prim_flags;
   prim_info.primitive_count = 1;
   prim_info.primitive_lengths = &draw_count;

   llvm_pipeline_generic( middle, &fetch_info, &prim_info );

   return TRUE;
}


static void
llvm_middle_end_finish(struct draw_pt_middle_end *middle)
{
   /* nothing to do */
}


static void
llvm_middle_end_destroy(struct draw_pt_middle_end *middle)
{
   struct llvm_middle_end *fpme = llvm_middle_end(middle);

   if (fpme->fetch)
      draw_pt_fetch_destroy( fpme->fetch );

   if (fpme->emit)
      draw_pt_emit_destroy( fpme->emit );

   if (fpme->so_emit)
      draw_pt_so_emit_destroy( fpme->so_emit );

   if (fpme->post_vs)
      draw_pt_post_vs_destroy( fpme->post_vs );

   FREE(middle);
}


struct draw_pt_middle_end *
draw_pt_fetch_pipeline_or_emit_llvm(struct draw_context *draw)
{
   struct llvm_middle_end *fpme = 0;

   if (!draw->llvm)
      return NULL;

   fpme = CALLOC_STRUCT( llvm_middle_end );
   if (!fpme)
      goto fail;

   fpme->base.prepare         = llvm_middle_end_prepare;
   fpme->base.bind_parameters = llvm_middle_end_bind_parameters;
   fpme->base.run             = llvm_middle_end_run;
   fpme->base.run_linear      = llvm_middle_end_linear_run;
   fpme->base.run_linear_elts = llvm_middle_end_linear_run_elts;
   fpme->base.finish          = llvm_middle_end_finish;
   fpme->base.destroy         = llvm_middle_end_destroy;

   fpme->draw = draw;

   fpme->fetch = draw_pt_fetch_create( draw );
   if (!fpme->fetch)
      goto fail;

   fpme->post_vs = draw_pt_post_vs_create( draw );
   if (!fpme->post_vs)
      goto fail;

   fpme->emit = draw_pt_emit_create( draw );
   if (!fpme->emit)
      goto fail;

   fpme->so_emit = draw_pt_so_emit_create( draw );
   if (!fpme->so_emit)
      goto fail;

   fpme->llvm = draw->llvm;
   if (!fpme->llvm)
      goto fail;

   fpme->current_variant = NULL;

   return &fpme->base;

 fail:
   if (fpme)
      llvm_middle_end_destroy( &fpme->base );

   return NULL;
}