Lines Matching refs:loop_vinfo
417 vect_set_loop_masks_directly (class loop *loop, loop_vec_info loop_vinfo, in vect_set_loop_masks_directly() argument
423 tree compare_type = LOOP_VINFO_MASK_COMPARE_TYPE (loop_vinfo); in vect_set_loop_masks_directly()
424 tree iv_type = LOOP_VINFO_MASK_IV_TYPE (loop_vinfo); in vect_set_loop_masks_directly()
428 poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); in vect_set_loop_masks_directly()
511 LOOP_VINFO_VECT_FACTOR (loop_vinfo) in vect_set_loop_masks_directly()
646 vect_set_loop_condition_masked (class loop *loop, loop_vec_info loop_vinfo, in vect_set_loop_condition_masked() argument
654 tree compare_type = LOOP_VINFO_MASK_COMPARE_TYPE (loop_vinfo); in vect_set_loop_condition_masked()
661 tree niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo); in vect_set_loop_condition_masked()
680 widest_int iv_limit = vect_iv_limit_for_full_masking (loop_vinfo); in vect_set_loop_condition_masked()
688 vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo); in vect_set_loop_condition_masked()
713 test_mask = vect_set_loop_masks_directly (loop, loop_vinfo, in vect_set_loop_condition_masked()
735 LOOP_VINFO_VECT_FACTOR (loop_vinfo)); in vect_set_loop_condition_masked()
907 vect_set_loop_condition (class loop *loop, loop_vec_info loop_vinfo, in vect_set_loop_condition() argument
915 if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)) in vect_set_loop_condition()
916 cond_stmt = vect_set_loop_condition_masked (loop, loop_vinfo, niters, in vect_set_loop_condition()
926 if (loop_vinfo in vect_set_loop_condition()
927 && (orig_cond_info = loop_vinfo->lookup_stmt (orig_cond))) in vect_set_loop_condition()
928 loop_vinfo->remove_stmt (orig_cond_info); in vect_set_loop_condition()
1365 vect_can_advance_ivs_p (loop_vec_info loop_vinfo) in vect_can_advance_ivs_p() argument
1367 class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); in vect_can_advance_ivs_p()
1380 stmt_vec_info phi_info = loop_vinfo->lookup_stmt (phi); in vect_can_advance_ivs_p()
1477 vect_update_ivs_after_vectorizer (loop_vec_info loop_vinfo, in vect_update_ivs_after_vectorizer() argument
1481 class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); in vect_update_ivs_after_vectorizer()
1501 stmt_vec_info phi_info = loop_vinfo->lookup_stmt (phi); in vect_update_ivs_after_vectorizer()
1556 get_misalign_in_elems (gimple **seq, loop_vec_info loop_vinfo) in get_misalign_in_elems() argument
1558 dr_vec_info *dr_info = LOOP_VINFO_UNALIGNED_DR (loop_vinfo); in get_misalign_in_elems()
1634 vect_gen_prolog_loop_niters (loop_vec_info loop_vinfo, in vect_gen_prolog_loop_niters() argument
1637 dr_vec_info *dr_info = LOOP_VINFO_UNALIGNED_DR (loop_vinfo); in vect_gen_prolog_loop_niters()
1639 tree niters_type = TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)); in vect_gen_prolog_loop_niters()
1646 if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) > 0) in vect_gen_prolog_loop_niters()
1648 int npeel = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo); in vect_gen_prolog_loop_niters()
1655 *bound = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo); in vect_gen_prolog_loop_niters()
1659 tree misalign_in_elems = get_misalign_in_elems (&stmts, loop_vinfo); in vect_gen_prolog_loop_niters()
1744 vect_update_inits_of_drs (loop_vec_info loop_vinfo, tree niters, in vect_update_inits_of_drs() argument
1748 vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo); in vect_update_inits_of_drs()
1762 dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr); in vect_update_inits_of_drs()
1773 vect_prepare_for_masked_peels (loop_vec_info loop_vinfo) in vect_prepare_for_masked_peels() argument
1776 tree type = LOOP_VINFO_MASK_COMPARE_TYPE (loop_vinfo); in vect_prepare_for_masked_peels()
1778 gcc_assert (vect_use_loop_mask_for_alignment_p (loop_vinfo)); in vect_prepare_for_masked_peels()
1782 if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) > 0) in vect_prepare_for_masked_peels()
1784 poly_int64 misalign = (LOOP_VINFO_VECT_FACTOR (loop_vinfo) in vect_prepare_for_masked_peels()
1785 - LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)); in vect_prepare_for_masked_peels()
1791 misalign_in_elems = get_misalign_in_elems (&seq1, loop_vinfo); in vect_prepare_for_masked_peels()
1798 edge pe = loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo)); in vect_prepare_for_masked_peels()
1809 LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo) = misalign_in_elems; in vect_prepare_for_masked_peels()
1811 vect_update_inits_of_drs (loop_vinfo, misalign_in_elems, MINUS_EXPR); in vect_prepare_for_masked_peels()
1819 vect_build_loop_niters (loop_vec_info loop_vinfo, bool *new_var_p) in vect_build_loop_niters() argument
1821 tree ni = unshare_expr (LOOP_VINFO_NITERS (loop_vinfo)); in vect_build_loop_niters()
1828 edge pe = loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo)); in vect_build_loop_niters()
1910 vect_gen_vector_loop_niters (loop_vec_info loop_vinfo, tree niters, in vect_gen_vector_loop_niters() argument
1916 poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); in vect_gen_vector_loop_niters()
1917 edge pe = loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo)); in vect_gen_vector_loop_niters()
1923 if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)) in vect_gen_vector_loop_niters()
1941 && !LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)) in vect_gen_vector_loop_niters()
2011 vect_gen_vector_loop_niters_mult_vf (loop_vec_info loop_vinfo, in vect_gen_vector_loop_niters_mult_vf() argument
2016 int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo).to_constant (); in vect_gen_vector_loop_niters_mult_vf()
2017 class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); in vect_gen_vector_loop_niters_mult_vf()
2107 slpeel_update_phi_nodes_for_loops (loop_vec_info loop_vinfo, in slpeel_update_phi_nodes_for_loops() argument
2112 class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); in slpeel_update_phi_nodes_for_loops()
2134 stmt_vec_info vect_phi_info = loop_vinfo->lookup_stmt (vect_phi); in slpeel_update_phi_nodes_for_loops()
2464 vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, in vect_do_peeling() argument
2476 bool vect_epilogues = loop_vinfo->epilogue_vinfos.length () > 0; in vect_do_peeling()
2480 dr_vec_info *dr_info = LOOP_VINFO_UNALIGNED_DR (loop_vinfo); in vect_do_peeling()
2484 if (!vect_use_loop_mask_for_alignment_p (loop_vinfo)) in vect_do_peeling()
2485 prolog_peeling = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo); in vect_do_peeling()
2487 poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); in vect_do_peeling()
2489 if (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo) in vect_do_peeling()
2490 && LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo)) in vect_do_peeling()
2492 if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)) in vect_do_peeling()
2502 class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); in vect_do_peeling()
2505 basic_block bb = LOOP_VINFO_BBS (loop_vinfo)[i]; in vect_do_peeling()
2540 estimated_vf = vect_vf_for_cost (loop_vinfo); in vect_do_peeling()
2579 if (loop_vec_info orig_loop_vinfo = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo)) in vect_do_peeling()
2601 niters_prolog = vect_gen_prolog_loop_niters (loop_vinfo, anchor, in vect_do_peeling()
2609 epilogue_vinfo = loop_vinfo->epilogue_vinfos[0]; in vect_do_peeling()
2610 loop_vinfo->epilogue_vinfos.ordered_remove (0); in vect_do_peeling()
2615 tree before_loop_niters = LOOP_VINFO_NITERS (loop_vinfo); in vect_do_peeling()
2622 && LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) in vect_do_peeling()
2627 = (LOOP_VINFO_INT_NITERS (loop_vinfo) in vect_do_peeling()
2628 - LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)); in vect_do_peeling()
2632 = eiters % lowest_vf + LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo); in vect_do_peeling()
2638 (GET_MODE_SIZE (loop_vinfo->vector_mode), in vect_do_peeling()
2644 if (loop_vinfo->epilogue_vinfos.length () == 0) in vect_do_peeling()
2649 epilogue_vinfo = loop_vinfo->epilogue_vinfos[0]; in vect_do_peeling()
2650 loop_vinfo->epilogue_vinfos.ordered_remove (0); in vect_do_peeling()
2668 bool skip_vector = (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) in vect_do_peeling()
2669 ? maybe_lt (LOOP_VINFO_INT_NITERS (loop_vinfo), in vect_do_peeling()
2671 : (!LOOP_REQUIRES_VERSIONING (loop_vinfo) in vect_do_peeling()
2677 || !LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) in vect_do_peeling()
2680 if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)) in vect_do_peeling()
2701 class loop *scalar_loop = LOOP_VINFO_SCALAR_LOOP (loop_vinfo); in vect_do_peeling()
2706 = LOOP_VINFO_SCALAR_LOOP (loop_vinfo); in vect_do_peeling()
2726 slpeel_update_phi_nodes_for_loops (loop_vinfo, prolog, loop, true); in vect_do_peeling()
2756 vect_update_inits_of_drs (loop_vinfo, niters_prolog, PLUS_EXPR); in vect_do_peeling()
2758 LOOP_VINFO_NITERS (loop_vinfo) in vect_do_peeling()
2760 LOOP_VINFO_NITERSM1 (loop_vinfo) in vect_do_peeling()
2762 LOOP_VINFO_NITERSM1 (loop_vinfo), niters_prolog); in vect_do_peeling()
2764 niters = vect_build_loop_niters (loop_vinfo, &new_var_p); in vect_do_peeling()
2825 slpeel_update_phi_nodes_for_loops (loop_vinfo, loop, epilog, false); in vect_do_peeling()
2872 vect_gen_vector_loop_niters (loop_vinfo, niters, in vect_do_peeling()
2885 vect_gen_vector_loop_niters_mult_vf (loop_vinfo, *niters_vector, in vect_do_peeling()
2889 gcc_checking_assert (vect_can_advance_ivs_p (loop_vinfo)); in vect_do_peeling()
2891 vect_update_ivs_after_vectorizer (loop_vinfo, niters_vector_mult_vf, in vect_do_peeling()
3023 vect_create_cond_for_niters_checks (loop_vec_info loop_vinfo, tree *cond_expr) in vect_create_cond_for_niters_checks() argument
3025 tree part_cond_expr = LOOP_VINFO_NITERS_ASSUMPTIONS (loop_vinfo); in vect_create_cond_for_niters_checks()
3073 vect_create_cond_for_align_checks (loop_vec_info loop_vinfo, in vect_create_cond_for_align_checks() argument
3078 = LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo); in vect_create_cond_for_align_checks()
3080 int mask = LOOP_VINFO_PTR_MASK (loop_vinfo); in vect_create_cond_for_align_checks()
3165 vect_create_cond_for_unequal_addrs (loop_vec_info loop_vinfo, tree *cond_expr) in vect_create_cond_for_unequal_addrs() argument
3167 vec<vec_object_pair> pairs = LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo); in vect_create_cond_for_unequal_addrs()
3184 vect_create_cond_for_lower_bounds (loop_vec_info loop_vinfo, tree *cond_expr) in vect_create_cond_for_lower_bounds() argument
3186 vec<vec_lower_bound> lower_bounds = LOOP_VINFO_LOWER_BOUNDS (loop_vinfo); in vect_create_cond_for_lower_bounds()
3226 vect_create_cond_for_alias_checks (loop_vec_info loop_vinfo, tree * cond_expr) in vect_create_cond_for_alias_checks() argument
3229 LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo); in vect_create_cond_for_alias_checks()
3234 create_runtime_alias_checks (LOOP_VINFO_LOOP (loop_vinfo), in vect_create_cond_for_alias_checks()
3263 vect_loop_versioning (loop_vec_info loop_vinfo, in vect_loop_versioning() argument
3266 class loop *loop = LOOP_VINFO_LOOP (loop_vinfo), *nloop; in vect_loop_versioning()
3267 class loop *scalar_loop = LOOP_VINFO_SCALAR_LOOP (loop_vinfo); in vect_loop_versioning()
3280 tree scalar_loop_iters = LOOP_VINFO_NITERSM1 (loop_vinfo); in vect_loop_versioning()
3281 bool version_align = LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo); in vect_loop_versioning()
3282 bool version_alias = LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo); in vect_loop_versioning()
3283 bool version_niter = LOOP_REQUIRES_VERSIONING_FOR_NITERS (loop_vinfo); in vect_loop_versioning()
3285 = LOOP_VINFO_VERSIONING_THRESHOLD (loop_vinfo); in vect_loop_versioning()
3287 = LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND (loop_vinfo); in vect_loop_versioning()
3288 unsigned th = LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo); in vect_loop_versioning()
3290 if (vect_apply_runtime_profitability_check_p (loop_vinfo) in vect_loop_versioning()
3308 vect_create_cond_for_niters_checks (loop_vinfo, &cond_expr); in vect_loop_versioning()
3316 vect_create_cond_for_align_checks (loop_vinfo, &cond_expr, in vect_loop_versioning()
3321 vect_create_cond_for_unequal_addrs (loop_vinfo, &cond_expr); in vect_loop_versioning()
3322 vect_create_cond_for_lower_bounds (loop_vinfo, &cond_expr); in vect_loop_versioning()
3323 vect_create_cond_for_alias_checks (loop_vinfo, &cond_expr); in vect_loop_versioning()
3423 LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo) = fe->probability; in vect_loop_versioning()