Lines Matching refs:jpp
45 : jpp(ajpp) {} in jit_uni_pool_kernel()
48 status_t jit_uni_pool_kernel<isa>::init_conf(jit_pool_conf_t &jpp, in init_conf() argument
60 jpp.is_training = pd.prop_kind == prop_kind::forward_training; in init_conf()
61 jpp.is_backward = pd.prop_kind == prop_kind::backward_data; in init_conf()
63 jpp.id = (ndims == 5) ? src_d.dims()[2] : 1; in init_conf()
64 jpp.ih = (ndims == 3) ? 1 : src_d.dims()[ndims - 2]; in init_conf()
65 jpp.iw = src_d.dims()[ndims - 1]; in init_conf()
66 jpp.od = (ndims == 5) ? dst_d.dims()[2] : 1; in init_conf()
67 jpp.ow = dst_d.dims()[ndims - 1]; in init_conf()
68 jpp.oh = (ndims == 3) ? 1 : dst_d.dims()[ndims - 2]; in init_conf()
70 jpp.ndims = ndims; in init_conf()
71 jpp.mb = src_d.dims()[0]; in init_conf()
72 jpp.c_without_padding = src_d.dims()[1]; in init_conf()
73 jpp.c_block = 16; in init_conf()
75 jpp.alg = pd.alg_kind; in init_conf()
88 = ((size_t)jpp.id * jpp.ih * jpp.iw + jpp.od * jpp.oh * jpp.ow) in init_conf()
89 * jpp.c_block * types::data_type_size(src_d.data_type()); in init_conf()
91 const bool forward_ncsp_allowed = !jpp.is_backward in init_conf()
92 && jpp.c_without_padding > 3 in init_conf()
93 && ((jpp.ih > 1 && jpp.iw > 1 in init_conf()
97 const bool backward_ncsp_allowed = jpp.is_backward in init_conf()
98 && ((jpp.ih > 1 && jpp.iw > 1 && jpp.c_without_padding > 1 in init_conf()
101 && !(jpp.alg == pooling_max in init_conf()
121 jpp.is_bf16 = false; in init_conf()
122 jpp.dt_size = types::data_type_size(data_type::f32); in init_conf()
123 jpp.tag_kind = jit_memory_tag_kind_t::ncsp; in init_conf()
125 jpp.is_bf16 = (src_d.data_type() == data_type::bf16 in init_conf()
127 jpp.dt_size = types::data_type_size(src_d.data_type()); in init_conf()
128 jpp.tag_kind = (fmt_tag == nspc_fmt_tag) in init_conf()
133 jpp.isa = isa; in init_conf()
136 && jpp.is_bf16 == false in init_conf()
141 jpp.c = jpp.tag_kind == jit_memory_tag_kind_t::blocked in init_conf()
142 ? utils::rnd_up(jpp.c_without_padding, jpp.c_block) in init_conf()
143 : jpp.c_without_padding; in init_conf()
144 if (jpp.tag_kind == jit_memory_tag_kind_t::blocked) in init_conf()
145 assert(src_d.padded_dims()[1] == jpp.c); in init_conf()
146 jpp.nb_c = utils::div_up(jpp.c, jpp.c_block); in init_conf()
147 jpp.c_tail = jpp.c_without_padding % jpp.c_block; in init_conf()
148 jpp.is_c_padded = jpp.tag_kind == jit_memory_tag_kind_t::blocked in init_conf()
149 && src_d.padded_dims()[1] != jpp.c_without_padding; in init_conf()
151 jpp.stride_d = (ndims == 5) ? pd.strides[0] : 1; in init_conf()
152 jpp.stride_h = (ndims == 3) ? 1 : pd.strides[ndims - 4]; in init_conf()
153 jpp.stride_w = pd.strides[ndims - 3]; in init_conf()
154 jpp.kd = (ndims == 5) ? pd.kernel[0] : 1; in init_conf()
155 jpp.kh = (ndims == 3) ? 1 : pd.kernel[ndims - 4]; in init_conf()
156 jpp.kw = pd.kernel[ndims - 3]; in init_conf()
158 jpp.f_pad = (ndims == 5) ? pd.padding[0][0] : 0; in init_conf()
159 jpp.t_pad = (ndims == 3) ? 0 : pd.padding[0][ndims - 4]; in init_conf()
160 jpp.l_pad = pd.padding[0][ndims - 3]; in init_conf()
163 jpp.f_pad, jpp.od, jpp.id, jpp.stride_d, jpp.kd); in init_conf()
165 jpp.t_pad, jpp.oh, jpp.ih, jpp.stride_h, jpp.kh); in init_conf()
167 jpp.l_pad, jpp.ow, jpp.iw, jpp.stride_w, jpp.kw); in init_conf()
169 if (jpp.f_pad >= jpp.kd || jpp.t_pad >= jpp.kh || jpp.l_pad >= jpp.kw in init_conf()
170 || back_pad >= jpp.kd || bottom_pad >= jpp.kh in init_conf()
171 || right_pad >= jpp.kw) in init_conf()
174 jpp.ind_dt = ppd->workspace_md() ? ppd->workspace_md()->data_type in init_conf()
177 jpp.simple_alg = jpp.is_training in init_conf()
178 || IMPLICATION(jpp.is_backward, jpp.kd <= jpp.stride_d); in init_conf()
180 jpp.ur = 0; in init_conf()
181 if (jpp.alg == pooling_max) { in init_conf()
182 jpp.ur = 16; in init_conf()
184 if (jpp.is_training) in init_conf()
185 jpp.ur = 9; in init_conf()
186 else if (jpp.is_backward) in init_conf()
187 jpp.ur = 6; in init_conf()
189 if (jpp.is_backward) in init_conf()
190 jpp.ur = 12; in init_conf()
192 jpp.ur = 24; in init_conf()
195 if (jpp.tag_kind == jit_memory_tag_kind_t::nspc) { in init_conf()
196 auto min_ur_w = nstl::max(1, utils::div_up(jpp.l_pad, jpp.stride_w)); in init_conf()
197 int min_ur_w1 = utils::div_up(right_pad, jpp.stride_w); in init_conf()
199 jpp.ur_bc = nstl::min(jpp.nb_c, nstl::max(1, jpp.ur / min_ur_w)); in init_conf()
202 for (int ur_bc = jpp.ur_bc; ur_bc > 0; ur_bc--) { in init_conf()
204 const auto nb2_c = utils::div_up(jpp.nb_c, ur_bc); in init_conf()
205 auto work = jpp.is_backward in init_conf()
206 ? (ndims == 5 && jpp.simple_alg ? jpp.od : 1) in init_conf()
207 : (ndims == 5 ? jpp.od : jpp.oh); in init_conf()
208 work *= jpp.mb * nb2_c; in init_conf()
213 jpp.ur_bc = ur_bc; in init_conf()
218 jpp.ur_bc_tail = jpp.nb_c % jpp.ur_bc; in init_conf()
220 jpp.ur_bc = 1; in init_conf()
221 jpp.ur_bc_tail = 0; in init_conf()
223 auto ur_w = nstl::min(jpp.ow, jpp.ur / jpp.ur_bc); in init_conf()
224 if (utils::div_up(jpp.l_pad, jpp.stride_w) > ur_w) in init_conf()
226 if (utils::div_up(right_pad, jpp.stride_w) > ur_w) in init_conf()
231 const int nscr = nstl::min(dnnl_get_max_threads(), jpp.mb * jpp.nb_c); in init_conf()
232 if (jpp.tag_kind == jit_memory_tag_kind_t::ncsp) { in init_conf()
234 jpp.c_block * jpp.id * jpp.ih * jpp.iw * nscr, jpp.dt_size); in init_conf()
236 jpp.c_block * jpp.od * jpp.oh * jpp.ow * nscr, jpp.dt_size); in init_conf()
238 jpp.c_block * jpp.od * jpp.oh * jpp.ow * nscr); in init_conf()
242 if (!post_ops_ok(jpp, attr, dst_d)) return status::unimplemented; in init_conf()
253 size_t c_tail_mask = (1ULL << jpp.c_tail) - 1ULL; in prepare_tail_mask()
303 if (is_c_tail_proccessing && !jpp.is_c_padded) { in load()
318 if (is_c_tail_proccessing && !jpp.is_c_padded) { in store()
330 bool jit_uni_pool_kernel<isa>::post_ops_ok(jit_pool_conf_t &jpp, in post_ops_ok() argument
333 jpp.with_postops = false; in post_ops_ok()
334 jpp.with_eltwise = false; in post_ops_ok()
335 jpp.with_binary = false; in post_ops_ok()
344 if (jpp.alg == pooling_avg_exclude_padding) { in maybe_recalculate_divisor()
345 int kw = jpp.kw; in maybe_recalculate_divisor()
346 int stride_w = jpp.stride_w; in maybe_recalculate_divisor()
372 auto iw = jpp.iw; in avg_step()
373 auto kw = jpp.kw; in avg_step()
374 auto stride_w = jpp.stride_w; in avg_step()
375 auto c_block = jpp.c_block; in avg_step()
376 auto dt_size = jpp.dt_size; in avg_step()
378 = (jpp.tag_kind == jit_memory_tag_kind_t::nspc) ? jpp.c : c_block; in avg_step()
386 if (jpp.is_backward) in avg_step()
392 if (jpp.is_backward) { in avg_step()
403 if (jpp.simple_alg && jpp.ndims == 5) { in avg_step()
436 if (jpp.is_backward) { in avg_step()
457 add_imm(aux_reg_input, aux_reg_input, (jpp.dt_size * iw * c_off), in avg_step()
464 if (jpp.simple_alg && jpp.ndims == 5) { in avg_step()
466 (jpp.dt_size * jpp.ih * iw * c_off), X_TMP_0); in avg_step()
475 if (!jpp.is_backward) { in avg_step()
501 int iw = jpp.iw; in max_step_fwd()
502 int kw = jpp.kw; in max_step_fwd()
503 int stride_w = jpp.stride_w; in max_step_fwd()
504 int c_block = jpp.c_block; in max_step_fwd()
506 = (jpp.tag_kind == jit_memory_tag_kind_t::nspc) ? jpp.c : c_block; in max_step_fwd()
527 if (jpp.is_training) { in max_step_fwd()
533 if (jpp.is_training) { in max_step_fwd()
542 if (jpp.ndims == 5) { in max_step_fwd()
578 int input_offset = jpp.dt_size * aux_input_offset; in max_step_fwd()
584 if (jpp.is_training) { in max_step_fwd()
588 if (jpp.is_training) add(vmm_k_offset, vmm_k_offset, vmm_one); in max_step_fwd()
591 add_imm(aux_reg_input, aux_reg_input, (jpp.dt_size * iw * c_off), in max_step_fwd()
598 if (jpp.ndims == 5) { in max_step_fwd()
600 (jpp.dt_size * jpp.ih * iw * c_off), X_TMP_0); in max_step_fwd()
601 if (jpp.is_training) { in max_step_fwd()
626 const auto output_offset = jpp.dt_size * (jj * c_off + bci * c_block); in max_step_fwd()
630 if (jpp.is_training) { in max_step_fwd()
632 * types::data_type_size(jpp.ind_dt); in max_step_fwd()
636 if (jpp.ind_dt == data_type::u8) { in max_step_fwd()
638 if (jpp.is_c_padded) { in max_step_fwd()
676 int iw = jpp.iw; in max_step_bwd()
677 int kw = jpp.kw; in max_step_bwd()
678 int stride_w = jpp.stride_w; in max_step_bwd()
679 int c_block = jpp.c_block; in max_step_bwd()
681 = (jpp.tag_kind == jit_memory_tag_kind_t::nspc) ? jpp.c : c_block; in max_step_bwd()
691 auto out_offset = jpp.dt_size * (jj * c_off + bci * c_block); in max_step_bwd()
694 * types::data_type_size(jpp.ind_dt); in max_step_bwd()
698 if (jpp.ind_dt == data_type::u8) { in max_step_bwd()
699 if (is_tail_processing(bci) && !jpp.is_c_padded) { in max_step_bwd()
725 if (jpp.simple_alg && jpp.ndims == 5) { in max_step_bwd()
761 int inp_offset = jpp.dt_size * aux_inp_offset; in max_step_bwd()
779 add_imm(aux_reg_input, aux_reg_input, (jpp.dt_size * iw * c_off), in max_step_bwd()
785 if (jpp.simple_alg && jpp.ndims == 5) { in max_step_bwd()
787 (jpp.dt_size * jpp.ih * iw * c_off), X_TMP_0); in max_step_bwd()
812 const int c_off = (jpp.tag_kind == jit_memory_tag_kind_t::nspc) in zero_diff_src()
813 ? jpp.c in zero_diff_src()
814 : jpp.c_block; in zero_diff_src()
844 const int width_size = jpp.iw * c_off * jpp.dt_size; in zero_diff_src()
854 const int step = c_off * jpp.dt_size; in zero_diff_src()
859 const int offs = i + bci * jpp.c_block * jpp.dt_size; in zero_diff_src()
868 add_imm(reg_zero_ptr, reg_zero_ptr, (width_size * jpp.ih), X_TMP_0); in zero_diff_src()
883 int ow = jpp.ow; in generate()
884 int iw = jpp.iw; in generate()
885 int kw = jpp.kw; in generate()
886 int kh = jpp.kh; in generate()
887 int c_block = jpp.c_block; in generate()
888 int stride_w = jpp.stride_w; in generate()
889 int l_pad = jpp.l_pad; in generate()
891 = (jpp.tag_kind == jit_memory_tag_kind_t::nspc) ? jpp.c : c_block; in generate()
901 if (jpp.alg == pooling_max && (jpp.is_training || jpp.is_backward)) { in generate()
928 auto dt_size = jpp.dt_size; in generate()
933 if (jpp.alg == pooling_max && (jpp.is_training || jpp.is_backward)) { in generate()
934 auto ind_dt_size = types::data_type_size(jpp.ind_dt); in generate()
943 if (jpp.is_backward && jpp.simple_alg) in generate()
946 if (jpp.alg == pooling_avg_exclude_padding) { in generate()
955 if (jpp.alg == pooling_avg_include_padding) { in generate()
956 mov_imm(tmp_gpr, float2int((float)(kw * kh * jpp.kd))); in generate()
966 if (jpp.alg == pooling_max && (jpp.is_training || jpp.is_backward)) { in generate()
971 auto ur_w = nstl::min(jpp.ow, jpp.ur / jpp.ur_bc); in generate()
972 auto ur_w_tail = jpp.ow % ur_w; in generate()
1010 if (jpp.ur_bc_tail > 0) { in generate()
1011 mov_imm(X_TMP_0, jpp.ur_bc); in generate()
1014 } else if (jpp.c_tail != 0) { in generate()
1023 mov_imm(X_TMP_0, jpp.nb_c); in generate()
1028 perform_ker(jpp.ur_bc, false); in generate()
1030 if (jpp.ur_bc_tail > 0) { in generate()
1039 if (jpp.c_tail != 0) prepare_tail_mask(); in generate()
1040 perform_ker(jpp.ur_bc_tail, jpp.c_tail != 0); in generate()
1044 } else if (jpp.c_tail != 0) { in generate()
1050 perform_ker(jpp.ur_bc, true); in generate()