| /dports/misc/mxnet/incubator-mxnet-1.9.0/3rdparty/mkldnn/src/cpu/rnn/ |
| H A D | cell_gru.cpp | 40 scratch_gates_aoc<scratch_t> scratch_gates(rnn, scratch_gates_);
52 src_layer_ld, 0.0f, scratch_gates_, rnn.scratch_gates_ld));
58 src_iter_ld, 1.0f, scratch_gates_, rnn.scratch_gates_ld));
61 rnn_postgemm_->execute(rnn, cell_position, ws_gates_, scratch_gates_,
92 src_data_t *ws_gates_, scratch_data_t *scratch_gates_, in gru_bwd_cell_exec_template() argument
101 scratch_gates_aoc<scratch_data_t> scratch_gates(rnn, scratch_gates_); in gru_bwd_cell_exec_template()
123 rnn_postgemm_->execute(rnn, cell_position, ws_gates_, scratch_gates_, in gru_bwd_cell_exec_template()
142 scratch_gates_, src_iter_, src_iter_ld, 1.0f, diff_w_iter_)); in gru_bwd_cell_exec_template()
150 scratch_gates_, 1.0f, diff_src_iter_)); in gru_bwd_cell_exec_template()
155 scratch_gates_, src_layer_, src_layer_ld, diff_w_layer_)); in gru_bwd_cell_exec_template()
[all …]
|
| H A D | ref_postgemm_gru.cpp | 44 scratch_data_t *scratch_gates_, src_data_t *dst_layer_, in gru_fwd_part1_postgemm_template() argument
47 scratch_gates_aoc<scratch_data_t> scratch_gates(rnn, scratch_gates_); in gru_fwd_part1_postgemm_template()
89 scratch_data_t *scratch_gates_, src_data_t *dst_layer_, in gru_fwd_part2_postgemm_template() argument
307 scratch_data_t *scratch_gates_, src_data_t *dst_layer_, in gru_bwd_part1_postgemm_template() argument
314 ws_gates_aoc<scratch_data_t> scratch_gates(rnn, scratch_gates_); in gru_bwd_part1_postgemm_template()
343 scratch_data_t *scratch_gates_, src_data_t *dst_layer_, in gru_bwd_part2_postgemm_template() argument
352 ws_gates_aoc<scratch_data_t> scratch_gates(rnn, scratch_gates_); in gru_bwd_part2_postgemm_template()
381 scratch_gates_, dst_layer_, src_iter_, diff_src_iter_, in rnn_postgemm_sig()
390 scratch_gates_, dst_layer_, src_iter_, diff_src_layer_, in rnn_postgemm_sig()
399 scratch_gates_, dst_layer_, src_iter_, diff_src_iter_, in rnn_postgemm_sig()
[all …]
|
| H A D | ref_postgemm_rnn.cpp | 78 scratch_data_t *scratch_gates_, src_data_t *dst_layer_, in rnn_fwd_postgemm_template() argument
83 scratch_gates_aoc<scratch_data_t> scratch_gates(rnn, scratch_gates_); in rnn_fwd_postgemm_template()
124 ws_gates_, scratch_gates_, dst_layer_, dst_iter_, src_iter_, in rnn_postgemm_sig()
128 ws_gates_, scratch_gates_, dst_layer_, dst_iter_, src_iter_, in rnn_postgemm_sig()
144 ws_gates_, scratch_gates_, dst_layer_, dst_iter_, src_iter_, in rnn_postgemm_sig()
148 ws_gates_, scratch_gates_, dst_layer_, dst_iter_, src_iter_, in rnn_postgemm_sig()
166 scratch_data_t *scratch_gates_, acc_data_t *diff_dst_iter_, in rnn_bwd_postgemm_template() argument
169 ws_gates_aoc<scratch_data_t> scratch_gates(rnn, scratch_gates_); in rnn_bwd_postgemm_template()
198 scratch_gates_, diff_dst_iter_, diff_dst_layer_); in rnn_postgemm_sig()
201 ws_gates_, scratch_gates_, diff_dst_iter_, diff_dst_layer_); in rnn_postgemm_sig()
[all …]
|
| H A D | cell_common.cpp | 47 src_layer_ld, 0.0f, scratch_gates_, rnn.scratch_gates_ld));
51 src_iter_ld, 1.0f, scratch_gates_, rnn.scratch_gates_ld));
57 rnn_postgemm_->execute(rnn, cell_position, ws_gates_, scratch_gates_,
72 : scratch_gates_;
99 const float *dst_iter_c_, const scratch_data_t *scratch_gates_, in lstm_bwd_weights_peephole_and_bias() argument
108 ws_gates_aoc<const scratch_data_t> scratch_gates(rnn, scratch_gates_); in lstm_bwd_weights_peephole_and_bias()
163 scratch_data_t *scratch_gates_, src_data_t *ws_ht_, in common_bwd_cell_exec_template() argument
181 rnn_postgemm->execute(rnn, cell_position, ws_gates_, scratch_gates_, in common_bwd_cell_exec_template()
188 CHECK(gemm_iter_f(w_iter_[0], scratch_gates_, diff_src_iter_)); in common_bwd_cell_exec_template()
203 dst_iter_c_, scratch_gates_, diff_weights_peephole_, in common_bwd_cell_exec_template()
[all …]
|
| H A D | ref_postgemm_lstm.cpp | 42 scratch_data_t *scratch_gates_, src_data_t *dst_layer_, in lstm_fwd_postgemm_template() argument
47 scratch_gates_aoc<scratch_data_t> scratch_gates(rnn, scratch_gates_); in lstm_fwd_postgemm_template()
140 cscale, rnn, cell_position, ws_gates_, scratch_gates_, in rnn_postgemm_sig()
145 cscale, rnn, cell_position, ws_gates_, scratch_gates_, in rnn_postgemm_sig()
166 scales, cscale, rnn, cell_position, ws_gates_, scratch_gates_, in rnn_postgemm_sig()
260 src_data_t *ws_gates_, scratch_data_t *scratch_gates_, in lstm_bwd_postgemm_template() argument
266 ws_gates_aoc<scratch_data_t> scratch_gates(rnn, scratch_gates_); in lstm_bwd_postgemm_template()
331 cell_position, ws_gates_, scratch_gates_, dst_iter_c_, in rnn_postgemm_sig()
336 cell_position, ws_gates_, scratch_gates_, dst_iter_c_, in rnn_postgemm_sig()
351 cell_position, ws_gates_, scratch_gates_, dst_iter_c_, in rnn_postgemm_sig()
[all …]
|
| H A D | ref_postgemm_gru_lbr.cpp | 40 scratch_data_t *scratch_gates_, src_data_t *dst_layer_, in gru_lbr_fwd_postgemm_template() argument
52 scratch_gates_aoc<scratch_data_t> scratch_gates(rnn, scratch_gates_); in gru_lbr_fwd_postgemm_template()
96 cell_position, ws_gates_, scratch_gates_, dst_layer_, dst_iter_, in rnn_postgemm_sig()
100 cell_position, ws_gates_, scratch_gates_, dst_layer_, dst_iter_, in rnn_postgemm_sig()
118 cell_position, ws_gates_, scratch_gates_, dst_layer_, dst_iter_, in rnn_postgemm_sig()
122 cell_position, ws_gates_, scratch_gates_, dst_layer_, dst_iter_, in rnn_postgemm_sig()
140 scratch_data_t *scratch_gates_, const src_data_t *src_iter_, in gru_lbr_bwd_postgemm_template() argument
147 ws_gates_aoc<scratch_data_t> scratch_gates(rnn, scratch_gates_); in gru_lbr_bwd_postgemm_template()
182 scratch_gates_, src_iter_, diff_src_iter_, diff_dst_iter_, in rnn_postgemm_sig()
190 scratch_gates_, src_iter_, diff_src_iter_, diff_dst_iter_, in rnn_postgemm_sig()
|
| H A D | cell_gru_lbr.cpp | 46 src_layer_ld, 0.0, scratch_gates_, rnn.scratch_gates_ld));
52 rnn_postgemm_->execute(rnn, cell_position, ws_gates_, scratch_gates_,
87 src_data_t *ws_grid_, scratch_data_t *scratch_gates_, in common_bwd_cell_exec_template() argument
94 rnn_postgemm->execute(rnn, cell_position, ws_gates_, scratch_gates_, in common_bwd_cell_exec_template()
102 scratch_gates_, src_layer_, src_layer_ld, diff_w_layer_)); in common_bwd_cell_exec_template()
106 CHECK(gemm_layer_f(w_layer_[0], scratch_gates_, diff_src_layer_)); in common_bwd_cell_exec_template()
117 gates_reduction(rnn, scratch_gates_, diff_bias_); in common_bwd_cell_exec_template()
159 scratch_gates_, scratch_cell_, dst_iter_); in rnn_cell_execution_sig()
193 scratch_gates_, scratch_cell_, dst_iter_); in rnn_cell_execution_sig()
|
| H A D | postgemm_dispatcher.hpp | 150 scratch_gates_, sizeof(*scratch_gates_) * gates_nelems); in rnn_postgemm_sig()
161 scratch_gates_, dst_layer_, dst_iter_c_, src_iter_, in rnn_postgemm_sig()
166 unpoison(rnn, cell_position, ws_gates_, scratch_gates_, dst_layer_, in rnn_postgemm_sig()
175 (this->*postgemm_func)(rnn, cell_position, ws_gates_, scratch_gates_, in rnn_postgemm_sig()
188 scratch_gates_, dst_layer_, dst_iter_c_, src_iter_, in rnn_postgemm_sig()
193 unpoison(rnn, cell_position, ws_gates_, scratch_gates_, dst_layer_, in rnn_postgemm_sig()
203 scratch_gates_, dst_layer_, dst_iter_c_, src_iter_, src_iter_c_, in rnn_postgemm_sig()
|
| H A D | brgemm_cell_common.cpp | 97 src_iter_, src_layer_, w_iter_[0], w_layer_[0], scratch_gates_,
104 rnn_postgemm_->execute(rnn, cell_position, ws_gates_, scratch_gates_,
118 : scratch_gates_;
189 rnn_postgemm_->execute(rnn, cell_position, ws_gates_, scratch_gates_,
202 scratch_gates_, w_iter_[0], w_layer_[0], diff_src_iter_,
206 scratch_src_layer_, scratch_gates_, scratch_gates_blocked_,
233 cell_position, scratch_gates_, src_iter_c_, dst_iter_c_,
|
| H A D | ref_postgemm_lstm_projection.cpp | 81 (float *)scratch_gates_ + i * rnn.scratch_gates_ld, n_elem); in rnn_postgemm_sig()
121 float tmp = dequantize_s32_f32(scratch_gates_[scratch_off], j); in rnn_postgemm_sig()
163 float tmp = dequantize_s32_f32(scratch_gates_[scratch_off], j); in rnn_postgemm_sig()
|
| H A D | ref_rnn.cpp | 227 (gemm_acc_t *)scratch_gates_, rnn.scratch_gates_ld));
312 ? scratch_gates_
313 : scratch_gates_
408 (gates_t *)scratch_gates_, rnn.scratch_gates_ld, 0.0,
412 rnn.mb * n_iter, 1.0, (weights_t *)scratch_gates_,
438 (weights_t *)scratch_gates_
453 (weights_t *)scratch_gates_, rnn.scratch_gates_ld,
|
| /dports/math/onednn/oneDNN-2.5.1/src/cpu/rnn/ |
| H A D | cell_gru.cpp | 40 const scratch_gates_aoc<scratch_t> scratch_gates(rnn, scratch_gates_);
51 src_layer_ld, 0.0f, scratch_gates_, rnn.scratch_gates_ld));
57 src_iter_ld, 1.0f, scratch_gates_, rnn.scratch_gates_ld));
60 rnn_postgemm_->execute(rnn, cell_position, ws_gates_, scratch_gates_,
91 src_data_t *ws_gates_, scratch_data_t *scratch_gates_, in gru_bwd_cell_exec_template() argument
123 rnn_postgemm_->execute(rnn, cell_position, ws_gates_, scratch_gates_, in gru_bwd_cell_exec_template()
142 scratch_gates_, src_iter_, src_iter_ld, 1.0f, diff_w_iter_)); in gru_bwd_cell_exec_template()
150 scratch_gates_, 1.0f, diff_src_iter_)); in gru_bwd_cell_exec_template()
155 scratch_gates_, src_layer_, src_layer_ld, diff_w_layer_)); in gru_bwd_cell_exec_template()
159 CHECK(gemm_layer_f(w_layer_[0], scratch_gates_, diff_src_layer_)); in gru_bwd_cell_exec_template()
[all …]
|
| H A D | ref_postgemm_gru.cpp | 44 scratch_data_t *scratch_gates_, src_data_t *dst_layer_, in gru_fwd_part1_postgemm_template() argument
95 scratch_data_t *scratch_gates_, src_data_t *dst_layer_, in gru_fwd_part2_postgemm_template() argument
259 scales, rnn, cell_position, ws_gates_, scratch_gates_, in rnn_postgemm_sig()
264 scales, rnn, cell_position, ws_gates_, scratch_gates_, in rnn_postgemm_sig()
304 scales, rnn, cell_position, ws_gates_, scratch_gates_, in rnn_postgemm_sig()
327 scratch_data_t *scratch_gates_, src_data_t *dst_layer_, in gru_bwd_part1_postgemm_template() argument
367 scratch_data_t *scratch_gates_, src_data_t *dst_layer_, in gru_bwd_part2_postgemm_template() argument
409 scratch_gates_, dst_layer_, src_iter_, diff_src_iter_, in rnn_postgemm_sig()
418 scratch_gates_, dst_layer_, src_iter_, diff_src_layer_, in rnn_postgemm_sig()
427 scratch_gates_, dst_layer_, src_iter_, diff_src_iter_, in rnn_postgemm_sig()
[all …]
|
| H A D | ref_postgemm_rnn.cpp | 78 scratch_data_t *scratch_gates_, src_data_t *dst_layer_, in rnn_fwd_postgemm_template() argument
83 const scratch_gates_aoc<scratch_data_t> scratch_gates(rnn, scratch_gates_); in rnn_fwd_postgemm_template()
128 ws_gates_, scratch_gates_, dst_layer_, dst_iter_, src_iter_, in rnn_postgemm_sig()
132 ws_gates_, scratch_gates_, dst_layer_, dst_iter_, src_iter_, in rnn_postgemm_sig()
148 ws_gates_, scratch_gates_, dst_layer_, dst_iter_, src_iter_, in rnn_postgemm_sig()
152 ws_gates_, scratch_gates_, dst_layer_, dst_iter_, src_iter_, in rnn_postgemm_sig()
170 scratch_data_t *scratch_gates_, acc_data_t *diff_dst_iter_, in rnn_bwd_postgemm_template() argument
173 const ws_gates_aoc<scratch_data_t> scratch_gates(rnn, scratch_gates_); in rnn_bwd_postgemm_template()
204 scratch_gates_, diff_dst_iter_, diff_dst_layer_); in rnn_postgemm_sig()
207 ws_gates_, scratch_gates_, diff_dst_iter_, diff_dst_layer_); in rnn_postgemm_sig()
[all …]
|
| H A D | cell_common.cpp | 47 src_layer_ld, 0.0f, scratch_gates_, rnn.scratch_gates_ld));
51 src_iter_ld, 1.0f, scratch_gates_, rnn.scratch_gates_ld));
57 rnn_postgemm_->execute(rnn, cell_position, ws_gates_, scratch_gates_,
72 : scratch_gates_;
99 const void *dst_iter_c_, const scratch_data_t *scratch_gates_, in lstm_bwd_weights_peephole_and_bias() argument
170 scratch_data_t *scratch_gates_, src_data_t *ws_ht_, in common_bwd_cell_exec_template() argument
188 rnn_postgemm->execute(rnn, cell_position, ws_gates_, scratch_gates_, in common_bwd_cell_exec_template()
195 CHECK(gemm_iter_f(w_iter_[0], scratch_gates_, diff_src_iter_)); in common_bwd_cell_exec_template()
202 CHECK(gemm_layer_f(w_layer_[0], scratch_gates_, diff_src_layer_)); in common_bwd_cell_exec_template()
210 dst_iter_c_, scratch_gates_, diff_weights_peephole_, in common_bwd_cell_exec_template()
[all …]
|
| H A D | ref_postgemm_gru_lbr.cpp | 40 scratch_data_t *scratch_gates_, src_data_t *dst_layer_, in gru_lbr_fwd_postgemm_template() argument
54 const scratch_gates_aoc<scratch_data_t> scratch_gates(rnn, scratch_gates_); in gru_lbr_fwd_postgemm_template()
104 cell_position, ws_gates_, scratch_gates_, dst_layer_, dst_iter_, in rnn_postgemm_sig()
108 cell_position, ws_gates_, scratch_gates_, dst_layer_, dst_iter_, in rnn_postgemm_sig()
127 cell_position, ws_gates_, scratch_gates_, dst_layer_, dst_iter_, in rnn_postgemm_sig()
131 cell_position, ws_gates_, scratch_gates_, dst_layer_, dst_iter_, in rnn_postgemm_sig()
149 scratch_data_t *scratch_gates_, const src_data_t *src_iter_, in gru_lbr_bwd_postgemm_template() argument
157 const ws_gates_aoc<scratch_data_t> scratch_gates(rnn, scratch_gates_); in gru_lbr_bwd_postgemm_template()
196 scratch_gates_, src_iter_, diff_src_iter_, diff_dst_iter_, in rnn_postgemm_sig()
204 scratch_gates_, src_iter_, diff_src_iter_, diff_dst_iter_, in rnn_postgemm_sig()
|
| H A D | ref_postgemm_lstm.cpp | 42 scratch_data_t *scratch_gates_, src_data_t *dst_layer_, in lstm_fwd_postgemm_template() argument
47 const scratch_gates_aoc<scratch_data_t> scratch_gates(rnn, scratch_gates_); in lstm_fwd_postgemm_template()
164 cscale, rnn, cell_position, ws_gates_, scratch_gates_, in rnn_postgemm_sig()
169 cscale, rnn, cell_position, ws_gates_, scratch_gates_, in rnn_postgemm_sig()
191 scales, cscale, rnn, cell_position, ws_gates_, scratch_gates_, in rnn_postgemm_sig()
287 src_data_t *ws_gates_, scratch_data_t *scratch_gates_, in lstm_bwd_postgemm_template() argument
293 const ws_gates_aoc<scratch_data_t> scratch_gates(rnn, scratch_gates_); in lstm_bwd_postgemm_template()
374 cell_position, ws_gates_, scratch_gates_, dst_iter_c_, in rnn_postgemm_sig()
379 cell_position, ws_gates_, scratch_gates_, dst_iter_c_, in rnn_postgemm_sig()
395 cell_position, ws_gates_, scratch_gates_, dst_iter_c_, in rnn_postgemm_sig()
[all …]
|
| H A D | cell_gru_lbr.cpp | 46 src_layer_ld, 0.0, scratch_gates_, rnn.scratch_gates_ld));
52 rnn_postgemm_->execute(rnn, cell_position, ws_gates_, scratch_gates_,
87 src_data_t *ws_grid_, scratch_data_t *scratch_gates_, in common_bwd_cell_exec_template() argument
94 rnn_postgemm->execute(rnn, cell_position, ws_gates_, scratch_gates_, in common_bwd_cell_exec_template()
102 scratch_gates_, src_layer_, src_layer_ld, diff_w_layer_)); in common_bwd_cell_exec_template()
106 CHECK(gemm_layer_f(w_layer_[0], scratch_gates_, diff_src_layer_)); in common_bwd_cell_exec_template()
117 gates_reduction(rnn, scratch_gates_, diff_bias_); in common_bwd_cell_exec_template()
159 scratch_gates_, scratch_cell_, dst_iter_); in rnn_cell_execution_sig()
195 scratch_gates_, scratch_cell_, dst_iter_); in rnn_cell_execution_sig()
|
| H A D | postgemm_dispatcher.hpp | 152 scratch_gates_, sizeof(*scratch_gates_) * gates_nelems); in rnn_postgemm_sig()
163 scratch_gates_, dst_layer_, dst_iter_c_, src_iter_, in rnn_postgemm_sig()
168 unpoison(rnn, cell_position, ws_gates_, scratch_gates_, dst_layer_, in rnn_postgemm_sig()
177 (this->*postgemm_func)(rnn, cell_position, ws_gates_, scratch_gates_, in rnn_postgemm_sig()
190 scratch_gates_, dst_layer_, dst_iter_c_, src_iter_, in rnn_postgemm_sig()
195 unpoison(rnn, cell_position, ws_gates_, scratch_gates_, dst_layer_, in rnn_postgemm_sig()
205 scratch_gates_, dst_layer_, dst_iter_c_, src_iter_, src_iter_c_, in rnn_postgemm_sig()
|
| H A D | brgemm_cell_common.cpp | 98 src_iter_, src_layer_, w_iter_[0], w_layer_[0], scratch_gates_,
105 rnn_postgemm_->execute(rnn, cell_position, ws_gates_, scratch_gates_,
119 : scratch_gates_;
190 rnn_postgemm_->execute(rnn, cell_position, ws_gates_, scratch_gates_,
203 scratch_gates_, w_iter_[0], w_layer_[0], diff_src_iter_,
207 scratch_src_layer_, scratch_gates_, scratch_gates_blocked_,
243 cell_position, scratch_gates_, src_iter_c_, dst_iter_c_,
|
| H A D | ref_postgemm_lstm_projection.cpp | 81 (float *)scratch_gates_ + i * rnn.scratch_gates_ld, n_elem); in rnn_postgemm_sig()
123 = dequantize_s32_f32(scratch_gates_[scratch_off], j); in rnn_postgemm_sig()
167 = dequantize_s32_f32(scratch_gates_[scratch_off], j); in rnn_postgemm_sig()
|
| /dports/misc/mxnet/incubator-mxnet-1.9.0/3rdparty/mkldnn/src/cpu/x64/rnn/ |
| H A D | jit_uni_rnn_common_postgemm.hpp | 82 execute_bwd(rnn, cell_position, ws_gates_, scratch_gates_, in rnn_postgemm_sig()
89 execute_fwd(rnn, cell_position, ws_gates_, scratch_gates_, in rnn_postgemm_sig()
104 scratch_gates_, dst_layer_, dst_iter_c_, src_iter_, in rnn_postgemm_sig()
112 scratch_gates_, dst_layer_, dst_iter_c_, src_iter_, in rnn_postgemm_sig()
123 scratch_t *scratch_gates_, dst_layer_t *dst_layer_, in postgemm_fwd_call()
131 rnn, scratch_gates_); in postgemm_fwd_call()
209 rnn_utils::ws_gates_aoc<scratch_t> scratch_gates(rnn, scratch_gates_); in rnn_postgemm_sig()
|
| H A D | brgemm_cell_common_bwd.hpp | 222 const scratch_t *scratch_gates_; member in dnnl::impl::cpu::x64::brgemm_diff_wei_peep_t
|
| /dports/math/onednn/oneDNN-2.5.1/src/cpu/x64/rnn/ |
| H A D | jit_uni_rnn_common_postgemm.hpp | 84 execute_bwd(rnn, cell_position, ws_gates_, scratch_gates_, in rnn_postgemm_sig()
91 execute_fwd(rnn, cell_position, ws_gates_, scratch_gates_, in rnn_postgemm_sig()
106 scratch_gates_, dst_layer_, dst_iter_c_, src_iter_, in rnn_postgemm_sig()
114 scratch_gates_, dst_layer_, dst_iter_c_, src_iter_, in rnn_postgemm_sig()
125 scratch_t *scratch_gates_, dst_layer_t *dst_layer_, in postgemm_fwd_call()
133 rnn, scratch_gates_); in postgemm_fwd_call()
216 rnn, scratch_gates_); in rnn_postgemm_sig()
|
| H A D | brgemm_cell_common_bwd.hpp | 222 const scratch_t *scratch_gates_; member in dnnl::impl::cpu::x64::brgemm_diff_wei_peep_t
|