1 /*
2 * Copyright (c) 2021, Alliance for Open Media. All rights reserved
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #include "av1/encoder/txb_rdopt.h"
13 #include "av1/encoder/txb_rdopt_utils.h"
14
15 #include "av1/common/idct.h"
16
update_coeff_general(int * accu_rate,int64_t * accu_dist,int si,int eob,TX_SIZE tx_size,TX_CLASS tx_class,int bwl,int height,int64_t rdmult,int shift,int dc_sign_ctx,const int16_t * dequant,const int16_t * scan,const LV_MAP_COEFF_COST * txb_costs,const tran_low_t * tcoeff,tran_low_t * qcoeff,tran_low_t * dqcoeff,uint8_t * levels,const qm_val_t * iqmatrix)17 static INLINE void update_coeff_general(
18 int *accu_rate, int64_t *accu_dist, int si, int eob, TX_SIZE tx_size,
19 TX_CLASS tx_class, int bwl, int height, int64_t rdmult, int shift,
20 int dc_sign_ctx, const int16_t *dequant, const int16_t *scan,
21 const LV_MAP_COEFF_COST *txb_costs, const tran_low_t *tcoeff,
22 tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels,
23 const qm_val_t *iqmatrix) {
24 const int dqv = get_dqv(dequant, scan[si], iqmatrix);
25 const int ci = scan[si];
26 const tran_low_t qc = qcoeff[ci];
27 const int is_last = si == (eob - 1);
28 const int coeff_ctx = get_lower_levels_ctx_general(
29 is_last, si, bwl, height, levels, ci, tx_size, tx_class);
30 if (qc == 0) {
31 *accu_rate += txb_costs->base_cost[coeff_ctx][0];
32 } else {
33 const int sign = (qc < 0) ? 1 : 0;
34 const tran_low_t abs_qc = abs(qc);
35 const tran_low_t tqc = tcoeff[ci];
36 const tran_low_t dqc = dqcoeff[ci];
37 const int64_t dist = get_coeff_dist(tqc, dqc, shift);
38 const int64_t dist0 = get_coeff_dist(tqc, 0, shift);
39 const int rate =
40 get_coeff_cost_general(is_last, ci, abs_qc, sign, coeff_ctx,
41 dc_sign_ctx, txb_costs, bwl, tx_class, levels);
42 const int64_t rd = RDCOST(rdmult, rate, dist);
43
44 tran_low_t qc_low, dqc_low;
45 tran_low_t abs_qc_low;
46 int64_t dist_low, rd_low;
47 int rate_low;
48 if (abs_qc == 1) {
49 abs_qc_low = qc_low = dqc_low = 0;
50 dist_low = dist0;
51 rate_low = txb_costs->base_cost[coeff_ctx][0];
52 } else {
53 get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low);
54 abs_qc_low = abs_qc - 1;
55 dist_low = get_coeff_dist(tqc, dqc_low, shift);
56 rate_low =
57 get_coeff_cost_general(is_last, ci, abs_qc_low, sign, coeff_ctx,
58 dc_sign_ctx, txb_costs, bwl, tx_class, levels);
59 }
60
61 rd_low = RDCOST(rdmult, rate_low, dist_low);
62 if (rd_low < rd) {
63 qcoeff[ci] = qc_low;
64 dqcoeff[ci] = dqc_low;
65 levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX);
66 *accu_rate += rate_low;
67 *accu_dist += dist_low - dist0;
68 } else {
69 *accu_rate += rate;
70 *accu_dist += dist - dist0;
71 }
72 }
73 }
74
update_coeff_simple(int * accu_rate,int si,int eob,TX_SIZE tx_size,TX_CLASS tx_class,int bwl,int64_t rdmult,int shift,const int16_t * dequant,const int16_t * scan,const LV_MAP_COEFF_COST * txb_costs,const tran_low_t * tcoeff,tran_low_t * qcoeff,tran_low_t * dqcoeff,uint8_t * levels,const qm_val_t * iqmatrix)75 static AOM_FORCE_INLINE void update_coeff_simple(
76 int *accu_rate, int si, int eob, TX_SIZE tx_size, TX_CLASS tx_class,
77 int bwl, int64_t rdmult, int shift, const int16_t *dequant,
78 const int16_t *scan, const LV_MAP_COEFF_COST *txb_costs,
79 const tran_low_t *tcoeff, tran_low_t *qcoeff, tran_low_t *dqcoeff,
80 uint8_t *levels, const qm_val_t *iqmatrix) {
81 const int dqv = get_dqv(dequant, scan[si], iqmatrix);
82 (void)eob;
83 // this simple version assumes the coeff's scan_idx is not DC (scan_idx != 0)
84 // and not the last (scan_idx != eob - 1)
85 assert(si != eob - 1);
86 assert(si > 0);
87 const int ci = scan[si];
88 const tran_low_t qc = qcoeff[ci];
89 const int coeff_ctx =
90 get_lower_levels_ctx(levels, ci, bwl, tx_size, tx_class);
91 if (qc == 0) {
92 *accu_rate += txb_costs->base_cost[coeff_ctx][0];
93 } else {
94 const tran_low_t abs_qc = abs(qc);
95 const tran_low_t abs_tqc = abs(tcoeff[ci]);
96 const tran_low_t abs_dqc = abs(dqcoeff[ci]);
97 int rate_low = 0;
98 const int rate = get_two_coeff_cost_simple(
99 ci, abs_qc, coeff_ctx, txb_costs, bwl, tx_class, levels, &rate_low);
100 if (abs_dqc < abs_tqc) {
101 *accu_rate += rate;
102 return;
103 }
104
105 const int64_t dist = get_coeff_dist(abs_tqc, abs_dqc, shift);
106 const int64_t rd = RDCOST(rdmult, rate, dist);
107
108 const tran_low_t abs_qc_low = abs_qc - 1;
109 const tran_low_t abs_dqc_low = (abs_qc_low * dqv) >> shift;
110 const int64_t dist_low = get_coeff_dist(abs_tqc, abs_dqc_low, shift);
111 const int64_t rd_low = RDCOST(rdmult, rate_low, dist_low);
112
113 if (rd_low < rd) {
114 const int sign = (qc < 0) ? 1 : 0;
115 qcoeff[ci] = (-sign ^ abs_qc_low) + sign;
116 dqcoeff[ci] = (-sign ^ abs_dqc_low) + sign;
117 levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX);
118 *accu_rate += rate_low;
119 } else {
120 *accu_rate += rate;
121 }
122 }
123 }
124
update_coeff_eob(int * accu_rate,int64_t * accu_dist,int * eob,int * nz_num,int * nz_ci,int si,TX_SIZE tx_size,TX_CLASS tx_class,int bwl,int height,int dc_sign_ctx,int64_t rdmult,int shift,const int16_t * dequant,const int16_t * scan,const LV_MAP_EOB_COST * txb_eob_costs,const LV_MAP_COEFF_COST * txb_costs,const tran_low_t * tcoeff,tran_low_t * qcoeff,tran_low_t * dqcoeff,uint8_t * levels,int sharpness,const qm_val_t * iqmatrix)125 static AOM_FORCE_INLINE void update_coeff_eob(
126 int *accu_rate, int64_t *accu_dist, int *eob, int *nz_num, int *nz_ci,
127 int si, TX_SIZE tx_size, TX_CLASS tx_class, int bwl, int height,
128 int dc_sign_ctx, int64_t rdmult, int shift, const int16_t *dequant,
129 const int16_t *scan, const LV_MAP_EOB_COST *txb_eob_costs,
130 const LV_MAP_COEFF_COST *txb_costs, const tran_low_t *tcoeff,
131 tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels, int sharpness,
132 const qm_val_t *iqmatrix) {
133 const int dqv = get_dqv(dequant, scan[si], iqmatrix);
134 assert(si != *eob - 1);
135 const int ci = scan[si];
136 const tran_low_t qc = qcoeff[ci];
137 const int coeff_ctx =
138 get_lower_levels_ctx(levels, ci, bwl, tx_size, tx_class);
139 if (qc == 0) {
140 *accu_rate += txb_costs->base_cost[coeff_ctx][0];
141 } else {
142 int lower_level = 0;
143 const tran_low_t abs_qc = abs(qc);
144 const tran_low_t tqc = tcoeff[ci];
145 const tran_low_t dqc = dqcoeff[ci];
146 const int sign = (qc < 0) ? 1 : 0;
147 const int64_t dist0 = get_coeff_dist(tqc, 0, shift);
148 int64_t dist = get_coeff_dist(tqc, dqc, shift) - dist0;
149 int rate =
150 get_coeff_cost_general(0, ci, abs_qc, sign, coeff_ctx, dc_sign_ctx,
151 txb_costs, bwl, tx_class, levels);
152 int64_t rd = RDCOST(rdmult, *accu_rate + rate, *accu_dist + dist);
153
154 tran_low_t qc_low, dqc_low;
155 tran_low_t abs_qc_low;
156 int64_t dist_low, rd_low;
157 int rate_low;
158
159 if (abs_qc == 1) {
160 abs_qc_low = 0;
161 dqc_low = qc_low = 0;
162 dist_low = 0;
163 rate_low = txb_costs->base_cost[coeff_ctx][0];
164 rd_low = RDCOST(rdmult, *accu_rate + rate_low, *accu_dist);
165 } else {
166 get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low);
167 abs_qc_low = abs_qc - 1;
168 dist_low = get_coeff_dist(tqc, dqc_low, shift) - dist0;
169 rate_low =
170 get_coeff_cost_general(0, ci, abs_qc_low, sign, coeff_ctx,
171 dc_sign_ctx, txb_costs, bwl, tx_class, levels);
172 rd_low = RDCOST(rdmult, *accu_rate + rate_low, *accu_dist + dist_low);
173 }
174
175 int lower_level_new_eob = 0;
176 const int new_eob = si + 1;
177 const int coeff_ctx_new_eob = get_lower_levels_ctx_eob(bwl, height, si);
178 const int new_eob_cost =
179 get_eob_cost(new_eob, txb_eob_costs, txb_costs, tx_class);
180 int rate_coeff_eob =
181 new_eob_cost + get_coeff_cost_eob(ci, abs_qc, sign, coeff_ctx_new_eob,
182 dc_sign_ctx, txb_costs, bwl,
183 tx_class);
184 int64_t dist_new_eob = dist;
185 int64_t rd_new_eob = RDCOST(rdmult, rate_coeff_eob, dist_new_eob);
186
187 if (abs_qc_low > 0) {
188 const int rate_coeff_eob_low =
189 new_eob_cost + get_coeff_cost_eob(ci, abs_qc_low, sign,
190 coeff_ctx_new_eob, dc_sign_ctx,
191 txb_costs, bwl, tx_class);
192 const int64_t dist_new_eob_low = dist_low;
193 const int64_t rd_new_eob_low =
194 RDCOST(rdmult, rate_coeff_eob_low, dist_new_eob_low);
195 if (rd_new_eob_low < rd_new_eob) {
196 lower_level_new_eob = 1;
197 rd_new_eob = rd_new_eob_low;
198 rate_coeff_eob = rate_coeff_eob_low;
199 dist_new_eob = dist_new_eob_low;
200 }
201 }
202
203 if (sharpness == 0 || abs_qc > 1) {
204 if (rd_low < rd) {
205 lower_level = 1;
206 rd = rd_low;
207 rate = rate_low;
208 dist = dist_low;
209 }
210 }
211
212 if (sharpness == 0 && rd_new_eob < rd) {
213 for (int ni = 0; ni < *nz_num; ++ni) {
214 int last_ci = nz_ci[ni];
215 levels[get_padded_idx(last_ci, bwl)] = 0;
216 qcoeff[last_ci] = 0;
217 dqcoeff[last_ci] = 0;
218 }
219 *eob = new_eob;
220 *nz_num = 0;
221 *accu_rate = rate_coeff_eob;
222 *accu_dist = dist_new_eob;
223 lower_level = lower_level_new_eob;
224 } else {
225 *accu_rate += rate;
226 *accu_dist += dist;
227 }
228
229 if (lower_level) {
230 qcoeff[ci] = qc_low;
231 dqcoeff[ci] = dqc_low;
232 levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX);
233 }
234 if (qcoeff[ci]) {
235 nz_ci[*nz_num] = ci;
236 ++*nz_num;
237 }
238 }
239 }
240
update_skip(int * accu_rate,int64_t accu_dist,int * eob,int nz_num,int * nz_ci,int64_t rdmult,int skip_cost,int non_skip_cost,tran_low_t * qcoeff,tran_low_t * dqcoeff)241 static INLINE void update_skip(int *accu_rate, int64_t accu_dist, int *eob,
242 int nz_num, int *nz_ci, int64_t rdmult,
243 int skip_cost, int non_skip_cost,
244 tran_low_t *qcoeff, tran_low_t *dqcoeff) {
245 const int64_t rd = RDCOST(rdmult, *accu_rate + non_skip_cost, accu_dist);
246 const int64_t rd_new_eob = RDCOST(rdmult, skip_cost, 0);
247 if (rd_new_eob < rd) {
248 for (int i = 0; i < nz_num; ++i) {
249 const int ci = nz_ci[i];
250 qcoeff[ci] = 0;
251 dqcoeff[ci] = 0;
252 // no need to set up levels because this is the last step
253 // levels[get_padded_idx(ci, bwl)] = 0;
254 }
255 *accu_rate = 0;
256 *eob = 0;
257 }
258 }
259
260 // TODO(angiebird): use this function whenever it's possible
get_tx_type_cost(const MACROBLOCK * x,const MACROBLOCKD * xd,int plane,TX_SIZE tx_size,TX_TYPE tx_type,int reduced_tx_set_used)261 static int get_tx_type_cost(const MACROBLOCK *x, const MACROBLOCKD *xd,
262 int plane, TX_SIZE tx_size, TX_TYPE tx_type,
263 int reduced_tx_set_used) {
264 if (plane > 0) return 0;
265
266 const TX_SIZE square_tx_size = txsize_sqr_map[tx_size];
267
268 const MB_MODE_INFO *mbmi = xd->mi[0];
269 const int is_inter = is_inter_block(mbmi);
270 if (get_ext_tx_types(tx_size, is_inter, reduced_tx_set_used) > 1 &&
271 !xd->lossless[xd->mi[0]->segment_id]) {
272 const int ext_tx_set =
273 get_ext_tx_set(tx_size, is_inter, reduced_tx_set_used);
274 if (is_inter) {
275 if (ext_tx_set > 0)
276 return x->mode_costs
277 .inter_tx_type_costs[ext_tx_set][square_tx_size][tx_type];
278 } else {
279 if (ext_tx_set > 0) {
280 PREDICTION_MODE intra_dir;
281 if (mbmi->filter_intra_mode_info.use_filter_intra)
282 intra_dir = fimode_to_intradir[mbmi->filter_intra_mode_info
283 .filter_intra_mode];
284 else
285 intra_dir = mbmi->mode;
286 return x->mode_costs.intra_tx_type_costs[ext_tx_set][square_tx_size]
287 [intra_dir][tx_type];
288 }
289 }
290 }
291 return 0;
292 }
293
av1_optimize_txb(const struct AV1_COMP * cpi,MACROBLOCK * x,int plane,int block,TX_SIZE tx_size,TX_TYPE tx_type,const TXB_CTX * const txb_ctx,int * rate_cost,int sharpness)294 int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane,
295 int block, TX_SIZE tx_size, TX_TYPE tx_type,
296 const TXB_CTX *const txb_ctx, int *rate_cost,
297 int sharpness) {
298 MACROBLOCKD *xd = &x->e_mbd;
299 const struct macroblock_plane *p = &x->plane[plane];
300 const SCAN_ORDER *scan_order = get_scan(tx_size, tx_type);
301 const int16_t *scan = scan_order->scan;
302 const int shift = av1_get_tx_scale(tx_size);
303 int eob = p->eobs[block];
304 const int16_t *dequant = p->dequant_QTX;
305 const qm_val_t *iqmatrix =
306 av1_get_iqmatrix(&cpi->common.quant_params, xd, plane, tx_size, tx_type);
307 const int block_offset = BLOCK_OFFSET(block);
308 tran_low_t *qcoeff = p->qcoeff + block_offset;
309 tran_low_t *dqcoeff = p->dqcoeff + block_offset;
310 const tran_low_t *tcoeff = p->coeff + block_offset;
311 const CoeffCosts *coeff_costs = &x->coeff_costs;
312
313 // This function is not called if eob = 0.
314 assert(eob > 0);
315
316 const AV1_COMMON *cm = &cpi->common;
317 const PLANE_TYPE plane_type = get_plane_type(plane);
318 const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
319 const TX_CLASS tx_class = tx_type_to_class[tx_type];
320 const MB_MODE_INFO *mbmi = xd->mi[0];
321 const int bwl = get_txb_bwl(tx_size);
322 const int width = get_txb_wide(tx_size);
323 const int height = get_txb_high(tx_size);
324 assert(width == (1 << bwl));
325 const int is_inter = is_inter_block(mbmi);
326 const LV_MAP_COEFF_COST *txb_costs =
327 &coeff_costs->coeff_costs[txs_ctx][plane_type];
328 const int eob_multi_size = txsize_log2_minus4[tx_size];
329 const LV_MAP_EOB_COST *txb_eob_costs =
330 &coeff_costs->eob_costs[eob_multi_size][plane_type];
331
332 const int rshift = 2;
333
334 const int64_t rdmult =
335 (((int64_t)x->rdmult *
336 (plane_rd_mult[is_inter][plane_type] << (2 * (xd->bd - 8)))) +
337 2) >>
338 rshift;
339
340 uint8_t levels_buf[TX_PAD_2D];
341 uint8_t *const levels = set_levels(levels_buf, width);
342
343 if (eob > 1) av1_txb_init_levels(qcoeff, width, height, levels);
344
345 // TODO(angirbird): check iqmatrix
346
347 const int non_skip_cost = txb_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][0];
348 const int skip_cost = txb_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1];
349 const int eob_cost = get_eob_cost(eob, txb_eob_costs, txb_costs, tx_class);
350 int accu_rate = eob_cost;
351 int64_t accu_dist = 0;
352 int si = eob - 1;
353 const int ci = scan[si];
354 const tran_low_t qc = qcoeff[ci];
355 const tran_low_t abs_qc = abs(qc);
356 const int sign = qc < 0;
357 const int max_nz_num = 2;
358 int nz_num = 1;
359 int nz_ci[3] = { ci, 0, 0 };
360 if (abs_qc >= 2) {
361 update_coeff_general(&accu_rate, &accu_dist, si, eob, tx_size, tx_class,
362 bwl, height, rdmult, shift, txb_ctx->dc_sign_ctx,
363 dequant, scan, txb_costs, tcoeff, qcoeff, dqcoeff,
364 levels, iqmatrix);
365 --si;
366 } else {
367 assert(abs_qc == 1);
368 const int coeff_ctx = get_lower_levels_ctx_eob(bwl, height, si);
369 accu_rate +=
370 get_coeff_cost_eob(ci, abs_qc, sign, coeff_ctx, txb_ctx->dc_sign_ctx,
371 txb_costs, bwl, tx_class);
372 const tran_low_t tqc = tcoeff[ci];
373 const tran_low_t dqc = dqcoeff[ci];
374 const int64_t dist = get_coeff_dist(tqc, dqc, shift);
375 const int64_t dist0 = get_coeff_dist(tqc, 0, shift);
376 accu_dist += dist - dist0;
377 --si;
378 }
379
380 #define UPDATE_COEFF_EOB_CASE(tx_class_literal) \
381 case tx_class_literal: \
382 for (; si >= 0 && nz_num <= max_nz_num; --si) { \
383 update_coeff_eob(&accu_rate, &accu_dist, &eob, &nz_num, nz_ci, si, \
384 tx_size, tx_class_literal, bwl, height, \
385 txb_ctx->dc_sign_ctx, rdmult, shift, dequant, scan, \
386 txb_eob_costs, txb_costs, tcoeff, qcoeff, dqcoeff, \
387 levels, sharpness, iqmatrix); \
388 } \
389 break;
390 switch (tx_class) {
391 UPDATE_COEFF_EOB_CASE(TX_CLASS_2D);
392 UPDATE_COEFF_EOB_CASE(TX_CLASS_HORIZ);
393 UPDATE_COEFF_EOB_CASE(TX_CLASS_VERT);
394 #undef UPDATE_COEFF_EOB_CASE
395 default: assert(false);
396 }
397
398 if (si == -1 && nz_num <= max_nz_num && sharpness == 0) {
399 update_skip(&accu_rate, accu_dist, &eob, nz_num, nz_ci, rdmult, skip_cost,
400 non_skip_cost, qcoeff, dqcoeff);
401 }
402
403 #define UPDATE_COEFF_SIMPLE_CASE(tx_class_literal) \
404 case tx_class_literal: \
405 for (; si >= 1; --si) { \
406 update_coeff_simple(&accu_rate, si, eob, tx_size, tx_class_literal, bwl, \
407 rdmult, shift, dequant, scan, txb_costs, tcoeff, \
408 qcoeff, dqcoeff, levels, iqmatrix); \
409 } \
410 break;
411 switch (tx_class) {
412 UPDATE_COEFF_SIMPLE_CASE(TX_CLASS_2D);
413 UPDATE_COEFF_SIMPLE_CASE(TX_CLASS_HORIZ);
414 UPDATE_COEFF_SIMPLE_CASE(TX_CLASS_VERT);
415 #undef UPDATE_COEFF_SIMPLE_CASE
416 default: assert(false);
417 }
418
419 // DC position
420 if (si == 0) {
421 // no need to update accu_dist because it's not used after this point
422 int64_t dummy_dist = 0;
423 update_coeff_general(&accu_rate, &dummy_dist, si, eob, tx_size, tx_class,
424 bwl, height, rdmult, shift, txb_ctx->dc_sign_ctx,
425 dequant, scan, txb_costs, tcoeff, qcoeff, dqcoeff,
426 levels, iqmatrix);
427 }
428
429 const int tx_type_cost = get_tx_type_cost(x, xd, plane, tx_size, tx_type,
430 cm->features.reduced_tx_set_used);
431 if (eob == 0)
432 accu_rate += skip_cost;
433 else
434 accu_rate += non_skip_cost + tx_type_cost;
435
436 p->eobs[block] = eob;
437 p->txb_entropy_ctx[block] =
438 av1_get_txb_entropy_context(qcoeff, scan_order, p->eobs[block]);
439
440 *rate_cost = accu_rate;
441 return eob;
442 }
443
warehouse_efficients_txb(const MACROBLOCK * x,const int plane,const int block,const TX_SIZE tx_size,const TXB_CTX * const txb_ctx,const struct macroblock_plane * p,const int eob,const PLANE_TYPE plane_type,const LV_MAP_COEFF_COST * const coeff_costs,const MACROBLOCKD * const xd,const TX_TYPE tx_type,const TX_CLASS tx_class,int reduced_tx_set_used)444 static AOM_FORCE_INLINE int warehouse_efficients_txb(
445 const MACROBLOCK *x, const int plane, const int block,
446 const TX_SIZE tx_size, const TXB_CTX *const txb_ctx,
447 const struct macroblock_plane *p, const int eob,
448 const PLANE_TYPE plane_type, const LV_MAP_COEFF_COST *const coeff_costs,
449 const MACROBLOCKD *const xd, const TX_TYPE tx_type, const TX_CLASS tx_class,
450 int reduced_tx_set_used) {
451 const tran_low_t *const qcoeff = p->qcoeff + BLOCK_OFFSET(block);
452 const int txb_skip_ctx = txb_ctx->txb_skip_ctx;
453 const int bwl = get_txb_bwl(tx_size);
454 const int width = get_txb_wide(tx_size);
455 const int height = get_txb_high(tx_size);
456 const SCAN_ORDER *const scan_order = get_scan(tx_size, tx_type);
457 const int16_t *const scan = scan_order->scan;
458 uint8_t levels_buf[TX_PAD_2D];
459 uint8_t *const levels = set_levels(levels_buf, width);
460 DECLARE_ALIGNED(16, int8_t, coeff_contexts[MAX_TX_SQUARE]);
461 const int eob_multi_size = txsize_log2_minus4[tx_size];
462 const LV_MAP_EOB_COST *const eob_costs =
463 &x->coeff_costs.eob_costs[eob_multi_size][plane_type];
464 int cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0];
465
466 av1_txb_init_levels(qcoeff, width, height, levels);
467
468 cost += get_tx_type_cost(x, xd, plane, tx_size, tx_type, reduced_tx_set_used);
469
470 cost += get_eob_cost(eob, eob_costs, coeff_costs, tx_class);
471
472 av1_get_nz_map_contexts(levels, scan, eob, tx_size, tx_class, coeff_contexts);
473
474 const int(*lps_cost)[COEFF_BASE_RANGE + 1 + COEFF_BASE_RANGE + 1] =
475 coeff_costs->lps_cost;
476 int c = eob - 1;
477 {
478 const int pos = scan[c];
479 const tran_low_t v = qcoeff[pos];
480 const int sign = AOMSIGN(v);
481 const int level = (v ^ sign) - sign;
482 const int coeff_ctx = coeff_contexts[pos];
483 cost += coeff_costs->base_eob_cost[coeff_ctx][AOMMIN(level, 3) - 1];
484
485 if (v) {
486 // sign bit cost
487 if (level > NUM_BASE_LEVELS) {
488 const int ctx = get_br_ctx_eob(pos, bwl, tx_class);
489 cost += get_br_cost(level, lps_cost[ctx]);
490 }
491 if (c) {
492 cost += av1_cost_literal(1);
493 } else {
494 const int sign01 = (sign ^ sign) - sign;
495 const int dc_sign_ctx = txb_ctx->dc_sign_ctx;
496 cost += coeff_costs->dc_sign_cost[dc_sign_ctx][sign01];
497 return cost;
498 }
499 }
500 }
501 const int(*base_cost)[8] = coeff_costs->base_cost;
502 for (c = eob - 2; c >= 1; --c) {
503 const int pos = scan[c];
504 const int coeff_ctx = coeff_contexts[pos];
505 const tran_low_t v = qcoeff[pos];
506 const int level = abs(v);
507 cost += base_cost[coeff_ctx][AOMMIN(level, 3)];
508 if (v) {
509 // sign bit cost
510 cost += av1_cost_literal(1);
511 if (level > NUM_BASE_LEVELS) {
512 const int ctx = get_br_ctx(levels, pos, bwl, tx_class);
513 cost += get_br_cost(level, lps_cost[ctx]);
514 }
515 }
516 }
517 // c == 0 after previous loop
518 {
519 const int pos = scan[c];
520 const tran_low_t v = qcoeff[pos];
521 const int coeff_ctx = coeff_contexts[pos];
522 const int sign = AOMSIGN(v);
523 const int level = (v ^ sign) - sign;
524 cost += base_cost[coeff_ctx][AOMMIN(level, 3)];
525
526 if (v) {
527 // sign bit cost
528 const int sign01 = (sign ^ sign) - sign;
529 const int dc_sign_ctx = txb_ctx->dc_sign_ctx;
530 cost += coeff_costs->dc_sign_cost[dc_sign_ctx][sign01];
531 if (level > NUM_BASE_LEVELS) {
532 const int ctx = get_br_ctx(levels, pos, bwl, tx_class);
533 cost += get_br_cost(level, lps_cost[ctx]);
534 }
535 }
536 }
537 return cost;
538 }
539
av1_cost_coeffs_txb_estimate(const MACROBLOCK * x,const int plane,const int block,const TX_SIZE tx_size,const TX_TYPE tx_type)540 int av1_cost_coeffs_txb_estimate(const MACROBLOCK *x, const int plane,
541 const int block, const TX_SIZE tx_size,
542 const TX_TYPE tx_type) {
543 assert(plane == 0);
544
545 int cost = 0;
546 const struct macroblock_plane *p = &x->plane[plane];
547 const SCAN_ORDER *scan_order = get_scan(tx_size, tx_type);
548 const int16_t *scan = scan_order->scan;
549 tran_low_t *qcoeff = p->qcoeff + BLOCK_OFFSET(block);
550
551 int eob = p->eobs[block];
552
553 // coeffs
554 int c = eob - 1;
555 // eob
556 {
557 const int pos = scan[c];
558 const tran_low_t v = abs(qcoeff[pos]) - 1;
559 cost += (v << (AV1_PROB_COST_SHIFT + 2));
560 }
561 // other coeffs
562 for (c = eob - 2; c >= 0; c--) {
563 const int pos = scan[c];
564 const tran_low_t v = abs(qcoeff[pos]);
565 const int idx = AOMMIN(v, 14);
566
567 cost += costLUT[idx];
568 }
569
570 // const_term does not contain DC, and log(e) does not contain eob, so both
571 // (eob-1)
572 cost += (const_term + loge_par) * (eob - 1);
573
574 return cost;
575 }
576
warehouse_efficients_txb_laplacian(const MACROBLOCK * x,const int plane,const int block,const TX_SIZE tx_size,const TXB_CTX * const txb_ctx,const int eob,const PLANE_TYPE plane_type,const LV_MAP_COEFF_COST * const coeff_costs,const MACROBLOCKD * const xd,const TX_TYPE tx_type,const TX_CLASS tx_class,int reduced_tx_set_used)577 static AOM_FORCE_INLINE int warehouse_efficients_txb_laplacian(
578 const MACROBLOCK *x, const int plane, const int block,
579 const TX_SIZE tx_size, const TXB_CTX *const txb_ctx, const int eob,
580 const PLANE_TYPE plane_type, const LV_MAP_COEFF_COST *const coeff_costs,
581 const MACROBLOCKD *const xd, const TX_TYPE tx_type, const TX_CLASS tx_class,
582 int reduced_tx_set_used) {
583 const int txb_skip_ctx = txb_ctx->txb_skip_ctx;
584
585 const int eob_multi_size = txsize_log2_minus4[tx_size];
586 const LV_MAP_EOB_COST *const eob_costs =
587 &x->coeff_costs.eob_costs[eob_multi_size][plane_type];
588 int cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0];
589
590 cost += get_tx_type_cost(x, xd, plane, tx_size, tx_type, reduced_tx_set_used);
591
592 cost += get_eob_cost(eob, eob_costs, coeff_costs, tx_class);
593
594 cost += av1_cost_coeffs_txb_estimate(x, plane, block, tx_size, tx_type);
595 return cost;
596 }
597
av1_cost_coeffs_txb(const MACROBLOCK * x,const int plane,const int block,const TX_SIZE tx_size,const TX_TYPE tx_type,const TXB_CTX * const txb_ctx,int reduced_tx_set_used)598 int av1_cost_coeffs_txb(const MACROBLOCK *x, const int plane, const int block,
599 const TX_SIZE tx_size, const TX_TYPE tx_type,
600 const TXB_CTX *const txb_ctx, int reduced_tx_set_used) {
601 const struct macroblock_plane *p = &x->plane[plane];
602 const int eob = p->eobs[block];
603 const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
604 const PLANE_TYPE plane_type = get_plane_type(plane);
605 const LV_MAP_COEFF_COST *const coeff_costs =
606 &x->coeff_costs.coeff_costs[txs_ctx][plane_type];
607 if (eob == 0) {
608 return coeff_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1];
609 }
610
611 const MACROBLOCKD *const xd = &x->e_mbd;
612 const TX_CLASS tx_class = tx_type_to_class[tx_type];
613
614 return warehouse_efficients_txb(x, plane, block, tx_size, txb_ctx, p, eob,
615 plane_type, coeff_costs, xd, tx_type,
616 tx_class, reduced_tx_set_used);
617 }
618
av1_cost_coeffs_txb_laplacian(const MACROBLOCK * x,const int plane,const int block,const TX_SIZE tx_size,const TX_TYPE tx_type,const TXB_CTX * const txb_ctx,const int reduced_tx_set_used,const int adjust_eob)619 int av1_cost_coeffs_txb_laplacian(const MACROBLOCK *x, const int plane,
620 const int block, const TX_SIZE tx_size,
621 const TX_TYPE tx_type,
622 const TXB_CTX *const txb_ctx,
623 const int reduced_tx_set_used,
624 const int adjust_eob) {
625 const struct macroblock_plane *p = &x->plane[plane];
626 int eob = p->eobs[block];
627
628 if (adjust_eob) {
629 const SCAN_ORDER *scan_order = get_scan(tx_size, tx_type);
630 const int16_t *scan = scan_order->scan;
631 tran_low_t *tcoeff = p->coeff + BLOCK_OFFSET(block);
632 tran_low_t *qcoeff = p->qcoeff + BLOCK_OFFSET(block);
633 tran_low_t *dqcoeff = p->dqcoeff + BLOCK_OFFSET(block);
634 update_coeff_eob_fast(&eob, av1_get_tx_scale(tx_size), p->dequant_QTX, scan,
635 tcoeff, qcoeff, dqcoeff);
636 p->eobs[block] = eob;
637 }
638
639 const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
640 const PLANE_TYPE plane_type = get_plane_type(plane);
641 const LV_MAP_COEFF_COST *const coeff_costs =
642 &x->coeff_costs.coeff_costs[txs_ctx][plane_type];
643 if (eob == 0) {
644 return coeff_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1];
645 }
646
647 const MACROBLOCKD *const xd = &x->e_mbd;
648 const TX_CLASS tx_class = tx_type_to_class[tx_type];
649
650 return warehouse_efficients_txb_laplacian(
651 x, plane, block, tx_size, txb_ctx, eob, plane_type, coeff_costs, xd,
652 tx_type, tx_class, reduced_tx_set_used);
653 }
654