1 /*
2 * Copyright (c) 2020, Alliance for Open Media. All rights reserved
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #include <assert.h>
13 #include <limits.h>
14 #include <math.h>
15
16 #include "config/aom_dsp_rtcd.h"
17 #include "aom_dsp/aom_dsp_common.h"
18 #include "aom_scale/yv12config.h"
19 #include "aom/aom_integer.h"
20 #include "av1/common/reconinter.h"
21 #include "av1/encoder/reconinter_enc.h"
22 #include "av1/encoder/context_tree.h"
23 #include "av1/encoder/av1_temporal_denoiser.h"
24 #include "av1/encoder/encoder.h"
25
26 #ifdef OUTPUT_YUV_DENOISED
27 static void make_grayscale(YV12_BUFFER_CONFIG *yuv);
28 #endif
29
absdiff_thresh(BLOCK_SIZE bs,int increase_denoising)30 static int absdiff_thresh(BLOCK_SIZE bs, int increase_denoising) {
31 (void)bs;
32 return 3 + (increase_denoising ? 1 : 0);
33 }
34
delta_thresh(BLOCK_SIZE bs,int increase_denoising)35 static int delta_thresh(BLOCK_SIZE bs, int increase_denoising) {
36 (void)bs;
37 (void)increase_denoising;
38 return 4;
39 }
40
noise_motion_thresh(BLOCK_SIZE bs,int increase_denoising)41 static int noise_motion_thresh(BLOCK_SIZE bs, int increase_denoising) {
42 (void)bs;
43 (void)increase_denoising;
44 return 625;
45 }
46
sse_thresh(BLOCK_SIZE bs,int increase_denoising)47 static unsigned int sse_thresh(BLOCK_SIZE bs, int increase_denoising) {
48 return (1 << num_pels_log2_lookup[bs]) * (increase_denoising ? 80 : 40);
49 }
50
sse_diff_thresh(BLOCK_SIZE bs,int increase_denoising,int motion_magnitude)51 static int sse_diff_thresh(BLOCK_SIZE bs, int increase_denoising,
52 int motion_magnitude) {
53 if (motion_magnitude > noise_motion_thresh(bs, increase_denoising)) {
54 if (increase_denoising)
55 return (1 << num_pels_log2_lookup[bs]) << 2;
56 else
57 return 0;
58 } else {
59 return (1 << num_pels_log2_lookup[bs]) << 4;
60 }
61 }
62
total_adj_weak_thresh(BLOCK_SIZE bs,int increase_denoising)63 static int total_adj_weak_thresh(BLOCK_SIZE bs, int increase_denoising) {
64 return (1 << num_pels_log2_lookup[bs]) * (increase_denoising ? 3 : 2);
65 }
66
67 // TODO(kyslov): If increase_denoising is enabled in the future,
68 // we might need to update the code for calculating 'total_adj' in
69 // case the C code is not bit-exact with corresponding sse2 code.
av1_denoiser_filter_c(const uint8_t * sig,int sig_stride,const uint8_t * mc_avg,int mc_avg_stride,uint8_t * avg,int avg_stride,int increase_denoising,BLOCK_SIZE bs,int motion_magnitude)70 int av1_denoiser_filter_c(const uint8_t *sig, int sig_stride,
71 const uint8_t *mc_avg, int mc_avg_stride,
72 uint8_t *avg, int avg_stride, int increase_denoising,
73 BLOCK_SIZE bs, int motion_magnitude) {
74 int r, c;
75 const uint8_t *sig_start = sig;
76 const uint8_t *mc_avg_start = mc_avg;
77 uint8_t *avg_start = avg;
78 int diff, adj, absdiff, delta;
79 int adj_val[] = { 3, 4, 6 };
80 int total_adj = 0;
81 int shift_inc = 1;
82
83 // If motion_magnitude is small, making the denoiser more aggressive by
84 // increasing the adjustment for each level. Add another increment for
85 // blocks that are labeled for increase denoising.
86 if (motion_magnitude <= MOTION_MAGNITUDE_THRESHOLD) {
87 if (increase_denoising) {
88 shift_inc = 2;
89 }
90 adj_val[0] += shift_inc;
91 adj_val[1] += shift_inc;
92 adj_val[2] += shift_inc;
93 }
94
95 // First attempt to apply a strong temporal denoising filter.
96 for (r = 0; r < block_size_high[bs]; ++r) {
97 for (c = 0; c < block_size_wide[bs]; ++c) {
98 diff = mc_avg[c] - sig[c];
99 absdiff = abs(diff);
100
101 if (absdiff <= absdiff_thresh(bs, increase_denoising)) {
102 avg[c] = mc_avg[c];
103 total_adj += diff;
104 } else {
105 switch (absdiff) {
106 case 4:
107 case 5:
108 case 6:
109 case 7: adj = adj_val[0]; break;
110 case 8:
111 case 9:
112 case 10:
113 case 11:
114 case 12:
115 case 13:
116 case 14:
117 case 15: adj = adj_val[1]; break;
118 default: adj = adj_val[2];
119 }
120 if (diff > 0) {
121 avg[c] = AOMMIN(UINT8_MAX, sig[c] + adj);
122 total_adj += adj;
123 } else {
124 avg[c] = AOMMAX(0, sig[c] - adj);
125 total_adj -= adj;
126 }
127 }
128 }
129 sig += sig_stride;
130 avg += avg_stride;
131 mc_avg += mc_avg_stride;
132 }
133
134 // If the strong filter did not modify the signal too much, we're all set.
135 if (abs(total_adj) <= total_adj_strong_thresh(bs, increase_denoising)) {
136 return FILTER_BLOCK;
137 }
138
139 // Otherwise, we try to dampen the filter if the delta is not too high.
140 delta = ((abs(total_adj) - total_adj_strong_thresh(bs, increase_denoising)) >>
141 num_pels_log2_lookup[bs]) +
142 1;
143
144 if (delta >= delta_thresh(bs, increase_denoising)) {
145 return COPY_BLOCK;
146 }
147
148 mc_avg = mc_avg_start;
149 avg = avg_start;
150 sig = sig_start;
151 for (r = 0; r < block_size_high[bs]; ++r) {
152 for (c = 0; c < block_size_wide[bs]; ++c) {
153 diff = mc_avg[c] - sig[c];
154 adj = abs(diff);
155 if (adj > delta) {
156 adj = delta;
157 }
158 if (diff > 0) {
159 // Diff positive means we made positive adjustment above
160 // (in first try/attempt), so now make negative adjustment to bring
161 // denoised signal down.
162 avg[c] = AOMMAX(0, avg[c] - adj);
163 total_adj -= adj;
164 } else {
165 // Diff negative means we made negative adjustment above
166 // (in first try/attempt), so now make positive adjustment to bring
167 // denoised signal up.
168 avg[c] = AOMMIN(UINT8_MAX, avg[c] + adj);
169 total_adj += adj;
170 }
171 }
172 sig += sig_stride;
173 avg += avg_stride;
174 mc_avg += mc_avg_stride;
175 }
176
177 // We can use the filter if it has been sufficiently dampened
178 if (abs(total_adj) <= total_adj_weak_thresh(bs, increase_denoising)) {
179 return FILTER_BLOCK;
180 }
181 return COPY_BLOCK;
182 }
183
block_start(uint8_t * framebuf,int stride,int mi_row,int mi_col)184 static uint8_t *block_start(uint8_t *framebuf, int stride, int mi_row,
185 int mi_col) {
186 return framebuf + (stride * mi_row << 2) + (mi_col << 2);
187 }
188
perform_motion_compensation(AV1_COMMON * const cm,AV1_DENOISER * denoiser,MACROBLOCK * mb,BLOCK_SIZE bs,int increase_denoising,int mi_row,int mi_col,PICK_MODE_CONTEXT * ctx,int motion_magnitude,int * zeromv_filter,int num_spatial_layers,int width,int lst_fb_idx,int gld_fb_idx,int use_svc,int spatial_layer,int use_gf_temporal_ref)189 static AV1_DENOISER_DECISION perform_motion_compensation(
190 AV1_COMMON *const cm, AV1_DENOISER *denoiser, MACROBLOCK *mb, BLOCK_SIZE bs,
191 int increase_denoising, int mi_row, int mi_col, PICK_MODE_CONTEXT *ctx,
192 int motion_magnitude, int *zeromv_filter, int num_spatial_layers, int width,
193 int lst_fb_idx, int gld_fb_idx, int use_svc, int spatial_layer,
194 int use_gf_temporal_ref) {
195 const int sse_diff = (ctx->newmv_sse == UINT_MAX)
196 ? 0
197 : ((int)ctx->zeromv_sse - (int)ctx->newmv_sse);
198 int frame;
199 int denoise_layer_idx = 0;
200 MACROBLOCKD *filter_mbd = &mb->e_mbd;
201 MB_MODE_INFO *mi = filter_mbd->mi[0];
202 MB_MODE_INFO saved_mi;
203 int i;
204 struct buf_2d saved_dst[MAX_MB_PLANE];
205 struct buf_2d saved_pre[MAX_MB_PLANE];
206 // const RefBuffer *saved_block_refs[2];
207 MV_REFERENCE_FRAME saved_frame;
208
209 frame = ctx->best_reference_frame;
210
211 saved_mi = *mi;
212
213 // Avoid denoising small blocks. When noise > kDenLow or frame width > 480,
214 // denoise 16x16 blocks.
215 if (bs == BLOCK_8X8 || bs == BLOCK_8X16 || bs == BLOCK_16X8 ||
216 (bs == BLOCK_16X16 && width > 480 &&
217 denoiser->denoising_level <= kDenLow))
218 return COPY_BLOCK;
219
220 // If the best reference frame uses inter-prediction and there is enough of a
221 // difference in sum-squared-error, use it.
222 if (frame != INTRA_FRAME && frame != ALTREF_FRAME && frame != GOLDEN_FRAME &&
223 sse_diff > sse_diff_thresh(bs, increase_denoising, motion_magnitude)) {
224 mi->ref_frame[0] = ctx->best_reference_frame;
225 mi->mode = ctx->best_sse_inter_mode;
226 mi->mv[0] = ctx->best_sse_mv;
227 } else {
228 // Otherwise, use the zero reference frame.
229 frame = ctx->best_zeromv_reference_frame;
230 ctx->newmv_sse = ctx->zeromv_sse;
231 // Bias to last reference.
232 if ((num_spatial_layers > 1 && !use_gf_temporal_ref) ||
233 frame == ALTREF_FRAME ||
234 (frame == GOLDEN_FRAME && use_gf_temporal_ref) ||
235 (frame != LAST_FRAME &&
236 ((ctx->zeromv_lastref_sse<(5 * ctx->zeromv_sse)>> 2) ||
237 denoiser->denoising_level >= kDenHigh))) {
238 frame = LAST_FRAME;
239 ctx->newmv_sse = ctx->zeromv_lastref_sse;
240 }
241 mi->ref_frame[0] = frame;
242 mi->mode = GLOBALMV;
243 mi->mv[0].as_int = 0;
244 ctx->best_sse_inter_mode = GLOBALMV;
245 ctx->best_sse_mv.as_int = 0;
246 *zeromv_filter = 1;
247 if (denoiser->denoising_level > kDenMedium) {
248 motion_magnitude = 0;
249 }
250 }
251
252 saved_frame = frame;
253 // When using SVC, we need to map REF_FRAME to the frame buffer index.
254 if (use_svc) {
255 if (frame == LAST_FRAME)
256 frame = lst_fb_idx + 1;
257 else if (frame == GOLDEN_FRAME)
258 frame = gld_fb_idx + 1;
259 // Shift for the second spatial layer.
260 if (num_spatial_layers - spatial_layer == 2)
261 frame = frame + denoiser->num_ref_frames;
262 denoise_layer_idx = num_spatial_layers - spatial_layer - 1;
263 }
264
265 // Force copy (no denoise, copy source in denoised buffer) if
266 // running_avg_y[frame] is NULL.
267 if (denoiser->running_avg_y[frame].buffer_alloc == NULL) {
268 // Restore everything to its original state
269 *mi = saved_mi;
270 return COPY_BLOCK;
271 }
272
273 if (ctx->newmv_sse > sse_thresh(bs, increase_denoising)) {
274 // Restore everything to its original state
275 *mi = saved_mi;
276 return COPY_BLOCK;
277 }
278 if (motion_magnitude > (noise_motion_thresh(bs, increase_denoising) << 3)) {
279 // Restore everything to its original state
280 *mi = saved_mi;
281 return COPY_BLOCK;
282 }
283
284 // We will restore these after motion compensation.
285 for (i = 0; i < MAX_MB_PLANE; ++i) {
286 saved_pre[i] = filter_mbd->plane[i].pre[0];
287 saved_dst[i] = filter_mbd->plane[i].dst;
288 }
289
290 // Set the pointers in the MACROBLOCKD to point to the buffers in the denoiser
291 // struct.
292 set_ref_ptrs(cm, filter_mbd, saved_frame, NONE);
293 av1_setup_pre_planes(filter_mbd, 0, &(denoiser->running_avg_y[frame]), mi_row,
294 mi_col, filter_mbd->block_ref_scale_factors[0], 1);
295 av1_setup_dst_planes(filter_mbd->plane, bs,
296 &(denoiser->mc_running_avg_y[denoise_layer_idx]), mi_row,
297 mi_col, 0, 1);
298
299 av1_enc_build_inter_predictor_y(filter_mbd, mi_row, mi_col);
300
301 // Restore everything to its original state
302 *mi = saved_mi;
303 for (i = 0; i < MAX_MB_PLANE; ++i) {
304 filter_mbd->plane[i].pre[0] = saved_pre[i];
305 filter_mbd->plane[i].dst = saved_dst[i];
306 }
307
308 return FILTER_BLOCK;
309 }
310
av1_denoiser_denoise(AV1_COMP * cpi,MACROBLOCK * mb,int mi_row,int mi_col,BLOCK_SIZE bs,PICK_MODE_CONTEXT * ctx,AV1_DENOISER_DECISION * denoiser_decision,int use_gf_temporal_ref)311 void av1_denoiser_denoise(AV1_COMP *cpi, MACROBLOCK *mb, int mi_row, int mi_col,
312 BLOCK_SIZE bs, PICK_MODE_CONTEXT *ctx,
313 AV1_DENOISER_DECISION *denoiser_decision,
314 int use_gf_temporal_ref) {
315 int mv_col, mv_row;
316 int motion_magnitude = 0;
317 int zeromv_filter = 0;
318 AV1_DENOISER *denoiser = &cpi->denoiser;
319 AV1_DENOISER_DECISION decision = COPY_BLOCK;
320
321 const int shift =
322 cpi->svc.number_spatial_layers - cpi->svc.spatial_layer_id == 2
323 ? denoiser->num_ref_frames
324 : 0;
325 YV12_BUFFER_CONFIG avg = denoiser->running_avg_y[INTRA_FRAME + shift];
326 const int denoise_layer_index =
327 cpi->svc.number_spatial_layers - cpi->svc.spatial_layer_id - 1;
328 YV12_BUFFER_CONFIG mc_avg = denoiser->mc_running_avg_y[denoise_layer_index];
329 uint8_t *avg_start = block_start(avg.y_buffer, avg.y_stride, mi_row, mi_col);
330
331 uint8_t *mc_avg_start =
332 block_start(mc_avg.y_buffer, mc_avg.y_stride, mi_row, mi_col);
333 struct buf_2d src = mb->plane[0].src;
334 int increase_denoising = 0;
335 int last_is_reference = cpi->ref_frame_flags & AOM_LAST_FLAG;
336 mv_col = ctx->best_sse_mv.as_mv.col;
337 mv_row = ctx->best_sse_mv.as_mv.row;
338 motion_magnitude = mv_row * mv_row + mv_col * mv_col;
339
340 if (denoiser->denoising_level == kDenHigh) increase_denoising = 1;
341
342 // Copy block if LAST_FRAME is not a reference.
343 // Last doesn't always exist when SVC layers are dynamically changed, e.g. top
344 // spatial layer doesn't have last reference when it's brought up for the
345 // first time on the fly.
346 if (last_is_reference && denoiser->denoising_level >= kDenLow &&
347 !ctx->sb_skip_denoising)
348 decision = perform_motion_compensation(
349 &cpi->common, denoiser, mb, bs, increase_denoising, mi_row, mi_col, ctx,
350 motion_magnitude, &zeromv_filter, cpi->svc.number_spatial_layers,
351 cpi->source->y_width, cpi->svc.ref_idx[0], cpi->svc.ref_idx[3],
352 cpi->use_svc, cpi->svc.spatial_layer_id, use_gf_temporal_ref);
353
354 if (decision == FILTER_BLOCK) {
355 decision = av1_denoiser_filter(src.buf, src.stride, mc_avg_start,
356 mc_avg.y_stride, avg_start, avg.y_stride,
357 increase_denoising, bs, motion_magnitude);
358 }
359
360 if (decision == FILTER_BLOCK) {
361 aom_convolve_copy(avg_start, avg.y_stride, src.buf, src.stride,
362 block_size_wide[bs], block_size_high[bs]);
363 } else { // COPY_BLOCK
364 aom_convolve_copy(src.buf, src.stride, avg_start, avg.y_stride,
365 block_size_wide[bs], block_size_high[bs]);
366 }
367 *denoiser_decision = decision;
368 if (decision == FILTER_BLOCK && zeromv_filter == 1)
369 *denoiser_decision = FILTER_ZEROMV_BLOCK;
370 }
371
copy_frame(YV12_BUFFER_CONFIG * const dest,const YV12_BUFFER_CONFIG * const src)372 static void copy_frame(YV12_BUFFER_CONFIG *const dest,
373 const YV12_BUFFER_CONFIG *const src) {
374 int r;
375 const uint8_t *srcbuf = src->y_buffer;
376 uint8_t *destbuf = dest->y_buffer;
377
378 assert(dest->y_width == src->y_width);
379 assert(dest->y_height == src->y_height);
380
381 for (r = 0; r < dest->y_height; ++r) {
382 memcpy(destbuf, srcbuf, dest->y_width);
383 destbuf += dest->y_stride;
384 srcbuf += src->y_stride;
385 }
386 }
387
swap_frame_buffer(YV12_BUFFER_CONFIG * const dest,YV12_BUFFER_CONFIG * const src)388 static void swap_frame_buffer(YV12_BUFFER_CONFIG *const dest,
389 YV12_BUFFER_CONFIG *const src) {
390 uint8_t *tmp_buf = dest->y_buffer;
391 assert(dest->y_width == src->y_width);
392 assert(dest->y_height == src->y_height);
393 dest->y_buffer = src->y_buffer;
394 src->y_buffer = tmp_buf;
395 }
396
av1_denoiser_update_frame_info(AV1_DENOISER * denoiser,YV12_BUFFER_CONFIG src,struct SVC * svc,FRAME_TYPE frame_type,int refresh_alt_ref_frame,int refresh_golden_frame,int refresh_last_frame,int alt_fb_idx,int gld_fb_idx,int lst_fb_idx,int resized,int svc_refresh_denoiser_buffers,int second_spatial_layer)397 void av1_denoiser_update_frame_info(
398 AV1_DENOISER *denoiser, YV12_BUFFER_CONFIG src, struct SVC *svc,
399 FRAME_TYPE frame_type, int refresh_alt_ref_frame, int refresh_golden_frame,
400 int refresh_last_frame, int alt_fb_idx, int gld_fb_idx, int lst_fb_idx,
401 int resized, int svc_refresh_denoiser_buffers, int second_spatial_layer) {
402 const int shift = second_spatial_layer ? denoiser->num_ref_frames : 0;
403 // Copy source into denoised reference buffers on KEY_FRAME or
404 // if the just encoded frame was resized. For SVC, copy source if the base
405 // spatial layer was key frame.
406 if (frame_type == KEY_FRAME || resized != 0 || denoiser->reset ||
407 svc_refresh_denoiser_buffers) {
408 int i;
409 // Start at 1 so as not to overwrite the INTRA_FRAME
410 for (i = 1; i < denoiser->num_ref_frames; ++i) {
411 if (denoiser->running_avg_y[i + shift].buffer_alloc != NULL)
412 copy_frame(&denoiser->running_avg_y[i + shift], &src);
413 }
414 denoiser->reset = 0;
415 return;
416 }
417
418 if (svc->external_ref_frame_config) {
419 int i;
420 for (i = 0; i < REF_FRAMES; i++) {
421 if (svc->refresh[svc->spatial_layer_id] & (1 << i))
422 copy_frame(&denoiser->running_avg_y[i + 1 + shift],
423 &denoiser->running_avg_y[INTRA_FRAME + shift]);
424 }
425 } else {
426 // If more than one refresh occurs, must copy frame buffer.
427 if ((refresh_alt_ref_frame + refresh_golden_frame + refresh_last_frame) >
428 1) {
429 if (refresh_alt_ref_frame) {
430 copy_frame(&denoiser->running_avg_y[alt_fb_idx + 1 + shift],
431 &denoiser->running_avg_y[INTRA_FRAME + shift]);
432 }
433 if (refresh_golden_frame) {
434 copy_frame(&denoiser->running_avg_y[gld_fb_idx + 1 + shift],
435 &denoiser->running_avg_y[INTRA_FRAME + shift]);
436 }
437 if (refresh_last_frame) {
438 copy_frame(&denoiser->running_avg_y[lst_fb_idx + 1 + shift],
439 &denoiser->running_avg_y[INTRA_FRAME + shift]);
440 }
441 } else {
442 if (refresh_alt_ref_frame) {
443 swap_frame_buffer(&denoiser->running_avg_y[alt_fb_idx + 1 + shift],
444 &denoiser->running_avg_y[INTRA_FRAME + shift]);
445 }
446 if (refresh_golden_frame) {
447 swap_frame_buffer(&denoiser->running_avg_y[gld_fb_idx + 1 + shift],
448 &denoiser->running_avg_y[INTRA_FRAME + shift]);
449 }
450 if (refresh_last_frame) {
451 swap_frame_buffer(&denoiser->running_avg_y[lst_fb_idx + 1 + shift],
452 &denoiser->running_avg_y[INTRA_FRAME + shift]);
453 }
454 }
455 }
456 }
457
av1_denoiser_reset_frame_stats(PICK_MODE_CONTEXT * ctx)458 void av1_denoiser_reset_frame_stats(PICK_MODE_CONTEXT *ctx) {
459 ctx->zeromv_sse = INT64_MAX;
460 ctx->newmv_sse = INT64_MAX;
461 ctx->zeromv_lastref_sse = INT64_MAX;
462 ctx->best_sse_mv.as_int = 0;
463 }
464
av1_denoiser_update_frame_stats(MB_MODE_INFO * mi,int64_t sse,PREDICTION_MODE mode,PICK_MODE_CONTEXT * ctx)465 void av1_denoiser_update_frame_stats(MB_MODE_INFO *mi, int64_t sse,
466 PREDICTION_MODE mode,
467 PICK_MODE_CONTEXT *ctx) {
468 if (mi->mv[0].as_int == 0 && sse < ctx->zeromv_sse) {
469 ctx->zeromv_sse = sse;
470 ctx->best_zeromv_reference_frame = mi->ref_frame[0];
471 if (mi->ref_frame[0] == LAST_FRAME) ctx->zeromv_lastref_sse = sse;
472 }
473
474 if (mi->mv[0].as_int != 0 && sse < ctx->newmv_sse) {
475 ctx->newmv_sse = sse;
476 ctx->best_sse_inter_mode = mode;
477 ctx->best_sse_mv = mi->mv[0];
478 ctx->best_reference_frame = mi->ref_frame[0];
479 }
480 }
481
av1_denoiser_realloc_svc_helper(AV1_COMMON * cm,AV1_DENOISER * denoiser,int fb_idx)482 static int av1_denoiser_realloc_svc_helper(AV1_COMMON *cm,
483 AV1_DENOISER *denoiser, int fb_idx) {
484 int fail = 0;
485 if (denoiser->running_avg_y[fb_idx].buffer_alloc == NULL) {
486 fail = aom_alloc_frame_buffer(
487 &denoiser->running_avg_y[fb_idx], cm->width, cm->height,
488 cm->seq_params.subsampling_x, cm->seq_params.subsampling_y,
489 cm->seq_params.use_highbitdepth, AOM_BORDER_IN_PIXELS,
490 cm->features.byte_alignment);
491 if (fail) {
492 av1_denoiser_free(denoiser);
493 return 1;
494 }
495 }
496 return 0;
497 }
498
av1_denoiser_realloc_svc(AV1_COMMON * cm,AV1_DENOISER * denoiser,struct SVC * svc,int svc_buf_shift,int refresh_alt,int refresh_gld,int refresh_lst,int alt_fb_idx,int gld_fb_idx,int lst_fb_idx)499 int av1_denoiser_realloc_svc(AV1_COMMON *cm, AV1_DENOISER *denoiser,
500 struct SVC *svc, int svc_buf_shift,
501 int refresh_alt, int refresh_gld, int refresh_lst,
502 int alt_fb_idx, int gld_fb_idx, int lst_fb_idx) {
503 int fail = 0;
504 if (svc->external_ref_frame_config) {
505 int i;
506 for (i = 0; i < REF_FRAMES; i++) {
507 if (cm->current_frame.frame_type == KEY_FRAME ||
508 svc->refresh[svc->spatial_layer_id] & (1 << i)) {
509 fail = av1_denoiser_realloc_svc_helper(cm, denoiser,
510 i + 1 + svc_buf_shift);
511 }
512 }
513 } else {
514 if (refresh_alt) {
515 // Increase the frame buffer index by 1 to map it to the buffer index in
516 // the denoiser.
517 fail = av1_denoiser_realloc_svc_helper(cm, denoiser,
518 alt_fb_idx + 1 + svc_buf_shift);
519 if (fail) return 1;
520 }
521 if (refresh_gld) {
522 fail = av1_denoiser_realloc_svc_helper(cm, denoiser,
523 gld_fb_idx + 1 + svc_buf_shift);
524 if (fail) return 1;
525 }
526 if (refresh_lst) {
527 fail = av1_denoiser_realloc_svc_helper(cm, denoiser,
528 lst_fb_idx + 1 + svc_buf_shift);
529 if (fail) return 1;
530 }
531 }
532 return 0;
533 }
534
av1_denoiser_alloc(AV1_COMMON * cm,struct SVC * svc,AV1_DENOISER * denoiser,int use_svc,int noise_sen,int width,int height,int ssx,int ssy,int use_highbitdepth,int border)535 int av1_denoiser_alloc(AV1_COMMON *cm, struct SVC *svc, AV1_DENOISER *denoiser,
536 int use_svc, int noise_sen, int width, int height,
537 int ssx, int ssy, int use_highbitdepth, int border) {
538 int i, layer, fail, init_num_ref_frames;
539 const int legacy_byte_alignment = 0;
540 int num_layers = 1;
541 int scaled_width = width;
542 int scaled_height = height;
543 if (use_svc) {
544 LAYER_CONTEXT *lc = &svc->layer_context[svc->spatial_layer_id *
545 svc->number_temporal_layers +
546 svc->temporal_layer_id];
547 av1_get_layer_resolution(width, height, lc->scaling_factor_num,
548 lc->scaling_factor_den, &scaled_width,
549 &scaled_height);
550 // For SVC: only denoise at most 2 spatial (highest) layers.
551 if (noise_sen >= 2)
552 // Denoise from one spatial layer below the top.
553 svc->first_layer_denoise = AOMMAX(svc->number_spatial_layers - 2, 0);
554 else
555 // Only denoise the top spatial layer.
556 svc->first_layer_denoise = AOMMAX(svc->number_spatial_layers - 1, 0);
557 num_layers = svc->number_spatial_layers - svc->first_layer_denoise;
558 }
559 assert(denoiser != NULL);
560 denoiser->num_ref_frames = use_svc ? SVC_REF_FRAMES : NONSVC_REF_FRAMES;
561 init_num_ref_frames = use_svc ? REF_FRAMES : NONSVC_REF_FRAMES;
562 denoiser->num_layers = num_layers;
563 CHECK_MEM_ERROR(cm, denoiser->running_avg_y,
564 aom_calloc(denoiser->num_ref_frames * num_layers,
565 sizeof(denoiser->running_avg_y[0])));
566 CHECK_MEM_ERROR(
567 cm, denoiser->mc_running_avg_y,
568 aom_calloc(num_layers, sizeof(denoiser->mc_running_avg_y[0])));
569
570 for (layer = 0; layer < num_layers; ++layer) {
571 const int denoise_width = (layer == 0) ? width : scaled_width;
572 const int denoise_height = (layer == 0) ? height : scaled_height;
573 for (i = 0; i < init_num_ref_frames; ++i) {
574 fail = aom_alloc_frame_buffer(
575 &denoiser->running_avg_y[i + denoiser->num_ref_frames * layer],
576 denoise_width, denoise_height, ssx, ssy, use_highbitdepth, border,
577 legacy_byte_alignment);
578 if (fail) {
579 av1_denoiser_free(denoiser);
580 return 1;
581 }
582 #ifdef OUTPUT_YUV_DENOISED
583 make_grayscale(&denoiser->running_avg_y[i]);
584 #endif
585 }
586
587 fail = aom_alloc_frame_buffer(
588 &denoiser->mc_running_avg_y[layer], denoise_width, denoise_height, ssx,
589 ssy, use_highbitdepth, border, legacy_byte_alignment);
590 if (fail) {
591 av1_denoiser_free(denoiser);
592 return 1;
593 }
594 }
595
596 // denoiser->last_source only used for noise_estimation, so only for top
597 // layer.
598 fail =
599 aom_alloc_frame_buffer(&denoiser->last_source, width, height, ssx, ssy,
600 use_highbitdepth, border, legacy_byte_alignment);
601 if (fail) {
602 av1_denoiser_free(denoiser);
603 return 1;
604 }
605 #ifdef OUTPUT_YUV_DENOISED
606 make_grayscale(&denoiser->running_avg_y[i]);
607 #endif
608 denoiser->frame_buffer_initialized = 1;
609 denoiser->denoising_level = kDenMedium;
610 denoiser->prev_denoising_level = kDenMedium;
611 denoiser->reset = 0;
612 denoiser->current_denoiser_frame = 0;
613 return 0;
614 }
615
av1_denoiser_free(AV1_DENOISER * denoiser)616 void av1_denoiser_free(AV1_DENOISER *denoiser) {
617 int i;
618 if (denoiser == NULL) {
619 return;
620 }
621 denoiser->frame_buffer_initialized = 0;
622 for (i = 0; i < denoiser->num_ref_frames * denoiser->num_layers; ++i) {
623 aom_free_frame_buffer(&denoiser->running_avg_y[i]);
624 }
625 aom_free(denoiser->running_avg_y);
626 denoiser->running_avg_y = NULL;
627
628 for (i = 0; i < denoiser->num_layers; ++i) {
629 aom_free_frame_buffer(&denoiser->mc_running_avg_y[i]);
630 }
631
632 aom_free(denoiser->mc_running_avg_y);
633 denoiser->mc_running_avg_y = NULL;
634 aom_free_frame_buffer(&denoiser->last_source);
635 }
636
637 // TODO(kyslov) Enable when SVC temporal denosing is implemented
638 #if 0
639 static void force_refresh_longterm_ref(AV1_COMP *const cpi) {
640 SVC *const svc = &cpi->svc;
641 // If long term reference is used, force refresh of that slot, so
642 // denoiser buffer for long term reference stays in sync.
643 if (svc->use_gf_temporal_ref_current_layer) {
644 int index = svc->spatial_layer_id;
645 if (svc->number_spatial_layers == 3) index = svc->spatial_layer_id - 1;
646 assert(index >= 0);
647 cpi->alt_fb_idx = svc->buffer_gf_temporal_ref[index].idx;
648 cpi->refresh_alt_ref_frame = 1;
649 }
650 }
651 #endif
652
av1_denoiser_set_noise_level(AV1_COMP * const cpi,int noise_level)653 void av1_denoiser_set_noise_level(AV1_COMP *const cpi, int noise_level) {
654 AV1_DENOISER *const denoiser = &cpi->denoiser;
655 denoiser->denoising_level = noise_level;
656 if (denoiser->denoising_level > kDenLowLow &&
657 denoiser->prev_denoising_level == kDenLowLow) {
658 denoiser->reset = 1;
659 // TODO(kyslov) Enable when SVC temporal denosing is implemented
660 #if 0
661 force_refresh_longterm_ref(cpi);
662 #endif
663 } else {
664 denoiser->reset = 0;
665 }
666 denoiser->prev_denoising_level = denoiser->denoising_level;
667 }
668
669 // Scale/increase the partition threshold
670 // for denoiser speed-up.
av1_scale_part_thresh(int64_t threshold,AV1_DENOISER_LEVEL noise_level,CONTENT_STATE_SB content_state,int temporal_layer_id)671 int64_t av1_scale_part_thresh(int64_t threshold, AV1_DENOISER_LEVEL noise_level,
672 CONTENT_STATE_SB content_state,
673 int temporal_layer_id) {
674 if ((content_state.source_sad == kLowSad && content_state.low_sumdiff) ||
675 (content_state.source_sad == kHighSad && content_state.low_sumdiff) ||
676 (content_state.lighting_change && !content_state.low_sumdiff) ||
677 (noise_level == kDenHigh) || (temporal_layer_id != 0)) {
678 int64_t scaled_thr =
679 (temporal_layer_id < 2) ? (3 * threshold) >> 1 : (7 * threshold) >> 2;
680 return scaled_thr;
681 } else {
682 return (5 * threshold) >> 2;
683 }
684 }
685
686 // Scale/increase the ac skip threshold for
687 // denoiser speed-up.
av1_scale_acskip_thresh(int64_t threshold,AV1_DENOISER_LEVEL noise_level,int abs_sumdiff,int temporal_layer_id)688 int64_t av1_scale_acskip_thresh(int64_t threshold,
689 AV1_DENOISER_LEVEL noise_level, int abs_sumdiff,
690 int temporal_layer_id) {
691 if (noise_level >= kDenLow && abs_sumdiff < 5)
692 return threshold *=
693 (noise_level == kDenLow) ? 2 : (temporal_layer_id == 2) ? 10 : 6;
694 else
695 return threshold;
696 }
697
av1_denoiser_reset_on_first_frame(AV1_COMP * const cpi)698 void av1_denoiser_reset_on_first_frame(AV1_COMP *const cpi) {
699 if (/*av1_denoise_svc_non_key(cpi) &&*/
700 cpi->denoiser.current_denoiser_frame == 0) {
701 cpi->denoiser.reset = 1;
702 // TODO(kyslov) Enable when SVC temporal denosing is implemented
703 #if 0
704 force_refresh_longterm_ref(cpi);
705 #endif
706 }
707 }
708
av1_denoiser_update_ref_frame(AV1_COMP * const cpi)709 void av1_denoiser_update_ref_frame(AV1_COMP *const cpi) {
710 AV1_COMMON *const cm = &cpi->common;
711 SVC *const svc = &cpi->svc;
712
713 if (cpi->oxcf.noise_sensitivity > 0 && denoise_svc(cpi) &&
714 cpi->denoiser.denoising_level > kDenLowLow) {
715 int svc_refresh_denoiser_buffers = 0;
716 int denoise_svc_second_layer = 0;
717 FRAME_TYPE frame_type = cm->current_frame.frame_type == INTRA_ONLY_FRAME
718 ? KEY_FRAME
719 : cm->current_frame.frame_type;
720 cpi->denoiser.current_denoiser_frame++;
721 const int resize_pending =
722 (cpi->resize_pending_params.width &&
723 cpi->resize_pending_params.height &&
724 (cpi->common.width != cpi->resize_pending_params.width ||
725 cpi->common.height != cpi->resize_pending_params.height));
726
727 if (cpi->use_svc) {
728 // TODO(kyslov) Enable when SVC temporal denosing is implemented
729 #if 0
730 const int svc_buf_shift =
731 svc->number_spatial_layers - svc->spatial_layer_id == 2
732 ? cpi->denoiser.num_ref_frames
733 : 0;
734 int layer =
735 LAYER_IDS_TO_IDX(svc->spatial_layer_id, svc->temporal_layer_id,
736 svc->number_temporal_layers);
737 LAYER_CONTEXT *const lc = &svc->layer_context[layer];
738 svc_refresh_denoiser_buffers =
739 lc->is_key_frame || svc->spatial_layer_sync[svc->spatial_layer_id];
740 denoise_svc_second_layer =
741 svc->number_spatial_layers - svc->spatial_layer_id == 2 ? 1 : 0;
742 // Check if we need to allocate extra buffers in the denoiser
743 // for refreshed frames.
744 if (av1_denoiser_realloc_svc(cm, &cpi->denoiser, svc, svc_buf_shift,
745 cpi->refresh_alt_ref_frame,
746 cpi->refresh_golden_frame,
747 cpi->refresh_last_frame, cpi->alt_fb_idx,
748 cpi->gld_fb_idx, cpi->lst_fb_idx))
749 aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR,
750 "Failed to re-allocate denoiser for SVC");
751 #endif
752 }
753 av1_denoiser_update_frame_info(
754 &cpi->denoiser, *cpi->source, svc, frame_type,
755 cpi->refresh_frame.alt_ref_frame, cpi->refresh_frame.golden_frame, 1,
756 svc->ref_idx[6], svc->ref_idx[3], svc->ref_idx[0], resize_pending,
757 svc_refresh_denoiser_buffers, denoise_svc_second_layer);
758 }
759 }
760
761 #ifdef OUTPUT_YUV_DENOISED
make_grayscale(YV12_BUFFER_CONFIG * yuv)762 static void make_grayscale(YV12_BUFFER_CONFIG *yuv) {
763 int r, c;
764 uint8_t *u = yuv->u_buffer;
765 uint8_t *v = yuv->v_buffer;
766
767 for (r = 0; r < yuv->uv_height; ++r) {
768 for (c = 0; c < yuv->uv_width; ++c) {
769 u[c] = UINT8_MAX / 2;
770 v[c] = UINT8_MAX / 2;
771 }
772 u += yuv->uv_stride;
773 v += yuv->uv_stride;
774 }
775 }
776
aom_write_yuv_frame(FILE * yuv_file,YV12_BUFFER_CONFIG * s)777 void aom_write_yuv_frame(FILE *yuv_file, YV12_BUFFER_CONFIG *s) {
778 unsigned char *src = s->y_buffer;
779 int h = s->y_crop_height;
780
781 do {
782 fwrite(src, s->y_width, 1, yuv_file);
783 src += s->y_stride;
784 } while (--h);
785
786 src = s->u_buffer;
787 h = s->uv_crop_height;
788
789 do {
790 fwrite(src, s->uv_width, 1, yuv_file);
791 src += s->uv_stride;
792 } while (--h);
793
794 src = s->v_buffer;
795 h = s->uv_crop_height;
796
797 do {
798 fwrite(src, s->uv_width, 1, yuv_file);
799 src += s->uv_stride;
800 } while (--h);
801 }
802 #endif
803