1 /*
2 * Copyright (c) 2020, Alliance for Open Media. All rights reserved
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #include <assert.h>
13 #include <limits.h>
14 #include <math.h>
15
16 #include "config/aom_dsp_rtcd.h"
17 #include "aom_dsp/aom_dsp_common.h"
18 #include "aom_scale/yv12config.h"
19 #include "aom/aom_integer.h"
20 #include "av1/encoder/context_tree.h"
21 #include "av1/encoder/av1_noise_estimate.h"
22 #include "av1/encoder/encoder.h"
23 #if CONFIG_AV1_TEMPORAL_DENOISING
24 #include "av1/encoder/av1_temporal_denoiser.h"
25 #endif
26
27 #if CONFIG_AV1_TEMPORAL_DENOISING
28 // For SVC: only do noise estimation on top spatial layer.
noise_est_svc(const struct AV1_COMP * const cpi)29 static INLINE int noise_est_svc(const struct AV1_COMP *const cpi) {
30 return (!cpi->ppi->use_svc ||
31 (cpi->ppi->use_svc &&
32 cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1));
33 }
34 #endif
35
av1_noise_estimate_init(NOISE_ESTIMATE * const ne,int width,int height)36 void av1_noise_estimate_init(NOISE_ESTIMATE *const ne, int width, int height) {
37 ne->enabled = 0;
38 ne->level = (width * height < 1280 * 720) ? kLowLow : kLow;
39 ne->value = 0;
40 ne->count = 0;
41 ne->thresh = 90;
42 ne->last_w = 0;
43 ne->last_h = 0;
44 if (width * height >= 1920 * 1080) {
45 ne->thresh = 200;
46 } else if (width * height >= 1280 * 720) {
47 ne->thresh = 140;
48 } else if (width * height >= 640 * 360) {
49 ne->thresh = 115;
50 }
51 ne->num_frames_estimate = 15;
52 ne->adapt_thresh = (3 * ne->thresh) >> 1;
53 }
54
enable_noise_estimation(AV1_COMP * const cpi)55 static int enable_noise_estimation(AV1_COMP *const cpi) {
56 ResizePendingParams *const resize_pending_params =
57 &cpi->resize_pending_params;
58 const int resize_pending =
59 (resize_pending_params->width && resize_pending_params->height &&
60 (cpi->common.width != resize_pending_params->width ||
61 cpi->common.height != resize_pending_params->height));
62
63 #if CONFIG_AV1_HIGHBITDEPTH
64 if (cpi->common.seq_params->use_highbitdepth) return 0;
65 #endif
66 // Enable noise estimation if denoising is on.
67 #if CONFIG_AV1_TEMPORAL_DENOISING
68 if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi) &&
69 cpi->common.width >= 320 && cpi->common.height >= 180)
70 return 1;
71 #endif
72 // Only allow noise estimate under certain encoding mode.
73 // Enabled for 1 pass CBR, speed >=5, and if resolution is same as original.
74 // Not enabled for SVC mode and screen_content_mode.
75 // Not enabled for low resolutions.
76 if (cpi->oxcf.pass == AOM_RC_ONE_PASS && cpi->oxcf.rc_cfg.mode == AOM_CBR &&
77 cpi->oxcf.q_cfg.aq_mode == CYCLIC_REFRESH_AQ && cpi->oxcf.speed >= 5 &&
78 resize_pending == 0 && !cpi->ppi->use_svc &&
79 cpi->oxcf.tune_cfg.content != AOM_CONTENT_SCREEN &&
80 cpi->common.width * cpi->common.height >= 640 * 360)
81 return 1;
82 else
83 return 0;
84 }
85
86 #if CONFIG_AV1_TEMPORAL_DENOISING
copy_frame(YV12_BUFFER_CONFIG * const dest,const YV12_BUFFER_CONFIG * const src)87 static void copy_frame(YV12_BUFFER_CONFIG *const dest,
88 const YV12_BUFFER_CONFIG *const src) {
89 const uint8_t *srcbuf = src->y_buffer;
90 uint8_t *destbuf = dest->y_buffer;
91
92 assert(dest->y_width == src->y_width);
93 assert(dest->y_height == src->y_height);
94
95 for (int r = 0; r < dest->y_height; ++r) {
96 memcpy(destbuf, srcbuf, dest->y_width);
97 destbuf += dest->y_stride;
98 srcbuf += src->y_stride;
99 }
100 }
101 #endif // CONFIG_AV1_TEMPORAL_DENOISING
102
av1_noise_estimate_extract_level(NOISE_ESTIMATE * const ne)103 NOISE_LEVEL av1_noise_estimate_extract_level(NOISE_ESTIMATE *const ne) {
104 int noise_level = kLowLow;
105 if (ne->value > (ne->thresh << 1)) {
106 noise_level = kHigh;
107 } else {
108 if (ne->value > ne->thresh)
109 noise_level = kMedium;
110 else if (ne->value > (ne->thresh >> 1))
111 noise_level = kLow;
112 else
113 noise_level = kLowLow;
114 }
115 return noise_level;
116 }
117
av1_update_noise_estimate(AV1_COMP * const cpi)118 void av1_update_noise_estimate(AV1_COMP *const cpi) {
119 const AV1_COMMON *const cm = &cpi->common;
120 const CommonModeInfoParams *const mi_params = &cm->mi_params;
121
122 NOISE_ESTIMATE *const ne = &cpi->noise_estimate;
123 const int low_res = (cm->width <= 352 && cm->height <= 288);
124 // Estimate of noise level every frame_period frames.
125 int frame_period = 8;
126 int thresh_consec_zeromv = 2;
127 int frame_counter = cm->current_frame.frame_number;
128 // Estimate is between current source and last source.
129 YV12_BUFFER_CONFIG *last_source = cpi->last_source;
130 #if CONFIG_AV1_TEMPORAL_DENOISING
131 if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi)) {
132 last_source = &cpi->denoiser.last_source;
133 // Tune these thresholds for different resolutions when denoising is
134 // enabled.
135 if (cm->width > 640 && cm->width <= 1920) {
136 thresh_consec_zeromv = 2;
137 }
138 }
139 #endif
140 ne->enabled = enable_noise_estimation(cpi);
141 if (cpi->svc.number_spatial_layers > 1)
142 frame_counter = cpi->svc.current_superframe;
143 if (!ne->enabled || frame_counter % frame_period != 0 ||
144 last_source == NULL ||
145 (cpi->svc.number_spatial_layers == 1 &&
146 (ne->last_w != cm->width || ne->last_h != cm->height))) {
147 #if CONFIG_AV1_TEMPORAL_DENOISING
148 if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi))
149 copy_frame(&cpi->denoiser.last_source, cpi->source);
150 #endif
151 if (last_source != NULL) {
152 ne->last_w = cm->width;
153 ne->last_h = cm->height;
154 }
155 return;
156 } else if (frame_counter > 60 && cpi->svc.num_encoded_top_layer > 1 &&
157 cpi->rc.frames_since_key > cpi->svc.number_spatial_layers &&
158 cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1 &&
159 cpi->rc.avg_frame_low_motion < (low_res ? 60 : 40)) {
160 // Force noise estimation to 0 and denoiser off if content has high motion.
161 ne->level = kLowLow;
162 ne->count = 0;
163 ne->num_frames_estimate = 10;
164 #if CONFIG_AV1_TEMPORAL_DENOISING
165 if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi) &&
166 cpi->svc.current_superframe > 1) {
167 av1_denoiser_set_noise_level(cpi, ne->level);
168 copy_frame(&cpi->denoiser.last_source, cpi->source);
169 }
170 #endif
171 return;
172 } else {
173 unsigned int bin_size = 100;
174 unsigned int hist[MAX_VAR_HIST_BINS] = { 0 };
175 unsigned int hist_avg[MAX_VAR_HIST_BINS];
176 unsigned int max_bin = 0;
177 unsigned int max_bin_count = 0;
178 unsigned int bin_cnt;
179 int bsize = BLOCK_16X16;
180 // Loop over sub-sample of 16x16 blocks of frame, and for blocks that have
181 // been encoded as zero/small mv at least x consecutive frames, compute
182 // the variance to update estimate of noise in the source.
183 const uint8_t *src_y = cpi->source->y_buffer;
184 const int src_ystride = cpi->source->y_stride;
185 const uint8_t *last_src_y = last_source->y_buffer;
186 const int last_src_ystride = last_source->y_stride;
187 int mi_row, mi_col;
188 int num_low_motion = 0;
189 int frame_low_motion = 1;
190 for (mi_row = 0; mi_row < mi_params->mi_rows; mi_row += 2) {
191 for (mi_col = 0; mi_col < mi_params->mi_cols; mi_col += 2) {
192 int bl_index =
193 (mi_row >> 1) * (mi_params->mi_cols >> 1) + (mi_col >> 1);
194 if (cpi->consec_zero_mv[bl_index] > thresh_consec_zeromv)
195 num_low_motion++;
196 }
197 }
198 if (num_low_motion <
199 (((3 * (mi_params->mi_rows * mi_params->mi_cols) >> 2)) >> 3))
200 frame_low_motion = 0;
201 for (mi_row = 0; mi_row < mi_params->mi_rows; mi_row++) {
202 for (mi_col = 0; mi_col < mi_params->mi_cols; mi_col++) {
203 // 16x16 blocks, 1/4 sample of frame.
204 if (mi_row % 8 == 0 && mi_col % 8 == 0 &&
205 mi_row < mi_params->mi_rows - 3 &&
206 mi_col < mi_params->mi_cols - 3) {
207 int bl_index =
208 (mi_row >> 1) * (mi_params->mi_cols >> 1) + (mi_col >> 1);
209 int bl_index1 = bl_index + 1;
210 int bl_index2 = bl_index + (mi_params->mi_cols >> 1);
211 int bl_index3 = bl_index2 + 1;
212 int consec_zeromv =
213 AOMMIN(cpi->consec_zero_mv[bl_index],
214 AOMMIN(cpi->consec_zero_mv[bl_index1],
215 AOMMIN(cpi->consec_zero_mv[bl_index2],
216 cpi->consec_zero_mv[bl_index3])));
217 // Only consider blocks that are likely steady background. i.e, have
218 // been encoded as zero/low motion x (= thresh_consec_zeromv) frames
219 // in a row. consec_zero_mv[] defined for 8x8 blocks, so consider all
220 // 4 sub-blocks for 16x16 block. And exclude this frame if
221 // high_source_sad is true (i.e., scene/content change).
222 if (frame_low_motion && consec_zeromv > thresh_consec_zeromv &&
223 !cpi->rc.high_source_sad) {
224 unsigned int sse;
225 // Compute variance between co-located blocks from current and
226 // last input frames.
227 unsigned int variance = cpi->ppi->fn_ptr[bsize].vf(
228 src_y, src_ystride, last_src_y, last_src_ystride, &sse);
229 unsigned int hist_index = variance / bin_size;
230 if (hist_index < MAX_VAR_HIST_BINS)
231 hist[hist_index]++;
232 else if (hist_index < 3 * (MAX_VAR_HIST_BINS >> 1))
233 hist[MAX_VAR_HIST_BINS - 1]++; // Account for the tail
234 }
235 }
236 src_y += 4;
237 last_src_y += 4;
238 }
239 src_y += (src_ystride << 2) - (mi_params->mi_cols << 2);
240 last_src_y += (last_src_ystride << 2) - (mi_params->mi_cols << 2);
241 }
242 ne->last_w = cm->width;
243 ne->last_h = cm->height;
244 // Adjust histogram to account for effect that histogram flattens
245 // and shifts to zero as scene darkens.
246 if (hist[0] > 10 && (hist[MAX_VAR_HIST_BINS - 1] > hist[0] >> 2)) {
247 hist[0] = 0;
248 hist[1] >>= 2;
249 hist[2] >>= 2;
250 hist[3] >>= 2;
251 hist[4] >>= 1;
252 hist[5] >>= 1;
253 hist[6] = 3 * hist[6] >> 1;
254 hist[MAX_VAR_HIST_BINS - 1] >>= 1;
255 }
256
257 // Average hist[] and find largest bin
258 for (bin_cnt = 0; bin_cnt < MAX_VAR_HIST_BINS; bin_cnt++) {
259 if (bin_cnt == 0)
260 hist_avg[bin_cnt] = (hist[0] + hist[1] + hist[2]) / 3;
261 else if (bin_cnt == MAX_VAR_HIST_BINS - 1)
262 hist_avg[bin_cnt] = hist[MAX_VAR_HIST_BINS - 1] >> 2;
263 else if (bin_cnt == MAX_VAR_HIST_BINS - 2)
264 hist_avg[bin_cnt] = (hist[bin_cnt - 1] + 2 * hist[bin_cnt] +
265 (hist[bin_cnt + 1] >> 1) + 2) >>
266 2;
267 else
268 hist_avg[bin_cnt] =
269 (hist[bin_cnt - 1] + 2 * hist[bin_cnt] + hist[bin_cnt + 1] + 2) >>
270 2;
271
272 if (hist_avg[bin_cnt] > max_bin_count) {
273 max_bin_count = hist_avg[bin_cnt];
274 max_bin = bin_cnt;
275 }
276 }
277 // Scale by 40 to work with existing thresholds
278 ne->value = (int)((3 * ne->value + max_bin * 40) >> 2);
279 // Quickly increase VNR strength when the noise level increases suddenly.
280 if (ne->level < kMedium && ne->value > ne->adapt_thresh) {
281 ne->count = ne->num_frames_estimate;
282 } else {
283 ne->count++;
284 }
285 if (ne->count == ne->num_frames_estimate) {
286 // Reset counter and check noise level condition.
287 ne->num_frames_estimate = 30;
288 ne->count = 0;
289 ne->level = av1_noise_estimate_extract_level(ne);
290 #if CONFIG_AV1_TEMPORAL_DENOISING
291 if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi))
292 av1_denoiser_set_noise_level(cpi, ne->level);
293 #endif
294 }
295 }
296 #if CONFIG_AV1_TEMPORAL_DENOISING
297 if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi))
298 copy_frame(&cpi->denoiser.last_source, cpi->source);
299 #endif
300 }
301