1 /*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "common_video/libyuv/include/webrtc_libyuv.h"
12 #include "modules/video_processing/video_denoiser.h"
13 #include "libyuv/planar_functions.h"
14
15 namespace webrtc {
16
17 #if DISPLAY || DISPLAYNEON
CopyMem8x8(const uint8_t * src,int src_stride,uint8_t * dst,int dst_stride)18 static void CopyMem8x8(const uint8_t* src,
19 int src_stride,
20 uint8_t* dst,
21 int dst_stride) {
22 for (int i = 0; i < 8; i++) {
23 memcpy(dst, src, 8);
24 src += src_stride;
25 dst += dst_stride;
26 }
27 }
28
ShowRect(const std::unique_ptr<DenoiserFilter> & filter,const std::unique_ptr<uint8_t[]> & d_status,const std::unique_ptr<uint8_t[]> & moving_edge_red,const std::unique_ptr<uint8_t[]> & x_density,const std::unique_ptr<uint8_t[]> & y_density,const uint8_t * u_src,int stride_u_src,const uint8_t * v_src,int stride_v_src,uint8_t * u_dst,int stride_u_dst,uint8_t * v_dst,int stride_v_dst,int mb_rows_,int mb_cols_)29 static void ShowRect(const std::unique_ptr<DenoiserFilter>& filter,
30 const std::unique_ptr<uint8_t[]>& d_status,
31 const std::unique_ptr<uint8_t[]>& moving_edge_red,
32 const std::unique_ptr<uint8_t[]>& x_density,
33 const std::unique_ptr<uint8_t[]>& y_density,
34 const uint8_t* u_src, int stride_u_src,
35 const uint8_t* v_src, int stride_v_src,
36 uint8_t* u_dst, int stride_u_dst,
37 uint8_t* v_dst, int stride_v_dst,
38 int mb_rows_,
39 int mb_cols_) {
40 for (int mb_row = 0; mb_row < mb_rows_; ++mb_row) {
41 for (int mb_col = 0; mb_col < mb_cols_; ++mb_col) {
42 int mb_index = mb_row * mb_cols_ + mb_col;
43 const uint8_t* mb_src_u =
44 u_src + (mb_row << 3) * stride_u_src + (mb_col << 3);
45 const uint8_t* mb_src_v =
46 v_src + (mb_row << 3) * stride_v_src + (mb_col << 3);
47 uint8_t* mb_dst_u = u_dst + (mb_row << 3) * stride_u_dst + (mb_col << 3);
48 uint8_t* mb_dst_v = v_dst + (mb_row << 3) * stride_v_dst + (mb_col << 3);
49 uint8_t uv_tmp[8 * 8];
50 memset(uv_tmp, 200, 8 * 8);
51 if (d_status[mb_index] == 1) {
52 // Paint to red.
53 CopyMem8x8(mb_src_u, stride_u_src, mb_dst_u, stride_u_dst);
54 CopyMem8x8(uv_tmp, 8, mb_dst_v, stride_v_dst);
55 } else if (moving_edge_red[mb_row * mb_cols_ + mb_col] &&
56 x_density[mb_col] * y_density[mb_row]) {
57 // Paint to blue.
58 CopyMem8x8(uv_tmp, 8, mb_dst_u, stride_u_dst);
59 CopyMem8x8(mb_src_v, stride_v_src, mb_dst_v, stride_v_dst);
60 } else {
61 CopyMem8x8(mb_src_u, stride_u_src, mb_dst_u, stride_u_dst);
62 CopyMem8x8(mb_src_v, stride_v_src, mb_dst_v, stride_v_dst);
63 }
64 }
65 }
66 }
67 #endif
68
VideoDenoiser(bool runtime_cpu_detection)69 VideoDenoiser::VideoDenoiser(bool runtime_cpu_detection)
70 : width_(0),
71 height_(0),
72 filter_(DenoiserFilter::Create(runtime_cpu_detection, &cpu_type_)),
73 ne_(new NoiseEstimation()) {}
74
DenoiserReset(rtc::scoped_refptr<I420BufferInterface> frame)75 void VideoDenoiser::DenoiserReset(
76 rtc::scoped_refptr<I420BufferInterface> frame) {
77 width_ = frame->width();
78 height_ = frame->height();
79 mb_cols_ = width_ >> 4;
80 mb_rows_ = height_ >> 4;
81
82 // Init noise estimator and allocate buffers.
83 ne_->Init(width_, height_, cpu_type_);
84 moving_edge_.reset(new uint8_t[mb_cols_ * mb_rows_]);
85 mb_filter_decision_.reset(new DenoiserDecision[mb_cols_ * mb_rows_]);
86 x_density_.reset(new uint8_t[mb_cols_]);
87 y_density_.reset(new uint8_t[mb_rows_]);
88 moving_object_.reset(new uint8_t[mb_cols_ * mb_rows_]);
89 }
90
PositionCheck(int mb_row,int mb_col,int noise_level)91 int VideoDenoiser::PositionCheck(int mb_row, int mb_col, int noise_level) {
92 if (noise_level == 0)
93 return 1;
94 if ((mb_row <= (mb_rows_ >> 4)) || (mb_col <= (mb_cols_ >> 4)) ||
95 (mb_col >= (15 * mb_cols_ >> 4)))
96 return 3;
97 else if ((mb_row <= (mb_rows_ >> 3)) || (mb_col <= (mb_cols_ >> 3)) ||
98 (mb_col >= (7 * mb_cols_ >> 3)))
99 return 2;
100 else
101 return 1;
102 }
103
ReduceFalseDetection(const std::unique_ptr<uint8_t[]> & d_status,std::unique_ptr<uint8_t[]> * moving_edge_red,int noise_level)104 void VideoDenoiser::ReduceFalseDetection(
105 const std::unique_ptr<uint8_t[]>& d_status,
106 std::unique_ptr<uint8_t[]>* moving_edge_red,
107 int noise_level) {
108 // From up left corner.
109 int mb_col_stop = mb_cols_ - 1;
110 for (int mb_row = 0; mb_row <= mb_rows_ - 1; ++mb_row) {
111 for (int mb_col = 0; mb_col <= mb_col_stop; ++mb_col) {
112 if (d_status[mb_row * mb_cols_ + mb_col]) {
113 mb_col_stop = mb_col - 1;
114 break;
115 }
116 (*moving_edge_red)[mb_row * mb_cols_ + mb_col] = 0;
117 }
118 }
119 // From bottom left corner.
120 mb_col_stop = mb_cols_ - 1;
121 for (int mb_row = mb_rows_ - 1; mb_row >= 0; --mb_row) {
122 for (int mb_col = 0; mb_col <= mb_col_stop; ++mb_col) {
123 if (d_status[mb_row * mb_cols_ + mb_col]) {
124 mb_col_stop = mb_col - 1;
125 break;
126 }
127 (*moving_edge_red)[mb_row * mb_cols_ + mb_col] = 0;
128 }
129 }
130 // From up right corner.
131 mb_col_stop = 0;
132 for (int mb_row = 0; mb_row <= mb_rows_ - 1; ++mb_row) {
133 for (int mb_col = mb_cols_ - 1; mb_col >= mb_col_stop; --mb_col) {
134 if (d_status[mb_row * mb_cols_ + mb_col]) {
135 mb_col_stop = mb_col + 1;
136 break;
137 }
138 (*moving_edge_red)[mb_row * mb_cols_ + mb_col] = 0;
139 }
140 }
141 // From bottom right corner.
142 mb_col_stop = 0;
143 for (int mb_row = mb_rows_ - 1; mb_row >= 0; --mb_row) {
144 for (int mb_col = mb_cols_ - 1; mb_col >= mb_col_stop; --mb_col) {
145 if (d_status[mb_row * mb_cols_ + mb_col]) {
146 mb_col_stop = mb_col + 1;
147 break;
148 }
149 (*moving_edge_red)[mb_row * mb_cols_ + mb_col] = 0;
150 }
151 }
152 }
153
IsTrailingBlock(const std::unique_ptr<uint8_t[]> & d_status,int mb_row,int mb_col)154 bool VideoDenoiser::IsTrailingBlock(const std::unique_ptr<uint8_t[]>& d_status,
155 int mb_row,
156 int mb_col) {
157 bool ret = false;
158 int mb_index = mb_row * mb_cols_ + mb_col;
159 if (!mb_row || !mb_col || mb_row == mb_rows_ - 1 || mb_col == mb_cols_ - 1)
160 ret = false;
161 else
162 ret = d_status[mb_index + 1] || d_status[mb_index - 1] ||
163 d_status[mb_index + mb_cols_] || d_status[mb_index - mb_cols_];
164 return ret;
165 }
166
CopySrcOnMOB(const uint8_t * y_src,int stride_src,uint8_t * y_dst,int stride_dst)167 void VideoDenoiser::CopySrcOnMOB(const uint8_t* y_src,
168 int stride_src,
169 uint8_t* y_dst,
170 int stride_dst) {
171 // Loop over to copy src block if the block is marked as moving object block
172 // or if the block may cause trailing artifacts.
173 for (int mb_row = 0; mb_row < mb_rows_; ++mb_row) {
174 const int mb_index_base = mb_row * mb_cols_;
175 const uint8_t* mb_src_base = y_src + (mb_row << 4) * stride_src;
176 uint8_t* mb_dst_base = y_dst + (mb_row << 4) * stride_dst;
177 for (int mb_col = 0; mb_col < mb_cols_; ++mb_col) {
178 const int mb_index = mb_index_base + mb_col;
179 const uint32_t offset_col = mb_col << 4;
180 const uint8_t* mb_src = mb_src_base + offset_col;
181 uint8_t* mb_dst = mb_dst_base + offset_col;
182 // Check if the block is a moving object block or may cause a trailing
183 // artifacts.
184 if (mb_filter_decision_[mb_index] != FILTER_BLOCK ||
185 IsTrailingBlock(moving_edge_, mb_row, mb_col) ||
186 (x_density_[mb_col] * y_density_[mb_row] &&
187 moving_object_[mb_row * mb_cols_ + mb_col])) {
188 // Copy y source.
189 filter_->CopyMem16x16(mb_src, stride_src, mb_dst, stride_dst);
190 }
191 }
192 }
193 }
194
CopyLumaOnMargin(const uint8_t * y_src,int stride_src,uint8_t * y_dst,int stride_dst)195 void VideoDenoiser::CopyLumaOnMargin(const uint8_t* y_src,
196 int stride_src,
197 uint8_t* y_dst,
198 int stride_dst) {
199 int height_margin = height_ - (mb_rows_ << 4);
200 if (height_margin > 0) {
201 const uint8_t* margin_y_src = y_src + (mb_rows_ << 4) * stride_src;
202 uint8_t* margin_y_dst = y_dst + (mb_rows_ << 4) * stride_dst;
203 libyuv::CopyPlane(margin_y_src, stride_src, margin_y_dst, stride_dst,
204 width_, height_margin);
205 }
206 int width_margin = width_ - (mb_cols_ << 4);
207 if (width_margin > 0) {
208 const uint8_t* margin_y_src = y_src + (mb_cols_ << 4);
209 uint8_t* margin_y_dst = y_dst + (mb_cols_ << 4);
210 libyuv::CopyPlane(margin_y_src, stride_src, margin_y_dst, stride_dst,
211 width_ - (mb_cols_ << 4), mb_rows_ << 4);
212 }
213 }
214
DenoiseFrame(rtc::scoped_refptr<I420BufferInterface> frame,bool noise_estimation_enabled)215 rtc::scoped_refptr<I420BufferInterface> VideoDenoiser::DenoiseFrame(
216 rtc::scoped_refptr<I420BufferInterface> frame,
217 bool noise_estimation_enabled) {
218 // If previous width and height are different from current frame's, need to
219 // reallocate the buffers and no denoising for the current frame.
220 if (!prev_buffer_ || width_ != frame->width() || height_ != frame->height()) {
221 DenoiserReset(frame);
222 prev_buffer_ = frame;
223 return frame;
224 }
225
226 // Set buffer pointers.
227 const uint8_t* y_src = frame->DataY();
228 int stride_y_src = frame->StrideY();
229 rtc::scoped_refptr<I420Buffer> dst =
230 buffer_pool_.CreateBuffer(width_, height_);
231
232 uint8_t* y_dst = dst->MutableDataY();
233 int stride_y_dst = dst->StrideY();
234
235 const uint8_t* y_dst_prev = prev_buffer_->DataY();
236 int stride_prev = prev_buffer_->StrideY();
237
238 memset(x_density_.get(), 0, mb_cols_);
239 memset(y_density_.get(), 0, mb_rows_);
240 memset(moving_object_.get(), 1, mb_cols_ * mb_rows_);
241
242 uint8_t noise_level = noise_estimation_enabled ? ne_->GetNoiseLevel() : 0;
243 int thr_var_base = 16 * 16 * 2;
244 // Loop over blocks to accumulate/extract noise level and update x/y_density
245 // factors for moving object detection.
246 for (int mb_row = 0; mb_row < mb_rows_; ++mb_row) {
247 const int mb_index_base = mb_row * mb_cols_;
248 const uint8_t* mb_src_base = y_src + (mb_row << 4) * stride_y_src;
249 uint8_t* mb_dst_base = y_dst + (mb_row << 4) * stride_y_dst;
250 const uint8_t* mb_dst_prev_base = y_dst_prev + (mb_row << 4) * stride_prev;
251 for (int mb_col = 0; mb_col < mb_cols_; ++mb_col) {
252 const int mb_index = mb_index_base + mb_col;
253 const bool ne_enable = (mb_index % NOISE_SUBSAMPLE_INTERVAL == 0);
254 const int pos_factor = PositionCheck(mb_row, mb_col, noise_level);
255 const uint32_t thr_var_adp = thr_var_base * pos_factor;
256 const uint32_t offset_col = mb_col << 4;
257 const uint8_t* mb_src = mb_src_base + offset_col;
258 uint8_t* mb_dst = mb_dst_base + offset_col;
259 const uint8_t* mb_dst_prev = mb_dst_prev_base + offset_col;
260
261 // TODO(jackychen): Need SSE2/NEON opt.
262 int luma = 0;
263 if (ne_enable) {
264 for (int i = 4; i < 12; ++i) {
265 for (int j = 4; j < 12; ++j) {
266 luma += mb_src[i * stride_y_src + j];
267 }
268 }
269 }
270
271 // Get the filtered block and filter_decision.
272 mb_filter_decision_[mb_index] =
273 filter_->MbDenoise(mb_dst_prev, stride_prev, mb_dst, stride_y_dst,
274 mb_src, stride_y_src, 0, noise_level);
275
276 // If filter decision is FILTER_BLOCK, no need to check moving edge.
277 // It is unlikely for a moving edge block to be filtered in current
278 // setting.
279 if (mb_filter_decision_[mb_index] == FILTER_BLOCK) {
280 uint32_t sse_t = 0;
281 if (ne_enable) {
282 // The variance used in noise estimation is based on the src block in
283 // time t (mb_src) and filtered block in time t-1 (mb_dist_prev).
284 uint32_t noise_var = filter_->Variance16x8(
285 mb_dst_prev, stride_y_dst, mb_src, stride_y_src, &sse_t);
286 ne_->GetNoise(mb_index, noise_var, luma);
287 }
288 moving_edge_[mb_index] = 0; // Not a moving edge block.
289 } else {
290 uint32_t sse_t = 0;
291 // The variance used in MOD is based on the filtered blocks in time
292 // T (mb_dst) and T-1 (mb_dst_prev).
293 uint32_t noise_var = filter_->Variance16x8(
294 mb_dst_prev, stride_prev, mb_dst, stride_y_dst, &sse_t);
295 if (noise_var > thr_var_adp) { // Moving edge checking.
296 if (ne_enable) {
297 ne_->ResetConsecLowVar(mb_index);
298 }
299 moving_edge_[mb_index] = 1; // Mark as moving edge block.
300 x_density_[mb_col] += (pos_factor < 3);
301 y_density_[mb_row] += (pos_factor < 3);
302 } else {
303 moving_edge_[mb_index] = 0;
304 if (ne_enable) {
305 // The variance used in noise estimation is based on the src block
306 // in time t (mb_src) and filtered block in time t-1 (mb_dist_prev).
307 uint32_t noise_var = filter_->Variance16x8(
308 mb_dst_prev, stride_prev, mb_src, stride_y_src, &sse_t);
309 ne_->GetNoise(mb_index, noise_var, luma);
310 }
311 }
312 }
313 } // End of for loop
314 } // End of for loop
315
316 ReduceFalseDetection(moving_edge_, &moving_object_, noise_level);
317
318 CopySrcOnMOB(y_src, stride_y_src, y_dst, stride_y_dst);
319
320 // When frame width/height not divisible by 16, copy the margin to
321 // denoised_frame.
322 if ((mb_rows_ << 4) != height_ || (mb_cols_ << 4) != width_)
323 CopyLumaOnMargin(y_src, stride_y_src, y_dst, stride_y_dst);
324
325 // Copy u/v planes.
326 libyuv::CopyPlane(frame->DataU(), frame->StrideU(),
327 dst->MutableDataU(), dst->StrideU(),
328 (width_ + 1) >> 1, (height_ + 1) >> 1);
329 libyuv::CopyPlane(frame->DataV(), frame->StrideV(),
330 dst->MutableDataV(), dst->StrideV(),
331 (width_ + 1) >> 1, (height_ + 1) >> 1);
332
333 #if DISPLAY || DISPLAYNEON
334 // Show rectangular region
335 ShowRect(filter_, moving_edge_, moving_object_, x_density_, y_density_,
336 frame->DataU(), frame->StrideU(), frame->DataV(), frame->StrideV(),
337 dst->MutableDataU(), dst->StrideU(),
338 dst->MutableDataV(), dst->StrideV(),
339 mb_rows_, mb_cols_);
340 #endif
341 prev_buffer_ = dst;
342 return dst;
343 }
344
345 } // namespace webrtc
346