1 /*
2 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #ifndef AOM_AV1_ENCODER_FIRSTPASS_H_
13 #define AOM_AV1_ENCODER_FIRSTPASS_H_
14
15 #include "av1/common/av1_common_int.h"
16 #include "av1/common/enums.h"
17 #include "av1/encoder/lookahead.h"
18 #include "av1/encoder/ratectrl.h"
19
20 #ifdef __cplusplus
21 extern "C" {
22 #endif
23
24 #define DOUBLE_DIVIDE_CHECK(x) ((x) < 0 ? (x)-0.000001 : (x) + 0.000001)
25
26 #define MIN_ZERO_MOTION 0.95
27 #define MAX_SR_CODED_ERROR 40
28 #define MAX_RAW_ERR_VAR 2000
29 #define MIN_MV_IN_OUT 0.4
30
31 #define VLOW_MOTION_THRESHOLD 950
32 struct ThreadData;
33
34 /*!
35 * \brief The stucture of acummulated frame stats in the first pass.
36 *
37 * Errors (coded_error, intra_error, etc.) and counters (new_mv_count) are
38 * normalized to each MB. MV related stats (MVc, MVr, etc.) are normalized to
39 * the frame width and height. See function normalize_firstpass_stats.
40 */
41 typedef struct {
42 /*!
43 * Frame number in display order, if stats are for a single frame.
44 * No real meaning for a collection of frames.
45 */
46 double frame;
47 /*!
48 * Weight assigned to this frame (or total weight for the collection of
49 * frames) currently based on intra factor and brightness factor. This is used
50 * to distribute bits betweeen easier and harder frames.
51 */
52 double weight;
53 /*!
54 * Intra prediction error.
55 */
56 double intra_error;
57 /*!
58 * Average wavelet energy computed using Discrete Wavelet Transform (DWT).
59 */
60 double frame_avg_wavelet_energy;
61 /*!
62 * Best of intra pred error and inter pred error using last frame as ref.
63 */
64 double coded_error;
65 /*!
66 * Best of intra pred error and inter pred error using golden frame as ref.
67 */
68 double sr_coded_error;
69 /*!
70 * Percentage of blocks with inter pred error < intra pred error.
71 */
72 double pcnt_inter;
73 /*!
74 * Percentage of blocks using (inter prediction and) non-zero motion vectors.
75 */
76 double pcnt_motion;
77 /*!
78 * Percentage of blocks where golden frame was better than last or intra:
79 * inter pred error using golden frame < inter pred error using last frame and
80 * inter pred error using golden frame < intra pred error
81 */
82 double pcnt_second_ref;
83 /*!
84 * Percentage of blocks where intra and inter prediction errors were very
85 * close. Note that this is a 'weighted count', that is, the so blocks may be
86 * weighted by how close the two errors were.
87 */
88 double pcnt_neutral;
89 /*!
90 * Percentage of blocks that have almost no intra error residual
91 * (i.e. are in effect completely flat and untextured in the intra
92 * domain). In natural videos this is uncommon, but it is much more
93 * common in animations, graphics and screen content, so may be used
94 * as a signal to detect these types of content.
95 */
96 double intra_skip_pct;
97 /*!
98 * Image mask rows top and bottom.
99 */
100 double inactive_zone_rows;
101 /*!
102 * Image mask columns at left and right edges.
103 */
104 double inactive_zone_cols;
105 /*!
106 * Average of row motion vectors.
107 */
108 double MVr;
109 /*!
110 * Mean of absolute value of row motion vectors.
111 */
112 double mvr_abs;
113 /*!
114 * Mean of column motion vectors.
115 */
116 double MVc;
117 /*!
118 * Mean of absolute value of column motion vectors.
119 */
120 double mvc_abs;
121 /*!
122 * Variance of row motion vectors.
123 */
124 double MVrv;
125 /*!
126 * Variance of column motion vectors.
127 */
128 double MVcv;
129 /*!
130 * Value in range [-1,1] indicating fraction of row and column motion vectors
131 * that point inwards (negative MV value) or outwards (positive MV value).
132 * For example, value of 1 indicates, all row/column MVs are inwards.
133 */
134 double mv_in_out_count;
135 /*!
136 * Count of unique non-zero motion vectors.
137 */
138 double new_mv_count;
139 /*!
140 * Duration of the frame / collection of frames.
141 */
142 double duration;
143 /*!
144 * 1.0 if stats are for a single frame, OR
145 * Number of frames in this collection for which the stats are accumulated.
146 */
147 double count;
148 /*!
149 * standard deviation for (0, 0) motion prediction error
150 */
151 double raw_error_stdev;
152 /*!
153 * Whether the frame contains a flash
154 */
155 int64_t is_flash;
156 /*!
157 * Estimated noise variance
158 */
159 double noise_var;
160 /*!
161 * Correlation coefficient with the previous frame
162 */
163 double cor_coeff;
164 } FIRSTPASS_STATS;
165
166 // We want to keep one past stats for key frame detection
167 // in test_candidate_kf()
168 #define FIRSTPASS_INFO_STATS_PAST_MIN 1
169
170 // The size of static buffer used in FIRSTPASS_INFO.
171 #define FIRSTPASS_INFO_STATIC_BUF_SIZE \
172 (MAX_LAP_BUFFERS + FIRSTPASS_INFO_STATS_PAST_MIN)
173
174 /*!
175 * \brief Data structure used for managing first pass stats
176 */
177 typedef struct {
178 /*!
179 * A static buffer that will be used when no ext_stats_buf is assigned. The
180 * ext_stats_buf is assigned through av1_firstpass_info_init() when the user
181 * already has a pre-existing firstpass stats that is stored in an external
182 * buffer. The ext_stats_buf is usually used in two pass mode. When using one
183 * pass mode, we generate "firstpass" stats and encode the video in the same
184 * pass. In this scenario, the stats will be pushed and popped from
185 * static_stats_buf.
186 */
187 FIRSTPASS_STATS static_stats_buf[FIRSTPASS_INFO_STATIC_BUF_SIZE];
188 /*!
189 * A pointer to first pass stats.
190 * Note that this buffer will be used as ring buffer.
191 */
192 FIRSTPASS_STATS *stats_buf;
193 /*!
194 * size of stats_buf
195 */
196 int stats_buf_size;
197 /*!
198 * start index of the available frame stats
199 * Note that start_index doesn't always point to
200 * current frame's stats because we need to
201 * keep past stats as well. To access current
202 * frame's stats, please use cur_index.
203 */
204 int start_index;
205
206 /*!
207 * count available stats stored in stats_buf
208 * the following condition should stay true
209 * stats_count = future_stats_count + past_stats_count
210 */
211 int stats_count;
212
213 /*!
214 * index of the current frame's stats
215 */
216 int cur_index;
217
218 /*!
219 * count available future stats including current stats
220 */
221 int future_stats_count;
222
223 /*!
224 * count available past stats EXCLUDING current stats
225 */
226 int past_stats_count;
227
228 /*!
229 * Accumulation of the stats being pushed into firstpass_info
230 */
231 FIRSTPASS_STATS total_stats;
232 } FIRSTPASS_INFO;
233
234 /*!\brief Init firstpass_info
235 *
236 * If using ext_stats_buf, the buffer needs to stay available during encoding
237 * process.
238 *
239 * \ingroup rate_control
240 * \param[out] firstpass_info struct of firstpass_info.
241 * \param[in] ext_stats_buf external stats buffer. Pass in NULL if
242 * choose to use internal static_stats_buf.
243 * \param[in] ext_stats_buf_size external stats buffer size. Pass in 0 if
244 * choose to use internal static_stats_buf. \return status
245 */
246 aom_codec_err_t av1_firstpass_info_init(FIRSTPASS_INFO *firstpass_info,
247 FIRSTPASS_STATS *ext_stats_buf,
248 int ext_stats_buf_size);
249
250 /*!\brief Move cur_index by 1
251 *
252 * \ingroup rate_control
253 * \param[out] firstpass_info struct of firstpass_info.
254 * \return status
255 */
256 aom_codec_err_t av1_firstpass_info_move_cur_index(
257 FIRSTPASS_INFO *firstpass_info);
258
259 /*!\brief Pop a stats from firstpass_info
260 *
261 * \ingroup rate_control
262 * \param[out] firstpass_info struct of firstpass_info.
263 * \return status
264 */
265 aom_codec_err_t av1_firstpass_info_pop(FIRSTPASS_INFO *firstpass_info);
266
267 /*!\brief Move cur_index by 1 and pop a stats from firstpass_info
268 *
269 * \ingroup rate_control
270 * \param[out] firstpass_info struct of firstpass_info.
271 * \return status
272 */
273 aom_codec_err_t av1_firstpass_info_move_cur_index_and_pop(
274 FIRSTPASS_INFO *firstpass_info);
275
276 /*!\brief Push a stats into firstpass_info
277 *
278 * Note that the input stats will be copied into firstpass_info.
279 * \ingroup rate_control
280 * \param[out] firstpass_info struct of firstpass_info.
281 * \param[in] input_stats input stats
282 * \return status
283 */
284 aom_codec_err_t av1_firstpass_info_push(FIRSTPASS_INFO *firstpass_info,
285 const FIRSTPASS_STATS *input_stats);
286
287 /*!\brief Peek at a stats from firstpass_info
288 *
289 * The target index is as follows.
290 * (cur_index + offset_from_cur) % firstpass_info->stats_buf_size
291 *
292 * \ingroup rate_control
293 * \param[in] firstpass_info struct of firstpass_info.
294 * \param[in] offset_from_cur index offset from cur_index.
295 * \return pointer to the stats. The pointer will be NULL if
296 * stats_index_offset is invalid.
297 */
298 const FIRSTPASS_STATS *av1_firstpass_info_peek(
299 const FIRSTPASS_INFO *firstpass_info, int offset_from_cur);
300
301 /*!\brief Count the future stats from the target in firstpass_info
302 * Note that the target stats will be counted as well.
303 * The target index is as follows.
304 * (cur_index + offset_from_cur) % firstpass_info->stats_buf_size
305 *
306 * \ingroup rate_control
307 * \param[in] firstpass_info struct of firstpass_info.
308 * \param[in] offset_from_cur target stats's inffset
309 * from cur_index.
310 * \return Number of stats in the future after the target stats
311 * including itself.
312 */
313 int av1_firstpass_info_future_count(const FIRSTPASS_INFO *firstpass_info,
314 int offset_from_cur);
315
316 /*!\brief Count the past stats before the target in firstpass_info
317 * Note that the target stats will NOT be counted.
318 * The target index is as follows.
319 * (cur_index + offset_from_cur) % firstpass_info->stats_buf_size
320 *
321 * \ingroup rate_control
322 * \param[in] firstpass_info struct of firstpass_info.
323 * \param[in] offset_from_cur target stats's index offset
324 * from cur_index.
325 * \return Number of stats in the past before the target stats
326 * excluding itself.
327 */
328 int av1_firstpass_info_past_count(const FIRSTPASS_INFO *firstpass_info,
329 int offset_from_cur);
330
331 /*!\cond */
332 #define FC_ANIMATION_THRESH 0.15
333 enum {
334 FC_NORMAL = 0,
335 FC_GRAPHICS_ANIMATION = 1,
336 FRAME_CONTENT_TYPES = 2
337 } UENUM1BYTE(FRAME_CONTENT_TYPE);
338 /*!\endcond */
339
340 /*!
341 * \brief Data related to the current GF/ARF group and the
342 * individual frames within the group
343 */
344 typedef struct GF_GROUP {
345 /*!\cond */
346 // Frame update type, e.g. ARF/GF/LF/Overlay
347 FRAME_UPDATE_TYPE update_type[MAX_STATIC_GF_GROUP_LENGTH];
348 unsigned char arf_src_offset[MAX_STATIC_GF_GROUP_LENGTH];
349 // The number of frames displayed so far within the GOP at a given coding
350 // frame.
351 unsigned char cur_frame_idx[MAX_STATIC_GF_GROUP_LENGTH];
352 int layer_depth[MAX_STATIC_GF_GROUP_LENGTH];
353 int arf_boost[MAX_STATIC_GF_GROUP_LENGTH];
354 int max_layer_depth;
355 int max_layer_depth_allowed;
356 // This is currently only populated for AOM_Q mode
357 int q_val[MAX_STATIC_GF_GROUP_LENGTH];
358 int bit_allocation[MAX_STATIC_GF_GROUP_LENGTH];
359 // The frame coding type - inter/intra frame
360 FRAME_TYPE frame_type[MAX_STATIC_GF_GROUP_LENGTH];
361 // The reference frame buffer control - update or reset
362 REFBUF_STATE refbuf_state[MAX_STATIC_GF_GROUP_LENGTH];
363 int arf_index; // the index in the gf group of ARF, if no arf, then -1
364 int size; // The total length of a GOP
365 #if CONFIG_FRAME_PARALLEL_ENCODE
366 // Indicates the level of parallelism in frame parallel encodes.
367 // 0 : frame is independently encoded (not part of parallel encodes).
368 // 1 : frame is the first in encode order in a given parallel encode set.
369 // 2 : frame occurs later in encode order in a given parallel encode set.
370 int frame_parallel_level[MAX_STATIC_GF_GROUP_LENGTH];
371 // Indicates whether a frame should act as non-reference frame.
372 // 0 : frame is a reference frame.
373 // 1 : frame is a non-reference frame.
374 int is_frame_non_ref[MAX_STATIC_GF_GROUP_LENGTH];
375
376 // The offset into lookahead_ctx for choosing
377 // source of frame parallel encodes.
378 int src_offset[MAX_STATIC_GF_GROUP_LENGTH];
379 #if CONFIG_FRAME_PARALLEL_ENCODE_2
380 // Stores the display order hint of each frame in the current GF_GROUP.
381 int display_idx[MAX_STATIC_GF_GROUP_LENGTH];
382 // Stores the display order hint of the frames not to be
383 // refreshed by the current frame.
384 int skip_frame_refresh[MAX_STATIC_GF_GROUP_LENGTH][REF_FRAMES];
385 // Stores the display order hint of the frame to be excluded during reference
386 // assignment.
387 int skip_frame_as_ref[MAX_STATIC_GF_GROUP_LENGTH];
388 #endif // CONFIG_FRAME_PARALLEL_ENCODE_2
389 #endif // CONFIG_FRAME_PARALLEL_ENCODE
390 /*!\endcond */
391 } GF_GROUP;
392 /*!\cond */
393
394 typedef struct {
395 // Track if the last frame in a GOP has higher quality.
396 int arf_gf_boost_lst;
397 } GF_STATE;
398
399 typedef struct {
400 FIRSTPASS_STATS *stats_in_start;
401 FIRSTPASS_STATS *stats_in_end;
402 FIRSTPASS_STATS *stats_in_buf_end;
403 FIRSTPASS_STATS *total_stats;
404 FIRSTPASS_STATS *total_left_stats;
405 } STATS_BUFFER_CTX;
406
407 /*!\endcond */
408
409 /*!
410 * \brief Two pass status and control data.
411 */
412 typedef struct {
413 /*!\cond */
414 unsigned int section_intra_rating;
415 // Circular queue of first pass stats stored for most recent frames.
416 // cpi->output_pkt_list[i].data.twopass_stats.buf points to actual data stored
417 // here.
418 FIRSTPASS_STATS *frame_stats_arr[MAX_LAP_BUFFERS + 1];
419 int frame_stats_next_idx; // Index to next unused element in frame_stats_arr.
420 STATS_BUFFER_CTX *stats_buf_ctx;
421 FIRSTPASS_INFO firstpass_info; // This is the first pass data structure
422 // intended to replace stats_in
423 int first_pass_done;
424 int64_t bits_left;
425 double modified_error_min;
426 double modified_error_max;
427 double modified_error_left;
428
429 // Projected total bits available for a key frame group of frames
430 int64_t kf_group_bits;
431
432 // Error score of frames still to be coded in kf group
433 double kf_group_error_left;
434
435 // Over time correction for bits per macro block estimation
436 double bpm_factor;
437
438 // Record of target and actual bits spent in current ARF group
439 int rolling_arf_group_target_bits;
440 int rolling_arf_group_actual_bits;
441
442 int sr_update_lag;
443
444 int kf_zeromotion_pct;
445 int last_kfgroup_zeromotion_pct;
446 int extend_minq;
447 int extend_maxq;
448 int extend_minq_fast;
449 /*!\endcond */
450 } TWO_PASS;
451
452 /*!
453 * \brief Frame level Two pass status and control data.
454 */
455 typedef struct {
456 /*!\cond */
457 const FIRSTPASS_STATS *stats_in;
458 // Pointer to the stats of the current frame.
459 const FIRSTPASS_STATS *this_frame;
460 double mb_av_energy;
461 // An indication of the content type of the current frame
462 FRAME_CONTENT_TYPE fr_content_type;
463 double frame_avg_haar_energy;
464 /*!\endcond */
465 } TWO_PASS_FRAME;
466
467 /*!\cond */
468
469 // This structure contains several key parameters to be accumulated for this
470 // frame.
471 typedef struct {
472 // Intra prediction error.
473 int64_t intra_error;
474 // Average wavelet energy computed using Discrete Wavelet Transform (DWT).
475 int64_t frame_avg_wavelet_energy;
476 // Best of intra pred error and inter pred error using last frame as ref.
477 int64_t coded_error;
478 // Best of intra pred error and inter pred error using golden frame as ref.
479 int64_t sr_coded_error;
480 // Count of motion vector.
481 int mv_count;
482 // Count of blocks that pick inter prediction (inter pred error is smaller
483 // than intra pred error).
484 int inter_count;
485 // Count of blocks that pick second ref (golden frame).
486 int second_ref_count;
487 // Count of blocks where the inter and intra are very close and very low.
488 double neutral_count;
489 // Count of blocks where intra error is very small.
490 int intra_skip_count;
491 // Start row.
492 int image_data_start_row;
493 // Count of unique non-zero motion vectors.
494 int new_mv_count;
495 // Sum of inward motion vectors.
496 int sum_in_vectors;
497 // Sum of motion vector row.
498 int sum_mvr;
499 // Sum of motion vector column.
500 int sum_mvc;
501 // Sum of absolute value of motion vector row.
502 int sum_mvr_abs;
503 // Sum of absolute value of motion vector column.
504 int sum_mvc_abs;
505 // Sum of the square of motion vector row.
506 int64_t sum_mvrs;
507 // Sum of the square of motion vector column.
508 int64_t sum_mvcs;
509 // A factor calculated using intra pred error.
510 double intra_factor;
511 // A factor that measures brightness.
512 double brightness_factor;
513 } FRAME_STATS;
514
515 // This structure contains first pass data.
516 typedef struct {
517 // Buffer holding frame stats for all MACROBLOCKs.
518 // mb_stats[i] stores the FRAME_STATS of the ith
519 // MB in raster scan order.
520 FRAME_STATS *mb_stats;
521 // Buffer to store the prediction error of the (0,0) motion
522 // vector using the last source frame as the reference.
523 // raw_motion_err_list[i] stores the raw_motion_err of
524 // the ith MB in raster scan order.
525 int *raw_motion_err_list;
526 } FirstPassData;
527
528 struct AV1_COMP;
529 struct EncodeFrameParams;
530 struct AV1EncoderConfig;
531 struct TileDataEnc;
532
is_fp_wavelet_energy_invalid(const FIRSTPASS_STATS * fp_stats)533 static INLINE int is_fp_wavelet_energy_invalid(
534 const FIRSTPASS_STATS *fp_stats) {
535 assert(fp_stats != NULL);
536 return (fp_stats->frame_avg_wavelet_energy < 0);
537 }
538
get_fp_block_size(int is_screen_content_type)539 static INLINE BLOCK_SIZE get_fp_block_size(int is_screen_content_type) {
540 return (is_screen_content_type ? BLOCK_8X8 : BLOCK_16X16);
541 }
542
543 int av1_get_unit_rows_in_tile(TileInfo tile, const BLOCK_SIZE fp_block_size);
544 int av1_get_unit_cols_in_tile(TileInfo tile, const BLOCK_SIZE fp_block_size);
545
546 void av1_rc_get_first_pass_params(struct AV1_COMP *cpi);
547 void av1_first_pass_row(struct AV1_COMP *cpi, struct ThreadData *td,
548 struct TileDataEnc *tile_data, const int mb_row,
549 const BLOCK_SIZE fp_block_size);
550 void av1_end_first_pass(struct AV1_COMP *cpi);
551
552 void av1_twopass_zero_stats(FIRSTPASS_STATS *section);
553 void av1_accumulate_stats(FIRSTPASS_STATS *section,
554 const FIRSTPASS_STATS *frame);
555 /*!\endcond */
556
557 /*!\brief AV1 first pass encoding.
558 *
559 * \ingroup rate_control
560 * This function is the first encoding pass for the two pass encoding mode.
561 * It encodes the whole video and collect essential information.
562 * Two pass encoding is an encoding mode in the reference software (libaom)
563 * of AV1 for high performance encoding. The first pass is a fast encoding
564 * process to collect essential information to help the second pass make
565 * encoding decisions and improve coding quality. The collected stats is used
566 * in rate control, for example, to determine frame cut, the position of
567 * alternative reference frame (ARF), etc.
568 *
569 * \param[in] cpi Top-level encoder structure
570 * \param[in] ts_duration Duration of the frame / collection of frames
571 *
572 * \return Nothing is returned. Instead, the "TWO_PASS" structure inside "cpi"
573 * is modified to store information computed in this function.
574 */
575 void av1_first_pass(struct AV1_COMP *cpi, const int64_t ts_duration);
576
577 #ifdef __cplusplus
578 } // extern "C"
579 #endif
580
581 #endif // AOM_AV1_ENCODER_FIRSTPASS_H_
582