1 /*
2 * Copyright (c) 2019, Alliance for Open Media. All rights reserved
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #include <stdint.h>
13
14 #include "av1/common/blockd.h"
15 #include "config/aom_config.h"
16 #include "config/aom_scale_rtcd.h"
17
18 #include "aom/aom_codec.h"
19 #include "aom/aom_encoder.h"
20
21 #include "aom_ports/system_state.h"
22
23 #if CONFIG_MISMATCH_DEBUG
24 #include "aom_util/debug_util.h"
25 #endif // CONFIG_MISMATCH_DEBUG
26
27 #include "av1/common/av1_common_int.h"
28 #include "av1/common/reconinter.h"
29
30 #include "av1/encoder/encoder.h"
31 #include "av1/encoder/encode_strategy.h"
32 #include "av1/encoder/encodeframe.h"
33 #include "av1/encoder/firstpass.h"
34 #include "av1/encoder/pass2_strategy.h"
35 #include "av1/encoder/temporal_filter.h"
36 #include "av1/encoder/tpl_model.h"
37
38 #if CONFIG_TUNE_VMAF
39 #include "av1/encoder/tune_vmaf.h"
40 #endif
41
42 #define TEMPORAL_FILTER_KEY_FRAME (CONFIG_REALTIME_ONLY ? 0 : 1)
43
set_refresh_frame_flags(RefreshFrameFlagsInfo * const refresh_frame_flags,bool refresh_gf,bool refresh_bwdref,bool refresh_arf)44 static INLINE void set_refresh_frame_flags(
45 RefreshFrameFlagsInfo *const refresh_frame_flags, bool refresh_gf,
46 bool refresh_bwdref, bool refresh_arf) {
47 refresh_frame_flags->golden_frame = refresh_gf;
48 refresh_frame_flags->bwd_ref_frame = refresh_bwdref;
49 refresh_frame_flags->alt_ref_frame = refresh_arf;
50 }
51
av1_configure_buffer_updates(AV1_COMP * const cpi,RefreshFrameFlagsInfo * const refresh_frame_flags,const FRAME_UPDATE_TYPE type,const FRAME_TYPE frame_type,int force_refresh_all)52 void av1_configure_buffer_updates(
53 AV1_COMP *const cpi, RefreshFrameFlagsInfo *const refresh_frame_flags,
54 const FRAME_UPDATE_TYPE type, const FRAME_TYPE frame_type,
55 int force_refresh_all) {
56 // NOTE(weitinglin): Should we define another function to take care of
57 // cpi->rc.is_$Source_Type to make this function as it is in the comment?
58
59 const ExtRefreshFrameFlagsInfo *const ext_refresh_frame_flags =
60 &cpi->ext_flags.refresh_frame;
61 cpi->rc.is_src_frame_alt_ref = 0;
62
63 switch (type) {
64 case KF_UPDATE:
65 set_refresh_frame_flags(refresh_frame_flags, true, true, true);
66 break;
67
68 case LF_UPDATE:
69 set_refresh_frame_flags(refresh_frame_flags, false, false, false);
70 break;
71
72 case GF_UPDATE:
73 set_refresh_frame_flags(refresh_frame_flags, true, false, false);
74 break;
75
76 case OVERLAY_UPDATE:
77 if (frame_type == KEY_FRAME && cpi->rc.frames_to_key == 0) {
78 set_refresh_frame_flags(refresh_frame_flags, true, true, true);
79 } else {
80 set_refresh_frame_flags(refresh_frame_flags, true, false, false);
81 }
82 cpi->rc.is_src_frame_alt_ref = 1;
83 break;
84
85 case ARF_UPDATE:
86 // NOTE: BWDREF does not get updated along with ALTREF_FRAME.
87 if (frame_type == KEY_FRAME && !cpi->no_show_fwd_kf) {
88 // TODO(bohanli): consider moving this to force_refresh_all?
89 // This is Keyframe as arf
90 set_refresh_frame_flags(refresh_frame_flags, true, true, true);
91 } else {
92 set_refresh_frame_flags(refresh_frame_flags, false, false, true);
93 }
94 break;
95
96 case INTNL_OVERLAY_UPDATE:
97 set_refresh_frame_flags(refresh_frame_flags, false, false, false);
98 cpi->rc.is_src_frame_alt_ref = 1;
99 break;
100
101 case INTNL_ARF_UPDATE:
102 set_refresh_frame_flags(refresh_frame_flags, false, true, false);
103 break;
104
105 default: assert(0); break;
106 }
107
108 if (ext_refresh_frame_flags->update_pending &&
109 (!is_stat_generation_stage(cpi)))
110 set_refresh_frame_flags(refresh_frame_flags,
111 ext_refresh_frame_flags->golden_frame,
112 ext_refresh_frame_flags->bwd_ref_frame,
113 ext_refresh_frame_flags->alt_ref_frame);
114
115 if (force_refresh_all)
116 set_refresh_frame_flags(refresh_frame_flags, true, true, true);
117 }
118
set_additional_frame_flags(const AV1_COMMON * const cm,unsigned int * const frame_flags)119 static void set_additional_frame_flags(const AV1_COMMON *const cm,
120 unsigned int *const frame_flags) {
121 if (frame_is_intra_only(cm)) {
122 *frame_flags |= FRAMEFLAGS_INTRAONLY;
123 }
124 if (frame_is_sframe(cm)) {
125 *frame_flags |= FRAMEFLAGS_SWITCH;
126 }
127 if (cm->features.error_resilient_mode) {
128 *frame_flags |= FRAMEFLAGS_ERROR_RESILIENT;
129 }
130 }
131
update_keyframe_counters(AV1_COMP * cpi)132 static INLINE void update_keyframe_counters(AV1_COMP *cpi) {
133 if (cpi->common.show_frame && cpi->rc.frames_to_key) {
134 cpi->rc.frames_since_key++;
135 cpi->rc.frames_to_key--;
136 }
137 }
138
is_frame_droppable(const SVC * const svc,const ExtRefreshFrameFlagsInfo * const ext_refresh_frame_flags)139 static INLINE int is_frame_droppable(
140 const SVC *const svc,
141 const ExtRefreshFrameFlagsInfo *const ext_refresh_frame_flags) {
142 // Droppable frame is only used by external refresh flags. VoD setting won't
143 // trigger its use case.
144 if (svc->external_ref_frame_config)
145 return svc->non_reference_frame;
146 else if (ext_refresh_frame_flags->update_pending)
147 return !(ext_refresh_frame_flags->alt_ref_frame ||
148 ext_refresh_frame_flags->alt2_ref_frame ||
149 ext_refresh_frame_flags->bwd_ref_frame ||
150 ext_refresh_frame_flags->golden_frame ||
151 ext_refresh_frame_flags->last_frame);
152 else
153 return 0;
154 }
155
update_frames_till_gf_update(AV1_COMP * cpi)156 static INLINE void update_frames_till_gf_update(AV1_COMP *cpi) {
157 // TODO(weitinglin): Updating this counter for is_frame_droppable
158 // is a work-around to handle the condition when a frame is drop.
159 // We should fix the cpi->common.show_frame flag
160 // instead of checking the other condition to update the counter properly.
161 if (cpi->common.show_frame ||
162 is_frame_droppable(&cpi->svc, &cpi->ext_flags.refresh_frame)) {
163 // Decrement count down till next gf
164 if (cpi->rc.frames_till_gf_update_due > 0)
165 cpi->rc.frames_till_gf_update_due--;
166 }
167 }
168
update_gf_group_index(AV1_COMP * cpi)169 static INLINE void update_gf_group_index(AV1_COMP *cpi) {
170 // Increment the gf group index ready for the next frame.
171 ++cpi->gf_group.index;
172 }
173
update_rc_counts(AV1_COMP * cpi)174 static void update_rc_counts(AV1_COMP *cpi) {
175 update_keyframe_counters(cpi);
176 update_frames_till_gf_update(cpi);
177 update_gf_group_index(cpi);
178 }
179
set_ext_overrides(AV1_COMMON * const cm,EncodeFrameParams * const frame_params,ExternalFlags * const ext_flags)180 static void set_ext_overrides(AV1_COMMON *const cm,
181 EncodeFrameParams *const frame_params,
182 ExternalFlags *const ext_flags) {
183 // Overrides the defaults with the externally supplied values with
184 // av1_update_reference() and av1_update_entropy() calls
185 // Note: The overrides are valid only for the next frame passed
186 // to av1_encode_lowlevel()
187
188 if (ext_flags->use_s_frame) {
189 frame_params->frame_type = S_FRAME;
190 }
191
192 if (ext_flags->refresh_frame_context_pending) {
193 cm->features.refresh_frame_context = ext_flags->refresh_frame_context;
194 ext_flags->refresh_frame_context_pending = 0;
195 }
196 cm->features.allow_ref_frame_mvs = ext_flags->use_ref_frame_mvs;
197
198 frame_params->error_resilient_mode = ext_flags->use_error_resilient;
199 // A keyframe is already error resilient and keyframes with
200 // error_resilient_mode interferes with the use of show_existing_frame
201 // when forward reference keyframes are enabled.
202 frame_params->error_resilient_mode &= frame_params->frame_type != KEY_FRAME;
203 // For bitstream conformance, s-frames must be error-resilient
204 frame_params->error_resilient_mode |= frame_params->frame_type == S_FRAME;
205 }
206
get_current_frame_ref_type(const AV1_COMP * const cpi,const EncodeFrameParams * const frame_params)207 static int get_current_frame_ref_type(
208 const AV1_COMP *const cpi, const EncodeFrameParams *const frame_params) {
209 // We choose the reference "type" of this frame from the flags which indicate
210 // which reference frames will be refreshed by it. More than one of these
211 // flags may be set, so the order here implies an order of precedence. This is
212 // just used to choose the primary_ref_frame (as the most recent reference
213 // buffer of the same reference-type as the current frame)
214
215 (void)frame_params;
216 // TODO(jingning): This table should be a lot simpler with the new
217 // ARF system in place. Keep frame_params for the time being as we are
218 // still evaluating a few design options.
219 switch (cpi->gf_group.layer_depth[cpi->gf_group.index]) {
220 case 0: return 0;
221 case 1: return 1;
222 case MAX_ARF_LAYERS:
223 case MAX_ARF_LAYERS + 1: return 4;
224 default: return 7;
225 }
226 }
227
choose_primary_ref_frame(const AV1_COMP * const cpi,const EncodeFrameParams * const frame_params)228 static int choose_primary_ref_frame(
229 const AV1_COMP *const cpi, const EncodeFrameParams *const frame_params) {
230 const AV1_COMMON *const cm = &cpi->common;
231
232 const int intra_only = frame_params->frame_type == KEY_FRAME ||
233 frame_params->frame_type == INTRA_ONLY_FRAME;
234 if (intra_only || frame_params->error_resilient_mode ||
235 cpi->ext_flags.use_primary_ref_none) {
236 return PRIMARY_REF_NONE;
237 }
238
239 // In large scale case, always use Last frame's frame contexts.
240 // Note(yunqing): In other cases, primary_ref_frame is chosen based on
241 // cpi->gf_group.layer_depth[cpi->gf_group.index], which also controls
242 // frame bit allocation.
243 if (cm->tiles.large_scale) return (LAST_FRAME - LAST_FRAME);
244
245 if (cpi->use_svc) return av1_svc_primary_ref_frame(cpi);
246
247 // Find the most recent reference frame with the same reference type as the
248 // current frame
249 const int current_ref_type = get_current_frame_ref_type(cpi, frame_params);
250 int wanted_fb = cpi->fb_of_context_type[current_ref_type];
251
252 int primary_ref_frame = PRIMARY_REF_NONE;
253 for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ref_frame++) {
254 if (get_ref_frame_map_idx(cm, ref_frame) == wanted_fb) {
255 primary_ref_frame = ref_frame - LAST_FRAME;
256 }
257 }
258
259 return primary_ref_frame;
260 }
261
update_fb_of_context_type(const AV1_COMP * const cpi,const EncodeFrameParams * const frame_params,int * const fb_of_context_type)262 static void update_fb_of_context_type(
263 const AV1_COMP *const cpi, const EncodeFrameParams *const frame_params,
264 int *const fb_of_context_type) {
265 const AV1_COMMON *const cm = &cpi->common;
266 const int current_frame_ref_type =
267 get_current_frame_ref_type(cpi, frame_params);
268
269 if (frame_is_intra_only(cm) || cm->features.error_resilient_mode ||
270 cpi->ext_flags.use_primary_ref_none) {
271 for (int i = 0; i < REF_FRAMES; i++) {
272 fb_of_context_type[i] = -1;
273 }
274 fb_of_context_type[current_frame_ref_type] =
275 cm->show_frame ? get_ref_frame_map_idx(cm, GOLDEN_FRAME)
276 : get_ref_frame_map_idx(cm, ALTREF_FRAME);
277 }
278
279 if (!encode_show_existing_frame(cm)) {
280 // Refresh fb_of_context_type[]: see encoder.h for explanation
281 if (cm->current_frame.frame_type == KEY_FRAME) {
282 // All ref frames are refreshed, pick one that will live long enough
283 fb_of_context_type[current_frame_ref_type] = 0;
284 } else {
285 // If more than one frame is refreshed, it doesn't matter which one we
286 // pick so pick the first. LST sometimes doesn't refresh any: this is ok
287
288 for (int i = 0; i < REF_FRAMES; i++) {
289 if (cm->current_frame.refresh_frame_flags & (1 << i)) {
290 fb_of_context_type[current_frame_ref_type] = i;
291 break;
292 }
293 }
294 }
295 }
296 }
297
adjust_frame_rate(AV1_COMP * cpi,int64_t ts_start,int64_t ts_end)298 static void adjust_frame_rate(AV1_COMP *cpi, int64_t ts_start, int64_t ts_end) {
299 TimeStamps *time_stamps = &cpi->time_stamps;
300 int64_t this_duration;
301 int step = 0;
302
303 // Clear down mmx registers
304 aom_clear_system_state();
305
306 if (cpi->use_svc && cpi->svc.spatial_layer_id > 0) {
307 cpi->framerate = cpi->svc.base_framerate;
308 av1_rc_update_framerate(cpi, cpi->common.width, cpi->common.height);
309 return;
310 }
311
312 if (ts_start == time_stamps->first_ts_start) {
313 this_duration = ts_end - ts_start;
314 step = 1;
315 } else {
316 int64_t last_duration =
317 time_stamps->prev_ts_end - time_stamps->prev_ts_start;
318
319 this_duration = ts_end - time_stamps->prev_ts_end;
320
321 // do a step update if the duration changes by 10%
322 if (last_duration)
323 step = (int)((this_duration - last_duration) * 10 / last_duration);
324 }
325
326 if (this_duration) {
327 if (step) {
328 av1_new_framerate(cpi, 10000000.0 / this_duration);
329 } else {
330 // Average this frame's rate into the last second's average
331 // frame rate. If we haven't seen 1 second yet, then average
332 // over the whole interval seen.
333 const double interval =
334 AOMMIN((double)(ts_end - time_stamps->first_ts_start), 10000000.0);
335 double avg_duration = 10000000.0 / cpi->framerate;
336 avg_duration *= (interval - avg_duration + this_duration);
337 avg_duration /= interval;
338
339 av1_new_framerate(cpi, 10000000.0 / avg_duration);
340 }
341 }
342 time_stamps->prev_ts_start = ts_start;
343 time_stamps->prev_ts_end = ts_end;
344 }
345
346 // Determine whether there is a forced keyframe pending in the lookahead buffer
is_forced_keyframe_pending(struct lookahead_ctx * lookahead,const int up_to_index,const COMPRESSOR_STAGE compressor_stage)347 int is_forced_keyframe_pending(struct lookahead_ctx *lookahead,
348 const int up_to_index,
349 const COMPRESSOR_STAGE compressor_stage) {
350 for (int i = 0; i <= up_to_index; i++) {
351 const struct lookahead_entry *e =
352 av1_lookahead_peek(lookahead, i, compressor_stage);
353 if (e == NULL) {
354 // We have reached the end of the lookahead buffer and not early-returned
355 // so there isn't a forced key-frame pending.
356 return -1;
357 } else if (e->flags == AOM_EFLAG_FORCE_KF) {
358 return i;
359 } else {
360 continue;
361 }
362 }
363 return -1; // Never reached
364 }
365
366 // Check if we should encode an ARF or internal ARF. If not, try a LAST
367 // Do some setup associated with the chosen source
368 // temporal_filtered, flush, and frame_update_type are outputs.
369 // Return the frame source, or NULL if we couldn't find one
choose_frame_source(AV1_COMP * const cpi,int * const flush,int * pop_lookahead,struct lookahead_entry ** last_source,EncodeFrameParams * const frame_params)370 static struct lookahead_entry *choose_frame_source(
371 AV1_COMP *const cpi, int *const flush, int *pop_lookahead,
372 struct lookahead_entry **last_source,
373 EncodeFrameParams *const frame_params) {
374 AV1_COMMON *const cm = &cpi->common;
375 const GF_GROUP *const gf_group = &cpi->gf_group;
376 struct lookahead_entry *source = NULL;
377
378 // Source index in lookahead buffer.
379 int src_index = gf_group->arf_src_offset[gf_group->index];
380
381 // TODO(Aasaipriya): Forced key frames need to be fixed when rc_mode != AOM_Q
382 if (src_index &&
383 (is_forced_keyframe_pending(cpi->lookahead, src_index,
384 cpi->compressor_stage) != -1) &&
385 cpi->oxcf.rc_cfg.mode != AOM_Q) {
386 src_index = 0;
387 *flush = 1;
388 }
389
390 // If the current frame is arf, then we should not pop from the lookahead
391 // buffer. If the current frame is not arf, then pop it. This assumes the
392 // first frame in the GF group is not arf. May need to change if it is not
393 // true.
394 *pop_lookahead = (src_index == 0);
395 // If this is a key frame and keyframe filtering is enabled with overlay,
396 // then do not pop.
397 if (*pop_lookahead && cpi->oxcf.kf_cfg.enable_keyframe_filtering > 1 &&
398 gf_group->update_type[gf_group->index] == ARF_UPDATE &&
399 !is_stat_generation_stage(cpi) && cpi->lookahead) {
400 if (cpi->lookahead->read_ctxs[cpi->compressor_stage].sz &&
401 (*flush ||
402 cpi->lookahead->read_ctxs[cpi->compressor_stage].sz ==
403 cpi->lookahead->read_ctxs[cpi->compressor_stage].pop_sz)) {
404 *pop_lookahead = 0;
405 }
406 }
407 frame_params->show_frame = *pop_lookahead;
408 if (*pop_lookahead) {
409 // show frame, pop from buffer
410 // Get last frame source.
411 if (cm->current_frame.frame_number > 0) {
412 *last_source =
413 av1_lookahead_peek(cpi->lookahead, -1, cpi->compressor_stage);
414 }
415 // Read in the source frame.
416 source = av1_lookahead_peek(cpi->lookahead, 0, cpi->compressor_stage);
417 } else {
418 // no show frames are arf frames
419 source =
420 av1_lookahead_peek(cpi->lookahead, src_index, cpi->compressor_stage);
421 // When src_index == rc->frames_to_key, it indicates a fwd_kf
422 if (src_index == cpi->rc.frames_to_key && src_index != 0) {
423 cpi->no_show_fwd_kf = 1;
424 }
425 if (source != NULL) {
426 cm->showable_frame = 1;
427 }
428 }
429 return source;
430 }
431
432 // Don't allow a show_existing_frame to coincide with an error resilient or
433 // S-Frame. An exception can be made in the case of a keyframe, since it does
434 // not depend on any previous frames.
allow_show_existing(const AV1_COMP * const cpi,unsigned int frame_flags)435 static int allow_show_existing(const AV1_COMP *const cpi,
436 unsigned int frame_flags) {
437 if (cpi->common.current_frame.frame_number == 0) return 0;
438
439 const struct lookahead_entry *lookahead_src =
440 av1_lookahead_peek(cpi->lookahead, 0, cpi->compressor_stage);
441 if (lookahead_src == NULL) return 1;
442
443 const int is_error_resilient =
444 cpi->oxcf.tool_cfg.error_resilient_mode ||
445 (lookahead_src->flags & AOM_EFLAG_ERROR_RESILIENT);
446 const int is_s_frame = cpi->oxcf.kf_cfg.enable_sframe ||
447 (lookahead_src->flags & AOM_EFLAG_SET_S_FRAME);
448 const int is_key_frame =
449 (cpi->rc.frames_to_key == 0) || (frame_flags & FRAMEFLAGS_KEY);
450 return !(is_error_resilient || is_s_frame) || is_key_frame;
451 }
452
453 // Update frame_flags to tell the encoder's caller what sort of frame was
454 // encoded.
update_frame_flags(const AV1_COMMON * const cm,const RefreshFrameFlagsInfo * const refresh_frame_flags,unsigned int * frame_flags)455 static void update_frame_flags(
456 const AV1_COMMON *const cm,
457 const RefreshFrameFlagsInfo *const refresh_frame_flags,
458 unsigned int *frame_flags) {
459 if (encode_show_existing_frame(cm)) {
460 *frame_flags &= ~FRAMEFLAGS_GOLDEN;
461 *frame_flags &= ~FRAMEFLAGS_BWDREF;
462 *frame_flags &= ~FRAMEFLAGS_ALTREF;
463 *frame_flags &= ~FRAMEFLAGS_KEY;
464 return;
465 }
466
467 if (refresh_frame_flags->golden_frame) {
468 *frame_flags |= FRAMEFLAGS_GOLDEN;
469 } else {
470 *frame_flags &= ~FRAMEFLAGS_GOLDEN;
471 }
472
473 if (refresh_frame_flags->alt_ref_frame) {
474 *frame_flags |= FRAMEFLAGS_ALTREF;
475 } else {
476 *frame_flags &= ~FRAMEFLAGS_ALTREF;
477 }
478
479 if (refresh_frame_flags->bwd_ref_frame) {
480 *frame_flags |= FRAMEFLAGS_BWDREF;
481 } else {
482 *frame_flags &= ~FRAMEFLAGS_BWDREF;
483 }
484
485 if (cm->current_frame.frame_type == KEY_FRAME) {
486 *frame_flags |= FRAMEFLAGS_KEY;
487 } else {
488 *frame_flags &= ~FRAMEFLAGS_KEY;
489 }
490 }
491
492 #define DUMP_REF_FRAME_IMAGES 0
493
494 #if DUMP_REF_FRAME_IMAGES == 1
dump_one_image(AV1_COMMON * cm,const YV12_BUFFER_CONFIG * const ref_buf,char * file_name)495 static int dump_one_image(AV1_COMMON *cm,
496 const YV12_BUFFER_CONFIG *const ref_buf,
497 char *file_name) {
498 int h;
499 FILE *f_ref = NULL;
500
501 if (ref_buf == NULL) {
502 printf("Frame data buffer is NULL.\n");
503 return AOM_CODEC_MEM_ERROR;
504 }
505
506 if ((f_ref = fopen(file_name, "wb")) == NULL) {
507 printf("Unable to open file %s to write.\n", file_name);
508 return AOM_CODEC_MEM_ERROR;
509 }
510
511 // --- Y ---
512 for (h = 0; h < cm->height; ++h) {
513 fwrite(&ref_buf->y_buffer[h * ref_buf->y_stride], 1, cm->width, f_ref);
514 }
515 // --- U ---
516 for (h = 0; h < (cm->height >> 1); ++h) {
517 fwrite(&ref_buf->u_buffer[h * ref_buf->uv_stride], 1, (cm->width >> 1),
518 f_ref);
519 }
520 // --- V ---
521 for (h = 0; h < (cm->height >> 1); ++h) {
522 fwrite(&ref_buf->v_buffer[h * ref_buf->uv_stride], 1, (cm->width >> 1),
523 f_ref);
524 }
525
526 fclose(f_ref);
527
528 return AOM_CODEC_OK;
529 }
530
dump_ref_frame_images(AV1_COMP * cpi)531 static void dump_ref_frame_images(AV1_COMP *cpi) {
532 AV1_COMMON *const cm = &cpi->common;
533 MV_REFERENCE_FRAME ref_frame;
534
535 for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
536 char file_name[256] = "";
537 snprintf(file_name, sizeof(file_name), "/tmp/enc_F%d_ref_%d.yuv",
538 cm->current_frame.frame_number, ref_frame);
539 dump_one_image(cm, get_ref_frame_yv12_buf(cpi, ref_frame), file_name);
540 }
541 }
542 #endif // DUMP_REF_FRAME_IMAGES == 1
543
av1_get_refresh_ref_frame_map(int refresh_frame_flags)544 int av1_get_refresh_ref_frame_map(int refresh_frame_flags) {
545 int ref_map_index;
546
547 for (ref_map_index = 0; ref_map_index < REF_FRAMES; ++ref_map_index)
548 if ((refresh_frame_flags >> ref_map_index) & 1) break;
549
550 if (ref_map_index == REF_FRAMES) ref_map_index = INVALID_IDX;
551 return ref_map_index;
552 }
553
update_arf_stack(int ref_map_index,RefBufferStack * ref_buffer_stack)554 static void update_arf_stack(int ref_map_index,
555 RefBufferStack *ref_buffer_stack) {
556 if (ref_buffer_stack->arf_stack_size >= 0) {
557 if (ref_buffer_stack->arf_stack[0] == ref_map_index)
558 stack_pop(ref_buffer_stack->arf_stack, &ref_buffer_stack->arf_stack_size);
559 }
560
561 if (ref_buffer_stack->lst_stack_size) {
562 for (int i = ref_buffer_stack->lst_stack_size - 1; i >= 0; --i) {
563 if (ref_buffer_stack->lst_stack[i] == ref_map_index) {
564 for (int idx = i; idx < ref_buffer_stack->lst_stack_size - 1; ++idx)
565 ref_buffer_stack->lst_stack[idx] =
566 ref_buffer_stack->lst_stack[idx + 1];
567 ref_buffer_stack->lst_stack[ref_buffer_stack->lst_stack_size - 1] =
568 INVALID_IDX;
569 --ref_buffer_stack->lst_stack_size;
570 }
571 }
572 }
573
574 if (ref_buffer_stack->gld_stack_size) {
575 for (int i = ref_buffer_stack->gld_stack_size - 1; i >= 0; --i) {
576 if (ref_buffer_stack->gld_stack[i] == ref_map_index) {
577 for (int idx = i; idx < ref_buffer_stack->gld_stack_size - 1; ++idx)
578 ref_buffer_stack->gld_stack[idx] =
579 ref_buffer_stack->gld_stack[idx + 1];
580 ref_buffer_stack->gld_stack[ref_buffer_stack->gld_stack_size - 1] =
581 INVALID_IDX;
582 --ref_buffer_stack->gld_stack_size;
583 }
584 }
585 }
586 }
587
588 // Update reference frame stack info.
av1_update_ref_frame_map(AV1_COMP * cpi,FRAME_UPDATE_TYPE frame_update_type,FRAME_TYPE frame_type,int show_existing_frame,int ref_map_index,RefBufferStack * ref_buffer_stack)589 void av1_update_ref_frame_map(AV1_COMP *cpi,
590 FRAME_UPDATE_TYPE frame_update_type,
591 FRAME_TYPE frame_type, int show_existing_frame,
592 int ref_map_index,
593 RefBufferStack *ref_buffer_stack) {
594 AV1_COMMON *const cm = &cpi->common;
595 // TODO(jingning): Consider the S-frame same as key frame for the
596 // reference frame tracking purpose. The logic might be better
597 // expressed than converting the frame update type.
598 if (frame_is_sframe(cm)) frame_update_type = KEY_FRAME;
599
600 if (is_frame_droppable(&cpi->svc, &cpi->ext_flags.refresh_frame)) return;
601
602 switch (frame_update_type) {
603 case KEY_FRAME:
604 if (show_existing_frame)
605 ref_map_index = stack_pop(ref_buffer_stack->arf_stack,
606 &ref_buffer_stack->arf_stack_size);
607 stack_reset(ref_buffer_stack->lst_stack,
608 &ref_buffer_stack->lst_stack_size);
609 stack_reset(ref_buffer_stack->gld_stack,
610 &ref_buffer_stack->gld_stack_size);
611 stack_reset(ref_buffer_stack->arf_stack,
612 &ref_buffer_stack->arf_stack_size);
613 stack_push(ref_buffer_stack->gld_stack, &ref_buffer_stack->gld_stack_size,
614 ref_map_index);
615 break;
616 case GF_UPDATE:
617 update_arf_stack(ref_map_index, ref_buffer_stack);
618 stack_push(ref_buffer_stack->gld_stack, &ref_buffer_stack->gld_stack_size,
619 ref_map_index);
620 // For nonrd_mode: update LAST as well on GF_UPDATE frame.
621 if (cpi->sf.rt_sf.use_nonrd_pick_mode)
622 stack_push(ref_buffer_stack->lst_stack,
623 &ref_buffer_stack->lst_stack_size, ref_map_index);
624 break;
625 case LF_UPDATE:
626 update_arf_stack(ref_map_index, ref_buffer_stack);
627 stack_push(ref_buffer_stack->lst_stack, &ref_buffer_stack->lst_stack_size,
628 ref_map_index);
629 break;
630 case ARF_UPDATE:
631 case INTNL_ARF_UPDATE:
632 if (frame_type == KEY_FRAME && !cpi->no_show_fwd_kf) {
633 stack_reset(ref_buffer_stack->lst_stack,
634 &ref_buffer_stack->lst_stack_size);
635 stack_reset(ref_buffer_stack->gld_stack,
636 &ref_buffer_stack->gld_stack_size);
637 stack_reset(ref_buffer_stack->arf_stack,
638 &ref_buffer_stack->arf_stack_size);
639 } else {
640 update_arf_stack(ref_map_index, ref_buffer_stack);
641 }
642 stack_push(ref_buffer_stack->arf_stack, &ref_buffer_stack->arf_stack_size,
643 ref_map_index);
644 break;
645 case OVERLAY_UPDATE:
646 if (frame_type == KEY_FRAME) {
647 ref_map_index = stack_pop(ref_buffer_stack->arf_stack,
648 &ref_buffer_stack->arf_stack_size);
649 stack_reset(ref_buffer_stack->lst_stack,
650 &ref_buffer_stack->lst_stack_size);
651 stack_reset(ref_buffer_stack->gld_stack,
652 &ref_buffer_stack->gld_stack_size);
653 stack_reset(ref_buffer_stack->arf_stack,
654 &ref_buffer_stack->arf_stack_size);
655 stack_push(ref_buffer_stack->gld_stack,
656 &ref_buffer_stack->gld_stack_size, ref_map_index);
657 } else {
658 if (ref_map_index != INVALID_IDX) {
659 update_arf_stack(ref_map_index, ref_buffer_stack);
660 stack_push(ref_buffer_stack->lst_stack,
661 &ref_buffer_stack->lst_stack_size, ref_map_index);
662 }
663 ref_map_index = stack_pop(ref_buffer_stack->arf_stack,
664 &ref_buffer_stack->arf_stack_size);
665 stack_push(ref_buffer_stack->gld_stack,
666 &ref_buffer_stack->gld_stack_size, ref_map_index);
667 }
668 break;
669 case INTNL_OVERLAY_UPDATE:
670 ref_map_index = stack_pop(ref_buffer_stack->arf_stack,
671 &ref_buffer_stack->arf_stack_size);
672 stack_push(ref_buffer_stack->lst_stack, &ref_buffer_stack->lst_stack_size,
673 ref_map_index);
674 break;
675 default: assert(0 && "unknown type");
676 }
677 return;
678 }
679
get_free_ref_map_index(const RefBufferStack * ref_buffer_stack)680 static int get_free_ref_map_index(const RefBufferStack *ref_buffer_stack) {
681 for (int idx = 0; idx < REF_FRAMES; ++idx) {
682 int is_free = 1;
683 for (int i = 0; i < ref_buffer_stack->arf_stack_size; ++i) {
684 if (ref_buffer_stack->arf_stack[i] == idx) {
685 is_free = 0;
686 break;
687 }
688 }
689
690 for (int i = 0; i < ref_buffer_stack->lst_stack_size; ++i) {
691 if (ref_buffer_stack->lst_stack[i] == idx) {
692 is_free = 0;
693 break;
694 }
695 }
696
697 for (int i = 0; i < ref_buffer_stack->gld_stack_size; ++i) {
698 if (ref_buffer_stack->gld_stack[i] == idx) {
699 is_free = 0;
700 break;
701 }
702 }
703
704 if (is_free) return idx;
705 }
706 return INVALID_IDX;
707 }
708
av1_get_refresh_frame_flags(const AV1_COMP * const cpi,const EncodeFrameParams * const frame_params,FRAME_UPDATE_TYPE frame_update_type,const RefBufferStack * const ref_buffer_stack)709 int av1_get_refresh_frame_flags(const AV1_COMP *const cpi,
710 const EncodeFrameParams *const frame_params,
711 FRAME_UPDATE_TYPE frame_update_type,
712 const RefBufferStack *const ref_buffer_stack) {
713 const AV1_COMMON *const cm = &cpi->common;
714 const ExtRefreshFrameFlagsInfo *const ext_refresh_frame_flags =
715 &cpi->ext_flags.refresh_frame;
716
717 const SVC *const svc = &cpi->svc;
718 // Switch frames and shown key-frames overwrite all reference slots
719 if ((frame_params->frame_type == KEY_FRAME && !cpi->no_show_fwd_kf) ||
720 frame_params->frame_type == S_FRAME)
721 return 0xFF;
722
723 // show_existing_frames don't actually send refresh_frame_flags so set the
724 // flags to 0 to keep things consistent.
725 if (frame_params->show_existing_frame &&
726 (!frame_params->error_resilient_mode ||
727 frame_params->frame_type == KEY_FRAME)) {
728 return 0;
729 }
730
731 if (is_frame_droppable(svc, ext_refresh_frame_flags)) return 0;
732
733 int refresh_mask = 0;
734
735 if (ext_refresh_frame_flags->update_pending) {
736 if (svc->external_ref_frame_config) {
737 for (unsigned int i = 0; i < INTER_REFS_PER_FRAME; i++) {
738 int ref_frame_map_idx = svc->ref_idx[i];
739 refresh_mask |= svc->refresh[ref_frame_map_idx] << ref_frame_map_idx;
740 }
741 return refresh_mask;
742 }
743 // Unfortunately the encoder interface reflects the old refresh_*_frame
744 // flags so we have to replicate the old refresh_frame_flags logic here in
745 // order to preserve the behaviour of the flag overrides.
746 int ref_frame_map_idx = get_ref_frame_map_idx(cm, LAST_FRAME);
747 if (ref_frame_map_idx != INVALID_IDX)
748 refresh_mask |= ext_refresh_frame_flags->last_frame << ref_frame_map_idx;
749
750 ref_frame_map_idx = get_ref_frame_map_idx(cm, EXTREF_FRAME);
751 if (ref_frame_map_idx != INVALID_IDX)
752 refresh_mask |= ext_refresh_frame_flags->bwd_ref_frame
753 << ref_frame_map_idx;
754
755 ref_frame_map_idx = get_ref_frame_map_idx(cm, ALTREF2_FRAME);
756 if (ref_frame_map_idx != INVALID_IDX)
757 refresh_mask |= ext_refresh_frame_flags->alt2_ref_frame
758 << ref_frame_map_idx;
759
760 if (frame_update_type == OVERLAY_UPDATE) {
761 ref_frame_map_idx = get_ref_frame_map_idx(cm, ALTREF_FRAME);
762 if (ref_frame_map_idx != INVALID_IDX)
763 refresh_mask |= ext_refresh_frame_flags->golden_frame
764 << ref_frame_map_idx;
765 } else {
766 ref_frame_map_idx = get_ref_frame_map_idx(cm, GOLDEN_FRAME);
767 if (ref_frame_map_idx != INVALID_IDX)
768 refresh_mask |= ext_refresh_frame_flags->golden_frame
769 << ref_frame_map_idx;
770
771 ref_frame_map_idx = get_ref_frame_map_idx(cm, ALTREF_FRAME);
772 if (ref_frame_map_idx != INVALID_IDX)
773 refresh_mask |= ext_refresh_frame_flags->alt_ref_frame
774 << ref_frame_map_idx;
775 }
776 return refresh_mask;
777 }
778
779 // Search for the open slot to store the current frame.
780 int free_fb_index = get_free_ref_map_index(ref_buffer_stack);
781 switch (frame_update_type) {
782 case KF_UPDATE:
783 case GF_UPDATE:
784 if (free_fb_index != INVALID_IDX) {
785 refresh_mask = 1 << free_fb_index;
786 } else {
787 if (ref_buffer_stack->gld_stack_size)
788 refresh_mask =
789 1 << ref_buffer_stack
790 ->gld_stack[ref_buffer_stack->gld_stack_size - 1];
791 else
792 refresh_mask =
793 1 << ref_buffer_stack
794 ->lst_stack[ref_buffer_stack->lst_stack_size - 1];
795 }
796 break;
797 case LF_UPDATE:
798 if (free_fb_index != INVALID_IDX) {
799 refresh_mask = 1 << free_fb_index;
800 } else {
801 if (ref_buffer_stack->lst_stack_size >= 2)
802 refresh_mask =
803 1 << ref_buffer_stack
804 ->lst_stack[ref_buffer_stack->lst_stack_size - 1];
805 else if (ref_buffer_stack->gld_stack_size >= 2)
806 refresh_mask =
807 1 << ref_buffer_stack
808 ->gld_stack[ref_buffer_stack->gld_stack_size - 1];
809 else
810 assert(0 && "No ref map index found");
811 }
812 break;
813 case ARF_UPDATE:
814 if (free_fb_index != INVALID_IDX) {
815 refresh_mask = 1 << free_fb_index;
816 } else {
817 if (ref_buffer_stack->gld_stack_size >= 3)
818 refresh_mask =
819 1 << ref_buffer_stack
820 ->gld_stack[ref_buffer_stack->gld_stack_size - 1];
821 else if (ref_buffer_stack->lst_stack_size >= 2)
822 refresh_mask =
823 1 << ref_buffer_stack
824 ->lst_stack[ref_buffer_stack->lst_stack_size - 1];
825 else
826 assert(0 && "No ref map index found");
827 }
828 break;
829 case INTNL_ARF_UPDATE:
830 if (free_fb_index != INVALID_IDX) {
831 refresh_mask = 1 << free_fb_index;
832 } else {
833 refresh_mask =
834 1 << ref_buffer_stack
835 ->lst_stack[ref_buffer_stack->lst_stack_size - 1];
836 }
837 break;
838 case OVERLAY_UPDATE:
839 if (free_fb_index != INVALID_IDX) refresh_mask = 1 << free_fb_index;
840 break;
841 case INTNL_OVERLAY_UPDATE: break;
842 default: assert(0); break;
843 }
844
845 return refresh_mask;
846 }
847
848 #if !CONFIG_REALTIME_ONLY
setup_mi(AV1_COMP * const cpi,YV12_BUFFER_CONFIG * src)849 void setup_mi(AV1_COMP *const cpi, YV12_BUFFER_CONFIG *src) {
850 AV1_COMMON *const cm = &cpi->common;
851 const int num_planes = av1_num_planes(cm);
852 MACROBLOCK *const x = &cpi->td.mb;
853 MACROBLOCKD *const xd = &x->e_mbd;
854
855 av1_setup_src_planes(x, src, 0, 0, num_planes, cm->seq_params.sb_size);
856
857 av1_setup_block_planes(xd, cm->seq_params.subsampling_x,
858 cm->seq_params.subsampling_y, num_planes);
859
860 set_mi_offsets(&cm->mi_params, xd, 0, 0);
861 }
862
863 // Apply temporal filtering to source frames and encode the filtered frame.
864 // If the current frame does not require filtering, this function is identical
865 // to av1_encode() except that tpl is not performed.
denoise_and_encode(AV1_COMP * const cpi,uint8_t * const dest,EncodeFrameInput * const frame_input,EncodeFrameParams * const frame_params,EncodeFrameResults * const frame_results)866 static int denoise_and_encode(AV1_COMP *const cpi, uint8_t *const dest,
867 EncodeFrameInput *const frame_input,
868 EncodeFrameParams *const frame_params,
869 EncodeFrameResults *const frame_results) {
870 #if CONFIG_COLLECT_COMPONENT_TIMING
871 if (cpi->oxcf.pass == 2) start_timing(cpi, denoise_and_encode_time);
872 #endif
873 const AV1EncoderConfig *const oxcf = &cpi->oxcf;
874 AV1_COMMON *const cm = &cpi->common;
875 const GF_GROUP *const gf_group = &cpi->gf_group;
876 FRAME_UPDATE_TYPE update_type = get_frame_update_type(&cpi->gf_group);
877
878 // Decide whether to apply temporal filtering to the source frame.
879 int apply_filtering = 0;
880 if (frame_params->frame_type == KEY_FRAME) {
881 // Decide whether it is allowed to perform key frame filtering
882 int allow_kf_filtering =
883 oxcf->kf_cfg.enable_keyframe_filtering &&
884 !is_stat_generation_stage(cpi) && !frame_params->show_existing_frame &&
885 cpi->rc.frames_to_key > cpi->oxcf.algo_cfg.arnr_max_frames &&
886 !is_lossless_requested(&oxcf->rc_cfg) &&
887 oxcf->algo_cfg.arnr_max_frames > 0 && oxcf->gf_cfg.lag_in_frames > 1;
888 if (allow_kf_filtering) {
889 const double y_noise_level = av1_estimate_noise_from_single_plane(
890 frame_input->source, 0, cm->seq_params.bit_depth);
891 apply_filtering = y_noise_level > 0;
892 } else {
893 apply_filtering = 0;
894 }
895 // If we are doing kf filtering, set up a few things.
896 if (apply_filtering) {
897 av1_setup_past_independence(cm);
898 }
899 } else if (update_type == ARF_UPDATE || update_type == INTNL_ARF_UPDATE) {
900 // ARF
901 apply_filtering = oxcf->algo_cfg.arnr_max_frames > 0;
902 }
903
904 #if CONFIG_COLLECT_COMPONENT_TIMING
905 if (cpi->oxcf.pass == 2) start_timing(cpi, apply_filtering_time);
906 #endif
907 // Save the pointer to the original source image.
908 YV12_BUFFER_CONFIG *source_buffer = frame_input->source;
909 // apply filtering to frame
910 if (apply_filtering) {
911 int show_existing_alt_ref = 0;
912 // TODO(bohanli): figure out why we need frame_type in cm here.
913 cm->current_frame.frame_type = frame_params->frame_type;
914 int arf_src_index = gf_group->arf_src_offset[gf_group->index];
915 int is_forward_keyframe = 0;
916 if (!frame_params->show_frame && cpi->no_show_fwd_kf) {
917 // TODO(angiebird): Figure out why this condition yields forward keyframe.
918 // fwd kf
919 is_forward_keyframe = 1;
920 }
921 const int code_arf =
922 av1_temporal_filter(cpi, arf_src_index, update_type,
923 is_forward_keyframe, &show_existing_alt_ref);
924 if (code_arf) {
925 aom_extend_frame_borders(&cpi->alt_ref_buffer, av1_num_planes(cm));
926 frame_input->source = &cpi->alt_ref_buffer;
927 aom_copy_metadata_to_frame_buffer(frame_input->source,
928 source_buffer->metadata);
929 }
930 // Currently INTNL_ARF_UPDATE only do show_existing.
931 if (update_type == ARF_UPDATE && !cpi->no_show_fwd_kf) {
932 cpi->show_existing_alt_ref = show_existing_alt_ref;
933 }
934 }
935 #if CONFIG_COLLECT_COMPONENT_TIMING
936 if (cpi->oxcf.pass == 2) end_timing(cpi, apply_filtering_time);
937 #endif
938
939 // perform tpl after filtering
940 int allow_tpl = oxcf->gf_cfg.lag_in_frames > 1 &&
941 !is_stat_generation_stage(cpi) &&
942 oxcf->algo_cfg.enable_tpl_model;
943 if (frame_params->frame_type == KEY_FRAME) {
944 // Don't do tpl for fwd key frames or fwd key frame overlays
945 allow_tpl = allow_tpl && !cpi->sf.tpl_sf.disable_filtered_key_tpl &&
946 !cpi->no_show_fwd_kf &&
947 gf_group->update_type[gf_group->index] != OVERLAY_UPDATE;
948 } else {
949 // Do tpl after ARF is filtered, or if no ARF, at the second frame of GF
950 // group.
951 // TODO(bohanli): if no ARF, just do it at the first frame.
952 int gf_index = gf_group->index;
953 allow_tpl = allow_tpl && (gf_group->update_type[gf_index] == ARF_UPDATE ||
954 gf_group->update_type[gf_index] == GF_UPDATE);
955 if (allow_tpl) {
956 // Need to set the size for TPL for ARF
957 // TODO(bohanli): Why is this? what part of it is necessary?
958 av1_set_frame_size(cpi, cm->superres_upscaled_width,
959 cm->superres_upscaled_height);
960 }
961 }
962
963 if (allow_tpl == 0) {
964 // Avoid the use of unintended TPL stats from previous GOP's results.
965 if (gf_group->index == 0) av1_init_tpl_stats(&cpi->tpl_data);
966 } else {
967 if (!cpi->tpl_data.skip_tpl_setup_stats)
968 av1_tpl_setup_stats(cpi, 0, frame_params, frame_input);
969 }
970
971 if (av1_encode(cpi, dest, frame_input, frame_params, frame_results) !=
972 AOM_CODEC_OK) {
973 return AOM_CODEC_ERROR;
974 }
975
976 // Set frame_input source to true source for psnr calculation.
977 if (apply_filtering && is_psnr_calc_enabled(cpi)) {
978 cpi->source =
979 av1_scale_if_required(cm, source_buffer, &cpi->scaled_source,
980 cm->features.interp_filter, 0, false, true);
981 cpi->unscaled_source = source_buffer;
982 }
983 #if CONFIG_COLLECT_COMPONENT_TIMING
984 if (cpi->oxcf.pass == 2) end_timing(cpi, denoise_and_encode_time);
985 #endif
986 return AOM_CODEC_OK;
987 }
988 #endif // !CONFIG_REALTIME_ONLY
989
find_unused_ref_frame(const int * used_ref_frames,const int * stack,int stack_size)990 static INLINE int find_unused_ref_frame(const int *used_ref_frames,
991 const int *stack, int stack_size) {
992 for (int i = 0; i < stack_size; ++i) {
993 const int this_ref = stack[i];
994 int ref_idx = 0;
995 for (ref_idx = 0; ref_idx <= ALTREF_FRAME - LAST_FRAME; ++ref_idx) {
996 if (this_ref == used_ref_frames[ref_idx]) break;
997 }
998
999 // not in use
1000 if (ref_idx > ALTREF_FRAME - LAST_FRAME) return this_ref;
1001 }
1002
1003 return INVALID_IDX;
1004 }
1005
av1_get_ref_frames(AV1_COMP * const cpi,RefBufferStack * ref_buffer_stack)1006 void av1_get_ref_frames(AV1_COMP *const cpi, RefBufferStack *ref_buffer_stack) {
1007 AV1_COMMON *cm = &cpi->common;
1008 int *const remapped_ref_idx = cm->remapped_ref_idx;
1009 int *const arf_stack = ref_buffer_stack->arf_stack;
1010 int *const lst_stack = ref_buffer_stack->lst_stack;
1011 int *const gld_stack = ref_buffer_stack->gld_stack;
1012 const int arf_stack_size = ref_buffer_stack->arf_stack_size;
1013 const int lst_stack_size = ref_buffer_stack->lst_stack_size;
1014 const int gld_stack_size = ref_buffer_stack->gld_stack_size;
1015
1016 // Initialization
1017 for (int i = 0; i < REF_FRAMES; ++i) remapped_ref_idx[i] = INVALID_IDX;
1018
1019 if (arf_stack_size) {
1020 remapped_ref_idx[ALTREF_FRAME - LAST_FRAME] = arf_stack[arf_stack_size - 1];
1021
1022 if (arf_stack_size > 1)
1023 remapped_ref_idx[BWDREF_FRAME - LAST_FRAME] = arf_stack[0];
1024
1025 if (arf_stack_size > 2)
1026 remapped_ref_idx[ALTREF2_FRAME - LAST_FRAME] = arf_stack[1];
1027 }
1028
1029 if (lst_stack_size) {
1030 remapped_ref_idx[LAST_FRAME - LAST_FRAME] = lst_stack[0];
1031
1032 if (lst_stack_size > 1)
1033 remapped_ref_idx[LAST2_FRAME - LAST_FRAME] = lst_stack[1];
1034 }
1035
1036 if (gld_stack_size) {
1037 remapped_ref_idx[GOLDEN_FRAME - LAST_FRAME] = gld_stack[0];
1038
1039 // If there are more frames in the golden stack, assign them to BWDREF,
1040 // ALTREF2, or LAST3.
1041 if (gld_stack_size > 1) {
1042 if (arf_stack_size <= 2) {
1043 if (arf_stack_size <= 1) {
1044 remapped_ref_idx[BWDREF_FRAME - LAST_FRAME] = gld_stack[1];
1045 if (gld_stack_size > 2)
1046 remapped_ref_idx[ALTREF2_FRAME - LAST_FRAME] = gld_stack[2];
1047 } else {
1048 remapped_ref_idx[ALTREF2_FRAME - LAST_FRAME] = gld_stack[1];
1049 }
1050 } else {
1051 remapped_ref_idx[LAST3_FRAME - LAST_FRAME] = gld_stack[1];
1052 }
1053 }
1054 }
1055
1056 for (int idx = ALTREF_FRAME - LAST_FRAME; idx >= 0; --idx) {
1057 int ref_map_index = remapped_ref_idx[idx];
1058
1059 if (ref_map_index != INVALID_IDX) continue;
1060
1061 ref_map_index =
1062 find_unused_ref_frame(remapped_ref_idx, arf_stack, arf_stack_size);
1063
1064 if (ref_map_index == INVALID_IDX) {
1065 ref_map_index =
1066 find_unused_ref_frame(remapped_ref_idx, gld_stack, gld_stack_size);
1067 }
1068
1069 if (ref_map_index == INVALID_IDX) {
1070 ref_map_index =
1071 find_unused_ref_frame(remapped_ref_idx, lst_stack, lst_stack_size);
1072 }
1073
1074 if (ref_map_index != INVALID_IDX) {
1075 remapped_ref_idx[idx] = ref_map_index;
1076 } else if (!gld_stack_size && arf_stack_size) {
1077 remapped_ref_idx[idx] = ref_buffer_stack->arf_stack[0];
1078 } else {
1079 remapped_ref_idx[idx] = ref_buffer_stack->gld_stack[0];
1080 }
1081 }
1082 }
1083
av1_encode_strategy(AV1_COMP * const cpi,size_t * const size,uint8_t * const dest,unsigned int * frame_flags,int64_t * const time_stamp,int64_t * const time_end,const aom_rational64_t * const timestamp_ratio,int flush)1084 int av1_encode_strategy(AV1_COMP *const cpi, size_t *const size,
1085 uint8_t *const dest, unsigned int *frame_flags,
1086 int64_t *const time_stamp, int64_t *const time_end,
1087 const aom_rational64_t *const timestamp_ratio,
1088 int flush) {
1089 AV1EncoderConfig *const oxcf = &cpi->oxcf;
1090 AV1_COMMON *const cm = &cpi->common;
1091 GF_GROUP *gf_group = &cpi->gf_group;
1092 ExternalFlags *const ext_flags = &cpi->ext_flags;
1093 GFConfig *const gf_cfg = &oxcf->gf_cfg;
1094
1095 EncodeFrameInput frame_input;
1096 EncodeFrameParams frame_params;
1097 EncodeFrameResults frame_results;
1098 memset(&frame_input, 0, sizeof(frame_input));
1099 memset(&frame_params, 0, sizeof(frame_params));
1100 memset(&frame_results, 0, sizeof(frame_results));
1101
1102 // Check if we need to stuff more src frames
1103 if (flush == 0) {
1104 int srcbuf_size =
1105 av1_lookahead_depth(cpi->lookahead, cpi->compressor_stage);
1106 int pop_size = av1_lookahead_pop_sz(cpi->lookahead, cpi->compressor_stage);
1107
1108 // Continue buffering look ahead buffer.
1109 if (srcbuf_size < pop_size) return -1;
1110 }
1111
1112 if (!av1_lookahead_peek(cpi->lookahead, 0, cpi->compressor_stage)) {
1113 #if !CONFIG_REALTIME_ONLY
1114 if (flush && oxcf->pass == 1 && !cpi->twopass.first_pass_done) {
1115 av1_end_first_pass(cpi); /* get last stats packet */
1116 cpi->twopass.first_pass_done = 1;
1117 }
1118 #endif
1119 return -1;
1120 }
1121
1122 // TODO(sarahparker) finish bit allocation for one pass pyramid
1123 if (has_no_stats_stage(cpi)) {
1124 gf_cfg->gf_max_pyr_height =
1125 AOMMIN(gf_cfg->gf_max_pyr_height, USE_ALTREF_FOR_ONE_PASS);
1126 gf_cfg->gf_min_pyr_height =
1127 AOMMIN(gf_cfg->gf_min_pyr_height, gf_cfg->gf_max_pyr_height);
1128 }
1129
1130 cpi->tpl_data.skip_tpl_setup_stats = 0;
1131 #if !CONFIG_REALTIME_ONLY
1132 const int use_one_pass_rt_params = has_no_stats_stage(cpi) &&
1133 oxcf->mode == REALTIME &&
1134 gf_cfg->lag_in_frames == 0;
1135 if (!use_one_pass_rt_params && !is_stat_generation_stage(cpi)) {
1136 #if CONFIG_COLLECT_COMPONENT_TIMING
1137 start_timing(cpi, av1_get_second_pass_params_time);
1138 #endif
1139 av1_get_second_pass_params(cpi, &frame_params, &frame_input, *frame_flags);
1140 #if CONFIG_COLLECT_COMPONENT_TIMING
1141 end_timing(cpi, av1_get_second_pass_params_time);
1142 #endif
1143 }
1144 #endif
1145
1146 if (!is_stat_generation_stage(cpi)) {
1147 // If this is a forward keyframe, mark as a show_existing_frame
1148 // TODO(bohanli): find a consistent condition for fwd keyframes
1149 if (oxcf->kf_cfg.fwd_kf_enabled &&
1150 gf_group->update_type[gf_group->index] == OVERLAY_UPDATE &&
1151 cpi->rc.frames_to_key == 0) {
1152 frame_params.show_existing_frame = 1;
1153 } else {
1154 frame_params.show_existing_frame =
1155 (cpi->show_existing_alt_ref &&
1156 gf_group->update_type[gf_group->index] == OVERLAY_UPDATE) ||
1157 gf_group->update_type[gf_group->index] == INTNL_OVERLAY_UPDATE;
1158 }
1159 frame_params.show_existing_frame &= allow_show_existing(cpi, *frame_flags);
1160
1161 // Reset show_existing_alt_ref decision to 0 after it is used.
1162 if (gf_group->update_type[gf_group->index] == OVERLAY_UPDATE) {
1163 cpi->show_existing_alt_ref = 0;
1164 }
1165 } else {
1166 frame_params.show_existing_frame = 0;
1167 }
1168
1169 struct lookahead_entry *source = NULL;
1170 struct lookahead_entry *last_source = NULL;
1171 int pop_lookahead = 0;
1172 if (frame_params.show_existing_frame) {
1173 source = av1_lookahead_peek(cpi->lookahead, 0, cpi->compressor_stage);
1174 pop_lookahead = 1;
1175 frame_params.show_frame = 1;
1176 } else {
1177 source = choose_frame_source(cpi, &flush, &pop_lookahead, &last_source,
1178 &frame_params);
1179 }
1180
1181 if (source == NULL) { // If no source was found, we can't encode a frame.
1182 #if !CONFIG_REALTIME_ONLY
1183 if (flush && oxcf->pass == 1 && !cpi->twopass.first_pass_done) {
1184 av1_end_first_pass(cpi); /* get last stats packet */
1185 cpi->twopass.first_pass_done = 1;
1186 }
1187 #endif
1188 return -1;
1189 }
1190 // Source may be changed if temporal filtered later.
1191 frame_input.source = &source->img;
1192 frame_input.last_source = last_source != NULL ? &last_source->img : NULL;
1193 frame_input.ts_duration = source->ts_end - source->ts_start;
1194 // Save unfiltered source. It is used in av1_get_second_pass_params().
1195 cpi->unfiltered_source = frame_input.source;
1196
1197 *time_stamp = source->ts_start;
1198 *time_end = source->ts_end;
1199 if (source->ts_start < cpi->time_stamps.first_ts_start) {
1200 cpi->time_stamps.first_ts_start = source->ts_start;
1201 cpi->time_stamps.prev_ts_end = source->ts_start;
1202 }
1203
1204 av1_apply_encoding_flags(cpi, source->flags);
1205 *frame_flags = (source->flags & AOM_EFLAG_FORCE_KF) ? FRAMEFLAGS_KEY : 0;
1206
1207 // Shown frames and arf-overlay frames need frame-rate considering
1208 if (frame_params.show_frame)
1209 adjust_frame_rate(cpi, source->ts_start, source->ts_end);
1210
1211 if (!frame_params.show_existing_frame) {
1212 if (cpi->film_grain_table) {
1213 cm->cur_frame->film_grain_params_present = aom_film_grain_table_lookup(
1214 cpi->film_grain_table, *time_stamp, *time_end, 0 /* =erase */,
1215 &cm->film_grain_params);
1216 } else {
1217 cm->cur_frame->film_grain_params_present =
1218 cm->seq_params.film_grain_params_present;
1219 }
1220 // only one operating point supported now
1221 const int64_t pts64 = ticks_to_timebase_units(timestamp_ratio, *time_stamp);
1222 if (pts64 < 0 || pts64 > UINT32_MAX) return AOM_CODEC_ERROR;
1223 cm->frame_presentation_time = (uint32_t)pts64;
1224 }
1225
1226 #if CONFIG_REALTIME_ONLY
1227 av1_get_one_pass_rt_params(cpi, &frame_params, *frame_flags);
1228 if (cpi->oxcf.speed >= 5 && cm->number_spatial_layers == 1 &&
1229 cm->number_temporal_layers == 1)
1230 av1_set_reference_structure_one_pass_rt(cpi, gf_group->index == 0);
1231 #else
1232 if (use_one_pass_rt_params) {
1233 av1_get_one_pass_rt_params(cpi, &frame_params, *frame_flags);
1234 if (cpi->oxcf.speed >= 5 && cm->number_spatial_layers == 1 &&
1235 cm->number_temporal_layers == 1)
1236 av1_set_reference_structure_one_pass_rt(cpi, gf_group->index == 0);
1237 }
1238 #endif
1239
1240 FRAME_UPDATE_TYPE frame_update_type = get_frame_update_type(gf_group);
1241
1242 if (frame_params.show_existing_frame &&
1243 frame_params.frame_type != KEY_FRAME) {
1244 // Force show-existing frames to be INTER, except forward keyframes
1245 frame_params.frame_type = INTER_FRAME;
1246 }
1247
1248 // TODO(david.turner@argondesign.com): Move all the encode strategy
1249 // (largely near av1_get_compressed_data) in here
1250
1251 // TODO(david.turner@argondesign.com): Change all the encode strategy to
1252 // modify frame_params instead of cm or cpi.
1253
1254 // Per-frame encode speed. In theory this can vary, but things may have
1255 // been written assuming speed-level will not change within a sequence, so
1256 // this parameter should be used with caution.
1257 frame_params.speed = oxcf->speed;
1258
1259 // Work out some encoding parameters specific to the pass:
1260 if (has_no_stats_stage(cpi) && oxcf->q_cfg.aq_mode == CYCLIC_REFRESH_AQ) {
1261 av1_cyclic_refresh_update_parameters(cpi);
1262 } else if (is_stat_generation_stage(cpi)) {
1263 cpi->td.mb.e_mbd.lossless[0] = is_lossless_requested(&oxcf->rc_cfg);
1264 const int kf_requested = (cm->current_frame.frame_number == 0 ||
1265 (*frame_flags & FRAMEFLAGS_KEY));
1266 if (kf_requested && frame_update_type != OVERLAY_UPDATE &&
1267 frame_update_type != INTNL_OVERLAY_UPDATE) {
1268 frame_params.frame_type = KEY_FRAME;
1269 } else {
1270 frame_params.frame_type = INTER_FRAME;
1271 }
1272 } else if (is_stat_consumption_stage(cpi)) {
1273 #if CONFIG_MISMATCH_DEBUG
1274 mismatch_move_frame_idx_w();
1275 #endif
1276 #if TXCOEFF_COST_TIMER
1277 cm->txcoeff_cost_timer = 0;
1278 cm->txcoeff_cost_count = 0;
1279 #endif
1280 }
1281
1282 if (!is_stat_generation_stage(cpi))
1283 set_ext_overrides(cm, &frame_params, ext_flags);
1284
1285 // Shown keyframes and S frames refresh all reference buffers
1286 const int force_refresh_all =
1287 ((frame_params.frame_type == KEY_FRAME && frame_params.show_frame) ||
1288 frame_params.frame_type == S_FRAME) &&
1289 !frame_params.show_existing_frame;
1290
1291 av1_configure_buffer_updates(cpi, &frame_params.refresh_frame,
1292 frame_update_type, frame_params.frame_type,
1293 force_refresh_all);
1294
1295 if (!is_stat_generation_stage(cpi)) {
1296 const RefCntBuffer *ref_frames[INTER_REFS_PER_FRAME];
1297 const YV12_BUFFER_CONFIG *ref_frame_buf[INTER_REFS_PER_FRAME];
1298
1299 if (!ext_flags->refresh_frame.update_pending) {
1300 av1_get_ref_frames(cpi, &cpi->ref_buffer_stack);
1301 } else if (cpi->svc.external_ref_frame_config) {
1302 for (unsigned int i = 0; i < INTER_REFS_PER_FRAME; i++)
1303 cm->remapped_ref_idx[i] = cpi->svc.ref_idx[i];
1304 }
1305
1306 // Get the reference frames
1307 for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
1308 ref_frames[i] = get_ref_frame_buf(cm, ref_frame_priority_order[i]);
1309 ref_frame_buf[i] = ref_frames[i] != NULL ? &ref_frames[i]->buf : NULL;
1310 }
1311
1312 // Work out which reference frame slots may be used.
1313 frame_params.ref_frame_flags = get_ref_frame_flags(
1314 &cpi->sf, ref_frame_buf, ext_flags->ref_frame_flags);
1315
1316 frame_params.primary_ref_frame =
1317 choose_primary_ref_frame(cpi, &frame_params);
1318 frame_params.order_offset = gf_group->arf_src_offset[gf_group->index];
1319
1320 frame_params.refresh_frame_flags = av1_get_refresh_frame_flags(
1321 cpi, &frame_params, frame_update_type, &cpi->ref_buffer_stack);
1322
1323 frame_params.existing_fb_idx_to_show =
1324 frame_params.show_existing_frame
1325 ? (frame_update_type == INTNL_OVERLAY_UPDATE
1326 ? get_ref_frame_map_idx(cm, BWDREF_FRAME)
1327 : get_ref_frame_map_idx(cm, ALTREF_FRAME))
1328 : INVALID_IDX;
1329 }
1330
1331 // The way frame_params->remapped_ref_idx is setup is a placeholder.
1332 // Currently, reference buffer assignment is done by update_ref_frame_map()
1333 // which is called by high-level strategy AFTER encoding a frame. It
1334 // modifies cm->remapped_ref_idx. If you want to use an alternative method
1335 // to determine reference buffer assignment, just put your assignments into
1336 // frame_params->remapped_ref_idx here and they will be used when encoding
1337 // this frame. If frame_params->remapped_ref_idx is setup independently of
1338 // cm->remapped_ref_idx then update_ref_frame_map() will have no effect.
1339 memcpy(frame_params.remapped_ref_idx, cm->remapped_ref_idx,
1340 REF_FRAMES * sizeof(*cm->remapped_ref_idx));
1341
1342 cpi->td.mb.delta_qindex = 0;
1343
1344 if (!frame_params.show_existing_frame) {
1345 cm->quant_params.using_qmatrix = oxcf->q_cfg.using_qm;
1346 }
1347
1348 #if CONFIG_REALTIME_ONLY
1349 if (av1_encode(cpi, dest, &frame_input, &frame_params, &frame_results) !=
1350 AOM_CODEC_OK) {
1351 return AOM_CODEC_ERROR;
1352 }
1353 #else
1354 if (has_no_stats_stage(cpi) && oxcf->mode == REALTIME &&
1355 gf_cfg->lag_in_frames == 0) {
1356 if (av1_encode(cpi, dest, &frame_input, &frame_params, &frame_results) !=
1357 AOM_CODEC_OK) {
1358 return AOM_CODEC_ERROR;
1359 }
1360 } else if (denoise_and_encode(cpi, dest, &frame_input, &frame_params,
1361 &frame_results) != AOM_CODEC_OK) {
1362 return AOM_CODEC_ERROR;
1363 }
1364 #endif // CONFIG_REALTIME_ONLY
1365
1366 if (!is_stat_generation_stage(cpi)) {
1367 // First pass doesn't modify reference buffer assignment or produce frame
1368 // flags
1369 update_frame_flags(&cpi->common, &cpi->refresh_frame, frame_flags);
1370 if (!ext_flags->refresh_frame.update_pending) {
1371 int ref_map_index =
1372 av1_get_refresh_ref_frame_map(cm->current_frame.refresh_frame_flags);
1373 av1_update_ref_frame_map(cpi, frame_update_type, frame_params.frame_type,
1374 cm->show_existing_frame, ref_map_index,
1375 &cpi->ref_buffer_stack);
1376 }
1377 }
1378
1379 #if !CONFIG_REALTIME_ONLY
1380 if (!is_stat_generation_stage(cpi)) {
1381 #if TXCOEFF_COST_TIMER
1382 cm->cum_txcoeff_cost_timer += cm->txcoeff_cost_timer;
1383 fprintf(stderr,
1384 "\ntxb coeff cost block number: %ld, frame time: %ld, cum time %ld "
1385 "in us\n",
1386 cm->txcoeff_cost_count, cm->txcoeff_cost_timer,
1387 cm->cum_txcoeff_cost_timer);
1388 #endif
1389 if (!has_no_stats_stage(cpi)) av1_twopass_postencode_update(cpi);
1390 }
1391 #endif // !CONFIG_REALTIME_ONLY
1392
1393 #if CONFIG_TUNE_VMAF
1394 if (!is_stat_generation_stage(cpi) &&
1395 (oxcf->tune_cfg.tuning >= AOM_TUNE_VMAF_WITH_PREPROCESSING &&
1396 oxcf->tune_cfg.tuning <= AOM_TUNE_VMAF_NEG_MAX_GAIN)) {
1397 av1_update_vmaf_curve(cpi);
1398 }
1399 #endif
1400 if (pop_lookahead == 1) {
1401 av1_lookahead_pop(cpi->lookahead, flush, cpi->compressor_stage);
1402 }
1403
1404 if (!is_stat_generation_stage(cpi)) {
1405 update_fb_of_context_type(cpi, &frame_params, cpi->fb_of_context_type);
1406 set_additional_frame_flags(cm, frame_flags);
1407 update_rc_counts(cpi);
1408 }
1409
1410 // Unpack frame_results:
1411 *size = frame_results.size;
1412
1413 // Leave a signal for a higher level caller about if this frame is droppable
1414 if (*size > 0) {
1415 cpi->droppable = is_frame_droppable(&cpi->svc, &ext_flags->refresh_frame);
1416 }
1417
1418 if (cpi->use_svc) av1_save_layer_context(cpi);
1419
1420 return AOM_CODEC_OK;
1421 }
1422