1 /*
2 * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/codecs/vp8/default_temporal_layers.h"
12 #include "modules/video_coding/codecs/vp8/vp8_impl.h"
13 #include "modules/video_coding/include/video_codec_interface.h"
14 #include "test/field_trial.h"
15 #include "test/gtest.h"
16 #include "vpx/vp8cx.h"
17 #include "vpx/vpx_encoder.h"
18
19 namespace webrtc {
20 namespace test {
21
22 enum {
23 kTemporalUpdateLast = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
24 VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF,
25 kTemporalUpdateGoldenWithoutDependency =
26 VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF |
27 VP8_EFLAG_NO_UPD_LAST,
28 kTemporalUpdateGolden =
29 VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
30 kTemporalUpdateAltrefWithoutDependency =
31 VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF |
32 VP8_EFLAG_NO_UPD_LAST,
33 kTemporalUpdateAltref = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_LAST,
34 kTemporalUpdateNone = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
35 VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
36 kTemporalUpdateNoneNoRefAltRef =
37 VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
38 VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
39 kTemporalUpdateNoneNoRefGolden =
40 VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
41 VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
42 kTemporalUpdateNoneNoRefGoldenAltRef =
43 VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_REF_ARF |
44 VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
45 kTemporalUpdateGoldenWithoutDependencyRefAltRef =
46 VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
47 kTemporalUpdateGoldenRefAltRef = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
48 kTemporalUpdateLastRefAltRef =
49 VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF,
50 kTemporalUpdateLastAndGoldenRefAltRef =
51 VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF,
52 };
53
54 TEST(TemporalLayersTest, 2Layers) {
55 DefaultTemporalLayers tl(2, 0);
56 DefaultTemporalLayersChecker checker(2, 0);
57 vpx_codec_enc_cfg_t cfg;
58 CodecSpecificInfoVP8 vp8_info;
59 tl.OnRatesUpdated(500, 500, 30);
60 tl.UpdateConfiguration(&cfg);
61
62 int expected_flags[16] = {
63 kTemporalUpdateLastRefAltRef,
64 kTemporalUpdateGoldenWithoutDependencyRefAltRef,
65 kTemporalUpdateLastRefAltRef,
66 kTemporalUpdateGoldenRefAltRef,
67 kTemporalUpdateLastRefAltRef,
68 kTemporalUpdateGoldenRefAltRef,
69 kTemporalUpdateLastRefAltRef,
70 kTemporalUpdateNone,
71 kTemporalUpdateLastRefAltRef,
72 kTemporalUpdateGoldenWithoutDependencyRefAltRef,
73 kTemporalUpdateLastRefAltRef,
74 kTemporalUpdateGoldenRefAltRef,
75 kTemporalUpdateLastRefAltRef,
76 kTemporalUpdateGoldenRefAltRef,
77 kTemporalUpdateLastRefAltRef,
78 kTemporalUpdateNone,
79 };
80 int expected_temporal_idx[16] = {0, 1, 0, 1, 0, 1, 0, 1,
81 0, 1, 0, 1, 0, 1, 0, 1};
82
83 bool expected_layer_sync[16] = {false, true, false, false, false, false,
84 false, false, false, true, false, false,
85 false, false, false, false};
86
87 uint32_t timestamp = 0;
88 for (int i = 0; i < 16; ++i) {
89 TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
90 EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)) << i;
91 tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, 0);
92 EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config));
93 EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx);
94 EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
95 EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
96 EXPECT_EQ(i == 0 || expected_layer_sync[i], vp8_info.layerSync);
97 EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
98 timestamp += 3000;
99 }
100 }
101
102 TEST(TemporalLayersTest, 3Layers) {
103 DefaultTemporalLayers tl(3, 0);
104 DefaultTemporalLayersChecker checker(3, 0);
105 vpx_codec_enc_cfg_t cfg;
106 CodecSpecificInfoVP8 vp8_info;
107 tl.OnRatesUpdated(500, 500, 30);
108 tl.UpdateConfiguration(&cfg);
109
110 int expected_flags[16] = {
111 kTemporalUpdateLastRefAltRef,
112 kTemporalUpdateNoneNoRefGolden,
113 kTemporalUpdateGoldenWithoutDependencyRefAltRef,
114 kTemporalUpdateNone,
115 kTemporalUpdateLastRefAltRef,
116 kTemporalUpdateNone,
117 kTemporalUpdateGoldenRefAltRef,
118 kTemporalUpdateNone,
119 kTemporalUpdateLastRefAltRef,
120 kTemporalUpdateNoneNoRefGolden,
121 kTemporalUpdateGoldenWithoutDependencyRefAltRef,
122 kTemporalUpdateNone,
123 kTemporalUpdateLastRefAltRef,
124 kTemporalUpdateNone,
125 kTemporalUpdateGoldenRefAltRef,
126 kTemporalUpdateNone,
127 };
128 int expected_temporal_idx[16] = {0, 2, 1, 2, 0, 2, 1, 2,
129 0, 2, 1, 2, 0, 2, 1, 2};
130
131 bool expected_layer_sync[16] = {false, true, true, false, false, false,
132 false, false, false, true, true, false,
133 false, false, false, false};
134
135 unsigned int timestamp = 0;
136 for (int i = 0; i < 16; ++i) {
137 TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
138 EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)) << i;
139 tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, 0);
140 EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config));
141 EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx);
142 EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
143 EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
144 EXPECT_EQ(i == 0 || expected_layer_sync[i], vp8_info.layerSync);
145 EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
146 timestamp += 3000;
147 }
148 }
149
TEST(TemporalLayersTest,Alternative3Layers)150 TEST(TemporalLayersTest, Alternative3Layers) {
151 ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
152 DefaultTemporalLayers tl(3, 0);
153 DefaultTemporalLayersChecker checker(3, 0);
154 vpx_codec_enc_cfg_t cfg;
155 CodecSpecificInfoVP8 vp8_info;
156 tl.OnRatesUpdated(500, 500, 30);
157 tl.UpdateConfiguration(&cfg);
158
159 int expected_flags[8] = {kTemporalUpdateLast,
160 kTemporalUpdateAltrefWithoutDependency,
161 kTemporalUpdateGoldenWithoutDependency,
162 kTemporalUpdateNone,
163 kTemporalUpdateLast,
164 kTemporalUpdateAltrefWithoutDependency,
165 kTemporalUpdateGoldenWithoutDependency,
166 kTemporalUpdateNone};
167 int expected_temporal_idx[8] = {0, 2, 1, 2, 0, 2, 1, 2};
168
169 bool expected_layer_sync[8] = {false, true, true, false,
170 false, true, true, false};
171
172 unsigned int timestamp = 0;
173 for (int i = 0; i < 8; ++i) {
174 TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
175 EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)) << i;
176 tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, 0);
177 EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config));
178 EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx);
179 EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
180 EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
181 EXPECT_EQ(i == 0 || expected_layer_sync[i], vp8_info.layerSync);
182 EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
183 timestamp += 3000;
184 }
185 }
186
187 TEST(TemporalLayersTest, 4Layers) {
188 DefaultTemporalLayers tl(4, 0);
189 DefaultTemporalLayersChecker checker(4, 0);
190 vpx_codec_enc_cfg_t cfg;
191 CodecSpecificInfoVP8 vp8_info;
192 tl.OnRatesUpdated(500, 500, 30);
193 tl.UpdateConfiguration(&cfg);
194 int expected_flags[16] = {
195 kTemporalUpdateLast,
196 kTemporalUpdateNoneNoRefGoldenAltRef,
197 kTemporalUpdateAltrefWithoutDependency,
198 kTemporalUpdateNoneNoRefGolden,
199 kTemporalUpdateGoldenWithoutDependency,
200 kTemporalUpdateNone,
201 kTemporalUpdateAltref,
202 kTemporalUpdateNone,
203 kTemporalUpdateLast,
204 kTemporalUpdateNone,
205 kTemporalUpdateAltref,
206 kTemporalUpdateNone,
207 kTemporalUpdateGolden,
208 kTemporalUpdateNone,
209 kTemporalUpdateAltref,
210 kTemporalUpdateNone,
211 };
212 int expected_temporal_idx[16] = {0, 3, 2, 3, 1, 3, 2, 3,
213 0, 3, 2, 3, 1, 3, 2, 3};
214
215 bool expected_layer_sync[16] = {false, true, true, false, true, false,
216 false, false, false, false, false, false,
217 false, false, false, false};
218
219 uint32_t timestamp = 0;
220 for (int i = 0; i < 16; ++i) {
221 TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
222 EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)) << i;
223 tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, 0);
224 EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config));
225 EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx);
226 EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
227 EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
228 EXPECT_EQ(i == 0 || expected_layer_sync[i], vp8_info.layerSync);
229 EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
230 timestamp += 3000;
231 }
232 }
233
TEST(TemporalLayersTest,KeyFrame)234 TEST(TemporalLayersTest, KeyFrame) {
235 DefaultTemporalLayers tl(3, 0);
236 DefaultTemporalLayersChecker checker(3, 0);
237 vpx_codec_enc_cfg_t cfg;
238 CodecSpecificInfoVP8 vp8_info;
239 tl.OnRatesUpdated(500, 500, 30);
240 tl.UpdateConfiguration(&cfg);
241
242 int expected_flags[8] = {
243 kTemporalUpdateLastRefAltRef,
244 kTemporalUpdateNoneNoRefGolden,
245 kTemporalUpdateGoldenWithoutDependencyRefAltRef,
246 kTemporalUpdateNone,
247 kTemporalUpdateLastRefAltRef,
248 kTemporalUpdateNone,
249 kTemporalUpdateGoldenRefAltRef,
250 kTemporalUpdateNone,
251 };
252 int expected_temporal_idx[8] = {0, 2, 1, 2, 0, 2, 1, 2};
253 bool expected_layer_sync[8] = {false, true, true, false,
254 false, false, false, false};
255
256 uint32_t timestamp = 0;
257 for (int i = 0; i < 7; ++i) {
258 TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
259 EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)) << i;
260 tl.PopulateCodecSpecific(true, tl_config, &vp8_info, 0);
261 EXPECT_TRUE(checker.CheckTemporalConfig(true, tl_config));
262 EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
263 EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
264 EXPECT_EQ(0, vp8_info.temporalIdx)
265 << "Key frame should always be packetized as layer 0";
266 EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
267 EXPECT_TRUE(vp8_info.layerSync) << "Key frame should be marked layer sync.";
268 timestamp += 3000;
269 }
270 TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
271 EXPECT_EQ(expected_flags[7], VP8EncoderImpl::EncodeFlags(tl_config));
272 tl.PopulateCodecSpecific(false, tl_config, &vp8_info, 0);
273 EXPECT_TRUE(checker.CheckTemporalConfig(false, tl_config));
274 EXPECT_NE(0, vp8_info.temporalIdx)
275 << "To test something useful, this frame should not use layer 0.";
276 EXPECT_EQ(expected_temporal_idx[7], vp8_info.temporalIdx)
277 << "Non-keyframe, should use frame temporal index.";
278 EXPECT_EQ(expected_temporal_idx[7], tl_config.packetizer_temporal_idx);
279 EXPECT_EQ(expected_temporal_idx[7], tl_config.encoder_layer_id);
280 EXPECT_FALSE(tl_config.layer_sync);
281 EXPECT_TRUE(vp8_info.layerSync) << "Frame after keyframe should always be "
282 "marked layer sync since it only depends "
283 "on the base layer.";
284 }
285
286 class TemporalLayersReferenceTest : public ::testing::TestWithParam<int> {
287 public:
TemporalLayersReferenceTest()288 TemporalLayersReferenceTest()
289 : timestamp_(1),
290 last_sync_timestamp_(timestamp_),
291 tl0_reference_(nullptr) {}
~TemporalLayersReferenceTest()292 virtual ~TemporalLayersReferenceTest() {}
293
294 protected:
295 static const int kMaxPatternLength = 32;
296
297 struct BufferState {
BufferStatewebrtc::test::TemporalLayersReferenceTest::BufferState298 BufferState() : BufferState(-1, 0, false) {}
BufferStatewebrtc::test::TemporalLayersReferenceTest::BufferState299 BufferState(int temporal_idx, uint32_t timestamp, bool sync)
300 : temporal_idx(temporal_idx), timestamp(timestamp), sync(sync) {}
301 int temporal_idx;
302 uint32_t timestamp;
303 bool sync;
304 };
305
UpdateSyncRefState(const TemporalLayers::BufferFlags & flags,BufferState * buffer_state)306 bool UpdateSyncRefState(const TemporalLayers::BufferFlags& flags,
307 BufferState* buffer_state) {
308 if (flags & TemporalLayers::kReference) {
309 if (buffer_state->temporal_idx == -1)
310 return true; // References key-frame.
311 if (buffer_state->temporal_idx == 0) {
312 // No more than one reference to TL0 frame.
313 EXPECT_EQ(nullptr, tl0_reference_);
314 tl0_reference_ = buffer_state;
315 return true;
316 }
317 return false; // References higher layer.
318 }
319 return true; // No reference, does not affect sync frame status.
320 }
321
ValidateReference(const TemporalLayers::BufferFlags & flags,const BufferState & buffer_state,int temporal_layer)322 void ValidateReference(const TemporalLayers::BufferFlags& flags,
323 const BufferState& buffer_state,
324 int temporal_layer) {
325 if (flags & TemporalLayers::kReference) {
326 if (temporal_layer > 0 && buffer_state.timestamp > 0) {
327 // Check that high layer reference does not go past last sync frame.
328 EXPECT_GE(buffer_state.timestamp, last_sync_timestamp_);
329 }
330 // No reference to buffer in higher layer.
331 EXPECT_LE(buffer_state.temporal_idx, temporal_layer);
332 }
333 }
334
335 uint32_t timestamp_ = 1;
336 uint32_t last_sync_timestamp_ = timestamp_;
337 BufferState* tl0_reference_;
338
339 BufferState last_state;
340 BufferState golden_state;
341 BufferState altref_state;
342 };
343
344 INSTANTIATE_TEST_CASE_P(DefaultTemporalLayersTest,
345 TemporalLayersReferenceTest,
346 ::testing::Range(1, kMaxTemporalStreams + 1));
347
TEST_P(TemporalLayersReferenceTest,ValidFrameConfigs)348 TEST_P(TemporalLayersReferenceTest, ValidFrameConfigs) {
349 const int num_layers = GetParam();
350 DefaultTemporalLayers tl(num_layers, 0);
351 vpx_codec_enc_cfg_t cfg;
352 tl.OnRatesUpdated(500, 500, 30);
353 tl.UpdateConfiguration(&cfg);
354
355 // Run through the pattern and store the frame dependencies, plus keep track
356 // of the buffer state; which buffers references which temporal layers (if
357 // (any). If a given buffer is never updated, it is legal to reference it
358 // even for sync frames. In order to be general, don't assume TL0 always
359 // updates |last|.
360 std::vector<TemporalLayers::FrameConfig> tl_configs(kMaxPatternLength);
361 for (int i = 0; i < kMaxPatternLength; ++i) {
362 TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp_++);
363 EXPECT_FALSE(tl_config.drop_frame);
364 tl_configs.push_back(tl_config);
365 int temporal_idx = tl_config.encoder_layer_id;
366 // For the default layers, always keep encoder and rtp layers in sync.
367 EXPECT_EQ(tl_config.packetizer_temporal_idx, temporal_idx);
368
369 // Determine if this frame is in a higher layer but references only TL0
370 // or untouched buffers, if so verify it is marked as a layer sync.
371 bool is_sync_frame = true;
372 tl0_reference_ = nullptr;
373 if (temporal_idx <= 0) {
374 is_sync_frame = false; // TL0 by definition not a sync frame.
375 } else if (!UpdateSyncRefState(tl_config.last_buffer_flags, &last_state)) {
376 is_sync_frame = false;
377 } else if (!UpdateSyncRefState(tl_config.golden_buffer_flags,
378 &golden_state)) {
379 is_sync_frame = false;
380 } else if (!UpdateSyncRefState(tl_config.arf_buffer_flags, &altref_state)) {
381 is_sync_frame = false;
382 }
383 if (is_sync_frame) {
384 // Cache timestamp for last found sync frame, so that we can verify no
385 // references back past this frame.
386 ASSERT_TRUE(tl0_reference_);
387 last_sync_timestamp_ = tl0_reference_->timestamp;
388 }
389 EXPECT_EQ(tl_config.layer_sync, is_sync_frame);
390
391 // Validate no reference from lower to high temporal layer, or backwards
392 // past last reference frame.
393 ValidateReference(tl_config.last_buffer_flags, last_state, temporal_idx);
394 ValidateReference(tl_config.golden_buffer_flags, golden_state,
395 temporal_idx);
396 ValidateReference(tl_config.arf_buffer_flags, altref_state, temporal_idx);
397
398 // Update the current layer state.
399 BufferState state = {temporal_idx, timestamp_, is_sync_frame};
400 if (tl_config.last_buffer_flags & TemporalLayers::kUpdate)
401 last_state = state;
402 if (tl_config.golden_buffer_flags & TemporalLayers::kUpdate)
403 golden_state = state;
404 if (tl_config.arf_buffer_flags & TemporalLayers::kUpdate)
405 altref_state = state;
406 }
407 }
408 } // namespace test
409 } // namespace webrtc
410