1 /*
2  *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "modules/video_coding/codecs/vp8/default_temporal_layers.h"
12 #include "modules/video_coding/codecs/vp8/vp8_impl.h"
13 #include "modules/video_coding/include/video_codec_interface.h"
14 #include "test/field_trial.h"
15 #include "test/gtest.h"
16 #include "vpx/vp8cx.h"
17 #include "vpx/vpx_encoder.h"
18 
19 namespace webrtc {
20 namespace test {
21 
22 enum {
23   kTemporalUpdateLast = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
24                         VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF,
25   kTemporalUpdateGoldenWithoutDependency =
26       VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF |
27       VP8_EFLAG_NO_UPD_LAST,
28   kTemporalUpdateGolden =
29       VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
30   kTemporalUpdateAltrefWithoutDependency =
31       VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF |
32       VP8_EFLAG_NO_UPD_LAST,
33   kTemporalUpdateAltref = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_LAST,
34   kTemporalUpdateNone = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
35                         VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
36   kTemporalUpdateNoneNoRefAltRef =
37       VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
38       VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
39   kTemporalUpdateNoneNoRefGolden =
40       VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
41       VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
42   kTemporalUpdateNoneNoRefGoldenAltRef =
43       VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_REF_ARF |
44       VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
45   kTemporalUpdateGoldenWithoutDependencyRefAltRef =
46       VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
47   kTemporalUpdateGoldenRefAltRef = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
48   kTemporalUpdateLastRefAltRef =
49       VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF,
50   kTemporalUpdateLastAndGoldenRefAltRef =
51       VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF,
52 };
53 
54 TEST(TemporalLayersTest, 2Layers) {
55   DefaultTemporalLayers tl(2, 0);
56   DefaultTemporalLayersChecker checker(2, 0);
57   vpx_codec_enc_cfg_t cfg;
58   CodecSpecificInfoVP8 vp8_info;
59   tl.OnRatesUpdated(500, 500, 30);
60   tl.UpdateConfiguration(&cfg);
61 
62   int expected_flags[16] = {
63       kTemporalUpdateLastRefAltRef,
64       kTemporalUpdateGoldenWithoutDependencyRefAltRef,
65       kTemporalUpdateLastRefAltRef,
66       kTemporalUpdateGoldenRefAltRef,
67       kTemporalUpdateLastRefAltRef,
68       kTemporalUpdateGoldenRefAltRef,
69       kTemporalUpdateLastRefAltRef,
70       kTemporalUpdateNone,
71       kTemporalUpdateLastRefAltRef,
72       kTemporalUpdateGoldenWithoutDependencyRefAltRef,
73       kTemporalUpdateLastRefAltRef,
74       kTemporalUpdateGoldenRefAltRef,
75       kTemporalUpdateLastRefAltRef,
76       kTemporalUpdateGoldenRefAltRef,
77       kTemporalUpdateLastRefAltRef,
78       kTemporalUpdateNone,
79   };
80   int expected_temporal_idx[16] = {0, 1, 0, 1, 0, 1, 0, 1,
81                                    0, 1, 0, 1, 0, 1, 0, 1};
82 
83   bool expected_layer_sync[16] = {false, true,  false, false, false, false,
84                                   false, false, false, true,  false, false,
85                                   false, false, false, false};
86 
87   uint32_t timestamp = 0;
88   for (int i = 0; i < 16; ++i) {
89     TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
90     EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)) << i;
91     tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, 0);
92     EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config));
93     EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx);
94     EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
95     EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
96     EXPECT_EQ(i == 0 || expected_layer_sync[i], vp8_info.layerSync);
97     EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
98     timestamp += 3000;
99   }
100 }
101 
102 TEST(TemporalLayersTest, 3Layers) {
103   DefaultTemporalLayers tl(3, 0);
104   DefaultTemporalLayersChecker checker(3, 0);
105   vpx_codec_enc_cfg_t cfg;
106   CodecSpecificInfoVP8 vp8_info;
107   tl.OnRatesUpdated(500, 500, 30);
108   tl.UpdateConfiguration(&cfg);
109 
110   int expected_flags[16] = {
111       kTemporalUpdateLastRefAltRef,
112       kTemporalUpdateNoneNoRefGolden,
113       kTemporalUpdateGoldenWithoutDependencyRefAltRef,
114       kTemporalUpdateNone,
115       kTemporalUpdateLastRefAltRef,
116       kTemporalUpdateNone,
117       kTemporalUpdateGoldenRefAltRef,
118       kTemporalUpdateNone,
119       kTemporalUpdateLastRefAltRef,
120       kTemporalUpdateNoneNoRefGolden,
121       kTemporalUpdateGoldenWithoutDependencyRefAltRef,
122       kTemporalUpdateNone,
123       kTemporalUpdateLastRefAltRef,
124       kTemporalUpdateNone,
125       kTemporalUpdateGoldenRefAltRef,
126       kTemporalUpdateNone,
127   };
128   int expected_temporal_idx[16] = {0, 2, 1, 2, 0, 2, 1, 2,
129                                    0, 2, 1, 2, 0, 2, 1, 2};
130 
131   bool expected_layer_sync[16] = {false, true,  true,  false, false, false,
132                                   false, false, false, true,  true,  false,
133                                   false, false, false, false};
134 
135   unsigned int timestamp = 0;
136   for (int i = 0; i < 16; ++i) {
137     TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
138     EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)) << i;
139     tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, 0);
140     EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config));
141     EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx);
142     EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
143     EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
144     EXPECT_EQ(i == 0 || expected_layer_sync[i], vp8_info.layerSync);
145     EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
146     timestamp += 3000;
147   }
148 }
149 
TEST(TemporalLayersTest,Alternative3Layers)150 TEST(TemporalLayersTest, Alternative3Layers) {
151   ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
152   DefaultTemporalLayers tl(3, 0);
153   DefaultTemporalLayersChecker checker(3, 0);
154   vpx_codec_enc_cfg_t cfg;
155   CodecSpecificInfoVP8 vp8_info;
156   tl.OnRatesUpdated(500, 500, 30);
157   tl.UpdateConfiguration(&cfg);
158 
159   int expected_flags[8] = {kTemporalUpdateLast,
160                            kTemporalUpdateAltrefWithoutDependency,
161                            kTemporalUpdateGoldenWithoutDependency,
162                            kTemporalUpdateNone,
163                            kTemporalUpdateLast,
164                            kTemporalUpdateAltrefWithoutDependency,
165                            kTemporalUpdateGoldenWithoutDependency,
166                            kTemporalUpdateNone};
167   int expected_temporal_idx[8] = {0, 2, 1, 2, 0, 2, 1, 2};
168 
169   bool expected_layer_sync[8] = {false, true, true, false,
170                                  false, true, true, false};
171 
172   unsigned int timestamp = 0;
173   for (int i = 0; i < 8; ++i) {
174     TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
175     EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)) << i;
176     tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, 0);
177     EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config));
178     EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx);
179     EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
180     EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
181     EXPECT_EQ(i == 0 || expected_layer_sync[i], vp8_info.layerSync);
182     EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
183     timestamp += 3000;
184   }
185 }
186 
187 TEST(TemporalLayersTest, 4Layers) {
188   DefaultTemporalLayers tl(4, 0);
189   DefaultTemporalLayersChecker checker(4, 0);
190   vpx_codec_enc_cfg_t cfg;
191   CodecSpecificInfoVP8 vp8_info;
192   tl.OnRatesUpdated(500, 500, 30);
193   tl.UpdateConfiguration(&cfg);
194   int expected_flags[16] = {
195       kTemporalUpdateLast,
196       kTemporalUpdateNoneNoRefGoldenAltRef,
197       kTemporalUpdateAltrefWithoutDependency,
198       kTemporalUpdateNoneNoRefGolden,
199       kTemporalUpdateGoldenWithoutDependency,
200       kTemporalUpdateNone,
201       kTemporalUpdateAltref,
202       kTemporalUpdateNone,
203       kTemporalUpdateLast,
204       kTemporalUpdateNone,
205       kTemporalUpdateAltref,
206       kTemporalUpdateNone,
207       kTemporalUpdateGolden,
208       kTemporalUpdateNone,
209       kTemporalUpdateAltref,
210       kTemporalUpdateNone,
211   };
212   int expected_temporal_idx[16] = {0, 3, 2, 3, 1, 3, 2, 3,
213                                    0, 3, 2, 3, 1, 3, 2, 3};
214 
215   bool expected_layer_sync[16] = {false, true,  true,  false, true,  false,
216                                   false, false, false, false, false, false,
217                                   false, false, false, false};
218 
219   uint32_t timestamp = 0;
220   for (int i = 0; i < 16; ++i) {
221     TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
222     EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)) << i;
223     tl.PopulateCodecSpecific(i == 0, tl_config, &vp8_info, 0);
224     EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config));
225     EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx);
226     EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
227     EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
228     EXPECT_EQ(i == 0 || expected_layer_sync[i], vp8_info.layerSync);
229     EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
230     timestamp += 3000;
231   }
232 }
233 
TEST(TemporalLayersTest,KeyFrame)234 TEST(TemporalLayersTest, KeyFrame) {
235   DefaultTemporalLayers tl(3, 0);
236   DefaultTemporalLayersChecker checker(3, 0);
237   vpx_codec_enc_cfg_t cfg;
238   CodecSpecificInfoVP8 vp8_info;
239   tl.OnRatesUpdated(500, 500, 30);
240   tl.UpdateConfiguration(&cfg);
241 
242   int expected_flags[8] = {
243       kTemporalUpdateLastRefAltRef,
244       kTemporalUpdateNoneNoRefGolden,
245       kTemporalUpdateGoldenWithoutDependencyRefAltRef,
246       kTemporalUpdateNone,
247       kTemporalUpdateLastRefAltRef,
248       kTemporalUpdateNone,
249       kTemporalUpdateGoldenRefAltRef,
250       kTemporalUpdateNone,
251   };
252   int expected_temporal_idx[8] = {0, 2, 1, 2, 0, 2, 1, 2};
253   bool expected_layer_sync[8] = {false, true,  true,  false,
254                                  false, false, false, false};
255 
256   uint32_t timestamp = 0;
257   for (int i = 0; i < 7; ++i) {
258     TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
259     EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config)) << i;
260     tl.PopulateCodecSpecific(true, tl_config, &vp8_info, 0);
261     EXPECT_TRUE(checker.CheckTemporalConfig(true, tl_config));
262     EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
263     EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
264     EXPECT_EQ(0, vp8_info.temporalIdx)
265         << "Key frame should always be packetized as layer 0";
266     EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
267     EXPECT_TRUE(vp8_info.layerSync) << "Key frame should be marked layer sync.";
268     timestamp += 3000;
269   }
270   TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
271   EXPECT_EQ(expected_flags[7], VP8EncoderImpl::EncodeFlags(tl_config));
272   tl.PopulateCodecSpecific(false, tl_config, &vp8_info, 0);
273   EXPECT_TRUE(checker.CheckTemporalConfig(false, tl_config));
274   EXPECT_NE(0, vp8_info.temporalIdx)
275       << "To test something useful, this frame should not use layer 0.";
276   EXPECT_EQ(expected_temporal_idx[7], vp8_info.temporalIdx)
277       << "Non-keyframe, should use frame temporal index.";
278   EXPECT_EQ(expected_temporal_idx[7], tl_config.packetizer_temporal_idx);
279   EXPECT_EQ(expected_temporal_idx[7], tl_config.encoder_layer_id);
280   EXPECT_FALSE(tl_config.layer_sync);
281   EXPECT_TRUE(vp8_info.layerSync) << "Frame after keyframe should always be "
282                                      "marked layer sync since it only depends "
283                                      "on the base layer.";
284 }
285 
286 class TemporalLayersReferenceTest : public ::testing::TestWithParam<int> {
287  public:
TemporalLayersReferenceTest()288   TemporalLayersReferenceTest()
289       : timestamp_(1),
290         last_sync_timestamp_(timestamp_),
291         tl0_reference_(nullptr) {}
~TemporalLayersReferenceTest()292   virtual ~TemporalLayersReferenceTest() {}
293 
294  protected:
295   static const int kMaxPatternLength = 32;
296 
297   struct BufferState {
BufferStatewebrtc::test::TemporalLayersReferenceTest::BufferState298     BufferState() : BufferState(-1, 0, false) {}
BufferStatewebrtc::test::TemporalLayersReferenceTest::BufferState299     BufferState(int temporal_idx, uint32_t timestamp, bool sync)
300         : temporal_idx(temporal_idx), timestamp(timestamp), sync(sync) {}
301     int temporal_idx;
302     uint32_t timestamp;
303     bool sync;
304   };
305 
UpdateSyncRefState(const TemporalLayers::BufferFlags & flags,BufferState * buffer_state)306   bool UpdateSyncRefState(const TemporalLayers::BufferFlags& flags,
307                           BufferState* buffer_state) {
308     if (flags & TemporalLayers::kReference) {
309       if (buffer_state->temporal_idx == -1)
310         return true;  // References key-frame.
311       if (buffer_state->temporal_idx == 0) {
312         // No more than one reference to TL0 frame.
313         EXPECT_EQ(nullptr, tl0_reference_);
314         tl0_reference_ = buffer_state;
315         return true;
316       }
317       return false;  // References higher layer.
318     }
319     return true;  // No reference, does not affect sync frame status.
320   }
321 
ValidateReference(const TemporalLayers::BufferFlags & flags,const BufferState & buffer_state,int temporal_layer)322   void ValidateReference(const TemporalLayers::BufferFlags& flags,
323                          const BufferState& buffer_state,
324                          int temporal_layer) {
325     if (flags & TemporalLayers::kReference) {
326       if (temporal_layer > 0 && buffer_state.timestamp > 0) {
327         // Check that high layer reference does not go past last sync frame.
328         EXPECT_GE(buffer_state.timestamp, last_sync_timestamp_);
329       }
330       // No reference to buffer in higher layer.
331       EXPECT_LE(buffer_state.temporal_idx, temporal_layer);
332     }
333   }
334 
335   uint32_t timestamp_ = 1;
336   uint32_t last_sync_timestamp_ = timestamp_;
337   BufferState* tl0_reference_;
338 
339   BufferState last_state;
340   BufferState golden_state;
341   BufferState altref_state;
342 };
343 
344 INSTANTIATE_TEST_CASE_P(DefaultTemporalLayersTest,
345                         TemporalLayersReferenceTest,
346                         ::testing::Range(1, kMaxTemporalStreams + 1));
347 
TEST_P(TemporalLayersReferenceTest,ValidFrameConfigs)348 TEST_P(TemporalLayersReferenceTest, ValidFrameConfigs) {
349   const int num_layers = GetParam();
350   DefaultTemporalLayers tl(num_layers, 0);
351   vpx_codec_enc_cfg_t cfg;
352   tl.OnRatesUpdated(500, 500, 30);
353   tl.UpdateConfiguration(&cfg);
354 
355   // Run through the pattern and store the frame dependencies, plus keep track
356   // of the buffer state; which buffers references which temporal layers (if
357   // (any). If a given buffer is never updated, it is legal to reference it
358   // even for sync frames. In order to be general, don't assume TL0 always
359   // updates |last|.
360   std::vector<TemporalLayers::FrameConfig> tl_configs(kMaxPatternLength);
361   for (int i = 0; i < kMaxPatternLength; ++i) {
362     TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp_++);
363     EXPECT_FALSE(tl_config.drop_frame);
364     tl_configs.push_back(tl_config);
365     int temporal_idx = tl_config.encoder_layer_id;
366     // For the default layers, always keep encoder and rtp layers in sync.
367     EXPECT_EQ(tl_config.packetizer_temporal_idx, temporal_idx);
368 
369     // Determine if this frame is in a higher layer but references only TL0
370     // or untouched buffers, if so verify it is marked as a layer sync.
371     bool is_sync_frame = true;
372     tl0_reference_ = nullptr;
373     if (temporal_idx <= 0) {
374       is_sync_frame = false;  // TL0 by definition not a sync frame.
375     } else if (!UpdateSyncRefState(tl_config.last_buffer_flags, &last_state)) {
376       is_sync_frame = false;
377     } else if (!UpdateSyncRefState(tl_config.golden_buffer_flags,
378                                    &golden_state)) {
379       is_sync_frame = false;
380     } else if (!UpdateSyncRefState(tl_config.arf_buffer_flags, &altref_state)) {
381       is_sync_frame = false;
382     }
383     if (is_sync_frame) {
384       // Cache timestamp for last found sync frame, so that we can verify no
385       // references back past this frame.
386       ASSERT_TRUE(tl0_reference_);
387       last_sync_timestamp_ = tl0_reference_->timestamp;
388     }
389     EXPECT_EQ(tl_config.layer_sync, is_sync_frame);
390 
391     // Validate no reference from lower to high temporal layer, or backwards
392     // past last reference frame.
393     ValidateReference(tl_config.last_buffer_flags, last_state, temporal_idx);
394     ValidateReference(tl_config.golden_buffer_flags, golden_state,
395                       temporal_idx);
396     ValidateReference(tl_config.arf_buffer_flags, altref_state, temporal_idx);
397 
398     // Update the current layer state.
399     BufferState state = {temporal_idx, timestamp_, is_sync_frame};
400     if (tl_config.last_buffer_flags & TemporalLayers::kUpdate)
401       last_state = state;
402     if (tl_config.golden_buffer_flags & TemporalLayers::kUpdate)
403       golden_state = state;
404     if (tl_config.arf_buffer_flags & TemporalLayers::kUpdate)
405       altref_state = state;
406   }
407 }
408 }  // namespace test
409 }  // namespace webrtc
410