1 /*
2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include <map>
15 #include <memory>
16 #include <ostream>
17 #include <tuple>
18 #include <vector>
19
20 #include "absl/types/optional.h"
21 #include "api/units/data_size.h"
22 #include "api/units/time_delta.h"
23 #include "api/video_codecs/video_codec.h"
24 #include "api/video_codecs/video_encoder.h"
25 #include "modules/video_coding/codecs/av1/create_scalability_structure.h"
26 #include "modules/video_coding/codecs/av1/libaom_av1_decoder.h"
27 #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
28 #include "modules/video_coding/codecs/av1/scalable_video_controller.h"
29 #include "modules/video_coding/codecs/av1/scalable_video_controller_no_layering.h"
30 #include "modules/video_coding/codecs/test/encoded_video_frame_producer.h"
31 #include "modules/video_coding/include/video_codec_interface.h"
32 #include "modules/video_coding/include/video_error_codes.h"
33 #include "test/gmock.h"
34 #include "test/gtest.h"
35
36 namespace webrtc {
37 namespace {
38
39 using ::testing::ContainerEq;
40 using ::testing::Each;
41 using ::testing::ElementsAreArray;
42 using ::testing::Ge;
43 using ::testing::IsEmpty;
44 using ::testing::Not;
45 using ::testing::NotNull;
46 using ::testing::Pointwise;
47 using ::testing::SizeIs;
48 using ::testing::Truly;
49 using ::testing::Values;
50
51 // Use small resolution for this test to make it faster.
52 constexpr int kWidth = 320;
53 constexpr int kHeight = 180;
54 constexpr int kFramerate = 30;
55
DefaultCodecSettings()56 VideoCodec DefaultCodecSettings() {
57 VideoCodec codec_settings;
58 codec_settings.width = kWidth;
59 codec_settings.height = kHeight;
60 codec_settings.maxFramerate = kFramerate;
61 codec_settings.maxBitrate = 1000;
62 codec_settings.qpMax = 63;
63 return codec_settings;
64 }
DefaultEncoderSettings()65 VideoEncoder::Settings DefaultEncoderSettings() {
66 return VideoEncoder::Settings(
67 VideoEncoder::Capabilities(/*loss_notification=*/false),
68 /*number_of_cores=*/1, /*max_payload_size=*/1200);
69 }
70
71 class TestAv1Decoder {
72 public:
TestAv1Decoder(int decoder_id)73 explicit TestAv1Decoder(int decoder_id)
74 : decoder_id_(decoder_id), decoder_(CreateLibaomAv1Decoder()) {
75 if (decoder_ == nullptr) {
76 ADD_FAILURE() << "Failed to create a decoder#" << decoder_id_;
77 return;
78 }
79 EXPECT_EQ(decoder_->InitDecode(/*codec_settings=*/nullptr,
80 /*number_of_cores=*/1),
81 WEBRTC_VIDEO_CODEC_OK);
82 EXPECT_EQ(decoder_->RegisterDecodeCompleteCallback(&callback_),
83 WEBRTC_VIDEO_CODEC_OK);
84 }
85 // This class requires pointer stability and thus not copyable nor movable.
86 TestAv1Decoder(const TestAv1Decoder&) = delete;
87 TestAv1Decoder& operator=(const TestAv1Decoder&) = delete;
88
Decode(int64_t frame_id,const EncodedImage & image)89 void Decode(int64_t frame_id, const EncodedImage& image) {
90 ASSERT_THAT(decoder_, NotNull());
91 int32_t error = decoder_->Decode(image, /*missing_frames=*/false,
92 /*render_time_ms=*/image.capture_time_ms_);
93 if (error != WEBRTC_VIDEO_CODEC_OK) {
94 ADD_FAILURE() << "Failed to decode frame id " << frame_id
95 << " with error code " << error << " by decoder#"
96 << decoder_id_;
97 return;
98 }
99 decoded_ids_.push_back(frame_id);
100 }
101
decoded_frame_ids() const102 const std::vector<int64_t>& decoded_frame_ids() const { return decoded_ids_; }
num_output_frames() const103 size_t num_output_frames() const { return callback_.num_called(); }
104
105 private:
106 // Decoder callback that only counts how many times it was called.
107 // While it is tempting to replace it with a simple mock, that one requires
108 // to set expectation on number of calls in advance. Tests below unsure about
109 // expected number of calls until after calls are done.
110 class DecoderCallback : public DecodedImageCallback {
111 public:
num_called() const112 size_t num_called() const { return num_called_; }
113
114 private:
Decoded(VideoFrame &)115 int32_t Decoded(VideoFrame& /*decoded_image*/) override {
116 ++num_called_;
117 return 0;
118 }
Decoded(VideoFrame &,absl::optional<int32_t>,absl::optional<uint8_t>)119 void Decoded(VideoFrame& /*decoded_image*/,
120 absl::optional<int32_t> /*decode_time_ms*/,
121 absl::optional<uint8_t> /*qp*/) override {
122 ++num_called_;
123 }
124
125 int num_called_ = 0;
126 };
127
128 const int decoder_id_;
129 std::vector<int64_t> decoded_ids_;
130 DecoderCallback callback_;
131 const std::unique_ptr<VideoDecoder> decoder_;
132 };
133
TEST(LibaomAv1Test,EncodeDecode)134 TEST(LibaomAv1Test, EncodeDecode) {
135 TestAv1Decoder decoder(0);
136 std::unique_ptr<VideoEncoder> encoder = CreateLibaomAv1Encoder();
137 VideoCodec codec_settings = DefaultCodecSettings();
138 ASSERT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()),
139 WEBRTC_VIDEO_CODEC_OK);
140
141 std::vector<EncodedVideoFrameProducer::EncodedFrame> encoded_frames =
142 EncodedVideoFrameProducer(*encoder).SetNumInputFrames(4).Encode();
143 for (size_t frame_id = 0; frame_id < encoded_frames.size(); ++frame_id) {
144 decoder.Decode(static_cast<int64_t>(frame_id),
145 encoded_frames[frame_id].encoded_image);
146 }
147
148 // Check encoder produced some frames for decoder to decode.
149 ASSERT_THAT(encoded_frames, Not(IsEmpty()));
150 // Check decoder found all of them valid.
151 EXPECT_THAT(decoder.decoded_frame_ids(), SizeIs(encoded_frames.size()));
152 // Check each of them produced an output frame.
153 EXPECT_EQ(decoder.num_output_frames(), decoder.decoded_frame_ids().size());
154 }
155
156 struct LayerId {
operator ==(const LayerId & lhs,const LayerId & rhs)157 friend bool operator==(const LayerId& lhs, const LayerId& rhs) {
158 return std::tie(lhs.spatial_id, lhs.temporal_id) ==
159 std::tie(rhs.spatial_id, rhs.temporal_id);
160 }
operator <(const LayerId & lhs,const LayerId & rhs)161 friend bool operator<(const LayerId& lhs, const LayerId& rhs) {
162 return std::tie(lhs.spatial_id, lhs.temporal_id) <
163 std::tie(rhs.spatial_id, rhs.temporal_id);
164 }
operator <<(std::ostream & s,const LayerId & layer)165 friend std::ostream& operator<<(std::ostream& s, const LayerId& layer) {
166 return s << "S" << layer.spatial_id << "T" << layer.temporal_id;
167 }
168
169 int spatial_id = 0;
170 int temporal_id = 0;
171 };
172
173 struct SvcTestParam {
174 std::string name;
175 int num_frames_to_generate;
176 std::map<LayerId, DataRate> configured_bitrates;
177 };
178
179 class LibaomAv1SvcTest : public ::testing::TestWithParam<SvcTestParam> {};
180
TEST_P(LibaomAv1SvcTest,EncodeAndDecodeAllDecodeTargets)181 TEST_P(LibaomAv1SvcTest, EncodeAndDecodeAllDecodeTargets) {
182 std::unique_ptr<ScalableVideoController> svc_controller =
183 CreateScalabilityStructure(GetParam().name);
184 size_t num_decode_targets =
185 svc_controller->DependencyStructure().num_decode_targets;
186
187 std::unique_ptr<VideoEncoder> encoder =
188 CreateLibaomAv1Encoder(std::move(svc_controller));
189 VideoCodec codec_settings = DefaultCodecSettings();
190 ASSERT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()),
191 WEBRTC_VIDEO_CODEC_OK);
192 std::vector<EncodedVideoFrameProducer::EncodedFrame> encoded_frames =
193 EncodedVideoFrameProducer(*encoder)
194 .SetNumInputFrames(GetParam().num_frames_to_generate)
195 .SetResolution({kWidth, kHeight})
196 .Encode();
197
198 ASSERT_THAT(
199 encoded_frames,
200 Each(Truly([&](const EncodedVideoFrameProducer::EncodedFrame& frame) {
201 return frame.codec_specific_info.generic_frame_info &&
202 frame.codec_specific_info.generic_frame_info
203 ->decode_target_indications.size() == num_decode_targets;
204 })));
205
206 for (size_t dt = 0; dt < num_decode_targets; ++dt) {
207 TestAv1Decoder decoder(dt);
208 std::vector<int64_t> requested_ids;
209 for (int64_t frame_id = 0;
210 frame_id < static_cast<int64_t>(encoded_frames.size()); ++frame_id) {
211 const EncodedVideoFrameProducer::EncodedFrame& frame =
212 encoded_frames[frame_id];
213 if (frame.codec_specific_info.generic_frame_info
214 ->decode_target_indications[dt] !=
215 DecodeTargetIndication::kNotPresent) {
216 requested_ids.push_back(frame_id);
217 decoder.Decode(frame_id, frame.encoded_image);
218 }
219 }
220
221 ASSERT_THAT(requested_ids, SizeIs(Ge(2u)));
222 // Check decoder found all of them valid.
223 EXPECT_THAT(decoder.decoded_frame_ids(), ContainerEq(requested_ids))
224 << "Decoder#" << dt;
225 // Check each of them produced an output frame.
226 EXPECT_EQ(decoder.num_output_frames(), decoder.decoded_frame_ids().size())
227 << "Decoder#" << dt;
228 }
229 }
230
231 MATCHER(SameLayerIdAndBitrateIsNear, "") {
232 // First check if layer id is the same.
233 return std::get<0>(arg).first == std::get<1>(arg).first &&
234 // check measured bitrate is not much lower than requested.
235 std::get<0>(arg).second >= std::get<1>(arg).second * 0.8 &&
236 // check measured bitrate is not much larger than requested.
237 std::get<0>(arg).second <= std::get<1>(arg).second * 1.1;
238 }
239
TEST_P(LibaomAv1SvcTest,SetRatesMatchMeasuredBitrate)240 TEST_P(LibaomAv1SvcTest, SetRatesMatchMeasuredBitrate) {
241 const SvcTestParam param = GetParam();
242 if (param.configured_bitrates.empty()) {
243 // Rates are not configured for this particular structure, skip the test.
244 return;
245 }
246 constexpr TimeDelta kDuration = TimeDelta::Seconds(5);
247
248 VideoBitrateAllocation allocation;
249 for (const auto& kv : param.configured_bitrates) {
250 allocation.SetBitrate(kv.first.spatial_id, kv.first.temporal_id,
251 kv.second.bps());
252 }
253
254 std::unique_ptr<VideoEncoder> encoder =
255 CreateLibaomAv1Encoder(CreateScalabilityStructure(param.name));
256 ASSERT_TRUE(encoder);
257 VideoCodec codec_settings = DefaultCodecSettings();
258 codec_settings.maxBitrate = allocation.get_sum_kbps();
259 codec_settings.maxFramerate = 30;
260 ASSERT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()),
261 WEBRTC_VIDEO_CODEC_OK);
262
263 encoder->SetRates(VideoEncoder::RateControlParameters(
264 allocation, codec_settings.maxFramerate));
265
266 std::vector<EncodedVideoFrameProducer::EncodedFrame> encoded_frames =
267 EncodedVideoFrameProducer(*encoder)
268 .SetNumInputFrames(codec_settings.maxFramerate * kDuration.seconds())
269 .SetResolution({codec_settings.width, codec_settings.height})
270 .SetFramerateFps(codec_settings.maxFramerate)
271 .Encode();
272
273 // Calculate size of each layer.
274 std::map<LayerId, DataSize> layer_size;
275 for (const auto& frame : encoded_frames) {
276 ASSERT_TRUE(frame.codec_specific_info.generic_frame_info);
277 const auto& layer = *frame.codec_specific_info.generic_frame_info;
278 LayerId layer_id = {layer.spatial_id, layer.temporal_id};
279 // This is almost same as
280 // layer_size[layer_id] += DataSize::Bytes(frame.encoded_image.size());
281 // but avoids calling deleted default constructor for DataSize.
282 layer_size.emplace(layer_id, DataSize::Zero()).first->second +=
283 DataSize::Bytes(frame.encoded_image.size());
284 }
285 // Convert size of the layer into bitrate of that layer.
286 std::vector<std::pair<LayerId, DataRate>> measured_bitrates;
287 for (const auto& kv : layer_size) {
288 measured_bitrates.emplace_back(kv.first, kv.second / kDuration);
289 }
290 EXPECT_THAT(measured_bitrates, Pointwise(SameLayerIdAndBitrateIsNear(),
291 param.configured_bitrates));
292 }
293
294 INSTANTIATE_TEST_SUITE_P(
295 Svc,
296 LibaomAv1SvcTest,
297 Values(SvcTestParam{"NONE", /*num_frames_to_generate=*/4},
298 SvcTestParam{"L1T2",
299 /*num_frames_to_generate=*/4,
300 /*configured_bitrates=*/
301 {{{0, 0}, DataRate::KilobitsPerSec(60)},
302 {{0, 1}, DataRate::KilobitsPerSec(40)}}},
303 SvcTestParam{"L1T3", /*num_frames_to_generate=*/8},
304 SvcTestParam{"L2T1",
305 /*num_frames_to_generate=*/3,
306 /*configured_bitrates=*/
307 {{{0, 0}, DataRate::KilobitsPerSec(30)},
308 {{1, 0}, DataRate::KilobitsPerSec(70)}}},
309 SvcTestParam{"L2T1h",
310 /*num_frames_to_generate=*/3,
311 /*configured_bitrates=*/
312 {{{0, 0}, DataRate::KilobitsPerSec(30)},
313 {{1, 0}, DataRate::KilobitsPerSec(70)}}},
314 SvcTestParam{"L2T1_KEY", /*num_frames_to_generate=*/3},
315 SvcTestParam{"L3T1", /*num_frames_to_generate=*/3},
316 SvcTestParam{"L3T3", /*num_frames_to_generate=*/8},
317 SvcTestParam{"S2T1", /*num_frames_to_generate=*/3},
318 SvcTestParam{"L2T2", /*num_frames_to_generate=*/4},
319 SvcTestParam{"L2T2_KEY", /*num_frames_to_generate=*/4},
320 SvcTestParam{"L2T2_KEY_SHIFT",
321 /*num_frames_to_generate=*/4,
322 /*configured_bitrates=*/
323 {{{0, 0}, DataRate::KilobitsPerSec(70)},
324 {{0, 1}, DataRate::KilobitsPerSec(30)},
325 {{1, 0}, DataRate::KilobitsPerSec(110)},
326 {{1, 1}, DataRate::KilobitsPerSec(80)}}}),
__anon715b9c6b0302(const testing::TestParamInfo<SvcTestParam>& info) 327 [](const testing::TestParamInfo<SvcTestParam>& info) {
328 return info.param.name;
329 });
330
331 } // namespace
332 } // namespace webrtc
333