1 /*
2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/codecs/test/videocodec_test_fixture_impl.h"
12
13 #include <stdint.h>
14 #include <stdio.h>
15
16 #include <algorithm>
17 #include <cmath>
18 #include <memory>
19 #include <string>
20 #include <utility>
21 #include <vector>
22
23 #include "absl/strings/str_replace.h"
24 #include "absl/types/optional.h"
25 #include "api/array_view.h"
26 #include "api/transport/field_trial_based_config.h"
27 #include "api/video/video_bitrate_allocation.h"
28 #include "api/video_codecs/sdp_video_format.h"
29 #include "api/video_codecs/video_codec.h"
30 #include "api/video_codecs/video_decoder.h"
31 #include "api/video_codecs/video_encoder_config.h"
32 #include "common_video/h264/h264_common.h"
33 #include "media/base/h264_profile_level_id.h"
34 #include "media/base/media_constants.h"
35 #include "media/engine/internal_decoder_factory.h"
36 #include "media/engine/internal_encoder_factory.h"
37 #include "media/engine/simulcast.h"
38 #include "modules/video_coding/codecs/h264/include/h264_globals.h"
39 #include "modules/video_coding/codecs/vp9/svc_config.h"
40 #include "modules/video_coding/utility/ivf_file_writer.h"
41 #include "rtc_base/checks.h"
42 #include "rtc_base/cpu_time.h"
43 #include "rtc_base/logging.h"
44 #include "rtc_base/strings/string_builder.h"
45 #include "rtc_base/time_utils.h"
46 #include "system_wrappers/include/cpu_info.h"
47 #include "system_wrappers/include/sleep.h"
48 #include "test/gtest.h"
49 #include "test/testsupport/file_utils.h"
50 #include "test/testsupport/frame_writer.h"
51 #include "test/testsupport/perf_test.h"
52 #include "test/video_codec_settings.h"
53
54 namespace webrtc {
55 namespace test {
56
57 using VideoStatistics = VideoCodecTestStats::VideoStatistics;
58
59 namespace {
60 const int kBaseKeyFrameInterval = 3000;
61 const double kBitratePriority = 1.0;
62 const int kDefaultMaxFramerateFps = 30;
63 const int kMaxQp = 56;
64
ConfigureSimulcast(VideoCodec * codec_settings)65 void ConfigureSimulcast(VideoCodec* codec_settings) {
66 FieldTrialBasedConfig trials;
67 const std::vector<webrtc::VideoStream> streams = cricket::GetSimulcastConfig(
68 /*min_layer=*/1, codec_settings->numberOfSimulcastStreams,
69 codec_settings->width, codec_settings->height, kBitratePriority, kMaxQp,
70 /* is_screenshare = */ false, true, trials);
71
72 for (size_t i = 0; i < streams.size(); ++i) {
73 SpatialLayer* ss = &codec_settings->simulcastStream[i];
74 ss->width = static_cast<uint16_t>(streams[i].width);
75 ss->height = static_cast<uint16_t>(streams[i].height);
76 ss->numberOfTemporalLayers =
77 static_cast<unsigned char>(*streams[i].num_temporal_layers);
78 ss->maxBitrate = streams[i].max_bitrate_bps / 1000;
79 ss->targetBitrate = streams[i].target_bitrate_bps / 1000;
80 ss->minBitrate = streams[i].min_bitrate_bps / 1000;
81 ss->qpMax = streams[i].max_qp;
82 ss->active = true;
83 }
84 }
85
ConfigureSvc(VideoCodec * codec_settings)86 void ConfigureSvc(VideoCodec* codec_settings) {
87 RTC_CHECK_EQ(kVideoCodecVP9, codec_settings->codecType);
88
89 const std::vector<SpatialLayer> layers = GetSvcConfig(
90 codec_settings->width, codec_settings->height, kDefaultMaxFramerateFps,
91 /*first_active_layer=*/0, codec_settings->VP9()->numberOfSpatialLayers,
92 codec_settings->VP9()->numberOfTemporalLayers,
93 /* is_screen_sharing = */ false);
94 ASSERT_EQ(codec_settings->VP9()->numberOfSpatialLayers, layers.size())
95 << "GetSvcConfig returned fewer spatial layers than configured.";
96
97 for (size_t i = 0; i < layers.size(); ++i) {
98 codec_settings->spatialLayers[i] = layers[i];
99 }
100 }
101
CodecSpecificToString(const VideoCodec & codec)102 std::string CodecSpecificToString(const VideoCodec& codec) {
103 char buf[1024];
104 rtc::SimpleStringBuilder ss(buf);
105 switch (codec.codecType) {
106 case kVideoCodecVP8:
107 ss << "complexity: " << static_cast<int>(codec.VP8().complexity);
108 ss << "\nnum_temporal_layers: "
109 << static_cast<int>(codec.VP8().numberOfTemporalLayers);
110 ss << "\ndenoising: " << codec.VP8().denoisingOn;
111 ss << "\nautomatic_resize: " << codec.VP8().automaticResizeOn;
112 ss << "\nframe_dropping: " << codec.VP8().frameDroppingOn;
113 ss << "\nkey_frame_interval: " << codec.VP8().keyFrameInterval;
114 break;
115 case kVideoCodecVP9:
116 ss << "complexity: " << static_cast<int>(codec.VP9().complexity);
117 ss << "\nnum_temporal_layers: "
118 << static_cast<int>(codec.VP9().numberOfTemporalLayers);
119 ss << "\nnum_spatial_layers: "
120 << static_cast<int>(codec.VP9().numberOfSpatialLayers);
121 ss << "\ndenoising: " << codec.VP9().denoisingOn;
122 ss << "\nframe_dropping: " << codec.VP9().frameDroppingOn;
123 ss << "\nkey_frame_interval: " << codec.VP9().keyFrameInterval;
124 ss << "\nadaptive_qp_mode: " << codec.VP9().adaptiveQpMode;
125 ss << "\nautomatic_resize: " << codec.VP9().automaticResizeOn;
126 ss << "\nflexible_mode: " << codec.VP9().flexibleMode;
127 break;
128 case kVideoCodecH264:
129 ss << "frame_dropping: " << codec.H264().frameDroppingOn;
130 ss << "\nkey_frame_interval: " << codec.H264().keyFrameInterval;
131 break;
132 default:
133 break;
134 }
135 return ss.str();
136 }
137
RunEncodeInRealTime(const VideoCodecTestFixtureImpl::Config & config)138 bool RunEncodeInRealTime(const VideoCodecTestFixtureImpl::Config& config) {
139 if (config.measure_cpu || config.encode_in_real_time) {
140 return true;
141 }
142 return false;
143 }
144
FilenameWithParams(const VideoCodecTestFixtureImpl::Config & config)145 std::string FilenameWithParams(
146 const VideoCodecTestFixtureImpl::Config& config) {
147 return config.filename + "_" + config.CodecName() + "_" +
148 std::to_string(config.codec_settings.startBitrate);
149 }
150
151 } // namespace
152
153 VideoCodecTestFixtureImpl::Config::Config() = default;
154
SetCodecSettings(std::string codec_name,size_t num_simulcast_streams,size_t num_spatial_layers,size_t num_temporal_layers,bool denoising_on,bool frame_dropper_on,bool spatial_resize_on,size_t width,size_t height)155 void VideoCodecTestFixtureImpl::Config::SetCodecSettings(
156 std::string codec_name,
157 size_t num_simulcast_streams,
158 size_t num_spatial_layers,
159 size_t num_temporal_layers,
160 bool denoising_on,
161 bool frame_dropper_on,
162 bool spatial_resize_on,
163 size_t width,
164 size_t height) {
165 this->codec_name = codec_name;
166 VideoCodecType codec_type = PayloadStringToCodecType(codec_name);
167 webrtc::test::CodecSettings(codec_type, &codec_settings);
168
169 // TODO(brandtr): Move the setting of |width| and |height| to the tests, and
170 // DCHECK that they are set before initializing the codec instead.
171 codec_settings.width = static_cast<uint16_t>(width);
172 codec_settings.height = static_cast<uint16_t>(height);
173
174 RTC_CHECK(num_simulcast_streams >= 1 &&
175 num_simulcast_streams <= kMaxSimulcastStreams);
176 RTC_CHECK(num_spatial_layers >= 1 && num_spatial_layers <= kMaxSpatialLayers);
177 RTC_CHECK(num_temporal_layers >= 1 &&
178 num_temporal_layers <= kMaxTemporalStreams);
179
180 // Simulcast is only available with VP8.
181 RTC_CHECK(num_simulcast_streams < 2 || codec_type == kVideoCodecVP8);
182
183 // Spatial scalability is only available with VP9.
184 RTC_CHECK(num_spatial_layers < 2 || codec_type == kVideoCodecVP9);
185
186 // Some base code requires numberOfSimulcastStreams to be set to zero
187 // when simulcast is not used.
188 codec_settings.numberOfSimulcastStreams =
189 num_simulcast_streams <= 1 ? 0
190 : static_cast<uint8_t>(num_simulcast_streams);
191
192 switch (codec_settings.codecType) {
193 case kVideoCodecVP8:
194 codec_settings.VP8()->numberOfTemporalLayers =
195 static_cast<uint8_t>(num_temporal_layers);
196 codec_settings.VP8()->denoisingOn = denoising_on;
197 codec_settings.VP8()->automaticResizeOn = spatial_resize_on;
198 codec_settings.VP8()->frameDroppingOn = frame_dropper_on;
199 codec_settings.VP8()->keyFrameInterval = kBaseKeyFrameInterval;
200 break;
201 case kVideoCodecVP9:
202 codec_settings.VP9()->numberOfTemporalLayers =
203 static_cast<uint8_t>(num_temporal_layers);
204 codec_settings.VP9()->denoisingOn = denoising_on;
205 codec_settings.VP9()->frameDroppingOn = frame_dropper_on;
206 codec_settings.VP9()->keyFrameInterval = kBaseKeyFrameInterval;
207 codec_settings.VP9()->automaticResizeOn = spatial_resize_on;
208 codec_settings.VP9()->numberOfSpatialLayers =
209 static_cast<uint8_t>(num_spatial_layers);
210 break;
211 case kVideoCodecAV1:
212 codec_settings.qpMax = 63;
213 break;
214 case kVideoCodecH264:
215 codec_settings.H264()->frameDroppingOn = frame_dropper_on;
216 codec_settings.H264()->keyFrameInterval = kBaseKeyFrameInterval;
217 break;
218 default:
219 break;
220 }
221
222 if (codec_settings.numberOfSimulcastStreams > 1) {
223 ConfigureSimulcast(&codec_settings);
224 } else if (codec_settings.codecType == kVideoCodecVP9 &&
225 codec_settings.VP9()->numberOfSpatialLayers > 1) {
226 ConfigureSvc(&codec_settings);
227 }
228 }
229
NumberOfCores() const230 size_t VideoCodecTestFixtureImpl::Config::NumberOfCores() const {
231 return use_single_core ? 1 : CpuInfo::DetectNumberOfCores();
232 }
233
NumberOfTemporalLayers() const234 size_t VideoCodecTestFixtureImpl::Config::NumberOfTemporalLayers() const {
235 if (codec_settings.codecType == kVideoCodecVP8) {
236 return codec_settings.VP8().numberOfTemporalLayers;
237 } else if (codec_settings.codecType == kVideoCodecVP9) {
238 return codec_settings.VP9().numberOfTemporalLayers;
239 } else {
240 return 1;
241 }
242 }
243
NumberOfSpatialLayers() const244 size_t VideoCodecTestFixtureImpl::Config::NumberOfSpatialLayers() const {
245 if (codec_settings.codecType == kVideoCodecVP9) {
246 return codec_settings.VP9().numberOfSpatialLayers;
247 } else {
248 return 1;
249 }
250 }
251
NumberOfSimulcastStreams() const252 size_t VideoCodecTestFixtureImpl::Config::NumberOfSimulcastStreams() const {
253 return codec_settings.numberOfSimulcastStreams;
254 }
255
ToString() const256 std::string VideoCodecTestFixtureImpl::Config::ToString() const {
257 std::string codec_type = CodecTypeToPayloadString(codec_settings.codecType);
258 rtc::StringBuilder ss;
259 ss << "test_name: " << test_name;
260 ss << "\nfilename: " << filename;
261 ss << "\nnum_frames: " << num_frames;
262 ss << "\nmax_payload_size_bytes: " << max_payload_size_bytes;
263 ss << "\ndecode: " << decode;
264 ss << "\nuse_single_core: " << use_single_core;
265 ss << "\nmeasure_cpu: " << measure_cpu;
266 ss << "\nnum_cores: " << NumberOfCores();
267 ss << "\ncodec_type: " << codec_type;
268 ss << "\n\n--> codec_settings";
269 ss << "\nwidth: " << codec_settings.width;
270 ss << "\nheight: " << codec_settings.height;
271 ss << "\nmax_framerate_fps: " << codec_settings.maxFramerate;
272 ss << "\nstart_bitrate_kbps: " << codec_settings.startBitrate;
273 ss << "\nmax_bitrate_kbps: " << codec_settings.maxBitrate;
274 ss << "\nmin_bitrate_kbps: " << codec_settings.minBitrate;
275 ss << "\nmax_qp: " << codec_settings.qpMax;
276 ss << "\nnum_simulcast_streams: "
277 << static_cast<int>(codec_settings.numberOfSimulcastStreams);
278 ss << "\n\n--> codec_settings." << codec_type;
279 ss << "\n" << CodecSpecificToString(codec_settings);
280 if (codec_settings.numberOfSimulcastStreams > 1) {
281 for (int i = 0; i < codec_settings.numberOfSimulcastStreams; ++i) {
282 ss << "\n\n--> codec_settings.simulcastStream[" << i << "]";
283 const SpatialLayer& simulcast_stream = codec_settings.simulcastStream[i];
284 ss << "\nwidth: " << simulcast_stream.width;
285 ss << "\nheight: " << simulcast_stream.height;
286 ss << "\nnum_temporal_layers: "
287 << static_cast<int>(simulcast_stream.numberOfTemporalLayers);
288 ss << "\nmin_bitrate_kbps: " << simulcast_stream.minBitrate;
289 ss << "\ntarget_bitrate_kbps: " << simulcast_stream.targetBitrate;
290 ss << "\nmax_bitrate_kbps: " << simulcast_stream.maxBitrate;
291 ss << "\nmax_qp: " << simulcast_stream.qpMax;
292 ss << "\nactive: " << simulcast_stream.active;
293 }
294 }
295 ss << "\n";
296 return ss.Release();
297 }
298
CodecName() const299 std::string VideoCodecTestFixtureImpl::Config::CodecName() const {
300 std::string name = codec_name;
301 if (name.empty()) {
302 name = CodecTypeToPayloadString(codec_settings.codecType);
303 }
304 if (codec_settings.codecType == kVideoCodecH264) {
305 if (h264_codec_settings.profile == H264::kProfileConstrainedHigh) {
306 return name + "-CHP";
307 } else {
308 RTC_DCHECK_EQ(h264_codec_settings.profile,
309 H264::kProfileConstrainedBaseline);
310 return name + "-CBP";
311 }
312 }
313 return name;
314 }
315
316 // TODO(kthelgason): Move this out of the test fixture impl and
317 // make available as a shared utility class.
CheckEncodedFrame(webrtc::VideoCodecType codec,const EncodedImage & encoded_frame) const318 void VideoCodecTestFixtureImpl::H264KeyframeChecker::CheckEncodedFrame(
319 webrtc::VideoCodecType codec,
320 const EncodedImage& encoded_frame) const {
321 EXPECT_EQ(kVideoCodecH264, codec);
322 bool contains_sps = false;
323 bool contains_pps = false;
324 bool contains_idr = false;
325 const std::vector<webrtc::H264::NaluIndex> nalu_indices =
326 webrtc::H264::FindNaluIndices(encoded_frame.data(), encoded_frame.size());
327 for (const webrtc::H264::NaluIndex& index : nalu_indices) {
328 webrtc::H264::NaluType nalu_type = webrtc::H264::ParseNaluType(
329 encoded_frame.data()[index.payload_start_offset]);
330 if (nalu_type == webrtc::H264::NaluType::kSps) {
331 contains_sps = true;
332 } else if (nalu_type == webrtc::H264::NaluType::kPps) {
333 contains_pps = true;
334 } else if (nalu_type == webrtc::H264::NaluType::kIdr) {
335 contains_idr = true;
336 }
337 }
338 if (encoded_frame._frameType == VideoFrameType::kVideoFrameKey) {
339 EXPECT_TRUE(contains_sps) << "Keyframe should contain SPS.";
340 EXPECT_TRUE(contains_pps) << "Keyframe should contain PPS.";
341 EXPECT_TRUE(contains_idr) << "Keyframe should contain IDR.";
342 } else if (encoded_frame._frameType == VideoFrameType::kVideoFrameDelta) {
343 EXPECT_FALSE(contains_sps) << "Delta frame should not contain SPS.";
344 EXPECT_FALSE(contains_pps) << "Delta frame should not contain PPS.";
345 EXPECT_FALSE(contains_idr) << "Delta frame should not contain IDR.";
346 } else {
347 RTC_NOTREACHED();
348 }
349 }
350
351 class VideoCodecTestFixtureImpl::CpuProcessTime final {
352 public:
CpuProcessTime(const Config & config)353 explicit CpuProcessTime(const Config& config) : config_(config) {}
~CpuProcessTime()354 ~CpuProcessTime() {}
355
Start()356 void Start() {
357 if (config_.measure_cpu) {
358 cpu_time_ -= rtc::GetProcessCpuTimeNanos();
359 wallclock_time_ -= rtc::SystemTimeNanos();
360 }
361 }
Stop()362 void Stop() {
363 if (config_.measure_cpu) {
364 cpu_time_ += rtc::GetProcessCpuTimeNanos();
365 wallclock_time_ += rtc::SystemTimeNanos();
366 }
367 }
Print() const368 void Print() const {
369 if (config_.measure_cpu) {
370 RTC_LOG(LS_INFO) << "cpu_usage_percent: "
371 << GetUsagePercent() / config_.NumberOfCores();
372 }
373 }
374
375 private:
GetUsagePercent() const376 double GetUsagePercent() const {
377 return static_cast<double>(cpu_time_) / wallclock_time_ * 100.0;
378 }
379
380 const Config config_;
381 int64_t cpu_time_ = 0;
382 int64_t wallclock_time_ = 0;
383 };
384
VideoCodecTestFixtureImpl(Config config)385 VideoCodecTestFixtureImpl::VideoCodecTestFixtureImpl(Config config)
386 : encoder_factory_(std::make_unique<InternalEncoderFactory>()),
387 decoder_factory_(std::make_unique<InternalDecoderFactory>()),
388 config_(config) {}
389
VideoCodecTestFixtureImpl(Config config,std::unique_ptr<VideoDecoderFactory> decoder_factory,std::unique_ptr<VideoEncoderFactory> encoder_factory)390 VideoCodecTestFixtureImpl::VideoCodecTestFixtureImpl(
391 Config config,
392 std::unique_ptr<VideoDecoderFactory> decoder_factory,
393 std::unique_ptr<VideoEncoderFactory> encoder_factory)
394 : encoder_factory_(std::move(encoder_factory)),
395 decoder_factory_(std::move(decoder_factory)),
396 config_(config) {}
397
398 VideoCodecTestFixtureImpl::~VideoCodecTestFixtureImpl() = default;
399
400 // Processes all frames in the clip and verifies the result.
RunTest(const std::vector<RateProfile> & rate_profiles,const std::vector<RateControlThresholds> * rc_thresholds,const std::vector<QualityThresholds> * quality_thresholds,const BitstreamThresholds * bs_thresholds)401 void VideoCodecTestFixtureImpl::RunTest(
402 const std::vector<RateProfile>& rate_profiles,
403 const std::vector<RateControlThresholds>* rc_thresholds,
404 const std::vector<QualityThresholds>* quality_thresholds,
405 const BitstreamThresholds* bs_thresholds) {
406 RTC_DCHECK(!rate_profiles.empty());
407
408 // To emulate operation on a production VideoStreamEncoder, we call the
409 // codecs on a task queue.
410 TaskQueueForTest task_queue("VidProc TQ");
411
412 bool is_setup_succeeded = SetUpAndInitObjects(
413 &task_queue, rate_profiles[0].target_kbps, rate_profiles[0].input_fps);
414 EXPECT_TRUE(is_setup_succeeded);
415 if (!is_setup_succeeded) {
416 ReleaseAndCloseObjects(&task_queue);
417 return;
418 }
419
420 PrintSettings(&task_queue);
421 ProcessAllFrames(&task_queue, rate_profiles);
422 ReleaseAndCloseObjects(&task_queue);
423
424 AnalyzeAllFrames(rate_profiles, rc_thresholds, quality_thresholds,
425 bs_thresholds);
426 }
427
ProcessAllFrames(TaskQueueForTest * task_queue,const std::vector<RateProfile> & rate_profiles)428 void VideoCodecTestFixtureImpl::ProcessAllFrames(
429 TaskQueueForTest* task_queue,
430 const std::vector<RateProfile>& rate_profiles) {
431 // Set initial rates.
432 auto rate_profile = rate_profiles.begin();
433 task_queue->PostTask([this, rate_profile] {
434 processor_->SetRates(rate_profile->target_kbps, rate_profile->input_fps);
435 });
436
437 cpu_process_time_->Start();
438
439 for (size_t frame_num = 0; frame_num < config_.num_frames; ++frame_num) {
440 auto next_rate_profile = std::next(rate_profile);
441 if (next_rate_profile != rate_profiles.end() &&
442 frame_num == next_rate_profile->frame_num) {
443 rate_profile = next_rate_profile;
444 task_queue->PostTask([this, rate_profile] {
445 processor_->SetRates(rate_profile->target_kbps,
446 rate_profile->input_fps);
447 });
448 }
449
450 task_queue->PostTask([this] { processor_->ProcessFrame(); });
451
452 if (RunEncodeInRealTime(config_)) {
453 // Roughly pace the frames.
454 const int frame_duration_ms =
455 std::ceil(rtc::kNumMillisecsPerSec / rate_profile->input_fps);
456 SleepMs(frame_duration_ms);
457 }
458 }
459
460 task_queue->PostTask([this] { processor_->Finalize(); });
461
462 // Wait until we know that the last frame has been sent for encode.
463 task_queue->SendTask([] {}, RTC_FROM_HERE);
464
465 // Give the VideoProcessor pipeline some time to process the last frame,
466 // and then release the codecs.
467 SleepMs(1 * rtc::kNumMillisecsPerSec);
468 cpu_process_time_->Stop();
469 }
470
AnalyzeAllFrames(const std::vector<RateProfile> & rate_profiles,const std::vector<RateControlThresholds> * rc_thresholds,const std::vector<QualityThresholds> * quality_thresholds,const BitstreamThresholds * bs_thresholds)471 void VideoCodecTestFixtureImpl::AnalyzeAllFrames(
472 const std::vector<RateProfile>& rate_profiles,
473 const std::vector<RateControlThresholds>* rc_thresholds,
474 const std::vector<QualityThresholds>* quality_thresholds,
475 const BitstreamThresholds* bs_thresholds) {
476
477 for (size_t rate_profile_idx = 0; rate_profile_idx < rate_profiles.size();
478 ++rate_profile_idx) {
479 const size_t first_frame_num = rate_profiles[rate_profile_idx].frame_num;
480 const size_t last_frame_num =
481 rate_profile_idx + 1 < rate_profiles.size()
482 ? rate_profiles[rate_profile_idx + 1].frame_num - 1
483 : config_.num_frames - 1;
484 RTC_CHECK(last_frame_num >= first_frame_num);
485
486 VideoStatistics send_stat = stats_.SliceAndCalcAggregatedVideoStatistic(
487 first_frame_num, last_frame_num);
488 RTC_LOG(LS_INFO) << "==> Send stats";
489 RTC_LOG(LS_INFO) << send_stat.ToString("send_") << "\n";
490
491 std::vector<VideoStatistics> layer_stats =
492 stats_.SliceAndCalcLayerVideoStatistic(first_frame_num, last_frame_num);
493 RTC_LOG(LS_INFO) << "==> Receive stats";
494 for (const auto& layer_stat : layer_stats) {
495 RTC_LOG(LS_INFO) << layer_stat.ToString("recv_") << "\n";
496
497 // For perf dashboard.
498 char modifier_buf[256];
499 rtc::SimpleStringBuilder modifier(modifier_buf);
500 modifier << "_r" << rate_profile_idx << "_sl" << layer_stat.spatial_idx;
501
502 auto PrintResultHelper = [&modifier, this](const std::string& measurement,
503 double value,
504 const std::string& units) {
505 PrintResult(measurement, modifier.str(), config_.test_name, value,
506 units, /*important=*/false);
507 };
508
509 if (layer_stat.temporal_idx == config_.NumberOfTemporalLayers() - 1) {
510 PrintResultHelper("enc_speed", layer_stat.enc_speed_fps, "fps");
511 PrintResultHelper("avg_key_frame_size",
512 layer_stat.avg_key_frame_size_bytes, "bytes");
513 PrintResultHelper("num_key_frames", layer_stat.num_key_frames,
514 "frames");
515 printf("\n");
516 }
517
518 modifier << "tl" << layer_stat.temporal_idx;
519 PrintResultHelper("dec_speed", layer_stat.dec_speed_fps, "fps");
520 PrintResultHelper("avg_delta_frame_size",
521 layer_stat.avg_delta_frame_size_bytes, "bytes");
522 PrintResultHelper("bitrate", layer_stat.bitrate_kbps, "kbps");
523 PrintResultHelper("framerate", layer_stat.framerate_fps, "fps");
524 PrintResultHelper("avg_psnr_y", layer_stat.avg_psnr_y, "dB");
525 PrintResultHelper("avg_psnr_u", layer_stat.avg_psnr_u, "dB");
526 PrintResultHelper("avg_psnr_v", layer_stat.avg_psnr_v, "dB");
527 PrintResultHelper("min_psnr_yuv", layer_stat.min_psnr, "dB");
528 PrintResultHelper("avg_qp", layer_stat.avg_qp, "");
529 printf("\n");
530 if (layer_stat.temporal_idx == config_.NumberOfTemporalLayers() - 1) {
531 printf("\n");
532 }
533 }
534
535 const RateControlThresholds* rc_threshold =
536 rc_thresholds ? &(*rc_thresholds)[rate_profile_idx] : nullptr;
537 const QualityThresholds* quality_threshold =
538 quality_thresholds ? &(*quality_thresholds)[rate_profile_idx] : nullptr;
539
540 VerifyVideoStatistic(send_stat, rc_threshold, quality_threshold,
541 bs_thresholds,
542 rate_profiles[rate_profile_idx].target_kbps,
543 rate_profiles[rate_profile_idx].input_fps);
544 }
545
546 if (config_.print_frame_level_stats) {
547 RTC_LOG(LS_INFO) << "==> Frame stats";
548 std::vector<VideoCodecTestStats::FrameStatistics> frame_stats =
549 stats_.GetFrameStatistics();
550 for (const auto& frame_stat : frame_stats) {
551 RTC_LOG(LS_INFO) << frame_stat.ToString();
552 }
553 }
554
555 cpu_process_time_->Print();
556 }
557
VerifyVideoStatistic(const VideoStatistics & video_stat,const RateControlThresholds * rc_thresholds,const QualityThresholds * quality_thresholds,const BitstreamThresholds * bs_thresholds,size_t target_bitrate_kbps,double input_framerate_fps)558 void VideoCodecTestFixtureImpl::VerifyVideoStatistic(
559 const VideoStatistics& video_stat,
560 const RateControlThresholds* rc_thresholds,
561 const QualityThresholds* quality_thresholds,
562 const BitstreamThresholds* bs_thresholds,
563 size_t target_bitrate_kbps,
564 double input_framerate_fps) {
565 if (rc_thresholds) {
566 const float bitrate_mismatch_percent =
567 100 * std::fabs(1.0f * video_stat.bitrate_kbps - target_bitrate_kbps) /
568 target_bitrate_kbps;
569 const float framerate_mismatch_percent =
570 100 * std::fabs(video_stat.framerate_fps - input_framerate_fps) /
571 input_framerate_fps;
572 EXPECT_LE(bitrate_mismatch_percent,
573 rc_thresholds->max_avg_bitrate_mismatch_percent);
574 EXPECT_LE(video_stat.time_to_reach_target_bitrate_sec,
575 rc_thresholds->max_time_to_reach_target_bitrate_sec);
576 EXPECT_LE(framerate_mismatch_percent,
577 rc_thresholds->max_avg_framerate_mismatch_percent);
578 EXPECT_LE(video_stat.avg_delay_sec,
579 rc_thresholds->max_avg_buffer_level_sec);
580 EXPECT_LE(video_stat.max_key_frame_delay_sec,
581 rc_thresholds->max_max_key_frame_delay_sec);
582 EXPECT_LE(video_stat.max_delta_frame_delay_sec,
583 rc_thresholds->max_max_delta_frame_delay_sec);
584 EXPECT_LE(video_stat.num_spatial_resizes,
585 rc_thresholds->max_num_spatial_resizes);
586 EXPECT_LE(video_stat.num_key_frames, rc_thresholds->max_num_key_frames);
587 }
588
589 if (quality_thresholds) {
590 EXPECT_GT(video_stat.avg_psnr, quality_thresholds->min_avg_psnr);
591 EXPECT_GT(video_stat.min_psnr, quality_thresholds->min_min_psnr);
592
593 // SSIM calculation is not optimized and thus it is disabled in real-time
594 // mode.
595 if (!config_.encode_in_real_time) {
596 EXPECT_GT(video_stat.avg_ssim, quality_thresholds->min_avg_ssim);
597 EXPECT_GT(video_stat.min_ssim, quality_thresholds->min_min_ssim);
598 }
599 }
600
601 if (bs_thresholds) {
602 EXPECT_LE(video_stat.max_nalu_size_bytes,
603 bs_thresholds->max_max_nalu_size_bytes);
604 }
605 }
606
CreateEncoderAndDecoder()607 bool VideoCodecTestFixtureImpl::CreateEncoderAndDecoder() {
608 SdpVideoFormat::Parameters params;
609 if (config_.codec_settings.codecType == kVideoCodecH264) {
610 const char* packetization_mode =
611 config_.h264_codec_settings.packetization_mode ==
612 H264PacketizationMode::NonInterleaved
613 ? "1"
614 : "0";
615 params = {{cricket::kH264FmtpProfileLevelId,
616 *H264::ProfileLevelIdToString(H264::ProfileLevelId(
617 config_.h264_codec_settings.profile, H264::kLevel3_1))},
618 {cricket::kH264FmtpPacketizationMode, packetization_mode}};
619 } else {
620 params = {};
621 }
622 SdpVideoFormat format(config_.codec_name, params);
623
624 encoder_ = encoder_factory_->CreateVideoEncoder(format);
625 EXPECT_TRUE(encoder_) << "Encoder not successfully created.";
626 if (encoder_ == nullptr) {
627 return false;
628 }
629
630 const size_t num_simulcast_or_spatial_layers = std::max(
631 config_.NumberOfSimulcastStreams(), config_.NumberOfSpatialLayers());
632 for (size_t i = 0; i < num_simulcast_or_spatial_layers; ++i) {
633 decoders_.push_back(std::unique_ptr<VideoDecoder>(
634 decoder_factory_->CreateVideoDecoder(format)));
635 }
636
637 for (const auto& decoder : decoders_) {
638 EXPECT_TRUE(decoder) << "Decoder not successfully created.";
639 if (decoder == nullptr) {
640 return false;
641 }
642 }
643
644 return true;
645 }
646
DestroyEncoderAndDecoder()647 void VideoCodecTestFixtureImpl::DestroyEncoderAndDecoder() {
648 decoders_.clear();
649 encoder_.reset();
650 }
651
GetStats()652 VideoCodecTestStats& VideoCodecTestFixtureImpl::GetStats() {
653 return stats_;
654 }
655
SetUpAndInitObjects(TaskQueueForTest * task_queue,size_t initial_bitrate_kbps,double initial_framerate_fps)656 bool VideoCodecTestFixtureImpl::SetUpAndInitObjects(
657 TaskQueueForTest* task_queue,
658 size_t initial_bitrate_kbps,
659 double initial_framerate_fps) {
660 config_.codec_settings.minBitrate = 0;
661 config_.codec_settings.startBitrate = static_cast<int>(initial_bitrate_kbps);
662 config_.codec_settings.maxFramerate = std::ceil(initial_framerate_fps);
663
664 int clip_width = config_.clip_width.value_or(config_.codec_settings.width);
665 int clip_height = config_.clip_height.value_or(config_.codec_settings.height);
666
667 // Create file objects for quality analysis.
668 source_frame_reader_.reset(new YuvFrameReaderImpl(
669 config_.filepath, clip_width, clip_height,
670 config_.reference_width.value_or(clip_width),
671 config_.reference_height.value_or(clip_height),
672 YuvFrameReaderImpl::RepeatMode::kPingPong, config_.clip_fps,
673 config_.codec_settings.maxFramerate));
674 EXPECT_TRUE(source_frame_reader_->Init());
675
676 RTC_DCHECK(encoded_frame_writers_.empty());
677 RTC_DCHECK(decoded_frame_writers_.empty());
678
679 stats_.Clear();
680
681 cpu_process_time_.reset(new CpuProcessTime(config_));
682
683 bool is_codec_created = false;
684 task_queue->SendTask(
685 [this, &is_codec_created]() {
686 is_codec_created = CreateEncoderAndDecoder();
687 },
688 RTC_FROM_HERE);
689
690 if (!is_codec_created) {
691 return false;
692 }
693
694 task_queue->SendTask(
695 [this]() {
696 processor_ = std::make_unique<VideoProcessor>(
697 encoder_.get(), &decoders_, source_frame_reader_.get(), config_,
698 &stats_, &encoded_frame_writers_,
699 decoded_frame_writers_.empty() ? nullptr : &decoded_frame_writers_);
700 },
701 RTC_FROM_HERE);
702
703 if (config_.visualization_params.save_encoded_ivf ||
704 config_.visualization_params.save_decoded_y4m) {
705 std::string encoder_name = GetCodecName(task_queue, /*is_encoder=*/true);
706 encoder_name = absl::StrReplaceAll(encoder_name, {{":", ""}, {" ", "-"}});
707
708 const size_t num_simulcast_or_spatial_layers = std::max(
709 config_.NumberOfSimulcastStreams(), config_.NumberOfSpatialLayers());
710 const size_t num_temporal_layers = config_.NumberOfTemporalLayers();
711 for (size_t simulcast_svc_idx = 0;
712 simulcast_svc_idx < num_simulcast_or_spatial_layers;
713 ++simulcast_svc_idx) {
714 const std::string output_filename_base =
715 JoinFilename(config_.output_path,
716 FilenameWithParams(config_) + "_" + encoder_name +
717 "_sl" + std::to_string(simulcast_svc_idx));
718
719 if (config_.visualization_params.save_encoded_ivf) {
720 for (size_t temporal_idx = 0; temporal_idx < num_temporal_layers;
721 ++temporal_idx) {
722 const std::string output_file_path = output_filename_base + "tl" +
723 std::to_string(temporal_idx) +
724 ".ivf";
725 FileWrapper ivf_file = FileWrapper::OpenWriteOnly(output_file_path);
726
727 const VideoProcessor::LayerKey layer_key(simulcast_svc_idx,
728 temporal_idx);
729 encoded_frame_writers_[layer_key] =
730 IvfFileWriter::Wrap(std::move(ivf_file), /*byte_limit=*/0);
731 }
732 }
733
734 if (config_.visualization_params.save_decoded_y4m) {
735 FrameWriter* decoded_frame_writer = new Y4mFrameWriterImpl(
736 output_filename_base + ".y4m", config_.codec_settings.width,
737 config_.codec_settings.height, config_.codec_settings.maxFramerate);
738 EXPECT_TRUE(decoded_frame_writer->Init());
739 decoded_frame_writers_.push_back(
740 std::unique_ptr<FrameWriter>(decoded_frame_writer));
741 }
742 }
743 }
744
745 return true;
746 }
747
ReleaseAndCloseObjects(TaskQueueForTest * task_queue)748 void VideoCodecTestFixtureImpl::ReleaseAndCloseObjects(
749 TaskQueueForTest* task_queue) {
750 task_queue->SendTask(
751 [this]() {
752 processor_.reset();
753 // The VideoProcessor must be destroyed before the codecs.
754 DestroyEncoderAndDecoder();
755 },
756 RTC_FROM_HERE);
757
758 source_frame_reader_->Close();
759
760 // Close visualization files.
761 for (auto& encoded_frame_writer : encoded_frame_writers_) {
762 EXPECT_TRUE(encoded_frame_writer.second->Close());
763 }
764 encoded_frame_writers_.clear();
765 for (auto& decoded_frame_writer : decoded_frame_writers_) {
766 decoded_frame_writer->Close();
767 }
768 decoded_frame_writers_.clear();
769 }
770
GetCodecName(TaskQueueForTest * task_queue,bool is_encoder) const771 std::string VideoCodecTestFixtureImpl::GetCodecName(
772 TaskQueueForTest* task_queue,
773 bool is_encoder) const {
774 std::string codec_name;
775 task_queue->SendTask(
776 [this, is_encoder, &codec_name] {
777 if (is_encoder) {
778 codec_name = encoder_->GetEncoderInfo().implementation_name;
779 } else {
780 codec_name = decoders_.at(0)->ImplementationName();
781 }
782 },
783 RTC_FROM_HERE);
784 return codec_name;
785 }
786
PrintSettings(TaskQueueForTest * task_queue) const787 void VideoCodecTestFixtureImpl::PrintSettings(
788 TaskQueueForTest* task_queue) const {
789 RTC_LOG(LS_INFO) << "==> Config";
790 RTC_LOG(LS_INFO) << config_.ToString();
791
792 RTC_LOG(LS_INFO) << "==> Codec names";
793 RTC_LOG(LS_INFO) << "enc_impl_name: "
794 << GetCodecName(task_queue, /*is_encoder=*/true);
795 RTC_LOG(LS_INFO) << "dec_impl_name: "
796 << GetCodecName(task_queue, /*is_encoder=*/false);
797 }
798
799 } // namespace test
800 } // namespace webrtc
801