1 /*
2 * Copyright 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "video/video_analyzer.h"
11
12 #include <algorithm>
13 #include <utility>
14
15 #include "absl/algorithm/container.h"
16 #include "absl/flags/flag.h"
17 #include "absl/flags/parse.h"
18 #include "common_video/libyuv/include/webrtc_libyuv.h"
19 #include "modules/rtp_rtcp/source/create_video_rtp_depacketizer.h"
20 #include "modules/rtp_rtcp/source/rtp_packet.h"
21 #include "rtc_base/cpu_time.h"
22 #include "rtc_base/format_macros.h"
23 #include "rtc_base/memory_usage.h"
24 #include "rtc_base/task_queue_for_test.h"
25 #include "rtc_base/task_utils/repeating_task.h"
26 #include "rtc_base/time_utils.h"
27 #include "system_wrappers/include/cpu_info.h"
28 #include "test/call_test.h"
29 #include "test/testsupport/file_utils.h"
30 #include "test/testsupport/frame_writer.h"
31 #include "test/testsupport/perf_test.h"
32 #include "test/testsupport/test_artifacts.h"
33
34 ABSL_FLAG(bool,
35 save_worst_frame,
36 false,
37 "Enable saving a frame with the lowest PSNR to a jpeg file in the "
38 "test_artifacts_dir");
39
40 namespace webrtc {
41 namespace {
42 constexpr TimeDelta kSendStatsPollingInterval = TimeDelta::Seconds(1);
43 constexpr size_t kMaxComparisons = 10;
44 // How often is keep alive message printed.
45 constexpr int kKeepAliveIntervalSeconds = 30;
46 // Interval between checking that the test is over.
47 constexpr int kProbingIntervalMs = 500;
48 constexpr int kKeepAliveIntervalIterations =
49 kKeepAliveIntervalSeconds * 1000 / kProbingIntervalMs;
50
IsFlexfec(int payload_type)51 bool IsFlexfec(int payload_type) {
52 return payload_type == test::CallTest::kFlexfecPayloadType;
53 }
54 } // namespace
55
VideoAnalyzer(test::LayerFilteringTransport * transport,const std::string & test_label,double avg_psnr_threshold,double avg_ssim_threshold,int duration_frames,TimeDelta test_duration,FILE * graph_data_output_file,const std::string & graph_title,uint32_t ssrc_to_analyze,uint32_t rtx_ssrc_to_analyze,size_t selected_stream,int selected_sl,int selected_tl,bool is_quick_test_enabled,Clock * clock,std::string rtp_dump_name,TaskQueueBase * task_queue)56 VideoAnalyzer::VideoAnalyzer(test::LayerFilteringTransport* transport,
57 const std::string& test_label,
58 double avg_psnr_threshold,
59 double avg_ssim_threshold,
60 int duration_frames,
61 TimeDelta test_duration,
62 FILE* graph_data_output_file,
63 const std::string& graph_title,
64 uint32_t ssrc_to_analyze,
65 uint32_t rtx_ssrc_to_analyze,
66 size_t selected_stream,
67 int selected_sl,
68 int selected_tl,
69 bool is_quick_test_enabled,
70 Clock* clock,
71 std::string rtp_dump_name,
72 TaskQueueBase* task_queue)
73 : transport_(transport),
74 receiver_(nullptr),
75 call_(nullptr),
76 send_stream_(nullptr),
77 receive_stream_(nullptr),
78 audio_receive_stream_(nullptr),
79 captured_frame_forwarder_(this, clock, duration_frames, test_duration),
80 test_label_(test_label),
81 graph_data_output_file_(graph_data_output_file),
82 graph_title_(graph_title),
83 ssrc_to_analyze_(ssrc_to_analyze),
84 rtx_ssrc_to_analyze_(rtx_ssrc_to_analyze),
85 selected_stream_(selected_stream),
86 selected_sl_(selected_sl),
87 selected_tl_(selected_tl),
88 mean_decode_time_ms_(0.0),
89 freeze_count_(0),
90 total_freezes_duration_ms_(0),
91 total_frames_duration_ms_(0),
92 sum_squared_frame_durations_(0),
93 decode_frame_rate_(0),
94 render_frame_rate_(0),
95 last_fec_bytes_(0),
96 frames_to_process_(duration_frames),
97 test_end_(clock->CurrentTime() + test_duration),
98 frames_recorded_(0),
99 frames_processed_(0),
100 captured_frames_(0),
101 dropped_frames_(0),
102 dropped_frames_before_first_encode_(0),
103 dropped_frames_before_rendering_(0),
104 last_render_time_(0),
105 last_render_delta_ms_(0),
106 last_unfreeze_time_ms_(0),
107 rtp_timestamp_delta_(0),
108 cpu_time_(0),
109 wallclock_time_(0),
110 avg_psnr_threshold_(avg_psnr_threshold),
111 avg_ssim_threshold_(avg_ssim_threshold),
112 is_quick_test_enabled_(is_quick_test_enabled),
113 quit_(false),
114 done_(true, false),
115 vp8_depacketizer_(CreateVideoRtpDepacketizer(kVideoCodecVP8)),
116 vp9_depacketizer_(CreateVideoRtpDepacketizer(kVideoCodecVP9)),
117 clock_(clock),
118 start_ms_(clock->TimeInMilliseconds()),
119 task_queue_(task_queue) {
120 // Create thread pool for CPU-expensive PSNR/SSIM calculations.
121
122 // Try to use about as many threads as cores, but leave kMinCoresLeft alone,
123 // so that we don't accidentally starve "real" worker threads (codec etc).
124 // Also, don't allocate more than kMaxComparisonThreads, even if there are
125 // spare cores.
126
127 uint32_t num_cores = CpuInfo::DetectNumberOfCores();
128 RTC_DCHECK_GE(num_cores, 1);
129 static const uint32_t kMinCoresLeft = 4;
130 static const uint32_t kMaxComparisonThreads = 8;
131
132 if (num_cores <= kMinCoresLeft) {
133 num_cores = 1;
134 } else {
135 num_cores -= kMinCoresLeft;
136 num_cores = std::min(num_cores, kMaxComparisonThreads);
137 }
138
139 for (uint32_t i = 0; i < num_cores; ++i) {
140 rtc::PlatformThread* thread =
141 new rtc::PlatformThread(&FrameComparisonThread, this, "Analyzer");
142 thread->Start();
143 comparison_thread_pool_.push_back(thread);
144 }
145
146 if (!rtp_dump_name.empty()) {
147 fprintf(stdout, "Writing rtp dump to %s\n", rtp_dump_name.c_str());
148 rtp_file_writer_.reset(test::RtpFileWriter::Create(
149 test::RtpFileWriter::kRtpDump, rtp_dump_name));
150 }
151 }
152
~VideoAnalyzer()153 VideoAnalyzer::~VideoAnalyzer() {
154 {
155 MutexLock lock(&comparison_lock_);
156 quit_ = true;
157 }
158 for (rtc::PlatformThread* thread : comparison_thread_pool_) {
159 thread->Stop();
160 delete thread;
161 }
162 }
163
SetReceiver(PacketReceiver * receiver)164 void VideoAnalyzer::SetReceiver(PacketReceiver* receiver) {
165 receiver_ = receiver;
166 }
167
SetSource(rtc::VideoSourceInterface<VideoFrame> * video_source,bool respect_sink_wants)168 void VideoAnalyzer::SetSource(
169 rtc::VideoSourceInterface<VideoFrame>* video_source,
170 bool respect_sink_wants) {
171 if (respect_sink_wants)
172 captured_frame_forwarder_.SetSource(video_source);
173 rtc::VideoSinkWants wants;
174 video_source->AddOrUpdateSink(InputInterface(), wants);
175 }
176
SetCall(Call * call)177 void VideoAnalyzer::SetCall(Call* call) {
178 MutexLock lock(&lock_);
179 RTC_DCHECK(!call_);
180 call_ = call;
181 }
182
SetSendStream(VideoSendStream * stream)183 void VideoAnalyzer::SetSendStream(VideoSendStream* stream) {
184 MutexLock lock(&lock_);
185 RTC_DCHECK(!send_stream_);
186 send_stream_ = stream;
187 }
188
SetReceiveStream(VideoReceiveStream * stream)189 void VideoAnalyzer::SetReceiveStream(VideoReceiveStream* stream) {
190 MutexLock lock(&lock_);
191 RTC_DCHECK(!receive_stream_);
192 receive_stream_ = stream;
193 }
194
SetAudioReceiveStream(AudioReceiveStream * recv_stream)195 void VideoAnalyzer::SetAudioReceiveStream(AudioReceiveStream* recv_stream) {
196 MutexLock lock(&lock_);
197 RTC_CHECK(!audio_receive_stream_);
198 audio_receive_stream_ = recv_stream;
199 }
200
InputInterface()201 rtc::VideoSinkInterface<VideoFrame>* VideoAnalyzer::InputInterface() {
202 return &captured_frame_forwarder_;
203 }
204
OutputInterface()205 rtc::VideoSourceInterface<VideoFrame>* VideoAnalyzer::OutputInterface() {
206 return &captured_frame_forwarder_;
207 }
208
DeliverPacket(MediaType media_type,rtc::CopyOnWriteBuffer packet,int64_t packet_time_us)209 PacketReceiver::DeliveryStatus VideoAnalyzer::DeliverPacket(
210 MediaType media_type,
211 rtc::CopyOnWriteBuffer packet,
212 int64_t packet_time_us) {
213 // Ignore timestamps of RTCP packets. They're not synchronized with
214 // RTP packet timestamps and so they would confuse wrap_handler_.
215 if (RtpHeaderParser::IsRtcp(packet.cdata(), packet.size())) {
216 return receiver_->DeliverPacket(media_type, std::move(packet),
217 packet_time_us);
218 }
219
220 if (rtp_file_writer_) {
221 test::RtpPacket p;
222 memcpy(p.data, packet.cdata(), packet.size());
223 p.length = packet.size();
224 p.original_length = packet.size();
225 p.time_ms = clock_->TimeInMilliseconds() - start_ms_;
226 rtp_file_writer_->WritePacket(&p);
227 }
228
229 RtpPacket rtp_packet;
230 rtp_packet.Parse(packet);
231 if (!IsFlexfec(rtp_packet.PayloadType()) &&
232 (rtp_packet.Ssrc() == ssrc_to_analyze_ ||
233 rtp_packet.Ssrc() == rtx_ssrc_to_analyze_)) {
234 // Ignore FlexFEC timestamps, to avoid collisions with media timestamps.
235 // (FlexFEC and media are sent on different SSRCs, which have different
236 // timestamps spaces.)
237 // Also ignore packets from wrong SSRC, but include retransmits.
238 MutexLock lock(&lock_);
239 int64_t timestamp =
240 wrap_handler_.Unwrap(rtp_packet.Timestamp() - rtp_timestamp_delta_);
241 recv_times_[timestamp] = clock_->CurrentNtpInMilliseconds();
242 }
243
244 return receiver_->DeliverPacket(media_type, std::move(packet),
245 packet_time_us);
246 }
247
PreEncodeOnFrame(const VideoFrame & video_frame)248 void VideoAnalyzer::PreEncodeOnFrame(const VideoFrame& video_frame) {
249 MutexLock lock(&lock_);
250 if (!first_encoded_timestamp_) {
251 while (frames_.front().timestamp() != video_frame.timestamp()) {
252 ++dropped_frames_before_first_encode_;
253 frames_.pop_front();
254 RTC_CHECK(!frames_.empty());
255 }
256 first_encoded_timestamp_ = video_frame.timestamp();
257 }
258 }
259
PostEncodeOnFrame(size_t stream_id,uint32_t timestamp)260 void VideoAnalyzer::PostEncodeOnFrame(size_t stream_id, uint32_t timestamp) {
261 MutexLock lock(&lock_);
262 if (!first_sent_timestamp_ && stream_id == selected_stream_) {
263 first_sent_timestamp_ = timestamp;
264 }
265 }
266
SendRtp(const uint8_t * packet,size_t length,const PacketOptions & options)267 bool VideoAnalyzer::SendRtp(const uint8_t* packet,
268 size_t length,
269 const PacketOptions& options) {
270 RtpPacket rtp_packet;
271 rtp_packet.Parse(packet, length);
272
273 int64_t current_time = clock_->CurrentNtpInMilliseconds();
274
275 bool result = transport_->SendRtp(packet, length, options);
276 {
277 MutexLock lock(&lock_);
278 if (rtp_timestamp_delta_ == 0 && rtp_packet.Ssrc() == ssrc_to_analyze_) {
279 RTC_CHECK(static_cast<bool>(first_sent_timestamp_));
280 rtp_timestamp_delta_ = rtp_packet.Timestamp() - *first_sent_timestamp_;
281 }
282
283 if (!IsFlexfec(rtp_packet.PayloadType()) &&
284 rtp_packet.Ssrc() == ssrc_to_analyze_) {
285 // Ignore FlexFEC timestamps, to avoid collisions with media timestamps.
286 // (FlexFEC and media are sent on different SSRCs, which have different
287 // timestamps spaces.)
288 // Also ignore packets from wrong SSRC and retransmits.
289 int64_t timestamp =
290 wrap_handler_.Unwrap(rtp_packet.Timestamp() - rtp_timestamp_delta_);
291 send_times_[timestamp] = current_time;
292
293 if (IsInSelectedSpatialAndTemporalLayer(rtp_packet)) {
294 encoded_frame_sizes_[timestamp] += rtp_packet.payload_size();
295 }
296 }
297 }
298 return result;
299 }
300
SendRtcp(const uint8_t * packet,size_t length)301 bool VideoAnalyzer::SendRtcp(const uint8_t* packet, size_t length) {
302 return transport_->SendRtcp(packet, length);
303 }
304
OnFrame(const VideoFrame & video_frame)305 void VideoAnalyzer::OnFrame(const VideoFrame& video_frame) {
306 int64_t render_time_ms = clock_->CurrentNtpInMilliseconds();
307
308 MutexLock lock(&lock_);
309
310 StartExcludingCpuThreadTime();
311
312 int64_t send_timestamp =
313 wrap_handler_.Unwrap(video_frame.timestamp() - rtp_timestamp_delta_);
314
315 while (wrap_handler_.Unwrap(frames_.front().timestamp()) < send_timestamp) {
316 if (!last_rendered_frame_) {
317 // No previous frame rendered, this one was dropped after sending but
318 // before rendering.
319 ++dropped_frames_before_rendering_;
320 } else {
321 AddFrameComparison(frames_.front(), *last_rendered_frame_, true,
322 render_time_ms);
323 }
324 frames_.pop_front();
325 RTC_DCHECK(!frames_.empty());
326 }
327
328 VideoFrame reference_frame = frames_.front();
329 frames_.pop_front();
330 int64_t reference_timestamp =
331 wrap_handler_.Unwrap(reference_frame.timestamp());
332 if (send_timestamp == reference_timestamp - 1) {
333 // TODO(ivica): Make this work for > 2 streams.
334 // Look at RTPSender::BuildRTPHeader.
335 ++send_timestamp;
336 }
337 ASSERT_EQ(reference_timestamp, send_timestamp);
338
339 AddFrameComparison(reference_frame, video_frame, false, render_time_ms);
340
341 last_rendered_frame_ = video_frame;
342
343 StopExcludingCpuThreadTime();
344 }
345
Wait()346 void VideoAnalyzer::Wait() {
347 // Frame comparisons can be very expensive. Wait for test to be done, but
348 // at time-out check if frames_processed is going up. If so, give it more
349 // time, otherwise fail. Hopefully this will reduce test flakiness.
350
351 RepeatingTaskHandle stats_polling_task = RepeatingTaskHandle::DelayedStart(
352 task_queue_, kSendStatsPollingInterval, [this] {
353 PollStats();
354 return kSendStatsPollingInterval;
355 });
356
357 int last_frames_processed = -1;
358 int last_frames_captured = -1;
359 int iteration = 0;
360
361 while (!done_.Wait(kProbingIntervalMs)) {
362 int frames_processed;
363 int frames_captured;
364 {
365 MutexLock lock(&comparison_lock_);
366 frames_processed = frames_processed_;
367 frames_captured = captured_frames_;
368 }
369
370 // Print some output so test infrastructure won't think we've crashed.
371 const char* kKeepAliveMessages[3] = {
372 "Uh, I'm-I'm not quite dead, sir.",
373 "Uh, I-I think uh, I could pull through, sir.",
374 "Actually, I think I'm all right to come with you--"};
375 if (++iteration % kKeepAliveIntervalIterations == 0) {
376 printf("- %s\n", kKeepAliveMessages[iteration % 3]);
377 }
378
379 if (last_frames_processed == -1) {
380 last_frames_processed = frames_processed;
381 last_frames_captured = frames_captured;
382 continue;
383 }
384 if (frames_processed == last_frames_processed &&
385 last_frames_captured == frames_captured &&
386 clock_->CurrentTime() > test_end_) {
387 done_.Set();
388 break;
389 }
390 last_frames_processed = frames_processed;
391 last_frames_captured = frames_captured;
392 }
393
394 if (iteration > 0)
395 printf("- Farewell, sweet Concorde!\n");
396
397 SendTask(RTC_FROM_HERE, task_queue_, [&] { stats_polling_task.Stop(); });
398
399 PrintResults();
400 if (graph_data_output_file_)
401 PrintSamplesToFile();
402 }
403
StartMeasuringCpuProcessTime()404 void VideoAnalyzer::StartMeasuringCpuProcessTime() {
405 MutexLock lock(&cpu_measurement_lock_);
406 cpu_time_ -= rtc::GetProcessCpuTimeNanos();
407 wallclock_time_ -= rtc::SystemTimeNanos();
408 }
409
StopMeasuringCpuProcessTime()410 void VideoAnalyzer::StopMeasuringCpuProcessTime() {
411 MutexLock lock(&cpu_measurement_lock_);
412 cpu_time_ += rtc::GetProcessCpuTimeNanos();
413 wallclock_time_ += rtc::SystemTimeNanos();
414 }
415
StartExcludingCpuThreadTime()416 void VideoAnalyzer::StartExcludingCpuThreadTime() {
417 MutexLock lock(&cpu_measurement_lock_);
418 cpu_time_ += rtc::GetThreadCpuTimeNanos();
419 }
420
StopExcludingCpuThreadTime()421 void VideoAnalyzer::StopExcludingCpuThreadTime() {
422 MutexLock lock(&cpu_measurement_lock_);
423 cpu_time_ -= rtc::GetThreadCpuTimeNanos();
424 }
425
GetCpuUsagePercent()426 double VideoAnalyzer::GetCpuUsagePercent() {
427 MutexLock lock(&cpu_measurement_lock_);
428 return static_cast<double>(cpu_time_) / wallclock_time_ * 100.0;
429 }
430
IsInSelectedSpatialAndTemporalLayer(const RtpPacket & rtp_packet)431 bool VideoAnalyzer::IsInSelectedSpatialAndTemporalLayer(
432 const RtpPacket& rtp_packet) {
433 if (rtp_packet.PayloadType() == test::CallTest::kPayloadTypeVP8) {
434 auto parsed_payload = vp8_depacketizer_->Parse(rtp_packet.PayloadBuffer());
435 RTC_DCHECK(parsed_payload);
436 const auto& vp8_header = absl::get<RTPVideoHeaderVP8>(
437 parsed_payload->video_header.video_type_header);
438 int temporal_idx = vp8_header.temporalIdx;
439 return selected_tl_ < 0 || temporal_idx == kNoTemporalIdx ||
440 temporal_idx <= selected_tl_;
441 }
442
443 if (rtp_packet.PayloadType() == test::CallTest::kPayloadTypeVP9) {
444 auto parsed_payload = vp9_depacketizer_->Parse(rtp_packet.PayloadBuffer());
445 RTC_DCHECK(parsed_payload);
446 const auto& vp9_header = absl::get<RTPVideoHeaderVP9>(
447 parsed_payload->video_header.video_type_header);
448 int temporal_idx = vp9_header.temporal_idx;
449 int spatial_idx = vp9_header.spatial_idx;
450 return (selected_tl_ < 0 || temporal_idx == kNoTemporalIdx ||
451 temporal_idx <= selected_tl_) &&
452 (selected_sl_ < 0 || spatial_idx == kNoSpatialIdx ||
453 spatial_idx <= selected_sl_);
454 }
455
456 return true;
457 }
458
PollStats()459 void VideoAnalyzer::PollStats() {
460 // Do not grab |comparison_lock_|, before |GetStats()| completes.
461 // Otherwise a deadlock may occur:
462 // 1) |comparison_lock_| is acquired after |lock_|
463 // 2) |lock_| is acquired after internal pacer lock in SendRtp()
464 // 3) internal pacer lock is acquired by GetStats().
465 Call::Stats call_stats = call_->GetStats();
466
467 MutexLock lock(&comparison_lock_);
468
469 send_bandwidth_bps_.AddSample(call_stats.send_bandwidth_bps);
470
471 VideoSendStream::Stats send_stats = send_stream_->GetStats();
472 // It's not certain that we yet have estimates for any of these stats.
473 // Check that they are positive before mixing them in.
474 if (send_stats.encode_frame_rate > 0)
475 encode_frame_rate_.AddSample(send_stats.encode_frame_rate);
476 if (send_stats.avg_encode_time_ms > 0)
477 encode_time_ms_.AddSample(send_stats.avg_encode_time_ms);
478 if (send_stats.encode_usage_percent > 0)
479 encode_usage_percent_.AddSample(send_stats.encode_usage_percent);
480 if (send_stats.media_bitrate_bps > 0)
481 media_bitrate_bps_.AddSample(send_stats.media_bitrate_bps);
482 size_t fec_bytes = 0;
483 for (const auto& kv : send_stats.substreams) {
484 fec_bytes += kv.second.rtp_stats.fec.payload_bytes +
485 kv.second.rtp_stats.fec.padding_bytes;
486 }
487 fec_bitrate_bps_.AddSample((fec_bytes - last_fec_bytes_) * 8);
488 last_fec_bytes_ = fec_bytes;
489
490 if (receive_stream_ != nullptr) {
491 VideoReceiveStream::Stats receive_stats = receive_stream_->GetStats();
492 // |total_decode_time_ms| gives a good estimate of the mean decode time,
493 // |decode_ms| is used to keep track of the standard deviation.
494 if (receive_stats.frames_decoded > 0)
495 mean_decode_time_ms_ =
496 static_cast<double>(receive_stats.total_decode_time_ms) /
497 receive_stats.frames_decoded;
498 if (receive_stats.decode_ms > 0)
499 decode_time_ms_.AddSample(receive_stats.decode_ms);
500 if (receive_stats.max_decode_ms > 0)
501 decode_time_max_ms_.AddSample(receive_stats.max_decode_ms);
502 if (receive_stats.width > 0 && receive_stats.height > 0) {
503 pixels_.AddSample(receive_stats.width * receive_stats.height);
504 }
505
506 // |frames_decoded| and |frames_rendered| are used because they are more
507 // accurate than |decode_frame_rate| and |render_frame_rate|.
508 // The latter two are calculated on a momentary basis.
509 const double total_frames_duration_sec_double =
510 static_cast<double>(receive_stats.total_frames_duration_ms) / 1000.0;
511 if (total_frames_duration_sec_double > 0) {
512 decode_frame_rate_ = static_cast<double>(receive_stats.frames_decoded) /
513 total_frames_duration_sec_double;
514 render_frame_rate_ = static_cast<double>(receive_stats.frames_rendered) /
515 total_frames_duration_sec_double;
516 }
517
518 // Freeze metrics.
519 freeze_count_ = receive_stats.freeze_count;
520 total_freezes_duration_ms_ = receive_stats.total_freezes_duration_ms;
521 total_frames_duration_ms_ = receive_stats.total_frames_duration_ms;
522 sum_squared_frame_durations_ = receive_stats.sum_squared_frame_durations;
523 }
524
525 if (audio_receive_stream_ != nullptr) {
526 AudioReceiveStream::Stats receive_stats =
527 audio_receive_stream_->GetStats(/*get_and_clear_legacy_stats=*/true);
528 audio_expand_rate_.AddSample(receive_stats.expand_rate);
529 audio_accelerate_rate_.AddSample(receive_stats.accelerate_rate);
530 audio_jitter_buffer_ms_.AddSample(receive_stats.jitter_buffer_ms);
531 }
532
533 memory_usage_.AddSample(rtc::GetProcessResidentSizeBytes());
534 }
535
FrameComparisonThread(void * obj)536 void VideoAnalyzer::FrameComparisonThread(void* obj) {
537 VideoAnalyzer* analyzer = static_cast<VideoAnalyzer*>(obj);
538 while (analyzer->CompareFrames()) {
539 }
540 }
541
CompareFrames()542 bool VideoAnalyzer::CompareFrames() {
543 if (AllFramesRecorded())
544 return false;
545
546 FrameComparison comparison;
547
548 if (!PopComparison(&comparison)) {
549 // Wait until new comparison task is available, or test is done.
550 // If done, wake up remaining threads waiting.
551 comparison_available_event_.Wait(1000);
552 if (AllFramesRecorded()) {
553 comparison_available_event_.Set();
554 return false;
555 }
556 return true; // Try again.
557 }
558
559 StartExcludingCpuThreadTime();
560
561 PerformFrameComparison(comparison);
562
563 StopExcludingCpuThreadTime();
564
565 if (FrameProcessed()) {
566 done_.Set();
567 comparison_available_event_.Set();
568 return false;
569 }
570
571 return true;
572 }
573
PopComparison(VideoAnalyzer::FrameComparison * comparison)574 bool VideoAnalyzer::PopComparison(VideoAnalyzer::FrameComparison* comparison) {
575 MutexLock lock(&comparison_lock_);
576 // If AllFramesRecorded() is true, it means we have already popped
577 // frames_to_process_ frames from comparisons_, so there is no more work
578 // for this thread to be done. frames_processed_ might still be lower if
579 // all comparisons are not done, but those frames are currently being
580 // worked on by other threads.
581 if (comparisons_.empty() || AllFramesRecordedLocked())
582 return false;
583
584 *comparison = comparisons_.front();
585 comparisons_.pop_front();
586
587 FrameRecorded();
588 return true;
589 }
590
FrameRecorded()591 void VideoAnalyzer::FrameRecorded() {
592 ++frames_recorded_;
593 }
594
AllFramesRecorded()595 bool VideoAnalyzer::AllFramesRecorded() {
596 MutexLock lock(&comparison_lock_);
597 return AllFramesRecordedLocked();
598 }
599
AllFramesRecordedLocked()600 bool VideoAnalyzer::AllFramesRecordedLocked() {
601 RTC_DCHECK(frames_recorded_ <= frames_to_process_);
602 return frames_recorded_ == frames_to_process_ ||
603 (clock_->CurrentTime() > test_end_ && comparisons_.empty()) || quit_;
604 }
605
FrameProcessed()606 bool VideoAnalyzer::FrameProcessed() {
607 MutexLock lock(&comparison_lock_);
608 ++frames_processed_;
609 assert(frames_processed_ <= frames_to_process_);
610 return frames_processed_ == frames_to_process_ ||
611 (clock_->CurrentTime() > test_end_ && comparisons_.empty());
612 }
613
PrintResults()614 void VideoAnalyzer::PrintResults() {
615 using ::webrtc::test::ImproveDirection;
616
617 StopMeasuringCpuProcessTime();
618 int dropped_frames_diff;
619 {
620 MutexLock lock(&lock_);
621 dropped_frames_diff = dropped_frames_before_first_encode_ +
622 dropped_frames_before_rendering_ + frames_.size();
623 }
624 MutexLock lock(&comparison_lock_);
625 PrintResult("psnr", psnr_, "dB", ImproveDirection::kBiggerIsBetter);
626 PrintResult("ssim", ssim_, "unitless", ImproveDirection::kBiggerIsBetter);
627 PrintResult("sender_time", sender_time_, "ms",
628 ImproveDirection::kSmallerIsBetter);
629 PrintResult("receiver_time", receiver_time_, "ms",
630 ImproveDirection::kSmallerIsBetter);
631 PrintResult("network_time", network_time_, "ms",
632 ImproveDirection::kSmallerIsBetter);
633 PrintResult("total_delay_incl_network", end_to_end_, "ms",
634 ImproveDirection::kSmallerIsBetter);
635 PrintResult("time_between_rendered_frames", rendered_delta_, "ms",
636 ImproveDirection::kSmallerIsBetter);
637 PrintResult("encode_frame_rate", encode_frame_rate_, "fps",
638 ImproveDirection::kBiggerIsBetter);
639 PrintResult("encode_time", encode_time_ms_, "ms",
640 ImproveDirection::kSmallerIsBetter);
641 PrintResult("media_bitrate", media_bitrate_bps_, "bps",
642 ImproveDirection::kNone);
643 PrintResult("fec_bitrate", fec_bitrate_bps_, "bps", ImproveDirection::kNone);
644 PrintResult("send_bandwidth", send_bandwidth_bps_, "bps",
645 ImproveDirection::kNone);
646 PrintResult("pixels_per_frame", pixels_, "count",
647 ImproveDirection::kBiggerIsBetter);
648
649 test::PrintResult("decode_frame_rate", "", test_label_.c_str(),
650 decode_frame_rate_, "fps", false,
651 ImproveDirection::kBiggerIsBetter);
652 test::PrintResult("render_frame_rate", "", test_label_.c_str(),
653 render_frame_rate_, "fps", false,
654 ImproveDirection::kBiggerIsBetter);
655
656 // Record the time from the last freeze until the last rendered frame to
657 // ensure we cover the full timespan of the session. Otherwise the metric
658 // would penalize an early freeze followed by no freezes until the end.
659 time_between_freezes_.AddSample(last_render_time_ - last_unfreeze_time_ms_);
660
661 // Freeze metrics.
662 PrintResult("time_between_freezes", time_between_freezes_, "ms",
663 ImproveDirection::kBiggerIsBetter);
664
665 const double freeze_count_double = static_cast<double>(freeze_count_);
666 const double total_freezes_duration_ms_double =
667 static_cast<double>(total_freezes_duration_ms_);
668 const double total_frames_duration_ms_double =
669 static_cast<double>(total_frames_duration_ms_);
670
671 if (total_frames_duration_ms_double > 0) {
672 test::PrintResult(
673 "freeze_duration_ratio", "", test_label_.c_str(),
674 total_freezes_duration_ms_double / total_frames_duration_ms_double,
675 "unitless", false, ImproveDirection::kSmallerIsBetter);
676 RTC_DCHECK_LE(total_freezes_duration_ms_double,
677 total_frames_duration_ms_double);
678
679 constexpr double ms_per_minute = 60 * 1000;
680 const double total_frames_duration_min =
681 total_frames_duration_ms_double / ms_per_minute;
682 if (total_frames_duration_min > 0) {
683 test::PrintResult("freeze_count_per_minute", "", test_label_.c_str(),
684 freeze_count_double / total_frames_duration_min,
685 "unitless", false, ImproveDirection::kSmallerIsBetter);
686 }
687 }
688
689 test::PrintResult("freeze_duration_average", "", test_label_.c_str(),
690 freeze_count_double > 0
691 ? total_freezes_duration_ms_double / freeze_count_double
692 : 0,
693 "ms", false, ImproveDirection::kSmallerIsBetter);
694
695 if (1000 * sum_squared_frame_durations_ > 0) {
696 test::PrintResult(
697 "harmonic_frame_rate", "", test_label_.c_str(),
698 total_frames_duration_ms_double / (1000 * sum_squared_frame_durations_),
699 "fps", false, ImproveDirection::kBiggerIsBetter);
700 }
701
702 if (worst_frame_) {
703 test::PrintResult("min_psnr", "", test_label_.c_str(), worst_frame_->psnr,
704 "dB", false, ImproveDirection::kBiggerIsBetter);
705 }
706
707 if (receive_stream_ != nullptr) {
708 PrintResultWithExternalMean("decode_time", mean_decode_time_ms_,
709 decode_time_ms_, "ms",
710 ImproveDirection::kSmallerIsBetter);
711 }
712 dropped_frames_ += dropped_frames_diff;
713 test::PrintResult("dropped_frames", "", test_label_.c_str(), dropped_frames_,
714 "count", false, ImproveDirection::kSmallerIsBetter);
715 test::PrintResult("cpu_usage", "", test_label_.c_str(), GetCpuUsagePercent(),
716 "%", false, ImproveDirection::kSmallerIsBetter);
717
718 #if defined(WEBRTC_WIN)
719 // On Linux and Mac in Resident Set some unused pages may be counted.
720 // Therefore this metric will depend on order in which tests are run and
721 // will be flaky.
722 PrintResult("memory_usage", memory_usage_, "sizeInBytes",
723 ImproveDirection::kSmallerIsBetter);
724 #endif
725
726 // Saving only the worst frame for manual analysis. Intention here is to
727 // only detect video corruptions and not to track picture quality. Thus,
728 // jpeg is used here.
729 if (absl::GetFlag(FLAGS_save_worst_frame) && worst_frame_) {
730 std::string output_dir;
731 test::GetTestArtifactsDir(&output_dir);
732 std::string output_path =
733 test::JoinFilename(output_dir, test_label_ + ".jpg");
734 RTC_LOG(LS_INFO) << "Saving worst frame to " << output_path;
735 test::JpegFrameWriter frame_writer(output_path);
736 RTC_CHECK(
737 frame_writer.WriteFrame(worst_frame_->frame, 100 /*best quality*/));
738 }
739
740 if (audio_receive_stream_ != nullptr) {
741 PrintResult("audio_expand_rate", audio_expand_rate_, "unitless",
742 ImproveDirection::kSmallerIsBetter);
743 PrintResult("audio_accelerate_rate", audio_accelerate_rate_, "unitless",
744 ImproveDirection::kSmallerIsBetter);
745 PrintResult("audio_jitter_buffer", audio_jitter_buffer_ms_, "ms",
746 ImproveDirection::kNone);
747 }
748
749 // Disable quality check for quick test, as quality checks may fail
750 // because too few samples were collected.
751 if (!is_quick_test_enabled_) {
752 EXPECT_GT(*psnr_.GetMean(), avg_psnr_threshold_);
753 EXPECT_GT(*ssim_.GetMean(), avg_ssim_threshold_);
754 }
755 }
756
PerformFrameComparison(const VideoAnalyzer::FrameComparison & comparison)757 void VideoAnalyzer::PerformFrameComparison(
758 const VideoAnalyzer::FrameComparison& comparison) {
759 // Perform expensive psnr and ssim calculations while not holding lock.
760 double psnr = -1.0;
761 double ssim = -1.0;
762 if (comparison.reference && !comparison.dropped) {
763 psnr = I420PSNR(&*comparison.reference, &*comparison.render);
764 ssim = I420SSIM(&*comparison.reference, &*comparison.render);
765 }
766
767 MutexLock lock(&comparison_lock_);
768
769 if (psnr >= 0.0 && (!worst_frame_ || worst_frame_->psnr > psnr)) {
770 worst_frame_.emplace(FrameWithPsnr{psnr, *comparison.render});
771 }
772
773 if (graph_data_output_file_) {
774 samples_.push_back(Sample(comparison.dropped, comparison.input_time_ms,
775 comparison.send_time_ms, comparison.recv_time_ms,
776 comparison.render_time_ms,
777 comparison.encoded_frame_size, psnr, ssim));
778 }
779 if (psnr >= 0.0)
780 psnr_.AddSample(psnr);
781 if (ssim >= 0.0)
782 ssim_.AddSample(ssim);
783
784 if (comparison.dropped) {
785 ++dropped_frames_;
786 return;
787 }
788 if (last_unfreeze_time_ms_ == 0)
789 last_unfreeze_time_ms_ = comparison.render_time_ms;
790 if (last_render_time_ != 0) {
791 const int64_t render_delta_ms =
792 comparison.render_time_ms - last_render_time_;
793 rendered_delta_.AddSample(render_delta_ms);
794 if (last_render_delta_ms_ != 0 &&
795 render_delta_ms - last_render_delta_ms_ > 150) {
796 time_between_freezes_.AddSample(last_render_time_ -
797 last_unfreeze_time_ms_);
798 last_unfreeze_time_ms_ = comparison.render_time_ms;
799 }
800 last_render_delta_ms_ = render_delta_ms;
801 }
802 last_render_time_ = comparison.render_time_ms;
803
804 sender_time_.AddSample(comparison.send_time_ms - comparison.input_time_ms);
805 if (comparison.recv_time_ms > 0) {
806 // If recv_time_ms == 0, this frame consisted of a packets which were all
807 // lost in the transport. Since we were able to render the frame, however,
808 // the dropped packets were recovered by FlexFEC. The FlexFEC recovery
809 // happens internally in Call, and we can therefore here not know which
810 // FEC packets that protected the lost media packets. Consequently, we
811 // were not able to record a meaningful recv_time_ms. We therefore skip
812 // this sample.
813 //
814 // The reasoning above does not hold for ULPFEC and RTX, as for those
815 // strategies the timestamp of the received packets is set to the
816 // timestamp of the protected/retransmitted media packet. I.e., then
817 // recv_time_ms != 0, even though the media packets were lost.
818 receiver_time_.AddSample(comparison.render_time_ms -
819 comparison.recv_time_ms);
820 network_time_.AddSample(comparison.recv_time_ms - comparison.send_time_ms);
821 }
822 end_to_end_.AddSample(comparison.render_time_ms - comparison.input_time_ms);
823 encoded_frame_size_.AddSample(comparison.encoded_frame_size);
824 }
825
PrintResult(const char * result_type,Statistics stats,const char * unit,webrtc::test::ImproveDirection improve_direction)826 void VideoAnalyzer::PrintResult(
827 const char* result_type,
828 Statistics stats,
829 const char* unit,
830 webrtc::test::ImproveDirection improve_direction) {
831 test::PrintResultMeanAndError(
832 result_type, "", test_label_.c_str(), stats.GetMean().value_or(0),
833 stats.GetStandardDeviation().value_or(0), unit, false, improve_direction);
834 }
835
PrintResultWithExternalMean(const char * result_type,double mean,Statistics stats,const char * unit,webrtc::test::ImproveDirection improve_direction)836 void VideoAnalyzer::PrintResultWithExternalMean(
837 const char* result_type,
838 double mean,
839 Statistics stats,
840 const char* unit,
841 webrtc::test::ImproveDirection improve_direction) {
842 // If the true mean is different than the sample mean, the sample variance is
843 // too low. The sample variance given a known mean is obtained by adding the
844 // squared error between the true mean and the sample mean.
845 double compensated_variance =
846 stats.Size() > 0
847 ? *stats.GetVariance() + pow(mean - *stats.GetMean(), 2.0)
848 : 0.0;
849 test::PrintResultMeanAndError(result_type, "", test_label_.c_str(), mean,
850 std::sqrt(compensated_variance), unit, false,
851 improve_direction);
852 }
853
PrintSamplesToFile()854 void VideoAnalyzer::PrintSamplesToFile() {
855 FILE* out = graph_data_output_file_;
856 MutexLock lock(&comparison_lock_);
857 absl::c_sort(samples_, [](const Sample& A, const Sample& B) -> bool {
858 return A.input_time_ms < B.input_time_ms;
859 });
860
861 fprintf(out, "%s\n", graph_title_.c_str());
862 fprintf(out, "%" RTC_PRIuS "\n", samples_.size());
863 fprintf(out,
864 "dropped "
865 "input_time_ms "
866 "send_time_ms "
867 "recv_time_ms "
868 "render_time_ms "
869 "encoded_frame_size "
870 "psnr "
871 "ssim "
872 "encode_time_ms\n");
873 for (const Sample& sample : samples_) {
874 fprintf(out,
875 "%d %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" RTC_PRIuS
876 " %lf %lf\n",
877 sample.dropped, sample.input_time_ms, sample.send_time_ms,
878 sample.recv_time_ms, sample.render_time_ms,
879 sample.encoded_frame_size, sample.psnr, sample.ssim);
880 }
881 }
882
AddCapturedFrameForComparison(const VideoFrame & video_frame)883 void VideoAnalyzer::AddCapturedFrameForComparison(
884 const VideoFrame& video_frame) {
885 bool must_capture = false;
886 {
887 MutexLock lock(&comparison_lock_);
888 must_capture = captured_frames_ < frames_to_process_;
889 if (must_capture) {
890 ++captured_frames_;
891 }
892 }
893 if (must_capture) {
894 MutexLock lock(&lock_);
895 frames_.push_back(video_frame);
896 }
897 }
898
AddFrameComparison(const VideoFrame & reference,const VideoFrame & render,bool dropped,int64_t render_time_ms)899 void VideoAnalyzer::AddFrameComparison(const VideoFrame& reference,
900 const VideoFrame& render,
901 bool dropped,
902 int64_t render_time_ms) {
903 int64_t reference_timestamp = wrap_handler_.Unwrap(reference.timestamp());
904 int64_t send_time_ms = send_times_[reference_timestamp];
905 send_times_.erase(reference_timestamp);
906 int64_t recv_time_ms = recv_times_[reference_timestamp];
907 recv_times_.erase(reference_timestamp);
908
909 // TODO(ivica): Make this work for > 2 streams.
910 auto it = encoded_frame_sizes_.find(reference_timestamp);
911 if (it == encoded_frame_sizes_.end())
912 it = encoded_frame_sizes_.find(reference_timestamp - 1);
913 size_t encoded_size = it == encoded_frame_sizes_.end() ? 0 : it->second;
914 if (it != encoded_frame_sizes_.end())
915 encoded_frame_sizes_.erase(it);
916
917 MutexLock lock(&comparison_lock_);
918 if (comparisons_.size() < kMaxComparisons) {
919 comparisons_.push_back(FrameComparison(
920 reference, render, dropped, reference.ntp_time_ms(), send_time_ms,
921 recv_time_ms, render_time_ms, encoded_size));
922 } else {
923 comparisons_.push_back(FrameComparison(dropped, reference.ntp_time_ms(),
924 send_time_ms, recv_time_ms,
925 render_time_ms, encoded_size));
926 }
927 comparison_available_event_.Set();
928 }
929
FrameComparison()930 VideoAnalyzer::FrameComparison::FrameComparison()
931 : dropped(false),
932 input_time_ms(0),
933 send_time_ms(0),
934 recv_time_ms(0),
935 render_time_ms(0),
936 encoded_frame_size(0) {}
937
FrameComparison(const VideoFrame & reference,const VideoFrame & render,bool dropped,int64_t input_time_ms,int64_t send_time_ms,int64_t recv_time_ms,int64_t render_time_ms,size_t encoded_frame_size)938 VideoAnalyzer::FrameComparison::FrameComparison(const VideoFrame& reference,
939 const VideoFrame& render,
940 bool dropped,
941 int64_t input_time_ms,
942 int64_t send_time_ms,
943 int64_t recv_time_ms,
944 int64_t render_time_ms,
945 size_t encoded_frame_size)
946 : reference(reference),
947 render(render),
948 dropped(dropped),
949 input_time_ms(input_time_ms),
950 send_time_ms(send_time_ms),
951 recv_time_ms(recv_time_ms),
952 render_time_ms(render_time_ms),
953 encoded_frame_size(encoded_frame_size) {}
954
FrameComparison(bool dropped,int64_t input_time_ms,int64_t send_time_ms,int64_t recv_time_ms,int64_t render_time_ms,size_t encoded_frame_size)955 VideoAnalyzer::FrameComparison::FrameComparison(bool dropped,
956 int64_t input_time_ms,
957 int64_t send_time_ms,
958 int64_t recv_time_ms,
959 int64_t render_time_ms,
960 size_t encoded_frame_size)
961 : dropped(dropped),
962 input_time_ms(input_time_ms),
963 send_time_ms(send_time_ms),
964 recv_time_ms(recv_time_ms),
965 render_time_ms(render_time_ms),
966 encoded_frame_size(encoded_frame_size) {}
967
Sample(int dropped,int64_t input_time_ms,int64_t send_time_ms,int64_t recv_time_ms,int64_t render_time_ms,size_t encoded_frame_size,double psnr,double ssim)968 VideoAnalyzer::Sample::Sample(int dropped,
969 int64_t input_time_ms,
970 int64_t send_time_ms,
971 int64_t recv_time_ms,
972 int64_t render_time_ms,
973 size_t encoded_frame_size,
974 double psnr,
975 double ssim)
976 : dropped(dropped),
977 input_time_ms(input_time_ms),
978 send_time_ms(send_time_ms),
979 recv_time_ms(recv_time_ms),
980 render_time_ms(render_time_ms),
981 encoded_frame_size(encoded_frame_size),
982 psnr(psnr),
983 ssim(ssim) {}
984
CapturedFrameForwarder(VideoAnalyzer * analyzer,Clock * clock,int frames_to_capture,TimeDelta test_duration)985 VideoAnalyzer::CapturedFrameForwarder::CapturedFrameForwarder(
986 VideoAnalyzer* analyzer,
987 Clock* clock,
988 int frames_to_capture,
989 TimeDelta test_duration)
990 : analyzer_(analyzer),
991 send_stream_input_(nullptr),
992 video_source_(nullptr),
993 clock_(clock),
994 captured_frames_(0),
995 frames_to_capture_(frames_to_capture),
996 test_end_(clock->CurrentTime() + test_duration) {}
997
SetSource(VideoSourceInterface<VideoFrame> * video_source)998 void VideoAnalyzer::CapturedFrameForwarder::SetSource(
999 VideoSourceInterface<VideoFrame>* video_source) {
1000 video_source_ = video_source;
1001 }
1002
OnFrame(const VideoFrame & video_frame)1003 void VideoAnalyzer::CapturedFrameForwarder::OnFrame(
1004 const VideoFrame& video_frame) {
1005 VideoFrame copy = video_frame;
1006 // Frames from the capturer does not have a rtp timestamp.
1007 // Create one so it can be used for comparison.
1008 RTC_DCHECK_EQ(0, video_frame.timestamp());
1009 if (video_frame.ntp_time_ms() == 0)
1010 copy.set_ntp_time_ms(clock_->CurrentNtpInMilliseconds());
1011 copy.set_timestamp(copy.ntp_time_ms() * 90);
1012 analyzer_->AddCapturedFrameForComparison(copy);
1013 MutexLock lock(&lock_);
1014 ++captured_frames_;
1015 if (send_stream_input_ && clock_->CurrentTime() <= test_end_ &&
1016 captured_frames_ <= frames_to_capture_) {
1017 send_stream_input_->OnFrame(copy);
1018 }
1019 }
1020
AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame> * sink,const rtc::VideoSinkWants & wants)1021 void VideoAnalyzer::CapturedFrameForwarder::AddOrUpdateSink(
1022 rtc::VideoSinkInterface<VideoFrame>* sink,
1023 const rtc::VideoSinkWants& wants) {
1024 {
1025 MutexLock lock(&lock_);
1026 RTC_DCHECK(!send_stream_input_ || send_stream_input_ == sink);
1027 send_stream_input_ = sink;
1028 }
1029 if (video_source_) {
1030 video_source_->AddOrUpdateSink(this, wants);
1031 }
1032 }
1033
RemoveSink(rtc::VideoSinkInterface<VideoFrame> * sink)1034 void VideoAnalyzer::CapturedFrameForwarder::RemoveSink(
1035 rtc::VideoSinkInterface<VideoFrame>* sink) {
1036 MutexLock lock(&lock_);
1037 RTC_DCHECK(sink == send_stream_input_);
1038 send_stream_input_ = nullptr;
1039 }
1040
1041 } // namespace webrtc
1042