1 /*
2  *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 #ifndef VIDEO_VIDEO_ANALYZER_H_
11 #define VIDEO_VIDEO_ANALYZER_H_
12 
13 #include <deque>
14 #include <map>
15 #include <memory>
16 #include <string>
17 #include <vector>
18 
19 #include "api/task_queue/task_queue_base.h"
20 #include "api/video/video_source_interface.h"
21 #include "modules/rtp_rtcp/source/rtp_packet.h"
22 #include "modules/rtp_rtcp/source/video_rtp_depacketizer.h"
23 #include "rtc_base/event.h"
24 #include "rtc_base/numerics/running_statistics.h"
25 #include "rtc_base/platform_thread.h"
26 #include "rtc_base/synchronization/mutex.h"
27 #include "rtc_base/time_utils.h"
28 #include "test/layer_filtering_transport.h"
29 #include "test/rtp_file_writer.h"
30 #include "test/testsupport/perf_test.h"
31 
32 namespace webrtc {
33 
34 class VideoAnalyzer : public PacketReceiver,
35                       public Transport,
36                       public rtc::VideoSinkInterface<VideoFrame> {
37  public:
38   using Statistics = webrtc_impl::RunningStatistics<double>;
39 
40   VideoAnalyzer(test::LayerFilteringTransport* transport,
41                 const std::string& test_label,
42                 double avg_psnr_threshold,
43                 double avg_ssim_threshold,
44                 int duration_frames,
45                 TimeDelta test_duration,
46                 FILE* graph_data_output_file,
47                 const std::string& graph_title,
48                 uint32_t ssrc_to_analyze,
49                 uint32_t rtx_ssrc_to_analyze,
50                 size_t selected_stream,
51                 int selected_sl,
52                 int selected_tl,
53                 bool is_quick_test_enabled,
54                 Clock* clock,
55                 std::string rtp_dump_name,
56                 TaskQueueBase* task_queue);
57   ~VideoAnalyzer();
58 
59   virtual void SetReceiver(PacketReceiver* receiver);
60   void SetSource(rtc::VideoSourceInterface<VideoFrame>* video_source,
61                  bool respect_sink_wants);
62   void SetCall(Call* call);
63   void SetSendStream(VideoSendStream* stream);
64   void SetReceiveStream(VideoReceiveStream* stream);
65   void SetAudioReceiveStream(AudioReceiveStream* recv_stream);
66 
67   rtc::VideoSinkInterface<VideoFrame>* InputInterface();
68   rtc::VideoSourceInterface<VideoFrame>* OutputInterface();
69 
70   DeliveryStatus DeliverPacket(MediaType media_type,
71                                rtc::CopyOnWriteBuffer packet,
72                                int64_t packet_time_us) override;
73 
74   void PreEncodeOnFrame(const VideoFrame& video_frame);
75   void PostEncodeOnFrame(size_t stream_id, uint32_t timestamp);
76 
77   bool SendRtp(const uint8_t* packet,
78                size_t length,
79                const PacketOptions& options) override;
80 
81   bool SendRtcp(const uint8_t* packet, size_t length) override;
82   void OnFrame(const VideoFrame& video_frame) override;
83   void Wait();
84 
85   void StartMeasuringCpuProcessTime();
86   void StopMeasuringCpuProcessTime();
87   void StartExcludingCpuThreadTime() RTC_LOCKS_EXCLUDED(cpu_measurement_lock_);
88   void StopExcludingCpuThreadTime() RTC_LOCKS_EXCLUDED(cpu_measurement_lock_);
89   double GetCpuUsagePercent() RTC_LOCKS_EXCLUDED(cpu_measurement_lock_);
90 
91   test::LayerFilteringTransport* const transport_;
92   PacketReceiver* receiver_;
93 
94  private:
95   struct FrameComparison {
96     FrameComparison();
97     FrameComparison(const VideoFrame& reference,
98                     const VideoFrame& render,
99                     bool dropped,
100                     int64_t input_time_ms,
101                     int64_t send_time_ms,
102                     int64_t recv_time_ms,
103                     int64_t render_time_ms,
104                     size_t encoded_frame_size);
105     FrameComparison(bool dropped,
106                     int64_t input_time_ms,
107                     int64_t send_time_ms,
108                     int64_t recv_time_ms,
109                     int64_t render_time_ms,
110                     size_t encoded_frame_size);
111 
112     absl::optional<VideoFrame> reference;
113     absl::optional<VideoFrame> render;
114     bool dropped;
115     int64_t input_time_ms;
116     int64_t send_time_ms;
117     int64_t recv_time_ms;
118     int64_t render_time_ms;
119     size_t encoded_frame_size;
120   };
121 
122   struct Sample {
123     Sample(int dropped,
124            int64_t input_time_ms,
125            int64_t send_time_ms,
126            int64_t recv_time_ms,
127            int64_t render_time_ms,
128            size_t encoded_frame_size,
129            double psnr,
130            double ssim);
131 
132     int dropped;
133     int64_t input_time_ms;
134     int64_t send_time_ms;
135     int64_t recv_time_ms;
136     int64_t render_time_ms;
137     size_t encoded_frame_size;
138     double psnr;
139     double ssim;
140   };
141 
142   // Implements VideoSinkInterface to receive captured frames from a
143   // FrameGeneratorCapturer. Implements VideoSourceInterface to be able to act
144   // as a source to VideoSendStream.
145   // It forwards all input frames to the VideoAnalyzer for later comparison and
146   // forwards the captured frames to the VideoSendStream.
147   class CapturedFrameForwarder : public rtc::VideoSinkInterface<VideoFrame>,
148                                  public rtc::VideoSourceInterface<VideoFrame> {
149    public:
150     CapturedFrameForwarder(VideoAnalyzer* analyzer,
151                            Clock* clock,
152                            int frames_to_capture,
153                            TimeDelta test_duration);
154     void SetSource(rtc::VideoSourceInterface<VideoFrame>* video_source);
155 
156    private:
157     void OnFrame(const VideoFrame& video_frame)
158         RTC_LOCKS_EXCLUDED(lock_) override;
159 
160     // Called when |send_stream_.SetSource()| is called.
161     void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
162                          const rtc::VideoSinkWants& wants)
163         RTC_LOCKS_EXCLUDED(lock_) override;
164 
165     // Called by |send_stream_| when |send_stream_.SetSource()| is called.
166     void RemoveSink(rtc::VideoSinkInterface<VideoFrame>* sink)
167         RTC_LOCKS_EXCLUDED(lock_) override;
168 
169     VideoAnalyzer* const analyzer_;
170     Mutex lock_;
171     rtc::VideoSinkInterface<VideoFrame>* send_stream_input_
172         RTC_GUARDED_BY(lock_);
173     VideoSourceInterface<VideoFrame>* video_source_;
174     Clock* clock_;
175     int captured_frames_ RTC_GUARDED_BY(lock_);
176     const int frames_to_capture_;
177     const Timestamp test_end_;
178   };
179 
180   struct FrameWithPsnr {
181     double psnr;
182     VideoFrame frame;
183   };
184 
185   bool IsInSelectedSpatialAndTemporalLayer(const RtpPacket& rtp_packet);
186 
187   void AddFrameComparison(const VideoFrame& reference,
188                           const VideoFrame& render,
189                           bool dropped,
190                           int64_t render_time_ms)
191       RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_);
192 
193   void PollStats() RTC_LOCKS_EXCLUDED(comparison_lock_);
194   static void FrameComparisonThread(void* obj);
195   bool CompareFrames();
196   bool PopComparison(FrameComparison* comparison);
197   // Increment counter for number of frames received for comparison.
198   void FrameRecorded() RTC_EXCLUSIVE_LOCKS_REQUIRED(comparison_lock_);
199   // Returns true if all frames to be compared have been taken from the queue.
200   bool AllFramesRecorded() RTC_LOCKS_EXCLUDED(comparison_lock_);
201   bool AllFramesRecordedLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(comparison_lock_);
202   // Increase count of number of frames processed. Returns true if this was the
203   // last frame to be processed.
204   bool FrameProcessed() RTC_LOCKS_EXCLUDED(comparison_lock_);
205   void PrintResults() RTC_LOCKS_EXCLUDED(lock_, comparison_lock_);
206   void PerformFrameComparison(const FrameComparison& comparison)
207       RTC_LOCKS_EXCLUDED(comparison_lock_);
208   void PrintResult(const char* result_type,
209                    Statistics stats,
210                    const char* unit,
211                    webrtc::test::ImproveDirection improve_direction);
212   void PrintResultWithExternalMean(
213       const char* result_type,
214       double mean,
215       Statistics stats,
216       const char* unit,
217       webrtc::test::ImproveDirection improve_direction);
218   void PrintSamplesToFile(void) RTC_LOCKS_EXCLUDED(comparison_lock_);
219   void AddCapturedFrameForComparison(const VideoFrame& video_frame)
220       RTC_LOCKS_EXCLUDED(lock_, comparison_lock_);
221 
222   Call* call_;
223   VideoSendStream* send_stream_;
224   VideoReceiveStream* receive_stream_;
225   AudioReceiveStream* audio_receive_stream_;
226   CapturedFrameForwarder captured_frame_forwarder_;
227   const std::string test_label_;
228   FILE* const graph_data_output_file_;
229   const std::string graph_title_;
230   const uint32_t ssrc_to_analyze_;
231   const uint32_t rtx_ssrc_to_analyze_;
232   const size_t selected_stream_;
233   const int selected_sl_;
234   const int selected_tl_;
235 
236   Mutex comparison_lock_;
237   std::vector<Sample> samples_ RTC_GUARDED_BY(comparison_lock_);
238   Statistics sender_time_ RTC_GUARDED_BY(comparison_lock_);
239   Statistics receiver_time_ RTC_GUARDED_BY(comparison_lock_);
240   Statistics network_time_ RTC_GUARDED_BY(comparison_lock_);
241   Statistics psnr_ RTC_GUARDED_BY(comparison_lock_);
242   Statistics ssim_ RTC_GUARDED_BY(comparison_lock_);
243   Statistics end_to_end_ RTC_GUARDED_BY(comparison_lock_);
244   Statistics rendered_delta_ RTC_GUARDED_BY(comparison_lock_);
245   Statistics encoded_frame_size_ RTC_GUARDED_BY(comparison_lock_);
246   Statistics encode_frame_rate_ RTC_GUARDED_BY(comparison_lock_);
247   Statistics encode_time_ms_ RTC_GUARDED_BY(comparison_lock_);
248   Statistics encode_usage_percent_ RTC_GUARDED_BY(comparison_lock_);
249   double mean_decode_time_ms_ RTC_GUARDED_BY(comparison_lock_);
250   Statistics decode_time_ms_ RTC_GUARDED_BY(comparison_lock_);
251   Statistics decode_time_max_ms_ RTC_GUARDED_BY(comparison_lock_);
252   Statistics media_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_);
253   Statistics fec_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_);
254   Statistics send_bandwidth_bps_ RTC_GUARDED_BY(comparison_lock_);
255   Statistics memory_usage_ RTC_GUARDED_BY(comparison_lock_);
256   Statistics audio_expand_rate_ RTC_GUARDED_BY(comparison_lock_);
257   Statistics audio_accelerate_rate_ RTC_GUARDED_BY(comparison_lock_);
258   Statistics audio_jitter_buffer_ms_ RTC_GUARDED_BY(comparison_lock_);
259   Statistics pixels_ RTC_GUARDED_BY(comparison_lock_);
260   // Rendered frame with worst PSNR is saved for further analysis.
261   absl::optional<FrameWithPsnr> worst_frame_ RTC_GUARDED_BY(comparison_lock_);
262   // Freeze metrics.
263   Statistics time_between_freezes_ RTC_GUARDED_BY(comparison_lock_);
264   uint32_t freeze_count_ RTC_GUARDED_BY(comparison_lock_);
265   uint32_t total_freezes_duration_ms_ RTC_GUARDED_BY(comparison_lock_);
266   uint32_t total_frames_duration_ms_ RTC_GUARDED_BY(comparison_lock_);
267   double sum_squared_frame_durations_ RTC_GUARDED_BY(comparison_lock_);
268 
269   double decode_frame_rate_ RTC_GUARDED_BY(comparison_lock_);
270   double render_frame_rate_ RTC_GUARDED_BY(comparison_lock_);
271 
272   size_t last_fec_bytes_;
273 
274   Mutex lock_ RTC_ACQUIRED_BEFORE(comparison_lock_)
275       RTC_ACQUIRED_BEFORE(cpu_measurement_lock_);
276   const int frames_to_process_;
277   const Timestamp test_end_;
278   int frames_recorded_ RTC_GUARDED_BY(comparison_lock_);
279   int frames_processed_ RTC_GUARDED_BY(comparison_lock_);
280   int captured_frames_ RTC_GUARDED_BY(comparison_lock_);
281   int dropped_frames_ RTC_GUARDED_BY(comparison_lock_);
282   int dropped_frames_before_first_encode_ RTC_GUARDED_BY(lock_);
283   int dropped_frames_before_rendering_ RTC_GUARDED_BY(lock_);
284   int64_t last_render_time_ RTC_GUARDED_BY(comparison_lock_);
285   int64_t last_render_delta_ms_ RTC_GUARDED_BY(comparison_lock_);
286   int64_t last_unfreeze_time_ms_ RTC_GUARDED_BY(comparison_lock_);
287   uint32_t rtp_timestamp_delta_ RTC_GUARDED_BY(lock_);
288 
289   Mutex cpu_measurement_lock_;
290   int64_t cpu_time_ RTC_GUARDED_BY(cpu_measurement_lock_);
291   int64_t wallclock_time_ RTC_GUARDED_BY(cpu_measurement_lock_);
292 
293   std::deque<VideoFrame> frames_ RTC_GUARDED_BY(lock_);
294   absl::optional<VideoFrame> last_rendered_frame_ RTC_GUARDED_BY(lock_);
295   rtc::TimestampWrapAroundHandler wrap_handler_ RTC_GUARDED_BY(lock_);
296   std::map<int64_t, int64_t> send_times_ RTC_GUARDED_BY(lock_);
297   std::map<int64_t, int64_t> recv_times_ RTC_GUARDED_BY(lock_);
298   std::map<int64_t, size_t> encoded_frame_sizes_ RTC_GUARDED_BY(lock_);
299   absl::optional<uint32_t> first_encoded_timestamp_ RTC_GUARDED_BY(lock_);
300   absl::optional<uint32_t> first_sent_timestamp_ RTC_GUARDED_BY(lock_);
301   const double avg_psnr_threshold_;
302   const double avg_ssim_threshold_;
303   bool is_quick_test_enabled_;
304 
305   std::vector<rtc::PlatformThread*> comparison_thread_pool_;
306   rtc::Event comparison_available_event_;
307   std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(comparison_lock_);
308   bool quit_ RTC_GUARDED_BY(comparison_lock_);
309   rtc::Event done_;
310 
311   std::unique_ptr<VideoRtpDepacketizer> vp8_depacketizer_;
312   std::unique_ptr<VideoRtpDepacketizer> vp9_depacketizer_;
313   std::unique_ptr<test::RtpFileWriter> rtp_file_writer_;
314   Clock* const clock_;
315   const int64_t start_ms_;
316   TaskQueueBase* task_queue_;
317 };
318 
319 }  // namespace webrtc
320 #endif  // VIDEO_VIDEO_ANALYZER_H_
321