1 /*
2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef VIDEO_SEND_STATISTICS_PROXY_H_
12 #define VIDEO_SEND_STATISTICS_PROXY_H_
13 
14 #include <array>
15 #include <map>
16 #include <memory>
17 #include <string>
18 #include <vector>
19 
20 #include "api/video/video_codec_constants.h"
21 #include "api/video/video_stream_encoder_observer.h"
22 #include "api/video_codecs/video_encoder_config.h"
23 #include "call/video_send_stream.h"
24 #include "modules/include/module_common_types_public.h"
25 #include "modules/rtp_rtcp/include/report_block_data.h"
26 #include "modules/video_coding/include/video_codec_interface.h"
27 #include "modules/video_coding/include/video_coding_defines.h"
28 #include "rtc_base/numerics/exp_filter.h"
29 #include "rtc_base/rate_tracker.h"
30 #include "rtc_base/synchronization/mutex.h"
31 #include "rtc_base/thread_annotations.h"
32 #include "system_wrappers/include/clock.h"
33 #include "video/quality_limitation_reason_tracker.h"
34 #include "video/report_block_stats.h"
35 #include "video/stats_counter.h"
36 
37 namespace webrtc {
38 
39 class SendStatisticsProxy : public VideoStreamEncoderObserver,
40                             public RtcpStatisticsCallback,
41                             public ReportBlockDataObserver,
42                             public RtcpPacketTypeCounterObserver,
43                             public StreamDataCountersCallback,
44                             public BitrateStatisticsObserver,
45                             public FrameCountObserver,
46                             public SendSideDelayObserver {
47  public:
48   static const int kStatsTimeoutMs;
49   // Number of required samples to be collected before a metric is added
50   // to a rtc histogram.
51   static const int kMinRequiredMetricsSamples = 200;
52 
53   SendStatisticsProxy(Clock* clock,
54                       const VideoSendStream::Config& config,
55                       VideoEncoderConfig::ContentType content_type);
56   ~SendStatisticsProxy() override;
57 
58   virtual VideoSendStream::Stats GetStats();
59 
60   void OnSendEncodedImage(const EncodedImage& encoded_image,
61                           const CodecSpecificInfo* codec_info) override;
62 
63   void OnEncoderImplementationChanged(
64       const std::string& implementation_name) override;
65 
66   // Used to update incoming frame rate.
67   void OnIncomingFrame(int width, int height) override;
68 
69   // Dropped frame stats.
70   void OnFrameDropped(DropReason) override;
71 
72   // Adaptation stats.
73   void OnAdaptationChanged(
74       VideoAdaptationReason reason,
75       const VideoAdaptationCounters& cpu_counters,
76       const VideoAdaptationCounters& quality_counters) override;
77   void ClearAdaptationStats() override;
78   void UpdateAdaptationSettings(AdaptationSettings cpu_settings,
79                                 AdaptationSettings quality_settings) override;
80 
81   void OnBitrateAllocationUpdated(
82       const VideoCodec& codec,
83       const VideoBitrateAllocation& allocation) override;
84 
85   void OnEncoderInternalScalerUpdate(bool is_scaled) override;
86 
87   void OnMinPixelLimitReached() override;
88   void OnInitialQualityResolutionAdaptDown() override;
89 
90   void OnSuspendChange(bool is_suspended) override;
91   void OnInactiveSsrc(uint32_t ssrc);
92 
93   // Used to indicate change in content type, which may require a change in
94   // how stats are collected.
95   void OnEncoderReconfigured(const VideoEncoderConfig& encoder_config,
96                              const std::vector<VideoStream>& streams) override;
97 
98   // Used to update the encoder target rate.
99   void OnSetEncoderTargetRate(uint32_t bitrate_bps);
100 
101   // Implements CpuOveruseMetricsObserver.
102   void OnEncodedFrameTimeMeasured(int encode_time_ms,
103                                   int encode_usage_percent) override;
104 
105   int GetInputFrameRate() const override;
106   int GetSendFrameRate() const;
107 
108  protected:
109   // From RtcpStatisticsCallback.
110   void StatisticsUpdated(const RtcpStatistics& statistics,
111                          uint32_t ssrc) override;
112   // From ReportBlockDataObserver.
113   void OnReportBlockDataUpdated(ReportBlockData report_block_data) override;
114   // From RtcpPacketTypeCounterObserver.
115   void RtcpPacketTypesCounterUpdated(
116       uint32_t ssrc,
117       const RtcpPacketTypeCounter& packet_counter) override;
118   // From StreamDataCountersCallback.
119   void DataCountersUpdated(const StreamDataCounters& counters,
120                            uint32_t ssrc) override;
121 
122   // From BitrateStatisticsObserver.
123   void Notify(uint32_t total_bitrate_bps,
124               uint32_t retransmit_bitrate_bps,
125               uint32_t ssrc) override;
126 
127   // From FrameCountObserver.
128   void FrameCountUpdated(const FrameCounts& frame_counts,
129                          uint32_t ssrc) override;
130 
131   void SendSideDelayUpdated(int avg_delay_ms,
132                             int max_delay_ms,
133                             uint64_t total_delay_ms,
134                             uint32_t ssrc) override;
135 
136  private:
137   class SampleCounter {
138    public:
SampleCounter()139     SampleCounter() : sum(0), num_samples(0) {}
~SampleCounter()140     ~SampleCounter() {}
141     void Add(int sample);
142     int Avg(int64_t min_required_samples) const;
143 
144    private:
145     int64_t sum;
146     int64_t num_samples;
147   };
148   class BoolSampleCounter {
149    public:
BoolSampleCounter()150     BoolSampleCounter() : sum(0), num_samples(0) {}
~BoolSampleCounter()151     ~BoolSampleCounter() {}
152     void Add(bool sample);
153     void Add(bool sample, int64_t count);
154     int Percent(int64_t min_required_samples) const;
155     int Permille(int64_t min_required_samples) const;
156 
157    private:
158     int Fraction(int64_t min_required_samples, float multiplier) const;
159     int64_t sum;
160     int64_t num_samples;
161   };
162   struct StatsUpdateTimes {
StatsUpdateTimesStatsUpdateTimes163     StatsUpdateTimes() : resolution_update_ms(0), bitrate_update_ms(0) {}
164     int64_t resolution_update_ms;
165     int64_t bitrate_update_ms;
166   };
167   struct TargetRateUpdates {
TargetRateUpdatesTargetRateUpdates168     TargetRateUpdates()
169         : pause_resume_events(0), last_paused_or_resumed(false), last_ms(-1) {}
170     int pause_resume_events;
171     bool last_paused_or_resumed;
172     int64_t last_ms;
173   };
174   struct FallbackEncoderInfo {
175     FallbackEncoderInfo();
176     bool is_possible = true;
177     bool is_active = false;
178     int on_off_events = 0;
179     int64_t elapsed_ms = 0;
180     absl::optional<int64_t> last_update_ms;
181     const int max_frame_diff_ms = 2000;
182   };
183   struct FallbackEncoderInfoDisabled {
184     bool is_possible = true;
185     bool min_pixel_limit_reached = false;
186   };
187   struct StatsTimer {
188     void Start(int64_t now_ms);
189     void Stop(int64_t now_ms);
190     void Restart(int64_t now_ms);
191     int64_t start_ms = -1;
192     int64_t total_ms = 0;
193   };
194   struct QpCounters {
195     SampleCounter vp8;   // QP range: 0-127.
196     SampleCounter vp9;   // QP range: 0-255.
197     SampleCounter h264;  // QP range: 0-51.
198   };
199   struct AdaptChanges {
200     int down = 0;
201     int up = 0;
202   };
203 
204   // Map holding encoded frames (mapped by timestamp).
205   // If simulcast layers are encoded on different threads, there is no guarantee
206   // that one frame of all layers are encoded before the next start.
207   struct TimestampOlderThan {
operatorTimestampOlderThan208     bool operator()(uint32_t ts1, uint32_t ts2) const {
209       return IsNewerTimestamp(ts2, ts1);
210     }
211   };
212   struct Frame {
FrameFrame213     Frame(int64_t send_ms, uint32_t width, uint32_t height, int simulcast_idx)
214         : send_ms(send_ms),
215           max_width(width),
216           max_height(height),
217           max_simulcast_idx(simulcast_idx) {}
218     const int64_t
219         send_ms;          // Time when first frame with this timestamp is sent.
220     uint32_t max_width;   // Max width with this timestamp.
221     uint32_t max_height;  // Max height with this timestamp.
222     int max_simulcast_idx;  // Max simulcast index with this timestamp.
223   };
224   typedef std::map<uint32_t, Frame, TimestampOlderThan> EncodedFrameMap;
225 
226   void PurgeOldStats() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
227   VideoSendStream::StreamStats* GetStatsEntry(uint32_t ssrc)
228       RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
229 
230   struct MaskedAdaptationCounts {
231     absl::optional<int> resolution_adaptations = absl::nullopt;
232     absl::optional<int> num_framerate_reductions = absl::nullopt;
233   };
234 
235   struct Adaptations {
236    public:
237     MaskedAdaptationCounts MaskedCpuCounts() const;
238     MaskedAdaptationCounts MaskedQualityCounts() const;
239 
240     void set_cpu_counts(const VideoAdaptationCounters& cpu_counts);
241     void set_quality_counts(const VideoAdaptationCounters& quality_counts);
242 
243     VideoAdaptationCounters cpu_counts() const;
244     VideoAdaptationCounters quality_counts() const;
245 
246     void UpdateMaskingSettings(AdaptationSettings cpu_settings,
247                                AdaptationSettings quality_settings);
248 
249    private:
250     VideoAdaptationCounters cpu_counts_;
251     AdaptationSettings cpu_settings_;
252     VideoAdaptationCounters quality_counts_;
253     AdaptationSettings quality_settings_;
254 
255     MaskedAdaptationCounts Mask(const VideoAdaptationCounters& counters,
256                                 const AdaptationSettings& settings) const;
257   };
258 
259   void SetAdaptTimer(const MaskedAdaptationCounts& counts, StatsTimer* timer)
260       RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
261   void UpdateAdaptationStats() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
262   void TryUpdateInitialQualityResolutionAdaptUp(
263       absl::optional<int> old_quality_downscales,
264       absl::optional<int> updated_quality_downscales)
265       RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
266 
267   void UpdateEncoderFallbackStats(const CodecSpecificInfo* codec_info,
268                                   int pixels,
269                                   int simulcast_index)
270       RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
271   void UpdateFallbackDisabledStats(const CodecSpecificInfo* codec_info,
272                                    int pixels,
273                                    int simulcast_index)
274       RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
275 
276   Clock* const clock_;
277   const std::string payload_name_;
278   const RtpConfig rtp_config_;
279   const absl::optional<int> fallback_max_pixels_;
280   const absl::optional<int> fallback_max_pixels_disabled_;
281   mutable Mutex mutex_;
282   VideoEncoderConfig::ContentType content_type_ RTC_GUARDED_BY(mutex_);
283   const int64_t start_ms_;
284   VideoSendStream::Stats stats_ RTC_GUARDED_BY(mutex_);
285   std::map<uint32_t, StatsUpdateTimes> update_times_ RTC_GUARDED_BY(mutex_);
286   rtc::ExpFilter encode_time_ RTC_GUARDED_BY(mutex_);
287   QualityLimitationReasonTracker quality_limitation_reason_tracker_
288       RTC_GUARDED_BY(mutex_);
289   rtc::RateTracker media_byte_rate_tracker_ RTC_GUARDED_BY(mutex_);
290   rtc::RateTracker encoded_frame_rate_tracker_ RTC_GUARDED_BY(mutex_);
291   std::map<uint32_t, std::unique_ptr<rtc::RateTracker>>
292       encoded_frame_rate_trackers_ RTC_GUARDED_BY(mutex_);
293 
294   absl::optional<int64_t> last_outlier_timestamp_ RTC_GUARDED_BY(mutex_);
295 
296   int last_num_spatial_layers_ RTC_GUARDED_BY(mutex_);
297   int last_num_simulcast_streams_ RTC_GUARDED_BY(mutex_);
298   std::array<bool, kMaxSpatialLayers> last_spatial_layer_use_
299       RTC_GUARDED_BY(mutex_);
300   // Indicates if the latest bitrate allocation had layers disabled by low
301   // available bandwidth.
302   bool bw_limited_layers_ RTC_GUARDED_BY(mutex_);
303   // Indicastes if the encoder internally downscales input image.
304   bool internal_encoder_scaler_ RTC_GUARDED_BY(mutex_);
305   Adaptations adaptation_limitations_ RTC_GUARDED_BY(mutex_);
306 
307   struct EncoderChangeEvent {
308     std::string previous_encoder_implementation;
309     std::string new_encoder_implementation;
310   };
311   // Stores the last change in encoder implementation in an optional, so that
312   // the event can be consumed.
313   absl::optional<EncoderChangeEvent> encoder_changed_;
314 
315   // Contains stats used for UMA histograms. These stats will be reset if
316   // content type changes between real-time video and screenshare, since these
317   // will be reported separately.
318   struct UmaSamplesContainer {
319     UmaSamplesContainer(const char* prefix,
320                         const VideoSendStream::Stats& start_stats,
321                         Clock* clock);
322     ~UmaSamplesContainer();
323 
324     void UpdateHistograms(const RtpConfig& rtp_config,
325                           const VideoSendStream::Stats& current_stats);
326 
327     void InitializeBitrateCounters(const VideoSendStream::Stats& stats);
328 
329     bool InsertEncodedFrame(const EncodedImage& encoded_frame,
330                             int simulcast_idx);
331     void RemoveOld(int64_t now_ms);
332 
333     const std::string uma_prefix_;
334     Clock* const clock_;
335     SampleCounter input_width_counter_;
336     SampleCounter input_height_counter_;
337     SampleCounter sent_width_counter_;
338     SampleCounter sent_height_counter_;
339     SampleCounter encode_time_counter_;
340     BoolSampleCounter key_frame_counter_;
341     BoolSampleCounter quality_limited_frame_counter_;
342     SampleCounter quality_downscales_counter_;
343     BoolSampleCounter cpu_limited_frame_counter_;
344     BoolSampleCounter bw_limited_frame_counter_;
345     SampleCounter bw_resolutions_disabled_counter_;
346     SampleCounter delay_counter_;
347     SampleCounter max_delay_counter_;
348     rtc::RateTracker input_frame_rate_tracker_;
349     RateCounter input_fps_counter_;
350     RateCounter sent_fps_counter_;
351     RateAccCounter total_byte_counter_;
352     RateAccCounter media_byte_counter_;
353     RateAccCounter rtx_byte_counter_;
354     RateAccCounter padding_byte_counter_;
355     RateAccCounter retransmit_byte_counter_;
356     RateAccCounter fec_byte_counter_;
357     int64_t first_rtcp_stats_time_ms_;
358     int64_t first_rtp_stats_time_ms_;
359     StatsTimer cpu_adapt_timer_;
360     StatsTimer quality_adapt_timer_;
361     BoolSampleCounter paused_time_counter_;
362     TargetRateUpdates target_rate_updates_;
363     BoolSampleCounter fallback_active_counter_;
364     FallbackEncoderInfo fallback_info_;
365     FallbackEncoderInfoDisabled fallback_info_disabled_;
366     ReportBlockStats report_block_stats_;
367     const VideoSendStream::Stats start_stats_;
368     size_t num_streams_;  // Number of configured streams to encoder.
369     size_t num_pixels_highest_stream_;
370     EncodedFrameMap encoded_frames_;
371     AdaptChanges initial_quality_changes_;
372 
373     std::map<int, QpCounters>
374         qp_counters_;  // QP counters mapped by spatial idx.
375   };
376 
377   std::unique_ptr<UmaSamplesContainer> uma_container_ RTC_GUARDED_BY(mutex_);
378 };
379 
380 }  // namespace webrtc
381 #endif  // VIDEO_SEND_STATISTICS_PROXY_H_
382