1 /*
2  *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_
12 #define VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_
13 
14 #include <list>
15 #include <memory>
16 
17 #include "absl/types/optional.h"
18 #include "api/task_queue/task_queue_base.h"
19 #include "api/video/video_stream_encoder_observer.h"
20 #include "modules/video_coding/utility/quality_scaler.h"
21 #include "rtc_base/constructor_magic.h"
22 #include "rtc_base/experiments/field_trial_parser.h"
23 #include "rtc_base/numerics/exp_filter.h"
24 #include "rtc_base/synchronization/sequence_checker.h"
25 #include "rtc_base/task_utils/repeating_task.h"
26 #include "rtc_base/thread_annotations.h"
27 
28 namespace webrtc {
29 
30 class VideoFrame;
31 
32 struct CpuOveruseOptions {
33   CpuOveruseOptions();
34 
35   int low_encode_usage_threshold_percent;  // Threshold for triggering underuse.
36   int high_encode_usage_threshold_percent;  // Threshold for triggering overuse.
37   // General settings.
38   int frame_timeout_interval_ms;  // The maximum allowed interval between two
39                                   // frames before resetting estimations.
40   int min_frame_samples;          // The minimum number of frames required.
41   int min_process_count;  // The number of initial process times required before
42                           // triggering an overuse/underuse.
43   int high_threshold_consecutive_count;  // The number of consecutive checks
44                                          // above the high threshold before
45                                          // triggering an overuse.
46   // New estimator enabled if this is set non-zero.
47   int filter_time_ms;  // Time constant for averaging
48 };
49 
50 // Use to detect system overuse based on the send-side processing time of
51 // incoming frames. All methods must be called on a single task queue but it can
52 // be created and destroyed on an arbitrary thread.
53 // OveruseFrameDetector::StartCheckForOveruse  must be called to periodically
54 // check for overuse.
55 class OveruseFrameDetector {
56  public:
57   explicit OveruseFrameDetector(CpuOveruseMetricsObserver* metrics_observer);
58   virtual ~OveruseFrameDetector();
59 
60   // Start to periodically check for overuse.
61   void StartCheckForOveruse(TaskQueueBase* task_queue_base,
62                             const CpuOveruseOptions& options,
63                             AdaptationObserverInterface* overuse_observer);
64 
65   // StopCheckForOveruse must be called before destruction if
66   // StartCheckForOveruse has been called.
67   void StopCheckForOveruse();
68 
69   // Defines the current maximum framerate targeted by the capturer. This is
70   // used to make sure the encode usage percent doesn't drop unduly if the
71   // capturer has quiet periods (for instance caused by screen capturers with
72   // variable capture rate depending on content updates), otherwise we might
73   // experience adaptation toggling.
74   virtual void OnTargetFramerateUpdated(int framerate_fps);
75 
76   // Called for each captured frame.
77   void FrameCaptured(const VideoFrame& frame, int64_t time_when_first_seen_us);
78 
79   // Called for each sent frame.
80   void FrameSent(uint32_t timestamp,
81                  int64_t time_sent_in_us,
82                  int64_t capture_time_us,
83                  absl::optional<int> encode_duration_us);
84 
85   // Interface for cpu load estimation. Intended for internal use only.
86   class ProcessingUsage {
87    public:
88     virtual void Reset() = 0;
89     virtual void SetMaxSampleDiffMs(float diff_ms) = 0;
90     virtual void FrameCaptured(const VideoFrame& frame,
91                                int64_t time_when_first_seen_us,
92                                int64_t last_capture_time_us) = 0;
93     // Returns encode_time in us, if there's a new measurement.
94     virtual absl::optional<int> FrameSent(
95         // These two argument used by old estimator.
96         uint32_t timestamp,
97         int64_t time_sent_in_us,
98         // And these two by the new estimator.
99         int64_t capture_time_us,
100         absl::optional<int> encode_duration_us) = 0;
101 
102     virtual int Value() = 0;
103     virtual ~ProcessingUsage() = default;
104   };
105 
106  protected:
107   // Protected for test purposes.
108   void CheckForOveruse(AdaptationObserverInterface* overuse_observer);
109   void SetOptions(const CpuOveruseOptions& options);
110 
111   CpuOveruseOptions options_;
112 
113  private:
114   void EncodedFrameTimeMeasured(int encode_duration_ms);
115   bool IsOverusing(int encode_usage_percent);
116   bool IsUnderusing(int encode_usage_percent, int64_t time_now);
117 
118   bool FrameTimeoutDetected(int64_t now) const;
119   bool FrameSizeChanged(int num_pixels) const;
120 
121   void ResetAll(int num_pixels);
122 
123   static std::unique_ptr<ProcessingUsage> CreateProcessingUsage(
124       const CpuOveruseOptions& options);
125 
126   SequenceChecker task_checker_;
127   // Owned by the task queue from where StartCheckForOveruse is called.
128   RepeatingTaskHandle check_overuse_task_ RTC_GUARDED_BY(task_checker_);
129 
130   // Stats metrics.
131   CpuOveruseMetricsObserver* const metrics_observer_;
132   absl::optional<int> encode_usage_percent_ RTC_GUARDED_BY(task_checker_);
133 
134   int64_t num_process_times_ RTC_GUARDED_BY(task_checker_);
135 
136   int64_t last_capture_time_us_ RTC_GUARDED_BY(task_checker_);
137 
138   // Number of pixels of last captured frame.
139   int num_pixels_ RTC_GUARDED_BY(task_checker_);
140   int max_framerate_ RTC_GUARDED_BY(task_checker_);
141   int64_t last_overuse_time_ms_ RTC_GUARDED_BY(task_checker_);
142   int checks_above_threshold_ RTC_GUARDED_BY(task_checker_);
143   int num_overuse_detections_ RTC_GUARDED_BY(task_checker_);
144   int64_t last_rampup_time_ms_ RTC_GUARDED_BY(task_checker_);
145   bool in_quick_rampup_ RTC_GUARDED_BY(task_checker_);
146   int current_rampup_delay_ms_ RTC_GUARDED_BY(task_checker_);
147 
148   std::unique_ptr<ProcessingUsage> usage_ RTC_PT_GUARDED_BY(task_checker_);
149 
150   // If set by field trial, overrides CpuOveruseOptions::filter_time_ms.
151   FieldTrialOptional<TimeDelta> filter_time_constant_{"tau"};
152 
153   RTC_DISALLOW_COPY_AND_ASSIGN(OveruseFrameDetector);
154 };
155 
156 }  // namespace webrtc
157 
158 #endif  // VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_
159