1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_ 12 #define VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_ 13 14 #include <list> 15 #include <memory> 16 17 #include "absl/types/optional.h" 18 #include "api/sequence_checker.h" 19 #include "api/task_queue/task_queue_base.h" 20 #include "api/video/video_stream_encoder_observer.h" 21 #include "rtc_base/constructor_magic.h" 22 #include "rtc_base/experiments/field_trial_parser.h" 23 #include "rtc_base/numerics/exp_filter.h" 24 #include "rtc_base/system/no_unique_address.h" 25 #include "rtc_base/task_utils/repeating_task.h" 26 #include "rtc_base/thread_annotations.h" 27 28 namespace webrtc { 29 30 class VideoFrame; 31 32 struct CpuOveruseOptions { 33 CpuOveruseOptions(); 34 35 int low_encode_usage_threshold_percent; // Threshold for triggering underuse. 36 int high_encode_usage_threshold_percent; // Threshold for triggering overuse. 37 // General settings. 38 int frame_timeout_interval_ms; // The maximum allowed interval between two 39 // frames before resetting estimations. 40 int min_frame_samples; // The minimum number of frames required. 41 int min_process_count; // The number of initial process times required before 42 // triggering an overuse/underuse. 43 int high_threshold_consecutive_count; // The number of consecutive checks 44 // above the high threshold before 45 // triggering an overuse. 46 // New estimator enabled if this is set non-zero. 47 int filter_time_ms; // Time constant for averaging 48 }; 49 50 class OveruseFrameDetectorObserverInterface { 51 public: 52 // Called to signal that we can handle larger or more frequent frames. 53 virtual void AdaptUp() = 0; 54 // Called to signal that the source should reduce the resolution or framerate. 55 virtual void AdaptDown() = 0; 56 57 protected: ~OveruseFrameDetectorObserverInterface()58 virtual ~OveruseFrameDetectorObserverInterface() {} 59 }; 60 61 // Use to detect system overuse based on the send-side processing time of 62 // incoming frames. All methods must be called on a single task queue but it can 63 // be created and destroyed on an arbitrary thread. 64 // OveruseFrameDetector::StartCheckForOveruse must be called to periodically 65 // check for overuse. 66 class OveruseFrameDetector { 67 public: 68 explicit OveruseFrameDetector(CpuOveruseMetricsObserver* metrics_observer); 69 virtual ~OveruseFrameDetector(); 70 71 // Start to periodically check for overuse. 72 void StartCheckForOveruse( 73 TaskQueueBase* task_queue_base, 74 const CpuOveruseOptions& options, 75 OveruseFrameDetectorObserverInterface* overuse_observer); 76 77 // StopCheckForOveruse must be called before destruction if 78 // StartCheckForOveruse has been called. 79 void StopCheckForOveruse(); 80 81 // Defines the current maximum framerate targeted by the capturer. This is 82 // used to make sure the encode usage percent doesn't drop unduly if the 83 // capturer has quiet periods (for instance caused by screen capturers with 84 // variable capture rate depending on content updates), otherwise we might 85 // experience adaptation toggling. 86 virtual void OnTargetFramerateUpdated(int framerate_fps); 87 88 // Called for each captured frame. 89 void FrameCaptured(const VideoFrame& frame, int64_t time_when_first_seen_us); 90 91 // Called for each sent frame. 92 void FrameSent(uint32_t timestamp, 93 int64_t time_sent_in_us, 94 int64_t capture_time_us, 95 absl::optional<int> encode_duration_us); 96 97 // Interface for cpu load estimation. Intended for internal use only. 98 class ProcessingUsage { 99 public: 100 virtual void Reset() = 0; 101 virtual void SetMaxSampleDiffMs(float diff_ms) = 0; 102 virtual void FrameCaptured(const VideoFrame& frame, 103 int64_t time_when_first_seen_us, 104 int64_t last_capture_time_us) = 0; 105 // Returns encode_time in us, if there's a new measurement. 106 virtual absl::optional<int> FrameSent( 107 // These two argument used by old estimator. 108 uint32_t timestamp, 109 int64_t time_sent_in_us, 110 // And these two by the new estimator. 111 int64_t capture_time_us, 112 absl::optional<int> encode_duration_us) = 0; 113 114 virtual int Value() = 0; 115 virtual ~ProcessingUsage() = default; 116 }; 117 118 protected: 119 // Protected for test purposes. 120 void CheckForOveruse(OveruseFrameDetectorObserverInterface* overuse_observer); 121 void SetOptions(const CpuOveruseOptions& options); 122 123 CpuOveruseOptions options_; 124 125 private: 126 void EncodedFrameTimeMeasured(int encode_duration_ms); 127 bool IsOverusing(int encode_usage_percent); 128 bool IsUnderusing(int encode_usage_percent, int64_t time_now); 129 130 bool FrameTimeoutDetected(int64_t now) const; 131 bool FrameSizeChanged(int num_pixels) const; 132 133 void ResetAll(int num_pixels); 134 135 static std::unique_ptr<ProcessingUsage> CreateProcessingUsage( 136 const CpuOveruseOptions& options); 137 138 RTC_NO_UNIQUE_ADDRESS SequenceChecker task_checker_; 139 // Owned by the task queue from where StartCheckForOveruse is called. 140 RepeatingTaskHandle check_overuse_task_ RTC_GUARDED_BY(task_checker_); 141 142 // Stats metrics. 143 CpuOveruseMetricsObserver* const metrics_observer_; 144 absl::optional<int> encode_usage_percent_ RTC_GUARDED_BY(task_checker_); 145 146 int64_t num_process_times_ RTC_GUARDED_BY(task_checker_); 147 148 int64_t last_capture_time_us_ RTC_GUARDED_BY(task_checker_); 149 150 // Number of pixels of last captured frame. 151 int num_pixels_ RTC_GUARDED_BY(task_checker_); 152 int max_framerate_ RTC_GUARDED_BY(task_checker_); 153 int64_t last_overuse_time_ms_ RTC_GUARDED_BY(task_checker_); 154 int checks_above_threshold_ RTC_GUARDED_BY(task_checker_); 155 int num_overuse_detections_ RTC_GUARDED_BY(task_checker_); 156 int64_t last_rampup_time_ms_ RTC_GUARDED_BY(task_checker_); 157 bool in_quick_rampup_ RTC_GUARDED_BY(task_checker_); 158 int current_rampup_delay_ms_ RTC_GUARDED_BY(task_checker_); 159 160 std::unique_ptr<ProcessingUsage> usage_ RTC_PT_GUARDED_BY(task_checker_); 161 162 // If set by field trial, overrides CpuOveruseOptions::filter_time_ms. 163 FieldTrialOptional<TimeDelta> filter_time_constant_{"tau"}; 164 165 RTC_DISALLOW_COPY_AND_ASSIGN(OveruseFrameDetector); 166 }; 167 168 } // namespace webrtc 169 170 #endif // VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_ 171