1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_VIDEO_STREAM_ENCODER_H_ 12 #define VIDEO_VIDEO_STREAM_ENCODER_H_ 13 14 #include <atomic> 15 #include <map> 16 #include <memory> 17 #include <string> 18 #include <vector> 19 20 #include "api/adaptation/resource.h" 21 #include "api/sequence_checker.h" 22 #include "api/units/data_rate.h" 23 #include "api/video/video_bitrate_allocator.h" 24 #include "api/video/video_rotation.h" 25 #include "api/video/video_sink_interface.h" 26 #include "api/video/video_stream_encoder_interface.h" 27 #include "api/video/video_stream_encoder_observer.h" 28 #include "api/video/video_stream_encoder_settings.h" 29 #include "api/video_codecs/video_codec.h" 30 #include "api/video_codecs/video_encoder.h" 31 #include "call/adaptation/adaptation_constraint.h" 32 #include "call/adaptation/resource_adaptation_processor.h" 33 #include "call/adaptation/resource_adaptation_processor_interface.h" 34 #include "call/adaptation/video_source_restrictions.h" 35 #include "call/adaptation/video_stream_input_state_provider.h" 36 #include "modules/video_coding/utility/frame_dropper.h" 37 #include "modules/video_coding/utility/qp_parser.h" 38 #include "rtc_base/experiments/rate_control_settings.h" 39 #include "rtc_base/numerics/exp_filter.h" 40 #include "rtc_base/race_checker.h" 41 #include "rtc_base/rate_statistics.h" 42 #include "rtc_base/task_queue.h" 43 #include "rtc_base/task_utils/pending_task_safety_flag.h" 44 #include "rtc_base/thread_annotations.h" 45 #include "system_wrappers/include/clock.h" 46 #include "video/adaptation/video_stream_encoder_resource_manager.h" 47 #include "video/encoder_bitrate_adjuster.h" 48 #include "video/frame_encode_metadata_writer.h" 49 #include "video/video_source_sink_controller.h" 50 51 namespace webrtc { 52 53 // VideoStreamEncoder represent a video encoder that accepts raw video frames as 54 // input and produces an encoded bit stream. 55 // Usage: 56 // Instantiate. 57 // Call SetSink. 58 // Call SetSource. 59 // Call ConfigureEncoder with the codec settings. 60 // Call Stop() when done. 61 class VideoStreamEncoder : public VideoStreamEncoderInterface, 62 private EncodedImageCallback, 63 public VideoSourceRestrictionsListener { 64 public: 65 // TODO(bugs.webrtc.org/12000): Reporting of VideoBitrateAllocation is being 66 // deprecated. Instead VideoLayersAllocation should be reported. 67 enum class BitrateAllocationCallbackType { 68 kVideoBitrateAllocation, 69 kVideoBitrateAllocationWhenScreenSharing, 70 kVideoLayersAllocation 71 }; 72 VideoStreamEncoder(Clock* clock, 73 uint32_t number_of_cores, 74 VideoStreamEncoderObserver* encoder_stats_observer, 75 const VideoStreamEncoderSettings& settings, 76 std::unique_ptr<OveruseFrameDetector> overuse_detector, 77 TaskQueueFactory* task_queue_factory, 78 BitrateAllocationCallbackType allocation_cb_type); 79 ~VideoStreamEncoder() override; 80 81 void AddAdaptationResource(rtc::scoped_refptr<Resource> resource) override; 82 std::vector<rtc::scoped_refptr<Resource>> GetAdaptationResources() override; 83 84 void SetSource(rtc::VideoSourceInterface<VideoFrame>* source, 85 const DegradationPreference& degradation_preference) override; 86 87 void SetSink(EncoderSink* sink, bool rotation_applied) override; 88 89 // TODO(perkj): Can we remove VideoCodec.startBitrate ? 90 void SetStartBitrate(int start_bitrate_bps) override; 91 92 void SetFecControllerOverride( 93 FecControllerOverride* fec_controller_override) override; 94 95 void ConfigureEncoder(VideoEncoderConfig config, 96 size_t max_data_payload_length) override; 97 98 // Permanently stop encoding. After this method has returned, it is 99 // guaranteed that no encoded frames will be delivered to the sink. 100 void Stop() override; 101 102 void SendKeyFrame() override; 103 104 void OnLossNotification( 105 const VideoEncoder::LossNotification& loss_notification) override; 106 107 void OnBitrateUpdated(DataRate target_bitrate, 108 DataRate stable_target_bitrate, 109 DataRate target_headroom, 110 uint8_t fraction_lost, 111 int64_t round_trip_time_ms, 112 double cwnd_reduce_ratio) override; 113 114 DataRate UpdateTargetBitrate(DataRate target_bitrate, 115 double cwnd_reduce_ratio); 116 117 protected: 118 // Used for testing. For example the |ScalingObserverInterface| methods must 119 // be called on |encoder_queue_|. encoder_queue()120 rtc::TaskQueue* encoder_queue() { return &encoder_queue_; } 121 122 void OnVideoSourceRestrictionsUpdated( 123 VideoSourceRestrictions restrictions, 124 const VideoAdaptationCounters& adaptation_counters, 125 rtc::scoped_refptr<Resource> reason, 126 const VideoSourceRestrictions& unfiltered_restrictions) override; 127 128 // Used for injected test resources. 129 // TODO(eshr): Move all adaptation tests out of VideoStreamEncoder tests. 130 void InjectAdaptationResource(rtc::scoped_refptr<Resource> resource, 131 VideoAdaptationReason reason); 132 void InjectAdaptationConstraint(AdaptationConstraint* adaptation_constraint); 133 134 void AddRestrictionsListenerForTesting( 135 VideoSourceRestrictionsListener* restrictions_listener); 136 void RemoveRestrictionsListenerForTesting( 137 VideoSourceRestrictionsListener* restrictions_listener); 138 139 private: 140 class VideoFrameInfo { 141 public: VideoFrameInfo(int width,int height,bool is_texture)142 VideoFrameInfo(int width, int height, bool is_texture) 143 : width(width), height(height), is_texture(is_texture) {} 144 int width; 145 int height; 146 bool is_texture; pixel_count()147 int pixel_count() const { return width * height; } 148 }; 149 150 struct EncoderRateSettings { 151 EncoderRateSettings(); 152 EncoderRateSettings(const VideoBitrateAllocation& bitrate, 153 double framerate_fps, 154 DataRate bandwidth_allocation, 155 DataRate encoder_target, 156 DataRate stable_encoder_target); 157 bool operator==(const EncoderRateSettings& rhs) const; 158 bool operator!=(const EncoderRateSettings& rhs) const; 159 160 VideoEncoder::RateControlParameters rate_control; 161 // This is the scalar target bitrate before the VideoBitrateAllocator, i.e. 162 // the |target_bitrate| argument of the OnBitrateUpdated() method. This is 163 // needed because the bitrate allocator may truncate the total bitrate and a 164 // later call to the same allocator instance, e.g. 165 // |using last_encoder_rate_setings_->bitrate.get_sum_bps()|, may trick it 166 // into thinking the available bitrate has decreased since the last call. 167 DataRate encoder_target; 168 DataRate stable_encoder_target; 169 }; 170 171 class DegradationPreferenceManager; 172 173 void ReconfigureEncoder() RTC_RUN_ON(&encoder_queue_); 174 void OnEncoderSettingsChanged() RTC_RUN_ON(&encoder_queue_); 175 176 // Implements VideoSinkInterface. 177 void OnFrame(const VideoFrame& video_frame) override; 178 void OnDiscardedFrame() override; 179 180 void MaybeEncodeVideoFrame(const VideoFrame& frame, 181 int64_t time_when_posted_in_ms); 182 183 void EncodeVideoFrame(const VideoFrame& frame, 184 int64_t time_when_posted_in_ms); 185 // Indicates whether frame should be dropped because the pixel count is too 186 // large for the current bitrate configuration. 187 bool DropDueToSize(uint32_t pixel_count) const RTC_RUN_ON(&encoder_queue_); 188 189 // Implements EncodedImageCallback. 190 EncodedImageCallback::Result OnEncodedImage( 191 const EncodedImage& encoded_image, 192 const CodecSpecificInfo* codec_specific_info) override; 193 194 void OnDroppedFrame(EncodedImageCallback::DropReason reason) override; 195 196 bool EncoderPaused() const; 197 void TraceFrameDropStart(); 198 void TraceFrameDropEnd(); 199 200 // Returns a copy of |rate_settings| with the |bitrate| field updated using 201 // the current VideoBitrateAllocator. 202 EncoderRateSettings UpdateBitrateAllocation( 203 const EncoderRateSettings& rate_settings) RTC_RUN_ON(&encoder_queue_); 204 205 uint32_t GetInputFramerateFps() RTC_RUN_ON(&encoder_queue_); 206 void SetEncoderRates(const EncoderRateSettings& rate_settings) 207 RTC_RUN_ON(&encoder_queue_); 208 209 void RunPostEncode(const EncodedImage& encoded_image, 210 int64_t time_sent_us, 211 int temporal_index, 212 DataSize frame_size); 213 bool HasInternalSource() const RTC_RUN_ON(&encoder_queue_); 214 void ReleaseEncoder() RTC_RUN_ON(&encoder_queue_); 215 // After calling this function |resource_adaptation_processor_| will be null. 216 void ShutdownResourceAdaptationQueue(); 217 218 void CheckForAnimatedContent(const VideoFrame& frame, 219 int64_t time_when_posted_in_ms) 220 RTC_RUN_ON(&encoder_queue_); 221 222 // TODO(bugs.webrtc.org/11341) : Remove this version of RequestEncoderSwitch. 223 void QueueRequestEncoderSwitch( 224 const EncoderSwitchRequestCallback::Config& conf) 225 RTC_RUN_ON(&encoder_queue_); 226 void QueueRequestEncoderSwitch(const webrtc::SdpVideoFormat& format) 227 RTC_RUN_ON(&encoder_queue_); 228 229 TaskQueueBase* const main_queue_; 230 231 const uint32_t number_of_cores_; 232 233 EncoderSink* sink_; 234 const VideoStreamEncoderSettings settings_; 235 const BitrateAllocationCallbackType allocation_cb_type_; 236 const RateControlSettings rate_control_settings_; 237 238 std::unique_ptr<VideoEncoderFactory::EncoderSelectorInterface> const 239 encoder_selector_; 240 VideoStreamEncoderObserver* const encoder_stats_observer_; 241 242 VideoEncoderConfig encoder_config_ RTC_GUARDED_BY(&encoder_queue_); 243 std::unique_ptr<VideoEncoder> encoder_ RTC_GUARDED_BY(&encoder_queue_) 244 RTC_PT_GUARDED_BY(&encoder_queue_); 245 bool encoder_initialized_; 246 std::unique_ptr<VideoBitrateAllocator> rate_allocator_ 247 RTC_GUARDED_BY(&encoder_queue_) RTC_PT_GUARDED_BY(&encoder_queue_); 248 int max_framerate_ RTC_GUARDED_BY(&encoder_queue_); 249 250 // Set when ConfigureEncoder has been called in order to lazy reconfigure the 251 // encoder on the next frame. 252 bool pending_encoder_reconfiguration_ RTC_GUARDED_BY(&encoder_queue_); 253 // Set when configuration must create a new encoder object, e.g., 254 // because of a codec change. 255 bool pending_encoder_creation_ RTC_GUARDED_BY(&encoder_queue_); 256 257 absl::optional<VideoFrameInfo> last_frame_info_ 258 RTC_GUARDED_BY(&encoder_queue_); 259 int crop_width_ RTC_GUARDED_BY(&encoder_queue_); 260 int crop_height_ RTC_GUARDED_BY(&encoder_queue_); 261 absl::optional<uint32_t> encoder_target_bitrate_bps_ 262 RTC_GUARDED_BY(&encoder_queue_); 263 size_t max_data_payload_length_ RTC_GUARDED_BY(&encoder_queue_); 264 absl::optional<EncoderRateSettings> last_encoder_rate_settings_ 265 RTC_GUARDED_BY(&encoder_queue_); 266 bool encoder_paused_and_dropped_frame_ RTC_GUARDED_BY(&encoder_queue_); 267 268 // Set to true if at least one frame was sent to encoder since last encoder 269 // initialization. 270 bool was_encode_called_since_last_initialization_ 271 RTC_GUARDED_BY(&encoder_queue_); 272 273 bool encoder_failed_ RTC_GUARDED_BY(&encoder_queue_); 274 Clock* const clock_; 275 276 rtc::RaceChecker incoming_frame_race_checker_ 277 RTC_GUARDED_BY(incoming_frame_race_checker_); 278 std::atomic<int> posted_frames_waiting_for_encode_; 279 // Used to make sure incoming time stamp is increasing for every frame. 280 int64_t last_captured_timestamp_ RTC_GUARDED_BY(incoming_frame_race_checker_); 281 // Delta used for translating between NTP and internal timestamps. 282 const int64_t delta_ntp_internal_ms_ 283 RTC_GUARDED_BY(incoming_frame_race_checker_); 284 285 int64_t last_frame_log_ms_ RTC_GUARDED_BY(incoming_frame_race_checker_); 286 int captured_frame_count_ RTC_GUARDED_BY(&encoder_queue_); 287 int dropped_frame_cwnd_pushback_count_ RTC_GUARDED_BY(&encoder_queue_); 288 int dropped_frame_encoder_block_count_ RTC_GUARDED_BY(&encoder_queue_); 289 absl::optional<VideoFrame> pending_frame_ RTC_GUARDED_BY(&encoder_queue_); 290 int64_t pending_frame_post_time_us_ RTC_GUARDED_BY(&encoder_queue_); 291 292 VideoFrame::UpdateRect accumulated_update_rect_ 293 RTC_GUARDED_BY(&encoder_queue_); 294 bool accumulated_update_rect_is_valid_ RTC_GUARDED_BY(&encoder_queue_); 295 296 // Used for automatic content type detection. 297 absl::optional<VideoFrame::UpdateRect> last_update_rect_ 298 RTC_GUARDED_BY(&encoder_queue_); 299 Timestamp animation_start_time_ RTC_GUARDED_BY(&encoder_queue_); 300 bool cap_resolution_due_to_video_content_ RTC_GUARDED_BY(&encoder_queue_); 301 // Used to correctly ignore changes in update_rect introduced by 302 // resize triggered by animation detection. 303 enum class ExpectResizeState { 304 kNoResize, // Normal operation. 305 kResize, // Resize was triggered by the animation detection. 306 kFirstFrameAfterResize // Resize observed. 307 } expect_resize_state_ RTC_GUARDED_BY(&encoder_queue_); 308 309 FecControllerOverride* fec_controller_override_ 310 RTC_GUARDED_BY(&encoder_queue_); 311 absl::optional<int64_t> last_parameters_update_ms_ 312 RTC_GUARDED_BY(&encoder_queue_); 313 absl::optional<int64_t> last_encode_info_ms_ RTC_GUARDED_BY(&encoder_queue_); 314 315 VideoEncoder::EncoderInfo encoder_info_ RTC_GUARDED_BY(&encoder_queue_); 316 VideoEncoderFactory::CodecInfo codec_info_ RTC_GUARDED_BY(&encoder_queue_); 317 VideoCodec send_codec_ RTC_GUARDED_BY(&encoder_queue_); 318 319 FrameDropper frame_dropper_ RTC_GUARDED_BY(&encoder_queue_); 320 // If frame dropper is not force disabled, frame dropping might still be 321 // disabled if VideoEncoder::GetEncoderInfo() indicates that the encoder has a 322 // trusted rate controller. This is determined on a per-frame basis, as the 323 // encoder behavior might dynamically change. 324 bool force_disable_frame_dropper_ RTC_GUARDED_BY(&encoder_queue_); 325 RateStatistics input_framerate_ RTC_GUARDED_BY(&encoder_queue_); 326 // Incremented on worker thread whenever |frame_dropper_| determines that a 327 // frame should be dropped. Decremented on whichever thread runs 328 // OnEncodedImage(), which is only called by one thread but not necessarily 329 // the worker thread. 330 std::atomic<int> pending_frame_drops_; 331 332 // Congestion window frame drop ratio (drop 1 in every 333 // cwnd_frame_drop_interval_ frames). 334 absl::optional<int> cwnd_frame_drop_interval_ RTC_GUARDED_BY(&encoder_queue_); 335 // Frame counter for congestion window frame drop. 336 int cwnd_frame_counter_ RTC_GUARDED_BY(&encoder_queue_); 337 338 std::unique_ptr<EncoderBitrateAdjuster> bitrate_adjuster_ 339 RTC_GUARDED_BY(&encoder_queue_); 340 341 // TODO(sprang): Change actually support keyframe per simulcast stream, or 342 // turn this into a simple bool |pending_keyframe_request_|. 343 std::vector<VideoFrameType> next_frame_types_ RTC_GUARDED_BY(&encoder_queue_); 344 345 FrameEncodeMetadataWriter frame_encode_metadata_writer_; 346 347 // Experiment groups parsed from field trials for realtime video ([0]) and 348 // screenshare ([1]). 0 means no group specified. Positive values are 349 // experiment group numbers incremented by 1. 350 const std::array<uint8_t, 2> experiment_groups_; 351 352 struct AutomaticAnimationDetectionExperiment { 353 bool enabled = false; 354 int min_duration_ms = 2000; 355 double min_area_ratio = 0.8; 356 int min_fps = 10; ParserAutomaticAnimationDetectionExperiment357 std::unique_ptr<StructParametersParser> Parser() { 358 return StructParametersParser::Create( 359 "enabled", &enabled, // 360 "min_duration_ms", &min_duration_ms, // 361 "min_area_ratio", &min_area_ratio, // 362 "min_fps", &min_fps); 363 } 364 }; 365 366 AutomaticAnimationDetectionExperiment 367 ParseAutomatincAnimationDetectionFieldTrial() const; 368 369 AutomaticAnimationDetectionExperiment 370 automatic_animation_detection_experiment_ RTC_GUARDED_BY(&encoder_queue_); 371 372 // Provides video stream input states: current resolution and frame rate. 373 VideoStreamInputStateProvider input_state_provider_; 374 375 std::unique_ptr<VideoStreamAdapter> video_stream_adapter_ 376 RTC_GUARDED_BY(&encoder_queue_); 377 // Responsible for adapting input resolution or frame rate to ensure resources 378 // (e.g. CPU or bandwidth) are not overused. Adding resources can occur on any 379 // thread. 380 std::unique_ptr<ResourceAdaptationProcessorInterface> 381 resource_adaptation_processor_; 382 std::unique_ptr<DegradationPreferenceManager> degradation_preference_manager_ 383 RTC_GUARDED_BY(&encoder_queue_); 384 std::vector<AdaptationConstraint*> adaptation_constraints_ 385 RTC_GUARDED_BY(&encoder_queue_); 386 // Handles input, output and stats reporting related to VideoStreamEncoder 387 // specific resources, such as "encode usage percent" measurements and "QP 388 // scaling". Also involved with various mitigations such as initial frame 389 // dropping. 390 // The manager primarily operates on the |encoder_queue_| but its lifetime is 391 // tied to the VideoStreamEncoder (which is destroyed off the encoder queue) 392 // and its resource list is accessible from any thread. 393 VideoStreamEncoderResourceManager stream_resource_manager_ 394 RTC_GUARDED_BY(&encoder_queue_); 395 std::vector<rtc::scoped_refptr<Resource>> additional_resources_ 396 RTC_GUARDED_BY(&encoder_queue_); 397 // Carries out the VideoSourceRestrictions provided by the 398 // ResourceAdaptationProcessor, i.e. reconfigures the source of video frames 399 // to provide us with different resolution or frame rate. 400 // This class is thread-safe. 401 VideoSourceSinkController video_source_sink_controller_ 402 RTC_GUARDED_BY(main_queue_); 403 404 // Default bitrate limits in EncoderInfoSettings allowed. 405 const bool default_limits_allowed_; 406 407 // QP parser is used to extract QP value from encoded frame when that is not 408 // provided by encoder. 409 QpParser qp_parser_; 410 const bool qp_parsing_allowed_; 411 412 // Public methods are proxied to the task queues. The queues must be destroyed 413 // first to make sure no tasks run that use other members. 414 rtc::TaskQueue encoder_queue_; 415 416 // Used to cancel any potentially pending tasks to the main thread. 417 ScopedTaskSafety task_safety_; 418 419 RTC_DISALLOW_COPY_AND_ASSIGN(VideoStreamEncoder); 420 }; 421 422 } // namespace webrtc 423 424 #endif // VIDEO_VIDEO_STREAM_ENCODER_H_ 425