1 /* 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 * 10 */ 11 12 #ifndef MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_ 13 #define MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_ 14 15 #include <list> 16 #include <memory> 17 #include <stack> 18 #include <string> 19 #include <utility> 20 #include <vector> 21 22 #include "absl/types/optional.h" 23 #include "api/fec_controller_override.h" 24 #include "api/sequence_checker.h" 25 #include "api/video_codecs/sdp_video_format.h" 26 #include "api/video_codecs/video_encoder.h" 27 #include "api/video_codecs/video_encoder_factory.h" 28 #include "modules/video_coding/include/video_codec_interface.h" 29 #include "modules/video_coding/utility/framerate_controller.h" 30 #include "rtc_base/atomic_ops.h" 31 #include "rtc_base/experiments/encoder_info_settings.h" 32 #include "rtc_base/system/no_unique_address.h" 33 #include "rtc_base/system/rtc_export.h" 34 35 namespace webrtc { 36 37 // SimulcastEncoderAdapter implements simulcast support by creating multiple 38 // webrtc::VideoEncoder instances with the given VideoEncoderFactory. 39 // The object is created and destroyed on the worker thread, but all public 40 // interfaces should be called from the encoder task queue. 41 class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder { 42 public: 43 // TODO(bugs.webrtc.org/11000): Remove when downstream usage is gone. 44 SimulcastEncoderAdapter(VideoEncoderFactory* primarty_factory, 45 const SdpVideoFormat& format); 46 // |primary_factory| produces the first-choice encoders to use. 47 // |fallback_factory|, if non-null, is used to create fallback encoder that 48 // will be used if InitEncode() fails for the primary encoder. 49 SimulcastEncoderAdapter(VideoEncoderFactory* primary_factory, 50 VideoEncoderFactory* fallback_factory, 51 const SdpVideoFormat& format); 52 ~SimulcastEncoderAdapter() override; 53 54 // Implements VideoEncoder. 55 void SetFecControllerOverride( 56 FecControllerOverride* fec_controller_override) override; 57 int Release() override; 58 int InitEncode(const VideoCodec* codec_settings, 59 const VideoEncoder::Settings& settings) override; 60 int Encode(const VideoFrame& input_image, 61 const std::vector<VideoFrameType>* frame_types) override; 62 int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override; 63 void SetRates(const RateControlParameters& parameters) override; 64 void OnPacketLossRateUpdate(float packet_loss_rate) override; 65 void OnRttUpdate(int64_t rtt_ms) override; 66 void OnLossNotification(const LossNotification& loss_notification) override; 67 68 EncoderInfo GetEncoderInfo() const override; 69 70 private: 71 class EncoderContext { 72 public: 73 EncoderContext(std::unique_ptr<VideoEncoder> encoder, 74 bool prefer_temporal_support); 75 EncoderContext& operator=(EncoderContext&&) = delete; 76 encoder()77 VideoEncoder& encoder() { return *encoder_; } prefer_temporal_support()78 bool prefer_temporal_support() { return prefer_temporal_support_; } 79 void Release(); 80 81 private: 82 std::unique_ptr<VideoEncoder> encoder_; 83 bool prefer_temporal_support_; 84 }; 85 86 class StreamContext : public EncodedImageCallback { 87 public: 88 StreamContext(SimulcastEncoderAdapter* parent, 89 std::unique_ptr<EncoderContext> encoder_context, 90 std::unique_ptr<FramerateController> framerate_controller, 91 int stream_idx, 92 uint16_t width, 93 uint16_t height, 94 bool send_stream); 95 StreamContext(StreamContext&& rhs); 96 StreamContext& operator=(StreamContext&&) = delete; 97 ~StreamContext() override; 98 99 Result OnEncodedImage( 100 const EncodedImage& encoded_image, 101 const CodecSpecificInfo* codec_specific_info) override; 102 void OnDroppedFrame(DropReason reason) override; 103 encoder()104 VideoEncoder& encoder() { return encoder_context_->encoder(); } encoder()105 const VideoEncoder& encoder() const { return encoder_context_->encoder(); } stream_idx()106 int stream_idx() const { return stream_idx_; } width()107 uint16_t width() const { return width_; } height()108 uint16_t height() const { return height_; } is_keyframe_needed()109 bool is_keyframe_needed() const { 110 return !is_paused_ && is_keyframe_needed_; 111 } set_is_keyframe_needed()112 void set_is_keyframe_needed() { is_keyframe_needed_ = true; } is_paused()113 bool is_paused() const { return is_paused_; } set_is_paused(bool is_paused)114 void set_is_paused(bool is_paused) { is_paused_ = is_paused; } target_fps()115 absl::optional<float> target_fps() const { 116 return framerate_controller_ == nullptr 117 ? absl::nullopt 118 : absl::optional<float>( 119 framerate_controller_->GetTargetRate()); 120 } 121 122 std::unique_ptr<EncoderContext> ReleaseEncoderContext() &&; 123 void OnKeyframe(Timestamp timestamp); 124 bool ShouldDropFrame(Timestamp timestamp); 125 126 private: 127 SimulcastEncoderAdapter* const parent_; 128 std::unique_ptr<EncoderContext> encoder_context_; 129 std::unique_ptr<FramerateController> framerate_controller_; 130 const int stream_idx_; 131 const uint16_t width_; 132 const uint16_t height_; 133 bool is_keyframe_needed_; 134 bool is_paused_; 135 }; 136 137 bool Initialized() const; 138 139 void DestroyStoredEncoders(); 140 141 std::unique_ptr<EncoderContext> FetchOrCreateEncoderContext( 142 bool is_lowest_quality_stream); 143 144 webrtc::VideoCodec MakeStreamCodec(const webrtc::VideoCodec& codec, 145 int stream_idx, 146 uint32_t start_bitrate_kbps, 147 bool is_lowest_quality_stream, 148 bool is_highest_quality_stream); 149 150 EncodedImageCallback::Result OnEncodedImage( 151 size_t stream_idx, 152 const EncodedImage& encoded_image, 153 const CodecSpecificInfo* codec_specific_info); 154 155 void OnDroppedFrame(size_t stream_idx); 156 157 void OverrideFromFieldTrial(VideoEncoder::EncoderInfo* info) const; 158 159 volatile int inited_; // Accessed atomically. 160 VideoEncoderFactory* const primary_encoder_factory_; 161 VideoEncoderFactory* const fallback_encoder_factory_; 162 const SdpVideoFormat video_format_; 163 VideoCodec codec_; 164 int total_streams_count_; 165 bool bypass_mode_; 166 std::vector<StreamContext> stream_contexts_; 167 EncodedImageCallback* encoded_complete_callback_; 168 169 // Used for checking the single-threaded access of the encoder interface. 170 RTC_NO_UNIQUE_ADDRESS SequenceChecker encoder_queue_; 171 172 // Store encoders in between calls to Release and InitEncode, so they don't 173 // have to be recreated. Remaining encoders are destroyed by the destructor. 174 std::list<std::unique_ptr<EncoderContext>> cached_encoder_contexts_; 175 176 const absl::optional<unsigned int> experimental_boosted_screenshare_qp_; 177 const bool boost_base_layer_quality_; 178 const bool prefer_temporal_support_on_base_layer_; 179 180 const SimulcastEncoderAdapterEncoderInfoSettings encoder_info_override_; 181 }; 182 183 } // namespace webrtc 184 185 #endif // MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_ 186