1 /*
2  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  *
10  */
11 
12 #ifndef MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_
13 #define MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_
14 
15 #include <list>
16 #include <memory>
17 #include <stack>
18 #include <string>
19 #include <utility>
20 #include <vector>
21 
22 #include "absl/types/optional.h"
23 #include "api/fec_controller_override.h"
24 #include "api/sequence_checker.h"
25 #include "api/video_codecs/sdp_video_format.h"
26 #include "api/video_codecs/video_encoder.h"
27 #include "api/video_codecs/video_encoder_factory.h"
28 #include "modules/video_coding/include/video_codec_interface.h"
29 #include "modules/video_coding/utility/framerate_controller.h"
30 #include "rtc_base/atomic_ops.h"
31 #include "rtc_base/experiments/encoder_info_settings.h"
32 #include "rtc_base/system/no_unique_address.h"
33 #include "rtc_base/system/rtc_export.h"
34 
35 namespace webrtc {
36 
37 // SimulcastEncoderAdapter implements simulcast support by creating multiple
38 // webrtc::VideoEncoder instances with the given VideoEncoderFactory.
39 // The object is created and destroyed on the worker thread, but all public
40 // interfaces should be called from the encoder task queue.
41 class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder {
42  public:
43   // TODO(bugs.webrtc.org/11000): Remove when downstream usage is gone.
44   SimulcastEncoderAdapter(VideoEncoderFactory* primarty_factory,
45                           const SdpVideoFormat& format);
46   // |primary_factory| produces the first-choice encoders to use.
47   // |fallback_factory|, if non-null, is used to create fallback encoder that
48   // will be used if InitEncode() fails for the primary encoder.
49   SimulcastEncoderAdapter(VideoEncoderFactory* primary_factory,
50                           VideoEncoderFactory* fallback_factory,
51                           const SdpVideoFormat& format);
52   ~SimulcastEncoderAdapter() override;
53 
54   // Implements VideoEncoder.
55   void SetFecControllerOverride(
56       FecControllerOverride* fec_controller_override) override;
57   int Release() override;
58   int InitEncode(const VideoCodec* codec_settings,
59                  const VideoEncoder::Settings& settings) override;
60   int Encode(const VideoFrame& input_image,
61              const std::vector<VideoFrameType>* frame_types) override;
62   int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override;
63   void SetRates(const RateControlParameters& parameters) override;
64   void OnPacketLossRateUpdate(float packet_loss_rate) override;
65   void OnRttUpdate(int64_t rtt_ms) override;
66   void OnLossNotification(const LossNotification& loss_notification) override;
67 
68   EncoderInfo GetEncoderInfo() const override;
69 
70  private:
71   class EncoderContext {
72    public:
73     EncoderContext(std::unique_ptr<VideoEncoder> encoder,
74                    bool prefer_temporal_support);
75     EncoderContext& operator=(EncoderContext&&) = delete;
76 
encoder()77     VideoEncoder& encoder() { return *encoder_; }
prefer_temporal_support()78     bool prefer_temporal_support() { return prefer_temporal_support_; }
79     void Release();
80 
81    private:
82     std::unique_ptr<VideoEncoder> encoder_;
83     bool prefer_temporal_support_;
84   };
85 
86   class StreamContext : public EncodedImageCallback {
87    public:
88     StreamContext(SimulcastEncoderAdapter* parent,
89                   std::unique_ptr<EncoderContext> encoder_context,
90                   std::unique_ptr<FramerateController> framerate_controller,
91                   int stream_idx,
92                   uint16_t width,
93                   uint16_t height,
94                   bool send_stream);
95     StreamContext(StreamContext&& rhs);
96     StreamContext& operator=(StreamContext&&) = delete;
97     ~StreamContext() override;
98 
99     Result OnEncodedImage(
100         const EncodedImage& encoded_image,
101         const CodecSpecificInfo* codec_specific_info) override;
102     void OnDroppedFrame(DropReason reason) override;
103 
encoder()104     VideoEncoder& encoder() { return encoder_context_->encoder(); }
encoder()105     const VideoEncoder& encoder() const { return encoder_context_->encoder(); }
stream_idx()106     int stream_idx() const { return stream_idx_; }
width()107     uint16_t width() const { return width_; }
height()108     uint16_t height() const { return height_; }
is_keyframe_needed()109     bool is_keyframe_needed() const {
110       return !is_paused_ && is_keyframe_needed_;
111     }
set_is_keyframe_needed()112     void set_is_keyframe_needed() { is_keyframe_needed_ = true; }
is_paused()113     bool is_paused() const { return is_paused_; }
set_is_paused(bool is_paused)114     void set_is_paused(bool is_paused) { is_paused_ = is_paused; }
target_fps()115     absl::optional<float> target_fps() const {
116       return framerate_controller_ == nullptr
117                  ? absl::nullopt
118                  : absl::optional<float>(
119                        framerate_controller_->GetTargetRate());
120     }
121 
122     std::unique_ptr<EncoderContext> ReleaseEncoderContext() &&;
123     void OnKeyframe(Timestamp timestamp);
124     bool ShouldDropFrame(Timestamp timestamp);
125 
126    private:
127     SimulcastEncoderAdapter* const parent_;
128     std::unique_ptr<EncoderContext> encoder_context_;
129     std::unique_ptr<FramerateController> framerate_controller_;
130     const int stream_idx_;
131     const uint16_t width_;
132     const uint16_t height_;
133     bool is_keyframe_needed_;
134     bool is_paused_;
135   };
136 
137   bool Initialized() const;
138 
139   void DestroyStoredEncoders();
140 
141   std::unique_ptr<EncoderContext> FetchOrCreateEncoderContext(
142       bool is_lowest_quality_stream);
143 
144   webrtc::VideoCodec MakeStreamCodec(const webrtc::VideoCodec& codec,
145                                      int stream_idx,
146                                      uint32_t start_bitrate_kbps,
147                                      bool is_lowest_quality_stream,
148                                      bool is_highest_quality_stream);
149 
150   EncodedImageCallback::Result OnEncodedImage(
151       size_t stream_idx,
152       const EncodedImage& encoded_image,
153       const CodecSpecificInfo* codec_specific_info);
154 
155   void OnDroppedFrame(size_t stream_idx);
156 
157   void OverrideFromFieldTrial(VideoEncoder::EncoderInfo* info) const;
158 
159   volatile int inited_;  // Accessed atomically.
160   VideoEncoderFactory* const primary_encoder_factory_;
161   VideoEncoderFactory* const fallback_encoder_factory_;
162   const SdpVideoFormat video_format_;
163   VideoCodec codec_;
164   int total_streams_count_;
165   bool bypass_mode_;
166   std::vector<StreamContext> stream_contexts_;
167   EncodedImageCallback* encoded_complete_callback_;
168 
169   // Used for checking the single-threaded access of the encoder interface.
170   RTC_NO_UNIQUE_ADDRESS SequenceChecker encoder_queue_;
171 
172   // Store encoders in between calls to Release and InitEncode, so they don't
173   // have to be recreated. Remaining encoders are destroyed by the destructor.
174   std::list<std::unique_ptr<EncoderContext>> cached_encoder_contexts_;
175 
176   const absl::optional<unsigned int> experimental_boosted_screenshare_qp_;
177   const bool boost_base_layer_quality_;
178   const bool prefer_temporal_support_on_base_layer_;
179 
180   const SimulcastEncoderAdapterEncoderInfoSettings encoder_info_override_;
181 };
182 
183 }  // namespace webrtc
184 
185 #endif  // MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_
186