1 /*
2  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef MEDIA_ENGINE_WEBRTCVIDEOENGINE_H_
12 #define MEDIA_ENGINE_WEBRTCVIDEOENGINE_H_
13 
14 #include <map>
15 #include <memory>
16 #include <set>
17 #include <string>
18 #include <vector>
19 
20 #include "api/call/transport.h"
21 #include "api/optional.h"
22 #include "api/video/video_frame.h"
23 #include "api/video_codecs/sdp_video_format.h"
24 #include "call/call.h"
25 #include "call/flexfec_receive_stream.h"
26 #include "call/video_receive_stream.h"
27 #include "call/video_send_stream.h"
28 #include "media/base/mediaengine.h"
29 #include "media/base/videosinkinterface.h"
30 #include "media/base/videosourceinterface.h"
31 #include "media/engine/webrtcvideodecoderfactory.h"
32 #include "media/engine/webrtcvideoencoderfactory.h"
33 #include "rtc_base/asyncinvoker.h"
34 #include "rtc_base/criticalsection.h"
35 #include "rtc_base/networkroute.h"
36 #include "rtc_base/thread_annotations.h"
37 #include "rtc_base/thread_checker.h"
38 
39 namespace webrtc {
40 class VideoDecoder;
41 class VideoDecoderFactory;
42 class VideoEncoder;
43 class VideoEncoderFactory;
44 struct MediaConfig;
45 }
46 
47 namespace rtc {
48 class Thread;
49 }  // namespace rtc
50 
51 namespace cricket {
52 
53 class DecoderFactoryAdapter;
54 class VideoCapturer;
55 class VideoProcessor;
56 class VideoRenderer;
57 class VoiceMediaChannel;
58 class WebRtcDecoderObserver;
59 class WebRtcEncoderObserver;
60 class WebRtcLocalStreamInfo;
61 class WebRtcRenderAdapter;
62 class WebRtcVideoChannel;
63 class WebRtcVideoChannelRecvInfo;
64 class WebRtcVideoChannelSendInfo;
65 class WebRtcVoiceEngine;
66 class WebRtcVoiceMediaChannel;
67 
68 class UnsignalledSsrcHandler {
69  public:
70   enum Action {
71     kDropPacket,
72     kDeliverPacket,
73   };
74   virtual Action OnUnsignalledSsrc(WebRtcVideoChannel* channel,
75                                    uint32_t ssrc) = 0;
76   virtual ~UnsignalledSsrcHandler() = default;
77 };
78 
79 // TODO(pbos): Remove, use external handlers only.
80 class DefaultUnsignalledSsrcHandler : public UnsignalledSsrcHandler {
81  public:
82   DefaultUnsignalledSsrcHandler();
83   Action OnUnsignalledSsrc(WebRtcVideoChannel* channel,
84                            uint32_t ssrc) override;
85 
86   rtc::VideoSinkInterface<webrtc::VideoFrame>* GetDefaultSink() const;
87   void SetDefaultSink(WebRtcVideoChannel* channel,
88                       rtc::VideoSinkInterface<webrtc::VideoFrame>* sink);
89 
90   virtual ~DefaultUnsignalledSsrcHandler() = default;
91 
92  private:
93   rtc::VideoSinkInterface<webrtc::VideoFrame>* default_sink_;
94 };
95 
96 // WebRtcVideoEngine is used for the new native WebRTC Video API (webrtc:1667).
97 class WebRtcVideoEngine {
98  public:
99   // Internal SW video codecs will be added on top of the external codecs.
100   WebRtcVideoEngine(
101       std::unique_ptr<WebRtcVideoEncoderFactory> external_video_encoder_factory,
102       std::unique_ptr<WebRtcVideoDecoderFactory>
103           external_video_decoder_factory);
104 
105   // These video codec factories represents all video codecs, i.e. both software
106   // and external hardware codecs.
107   WebRtcVideoEngine(
108       std::unique_ptr<webrtc::VideoEncoderFactory> video_encoder_factory,
109       std::unique_ptr<webrtc::VideoDecoderFactory> video_decoder_factory);
110 
111   virtual ~WebRtcVideoEngine();
112 
113   WebRtcVideoChannel* CreateChannel(webrtc::Call* call,
114                                     const MediaConfig& config,
115                                     const VideoOptions& options);
116 
117   std::vector<VideoCodec> codecs() const;
118   RtpCapabilities GetCapabilities() const;
119 
120  private:
121   const std::unique_ptr<DecoderFactoryAdapter> decoder_factory_;
122   const std::unique_ptr<webrtc::VideoEncoderFactory> encoder_factory_;
123 };
124 
125 class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport {
126  public:
127   WebRtcVideoChannel(webrtc::Call* call,
128                      const MediaConfig& config,
129                      const VideoOptions& options,
130                      webrtc::VideoEncoderFactory* encoder_factory,
131                      DecoderFactoryAdapter* decoder_factory);
132   ~WebRtcVideoChannel() override;
133 
134   // VideoMediaChannel implementation
135   rtc::DiffServCodePoint PreferredDscp() const override;
136 
137   bool SetSendParameters(const VideoSendParameters& params) override;
138   bool SetRecvParameters(const VideoRecvParameters& params) override;
139   webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const override;
140   bool SetRtpSendParameters(uint32_t ssrc,
141                             const webrtc::RtpParameters& parameters) override;
142   webrtc::RtpParameters GetRtpReceiveParameters(uint32_t ssrc) const override;
143   bool SetRtpReceiveParameters(
144       uint32_t ssrc,
145       const webrtc::RtpParameters& parameters) override;
146   bool GetSendCodec(VideoCodec* send_codec) override;
147   bool SetSend(bool send) override;
148   bool SetVideoSend(
149       uint32_t ssrc,
150       bool enable,
151       const VideoOptions* options,
152       rtc::VideoSourceInterface<webrtc::VideoFrame>* source) override;
153   bool AddSendStream(const StreamParams& sp) override;
154   bool RemoveSendStream(uint32_t ssrc) override;
155   bool AddRecvStream(const StreamParams& sp) override;
156   bool AddRecvStream(const StreamParams& sp, bool default_stream);
157   bool RemoveRecvStream(uint32_t ssrc) override;
158   bool SetSink(uint32_t ssrc,
159                rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override;
160   void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) override;
161   bool GetStats(VideoMediaInfo* info) override;
162 
163   void OnPacketReceived(rtc::CopyOnWriteBuffer* packet,
164                         const rtc::PacketTime& packet_time) override;
165   void OnRtcpReceived(rtc::CopyOnWriteBuffer* packet,
166                       const rtc::PacketTime& packet_time) override;
167   void OnReadyToSend(bool ready) override;
168   void OnNetworkRouteChanged(const std::string& transport_name,
169                              const rtc::NetworkRoute& network_route) override;
170   void SetInterface(NetworkInterface* iface) override;
171 
172   // Implemented for VideoMediaChannelTest.
sending()173   bool sending() const { return sending_; }
174 
175   rtc::Optional<uint32_t> GetDefaultReceiveStreamSsrc();
176 
177   // AdaptReason is used for expressing why a WebRtcVideoSendStream request
178   // a lower input frame size than the currently configured camera input frame
179   // size. There can be more than one reason OR:ed together.
180   enum AdaptReason {
181     ADAPTREASON_NONE = 0,
182     ADAPTREASON_CPU = 1,
183     ADAPTREASON_BANDWIDTH = 2,
184   };
185 
186   static constexpr int kDefaultQpMax = 56;
187 
188  private:
189   class WebRtcVideoReceiveStream;
190   struct VideoCodecSettings {
191     VideoCodecSettings();
192 
193     // Checks if all members of |*this| are equal to the corresponding members
194     // of |other|.
195     bool operator==(const VideoCodecSettings& other) const;
196     bool operator!=(const VideoCodecSettings& other) const;
197 
198     // Checks if all members of |a|, except |flexfec_payload_type|, are equal
199     // to the corresponding members of |b|.
200     static bool EqualsDisregardingFlexfec(const VideoCodecSettings& a,
201                                           const VideoCodecSettings& b);
202 
203     VideoCodec codec;
204     webrtc::UlpfecConfig ulpfec;
205     int flexfec_payload_type;
206     int rtx_payload_type;
207   };
208 
209   struct ChangedSendParameters {
210     // These optionals are unset if not changed.
211     rtc::Optional<VideoCodecSettings> codec;
212     rtc::Optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions;
213     rtc::Optional<int> max_bandwidth_bps;
214     rtc::Optional<bool> conference_mode;
215     rtc::Optional<webrtc::RtcpMode> rtcp_mode;
216   };
217 
218   struct ChangedRecvParameters {
219     // These optionals are unset if not changed.
220     rtc::Optional<std::vector<VideoCodecSettings>> codec_settings;
221     rtc::Optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions;
222     // Keep track of the FlexFEC payload type separately from |codec_settings|.
223     // This allows us to recreate the FlexfecReceiveStream separately from the
224     // VideoReceiveStream when the FlexFEC payload type is changed.
225     rtc::Optional<int> flexfec_payload_type;
226   };
227 
228   bool GetChangedSendParameters(const VideoSendParameters& params,
229                                 ChangedSendParameters* changed_params) const;
230   bool GetChangedRecvParameters(const VideoRecvParameters& params,
231                                 ChangedRecvParameters* changed_params) const;
232 
233   void SetMaxSendBandwidth(int bps);
234 
235   void ConfigureReceiverRtp(
236       webrtc::VideoReceiveStream::Config* config,
237       webrtc::FlexfecReceiveStream::Config* flexfec_config,
238       const StreamParams& sp) const;
239   bool ValidateSendSsrcAvailability(const StreamParams& sp) const
240       RTC_EXCLUSIVE_LOCKS_REQUIRED(stream_crit_);
241   bool ValidateReceiveSsrcAvailability(const StreamParams& sp) const
242       RTC_EXCLUSIVE_LOCKS_REQUIRED(stream_crit_);
243   void DeleteReceiveStream(WebRtcVideoReceiveStream* stream)
244       RTC_EXCLUSIVE_LOCKS_REQUIRED(stream_crit_);
245 
246   static std::string CodecSettingsVectorToString(
247       const std::vector<VideoCodecSettings>& codecs);
248 
249   // Wrapper for the sender part.
250   class WebRtcVideoSendStream
251       : public rtc::VideoSourceInterface<webrtc::VideoFrame> {
252    public:
253     WebRtcVideoSendStream(
254         webrtc::Call* call,
255         const StreamParams& sp,
256         webrtc::VideoSendStream::Config config,
257         const VideoOptions& options,
258         webrtc::VideoEncoderFactory* encoder_factory,
259         bool enable_cpu_overuse_detection,
260         int max_bitrate_bps,
261         const rtc::Optional<VideoCodecSettings>& codec_settings,
262         const rtc::Optional<std::vector<webrtc::RtpExtension>>& rtp_extensions,
263         const VideoSendParameters& send_params);
264     virtual ~WebRtcVideoSendStream();
265 
266     void SetSendParameters(const ChangedSendParameters& send_params);
267     bool SetRtpParameters(const webrtc::RtpParameters& parameters);
268     webrtc::RtpParameters GetRtpParameters() const;
269 
270     // Implements rtc::VideoSourceInterface<webrtc::VideoFrame>.
271     // WebRtcVideoSendStream acts as a source to the webrtc::VideoSendStream
272     // in |stream_|. This is done to proxy VideoSinkWants from the encoder to
273     // the worker thread.
274     void AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink,
275                          const rtc::VideoSinkWants& wants) override;
276     void RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override;
277 
278     bool SetVideoSend(bool mute,
279                       const VideoOptions* options,
280                       rtc::VideoSourceInterface<webrtc::VideoFrame>* source);
281 
282     void SetSend(bool send);
283 
284     const std::vector<uint32_t>& GetSsrcs() const;
285     VideoSenderInfo GetVideoSenderInfo(bool log_stats);
286     void FillBitrateInfo(BandwidthEstimationInfo* bwe_info);
287 
288    private:
289     // Parameters needed to reconstruct the underlying stream.
290     // webrtc::VideoSendStream doesn't support setting a lot of options on the
291     // fly, so when those need to be changed we tear down and reconstruct with
292     // similar parameters depending on which options changed etc.
293     struct VideoSendStreamParameters {
294       VideoSendStreamParameters(
295           webrtc::VideoSendStream::Config config,
296           const VideoOptions& options,
297           int max_bitrate_bps,
298           const rtc::Optional<VideoCodecSettings>& codec_settings);
299       webrtc::VideoSendStream::Config config;
300       VideoOptions options;
301       int max_bitrate_bps;
302       bool conference_mode;
303       rtc::Optional<VideoCodecSettings> codec_settings;
304       // Sent resolutions + bitrates etc. by the underlying VideoSendStream,
305       // typically changes when setting a new resolution or reconfiguring
306       // bitrates.
307       webrtc::VideoEncoderConfig encoder_config;
308     };
309 
310     rtc::scoped_refptr<webrtc::VideoEncoderConfig::EncoderSpecificSettings>
311     ConfigureVideoEncoderSettings(const VideoCodec& codec);
312     void SetCodec(const VideoCodecSettings& codec,
313                   bool force_encoder_allocation);
314     void RecreateWebRtcStream();
315     webrtc::VideoEncoderConfig CreateVideoEncoderConfig(
316         const VideoCodec& codec) const;
317     void ReconfigureEncoder();
318     bool ValidateRtpParameters(const webrtc::RtpParameters& parameters);
319 
320     // Calls Start or Stop according to whether or not |sending_| is true,
321     // and whether or not the encoding in |rtp_parameters_| is active.
322     void UpdateSendState();
323 
324     webrtc::VideoSendStream::DegradationPreference GetDegradationPreference()
325         const RTC_EXCLUSIVE_LOCKS_REQUIRED(&thread_checker_);
326 
327     rtc::ThreadChecker thread_checker_;
328     rtc::AsyncInvoker invoker_;
329     rtc::Thread* worker_thread_;
330     const std::vector<uint32_t> ssrcs_ RTC_ACCESS_ON(&thread_checker_);
331     const std::vector<SsrcGroup> ssrc_groups_ RTC_ACCESS_ON(&thread_checker_);
332     webrtc::Call* const call_;
333     const bool enable_cpu_overuse_detection_;
334     rtc::VideoSourceInterface<webrtc::VideoFrame>* source_
335         RTC_ACCESS_ON(&thread_checker_);
336     webrtc::VideoEncoderFactory* const encoder_factory_
337         RTC_ACCESS_ON(&thread_checker_);
338 
339     webrtc::VideoSendStream* stream_ RTC_ACCESS_ON(&thread_checker_);
340     rtc::VideoSinkInterface<webrtc::VideoFrame>* encoder_sink_
341         RTC_ACCESS_ON(&thread_checker_);
342     // Contains settings that are the same for all streams in the MediaChannel,
343     // such as codecs, header extensions, and the global bitrate limit for the
344     // entire channel.
345     VideoSendStreamParameters parameters_ RTC_ACCESS_ON(&thread_checker_);
346     // Contains settings that are unique for each stream, such as max_bitrate.
347     // Does *not* contain codecs, however.
348     // TODO(skvlad): Move ssrcs_ and ssrc_groups_ into rtp_parameters_.
349     // TODO(skvlad): Combine parameters_ and rtp_parameters_ once we have only
350     // one stream per MediaChannel.
351     webrtc::RtpParameters rtp_parameters_ RTC_ACCESS_ON(&thread_checker_);
352     std::unique_ptr<webrtc::VideoEncoder> allocated_encoder_
353         RTC_ACCESS_ON(&thread_checker_);
354     VideoCodec allocated_codec_ RTC_ACCESS_ON(&thread_checker_);
355 
356     bool sending_ RTC_ACCESS_ON(&thread_checker_);
357   };
358 
359   // Wrapper for the receiver part, contains configs etc. that are needed to
360   // reconstruct the underlying VideoReceiveStream.
361   class WebRtcVideoReceiveStream
362       : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
363    public:
364     WebRtcVideoReceiveStream(
365         webrtc::Call* call,
366         const StreamParams& sp,
367         webrtc::VideoReceiveStream::Config config,
368         DecoderFactoryAdapter* decoder_factory,
369         bool default_stream,
370         const std::vector<VideoCodecSettings>& recv_codecs,
371         const webrtc::FlexfecReceiveStream::Config& flexfec_config);
372     ~WebRtcVideoReceiveStream();
373 
374     const std::vector<uint32_t>& GetSsrcs() const;
375     rtc::Optional<uint32_t> GetFirstPrimarySsrc() const;
376 
377     void SetLocalSsrc(uint32_t local_ssrc);
378     // TODO(deadbeef): Move these feedback parameters into the recv parameters.
379     void SetFeedbackParameters(bool nack_enabled,
380                                bool remb_enabled,
381                                bool transport_cc_enabled,
382                                webrtc::RtcpMode rtcp_mode);
383     void SetRecvParameters(const ChangedRecvParameters& recv_params);
384 
385     void OnFrame(const webrtc::VideoFrame& frame) override;
386     bool IsDefaultStream() const;
387 
388     void SetSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink);
389 
390     VideoReceiverInfo GetVideoReceiverInfo(bool log_stats);
391 
392    private:
393     struct SdpVideoFormatCompare {
operatorSdpVideoFormatCompare394       bool operator()(const webrtc::SdpVideoFormat& lhs,
395                       const webrtc::SdpVideoFormat& rhs) const {
396         return std::tie(lhs.name, lhs.parameters) <
397                std::tie(rhs.name, rhs.parameters);
398       }
399     };
400     typedef std::map<webrtc::SdpVideoFormat,
401                      std::unique_ptr<webrtc::VideoDecoder>,
402                      SdpVideoFormatCompare>
403         DecoderMap;
404 
405     void RecreateWebRtcVideoStream();
406     void MaybeRecreateWebRtcFlexfecStream();
407 
408     void MaybeAssociateFlexfecWithVideo();
409     void MaybeDissociateFlexfecFromVideo();
410 
411     void ConfigureCodecs(const std::vector<VideoCodecSettings>& recv_codecs,
412                          DecoderMap* old_codecs);
413     void ConfigureFlexfecCodec(int flexfec_payload_type);
414 
415     std::string GetCodecNameFromPayloadType(int payload_type);
416 
417     webrtc::Call* const call_;
418     StreamParams stream_params_;
419 
420     // Both |stream_| and |flexfec_stream_| are managed by |this|. They are
421     // destroyed by calling call_->DestroyVideoReceiveStream and
422     // call_->DestroyFlexfecReceiveStream, respectively.
423     webrtc::VideoReceiveStream* stream_;
424     const bool default_stream_;
425     webrtc::VideoReceiveStream::Config config_;
426     webrtc::FlexfecReceiveStream::Config flexfec_config_;
427     webrtc::FlexfecReceiveStream* flexfec_stream_;
428 
429     DecoderFactoryAdapter* decoder_factory_;
430     DecoderMap allocated_decoders_;
431 
432     rtc::CriticalSection sink_lock_;
433     rtc::VideoSinkInterface<webrtc::VideoFrame>* sink_
434         RTC_GUARDED_BY(sink_lock_);
435     // Expands remote RTP timestamps to int64_t to be able to estimate how long
436     // the stream has been running.
437     rtc::TimestampWrapAroundHandler timestamp_wraparound_handler_
438         RTC_GUARDED_BY(sink_lock_);
439     int64_t first_frame_timestamp_ RTC_GUARDED_BY(sink_lock_);
440     // Start NTP time is estimated as current remote NTP time (estimated from
441     // RTCP) minus the elapsed time, as soon as remote NTP time is available.
442     int64_t estimated_remote_start_ntp_time_ms_ RTC_GUARDED_BY(sink_lock_);
443   };
444 
445   void Construct(webrtc::Call* call, WebRtcVideoEngine* engine);
446 
447   bool SendRtp(const uint8_t* data,
448                size_t len,
449                const webrtc::PacketOptions& options) override;
450   bool SendRtcp(const uint8_t* data, size_t len) override;
451 
452   static std::vector<VideoCodecSettings> MapCodecs(
453       const std::vector<VideoCodec>& codecs);
454   // Select what video codec will be used for sending, i.e. what codec is used
455   // for local encoding, based on supported remote codecs. The first remote
456   // codec that is supported locally will be selected.
457   rtc::Optional<VideoCodecSettings> SelectSendVideoCodec(
458       const std::vector<VideoCodecSettings>& remote_mapped_codecs) const;
459 
460   static bool NonFlexfecReceiveCodecsHaveChanged(
461       std::vector<VideoCodecSettings> before,
462       std::vector<VideoCodecSettings> after);
463 
464   void FillSenderStats(VideoMediaInfo* info, bool log_stats);
465   void FillReceiverStats(VideoMediaInfo* info, bool log_stats);
466   void FillBandwidthEstimationStats(const webrtc::Call::Stats& stats,
467                                     VideoMediaInfo* info);
468   void FillSendAndReceiveCodecStats(VideoMediaInfo* video_media_info);
469 
470   rtc::ThreadChecker thread_checker_;
471 
472   uint32_t rtcp_receiver_report_ssrc_;
473   bool sending_;
474   webrtc::Call* const call_;
475 
476   DefaultUnsignalledSsrcHandler default_unsignalled_ssrc_handler_;
477   UnsignalledSsrcHandler* const unsignalled_ssrc_handler_;
478 
479   const MediaConfig::Video video_config_;
480 
481   rtc::CriticalSection stream_crit_;
482   // Using primary-ssrc (first ssrc) as key.
483   std::map<uint32_t, WebRtcVideoSendStream*> send_streams_
484       RTC_GUARDED_BY(stream_crit_);
485   std::map<uint32_t, WebRtcVideoReceiveStream*> receive_streams_
486       RTC_GUARDED_BY(stream_crit_);
487   std::set<uint32_t> send_ssrcs_ RTC_GUARDED_BY(stream_crit_);
488   std::set<uint32_t> receive_ssrcs_ RTC_GUARDED_BY(stream_crit_);
489 
490   rtc::Optional<VideoCodecSettings> send_codec_;
491   rtc::Optional<std::vector<webrtc::RtpExtension>> send_rtp_extensions_;
492 
493   webrtc::VideoEncoderFactory* const encoder_factory_;
494   DecoderFactoryAdapter* const decoder_factory_;
495   std::vector<VideoCodecSettings> recv_codecs_;
496   std::vector<webrtc::RtpExtension> recv_rtp_extensions_;
497   // See reason for keeping track of the FlexFEC payload type separately in
498   // comment in WebRtcVideoChannel::ChangedRecvParameters.
499   int recv_flexfec_payload_type_;
500   webrtc::Call::Config::BitrateConfig bitrate_config_;
501   // TODO(deadbeef): Don't duplicate information between
502   // send_params/recv_params, rtp_extensions, options, etc.
503   VideoSendParameters send_params_;
504   VideoOptions default_send_options_;
505   VideoRecvParameters recv_params_;
506   int64_t last_stats_log_ms_;
507 };
508 
509 class EncoderStreamFactory
510     : public webrtc::VideoEncoderConfig::VideoStreamFactoryInterface {
511  public:
512   EncoderStreamFactory(std::string codec_name,
513                        int max_qp,
514                        int max_framerate,
515                        bool is_screencast,
516                        bool conference_mode);
517 
518  private:
519   std::vector<webrtc::VideoStream> CreateEncoderStreams(
520       int width,
521       int height,
522       const webrtc::VideoEncoderConfig& encoder_config) override;
523 
524   const std::string codec_name_;
525   const int max_qp_;
526   const int max_framerate_;
527   const bool is_screencast_;
528   const bool conference_mode_;
529 };
530 
531 }  // namespace cricket
532 
533 #endif  // MEDIA_ENGINE_WEBRTCVIDEOENGINE_H_
534