1 /* 2 * Copyright 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 // This file contains interfaces for MediaStream, MediaTrack and MediaSource. 12 // These interfaces are used for implementing MediaStream and MediaTrack as 13 // defined in http://dev.w3.org/2011/webrtc/editor/webrtc.html#stream-api. These 14 // interfaces must be used only with PeerConnection. PeerConnectionManager 15 // interface provides the factory methods to create MediaStream and MediaTracks. 16 17 #ifndef API_MEDIASTREAMINTERFACE_H_ 18 #define API_MEDIASTREAMINTERFACE_H_ 19 20 #include <stddef.h> 21 22 #include <string> 23 #include <vector> 24 25 #include "api/optional.h" 26 #include "api/video/video_frame.h" 27 // TODO(zhihuang): Remove unrelated headers once downstream applications stop 28 // relying on them; they were previously transitively included by 29 // mediachannel.h, which is no longer a dependency of this file. 30 #include "media/base/videosinkinterface.h" 31 #include "media/base/videosourceinterface.h" 32 #include "modules/audio_processing/include/audio_processing_statistics.h" 33 #include "rtc_base/ratetracker.h" 34 #include "rtc_base/refcount.h" 35 #include "rtc_base/scoped_ref_ptr.h" 36 #include "rtc_base/thread.h" 37 #include "rtc_base/timeutils.h" 38 39 namespace webrtc { 40 41 // Generic observer interface. 42 class ObserverInterface { 43 public: 44 virtual void OnChanged() = 0; 45 46 protected: ~ObserverInterface()47 virtual ~ObserverInterface() {} 48 }; 49 50 class NotifierInterface { 51 public: 52 virtual void RegisterObserver(ObserverInterface* observer) = 0; 53 virtual void UnregisterObserver(ObserverInterface* observer) = 0; 54 ~NotifierInterface()55 virtual ~NotifierInterface() {} 56 }; 57 58 // Base class for sources. A MediaStreamTrack has an underlying source that 59 // provides media. A source can be shared by multiple tracks. 60 class MediaSourceInterface : public rtc::RefCountInterface, 61 public NotifierInterface { 62 public: 63 enum SourceState { 64 kInitializing, 65 kLive, 66 kEnded, 67 kMuted 68 }; 69 70 virtual SourceState state() const = 0; 71 72 virtual bool remote() const = 0; 73 74 protected: ~MediaSourceInterface()75 virtual ~MediaSourceInterface() {} 76 }; 77 78 // C++ version of MediaStreamTrack. 79 // See: https://www.w3.org/TR/mediacapture-streams/#mediastreamtrack 80 class MediaStreamTrackInterface : public rtc::RefCountInterface, 81 public NotifierInterface { 82 public: 83 enum TrackState { 84 kLive, 85 kEnded, 86 }; 87 88 static const char kAudioKind[]; 89 static const char kVideoKind[]; 90 91 // The kind() method must return kAudioKind only if the object is a 92 // subclass of AudioTrackInterface, and kVideoKind only if the 93 // object is a subclass of VideoTrackInterface. It is typically used 94 // to protect a static_cast<> to the corresponding subclass. 95 virtual std::string kind() const = 0; 96 97 // Track identifier. 98 virtual std::string id() const = 0; 99 100 // A disabled track will produce silence (if audio) or black frames (if 101 // video). Can be disabled and re-enabled. 102 virtual bool enabled() const = 0; 103 virtual bool set_enabled(bool enable) = 0; 104 105 // Live or ended. A track will never be live again after becoming ended. 106 virtual TrackState state() const = 0; 107 108 protected: ~MediaStreamTrackInterface()109 virtual ~MediaStreamTrackInterface() {} 110 }; 111 112 // VideoTrackSourceInterface is a reference counted source used for 113 // VideoTracks. The same source can be used by multiple VideoTracks. 114 // VideoTrackSourceInterface is designed to be invoked on the signaling thread 115 // except for rtc::VideoSourceInterface<VideoFrame> methods that will be invoked 116 // on the worker thread via a VideoTrack. A custom implementation of a source 117 // can inherit AdaptedVideoTrackSource instead of directly implementing this 118 // interface. 119 class VideoTrackSourceInterface 120 : public MediaSourceInterface, 121 public rtc::VideoSourceInterface<VideoFrame> { 122 public: 123 struct Stats { 124 // Original size of captured frame, before video adaptation. 125 int input_width; 126 int input_height; 127 }; 128 129 // Indicates that parameters suitable for screencasts should be automatically 130 // applied to RtpSenders. 131 // TODO(perkj): Remove these once all known applications have moved to 132 // explicitly setting suitable parameters for screencasts and don't need this 133 // implicit behavior. 134 virtual bool is_screencast() const = 0; 135 136 // Indicates that the encoder should denoise video before encoding it. 137 // If it is not set, the default configuration is used which is different 138 // depending on video codec. 139 // TODO(perkj): Remove this once denoising is done by the source, and not by 140 // the encoder. 141 virtual rtc::Optional<bool> needs_denoising() const = 0; 142 143 // Returns false if no stats are available, e.g, for a remote source, or a 144 // source which has not seen its first frame yet. 145 // 146 // Implementation should avoid blocking. 147 virtual bool GetStats(Stats* stats) = 0; 148 149 protected: ~VideoTrackSourceInterface()150 virtual ~VideoTrackSourceInterface() {} 151 }; 152 153 // VideoTrackInterface is designed to be invoked on the signaling thread except 154 // for rtc::VideoSourceInterface<VideoFrame> methods that must be invoked 155 // on the worker thread. 156 // PeerConnectionFactory::CreateVideoTrack can be used for creating a VideoTrack 157 // that ensures thread safety and that all methods are called on the right 158 // thread. 159 class VideoTrackInterface 160 : public MediaStreamTrackInterface, 161 public rtc::VideoSourceInterface<VideoFrame> { 162 public: 163 // Video track content hint, used to override the source is_screencast 164 // property. 165 // See https://crbug.com/653531 and https://github.com/WICG/mst-content-hint. 166 enum class ContentHint { kNone, kFluid, kDetailed }; 167 168 // Register a video sink for this track. Used to connect the track to the 169 // underlying video engine. AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame> * sink,const rtc::VideoSinkWants & wants)170 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink, 171 const rtc::VideoSinkWants& wants) override {} RemoveSink(rtc::VideoSinkInterface<VideoFrame> * sink)172 void RemoveSink(rtc::VideoSinkInterface<VideoFrame>* sink) override {} 173 174 virtual VideoTrackSourceInterface* GetSource() const = 0; 175 content_hint()176 virtual ContentHint content_hint() const { return ContentHint::kNone; } set_content_hint(ContentHint hint)177 virtual void set_content_hint(ContentHint hint) {} 178 179 protected: ~VideoTrackInterface()180 virtual ~VideoTrackInterface() {} 181 }; 182 183 // Interface for receiving audio data from a AudioTrack. 184 class AudioTrackSinkInterface { 185 public: 186 virtual void OnData(const void* audio_data, 187 int bits_per_sample, 188 int sample_rate, 189 size_t number_of_channels, 190 size_t number_of_frames) = 0; 191 192 protected: ~AudioTrackSinkInterface()193 virtual ~AudioTrackSinkInterface() {} 194 }; 195 196 // AudioSourceInterface is a reference counted source used for AudioTracks. 197 // The same source can be used by multiple AudioTracks. 198 class AudioSourceInterface : public MediaSourceInterface { 199 public: 200 class AudioObserver { 201 public: 202 virtual void OnSetVolume(double volume) = 0; 203 204 protected: ~AudioObserver()205 virtual ~AudioObserver() {} 206 }; 207 208 // TODO(deadbeef): Makes all the interfaces pure virtual after they're 209 // implemented in chromium. 210 211 // Sets the volume of the source. |volume| is in the range of [0, 10]. 212 // TODO(tommi): This method should be on the track and ideally volume should 213 // be applied in the track in a way that does not affect clones of the track. SetVolume(double volume)214 virtual void SetVolume(double volume) {} 215 216 // Registers/unregisters observers to the audio source. RegisterAudioObserver(AudioObserver * observer)217 virtual void RegisterAudioObserver(AudioObserver* observer) {} UnregisterAudioObserver(AudioObserver * observer)218 virtual void UnregisterAudioObserver(AudioObserver* observer) {} 219 220 // TODO(tommi): Make pure virtual. AddSink(AudioTrackSinkInterface * sink)221 virtual void AddSink(AudioTrackSinkInterface* sink) {} RemoveSink(AudioTrackSinkInterface * sink)222 virtual void RemoveSink(AudioTrackSinkInterface* sink) {} 223 }; 224 225 // Interface of the audio processor used by the audio track to collect 226 // statistics. 227 class AudioProcessorInterface : public rtc::RefCountInterface { 228 public: 229 // Deprecated, use AudioProcessorStatistics instead. 230 // TODO(ivoc): Remove this when all implementations have switched to the new 231 // GetStats function. See b/67926135. 232 struct AudioProcessorStats { AudioProcessorStatsAudioProcessorStats233 AudioProcessorStats() 234 : typing_noise_detected(false), 235 echo_return_loss(0), 236 echo_return_loss_enhancement(0), 237 echo_delay_median_ms(0), 238 echo_delay_std_ms(0), 239 aec_quality_min(0.0), 240 residual_echo_likelihood(0.0f), 241 residual_echo_likelihood_recent_max(0.0f), 242 aec_divergent_filter_fraction(0.0) {} ~AudioProcessorStatsAudioProcessorStats243 ~AudioProcessorStats() {} 244 245 bool typing_noise_detected; 246 int echo_return_loss; 247 int echo_return_loss_enhancement; 248 int echo_delay_median_ms; 249 int echo_delay_std_ms; 250 float aec_quality_min; 251 float residual_echo_likelihood; 252 float residual_echo_likelihood_recent_max; 253 float aec_divergent_filter_fraction; 254 }; 255 // This struct maintains the optionality of the stats, and will replace the 256 // regular stats struct when all users have been updated. 257 struct AudioProcessorStatistics { 258 bool typing_noise_detected = false; 259 AudioProcessingStats apm_statistics; 260 }; 261 262 // Get audio processor statistics. 263 virtual void GetStats(AudioProcessorStats* stats) = 0; 264 265 // Get audio processor statistics. The |has_remote_tracks| argument should be 266 // set if there are active remote tracks (this would usually be true during 267 // a call). If there are no remote tracks some of the stats will not be set by 268 // the AudioProcessor, because they only make sense if there is at least one 269 // remote track. 270 // TODO(ivoc): Make pure virtual when all implementions are updated. 271 virtual AudioProcessorStatistics GetStats(bool has_remote_tracks); 272 273 protected: ~AudioProcessorInterface()274 virtual ~AudioProcessorInterface() {} 275 }; 276 277 class AudioTrackInterface : public MediaStreamTrackInterface { 278 public: 279 // TODO(deadbeef): Figure out if the following interface should be const or 280 // not. 281 virtual AudioSourceInterface* GetSource() const = 0; 282 283 // Add/Remove a sink that will receive the audio data from the track. 284 virtual void AddSink(AudioTrackSinkInterface* sink) = 0; 285 virtual void RemoveSink(AudioTrackSinkInterface* sink) = 0; 286 287 // Get the signal level from the audio track. 288 // Return true on success, otherwise false. 289 // TODO(deadbeef): Change the interface to int GetSignalLevel() and pure 290 // virtual after it's implemented in chromium. GetSignalLevel(int * level)291 virtual bool GetSignalLevel(int* level) { return false; } 292 293 // Get the audio processor used by the audio track. Return null if the track 294 // does not have any processor. 295 // TODO(deadbeef): Make the interface pure virtual. GetAudioProcessor()296 virtual rtc::scoped_refptr<AudioProcessorInterface> GetAudioProcessor() { 297 return nullptr; 298 } 299 300 protected: ~AudioTrackInterface()301 virtual ~AudioTrackInterface() {} 302 }; 303 304 typedef std::vector<rtc::scoped_refptr<AudioTrackInterface> > 305 AudioTrackVector; 306 typedef std::vector<rtc::scoped_refptr<VideoTrackInterface> > 307 VideoTrackVector; 308 309 // C++ version of https://www.w3.org/TR/mediacapture-streams/#mediastream. 310 // 311 // A major difference is that remote audio/video tracks (received by a 312 // PeerConnection/RtpReceiver) are not synchronized simply by adding them to 313 // the same stream; a session description with the correct "a=msid" attributes 314 // must be pushed down. 315 // 316 // Thus, this interface acts as simply a container for tracks. 317 class MediaStreamInterface : public rtc::RefCountInterface, 318 public NotifierInterface { 319 public: 320 // TODO(steveanton): This could be renamed to id() to match the spec. 321 virtual std::string label() const = 0; 322 323 virtual AudioTrackVector GetAudioTracks() = 0; 324 virtual VideoTrackVector GetVideoTracks() = 0; 325 virtual rtc::scoped_refptr<AudioTrackInterface> 326 FindAudioTrack(const std::string& track_id) = 0; 327 virtual rtc::scoped_refptr<VideoTrackInterface> 328 FindVideoTrack(const std::string& track_id) = 0; 329 330 virtual bool AddTrack(AudioTrackInterface* track) = 0; 331 virtual bool AddTrack(VideoTrackInterface* track) = 0; 332 virtual bool RemoveTrack(AudioTrackInterface* track) = 0; 333 virtual bool RemoveTrack(VideoTrackInterface* track) = 0; 334 335 protected: ~MediaStreamInterface()336 virtual ~MediaStreamInterface() {} 337 }; 338 339 } // namespace webrtc 340 341 #endif // API_MEDIASTREAMINTERFACE_H_ 342