1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* This Source Code Form is subject to the terms of the Mozilla Public 3 * License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 * You can obtain one at http://mozilla.org/MPL/2.0/. */ 5 6 // Original author: ekr@rtfm.com 7 8 #ifndef mediapipeline_h__ 9 #define mediapipeline_h__ 10 11 #include "sigslot.h" 12 13 #ifdef USE_FAKE_MEDIA_STREAMS 14 #include "FakeMediaStreams.h" 15 #endif 16 #include "MediaConduitInterface.h" 17 #include "mozilla/ReentrantMonitor.h" 18 #include "mozilla/Atomics.h" 19 #include "SrtpFlow.h" 20 #include "databuffer.h" 21 #include "runnable_utils.h" 22 #include "transportflow.h" 23 #include "AudioPacketizer.h" 24 #include "StreamTracks.h" 25 26 #include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" 27 28 // Should come from MediaEngine.h, but that's a pain to include here 29 // because of the MOZILLA_EXTERNAL_LINKAGE stuff. 30 #define WEBRTC_DEFAULT_SAMPLE_RATE 32000 31 32 class nsIPrincipal; 33 34 namespace mozilla { 35 class MediaPipelineFilter; 36 class PeerIdentity; 37 class AudioProxyThread; 38 #if !defined(MOZILLA_EXTERNAL_LINKAGE) 39 class VideoFrameConverter; 40 #endif 41 42 #ifndef USE_FAKE_MEDIA_STREAMS 43 namespace dom { 44 class MediaStreamTrack; 45 } // namespace dom 46 47 class SourceMediaStream; 48 #endif // USE_FAKE_MEDIA_STREAMS 49 50 // A class that represents the pipeline of audio and video 51 // The dataflow looks like: 52 // 53 // TRANSMIT 54 // CaptureDevice -> stream -> [us] -> conduit -> [us] -> transport -> network 55 // 56 // RECEIVE 57 // network -> transport -> [us] -> conduit -> [us] -> stream -> Playout 58 // 59 // The boxes labeled [us] are just bridge logic implemented in this class 60 // 61 // We have to deal with a number of threads: 62 // 63 // GSM: 64 // * Assembles the pipeline 65 // SocketTransportService 66 // * Receives notification that ICE and DTLS have completed 67 // * Processes incoming network data and passes it to the conduit 68 // * Processes outgoing RTP and RTCP 69 // MediaStreamGraph 70 // * Receives outgoing data from the MediaStreamGraph 71 // * Receives pull requests for more data from the 72 // MediaStreamGraph 73 // One or another GIPS threads 74 // * Receives RTCP messages to send to the other side 75 // * Processes video frames GIPS wants to render 76 // 77 // For a transmitting conduit, "output" is RTP and "input" is RTCP. 78 // For a receiving conduit, "input" is RTP and "output" is RTCP. 79 // 80 81 class MediaPipeline : public sigslot::has_slots<> { 82 public: 83 enum Direction { TRANSMIT, RECEIVE }; 84 enum State { MP_CONNECTING, MP_OPEN, MP_CLOSED }; 85 MediaPipeline(const std::string& pc, 86 Direction direction, 87 nsCOMPtr<nsIEventTarget> main_thread, 88 nsCOMPtr<nsIEventTarget> sts_thread, 89 const std::string& track_id, 90 int level, 91 RefPtr<MediaSessionConduit> conduit, 92 RefPtr<TransportFlow> rtp_transport, 93 RefPtr<TransportFlow> rtcp_transport, 94 nsAutoPtr<MediaPipelineFilter> filter); 95 96 // Must be called on the STS thread. Must be called after ShutdownMedia_m(). 97 void DetachTransport_s(); 98 99 // Must be called on the main thread. 100 void ShutdownMedia_m() 101 { 102 ASSERT_ON_THREAD(main_thread_); 103 104 if (direction_ == RECEIVE) { 105 conduit_->StopReceiving(); 106 } else { 107 conduit_->StopTransmitting(); 108 } 109 DetachMedia(); 110 } 111 112 virtual nsresult Init(); 113 114 void UpdateTransport_m(int level, 115 RefPtr<TransportFlow> rtp_transport, 116 RefPtr<TransportFlow> rtcp_transport, 117 nsAutoPtr<MediaPipelineFilter> filter); 118 119 void UpdateTransport_s(int level, 120 RefPtr<TransportFlow> rtp_transport, 121 RefPtr<TransportFlow> rtcp_transport, 122 nsAutoPtr<MediaPipelineFilter> filter); 123 124 // Used only for testing; installs a MediaPipelineFilter that filters 125 // everything but the nth ssrc 126 void SelectSsrc_m(size_t ssrc_index); 127 void SelectSsrc_s(size_t ssrc_index); 128 129 virtual Direction direction() const { return direction_; } 130 virtual const std::string& trackid() const { return track_id_; } 131 virtual int level() const { return level_; } 132 virtual bool IsVideo() const = 0; 133 134 bool IsDoingRtcpMux() const { 135 return (rtp_.type_ == MUX); 136 } 137 138 int32_t rtp_packets_sent() const { return rtp_packets_sent_; } 139 int64_t rtp_bytes_sent() const { return rtp_bytes_sent_; } 140 int32_t rtcp_packets_sent() const { return rtcp_packets_sent_; } 141 int32_t rtp_packets_received() const { return rtp_packets_received_; } 142 int64_t rtp_bytes_received() const { return rtp_bytes_received_; } 143 int32_t rtcp_packets_received() const { return rtcp_packets_received_; } 144 145 MediaSessionConduit *Conduit() const { return conduit_; } 146 147 // Thread counting 148 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaPipeline) 149 150 typedef enum { 151 RTP, 152 RTCP, 153 MUX, 154 MAX_RTP_TYPE 155 } RtpType; 156 157 protected: 158 virtual ~MediaPipeline(); 159 virtual void DetachMedia() {} 160 nsresult AttachTransport_s(); 161 162 // Separate class to allow ref counting 163 class PipelineTransport : public TransportInterface { 164 public: 165 // Implement the TransportInterface functions 166 explicit PipelineTransport(MediaPipeline *pipeline) 167 : pipeline_(pipeline), 168 sts_thread_(pipeline->sts_thread_) {} 169 170 void Attach(MediaPipeline *pipeline) { pipeline_ = pipeline; } 171 void Detach() { pipeline_ = nullptr; } 172 MediaPipeline *pipeline() const { return pipeline_; } 173 174 virtual nsresult SendRtpPacket(const void* data, int len); 175 virtual nsresult SendRtcpPacket(const void* data, int len); 176 177 private: 178 nsresult SendRtpRtcpPacket_s(nsAutoPtr<DataBuffer> data, 179 bool is_rtp); 180 181 MediaPipeline *pipeline_; // Raw pointer to avoid cycles 182 nsCOMPtr<nsIEventTarget> sts_thread_; 183 }; 184 friend class PipelineTransport; 185 186 class TransportInfo { 187 public: 188 TransportInfo(RefPtr<TransportFlow> flow, RtpType type) : 189 transport_(flow), 190 state_(MP_CONNECTING), 191 type_(type) { 192 MOZ_ASSERT(flow); 193 } 194 195 void Detach() 196 { 197 transport_ = nullptr; 198 send_srtp_ = nullptr; 199 recv_srtp_ = nullptr; 200 } 201 202 RefPtr<TransportFlow> transport_; 203 State state_; 204 RefPtr<SrtpFlow> send_srtp_; 205 RefPtr<SrtpFlow> recv_srtp_; 206 RtpType type_; 207 }; 208 209 // The transport is down 210 virtual nsresult TransportFailed_s(TransportInfo &info); 211 // The transport is ready 212 virtual nsresult TransportReady_s(TransportInfo &info); 213 void UpdateRtcpMuxState(TransportInfo &info); 214 215 // Unhooks from signals 216 void DisconnectTransport_s(TransportInfo &info); 217 nsresult ConnectTransport_s(TransportInfo &info); 218 219 TransportInfo* GetTransportInfo_s(TransportFlow *flow); 220 221 void increment_rtp_packets_sent(int bytes); 222 void increment_rtcp_packets_sent(); 223 void increment_rtp_packets_received(int bytes); 224 void increment_rtcp_packets_received(); 225 226 virtual nsresult SendPacket(TransportFlow *flow, const void *data, int len); 227 228 // Process slots on transports 229 void StateChange(TransportFlow *flow, TransportLayer::State); 230 void RtpPacketReceived(TransportLayer *layer, const unsigned char *data, 231 size_t len); 232 void RtcpPacketReceived(TransportLayer *layer, const unsigned char *data, 233 size_t len); 234 void PacketReceived(TransportLayer *layer, const unsigned char *data, 235 size_t len); 236 237 Direction direction_; 238 std::string track_id_; // The track on the stream. 239 // Written on the main thread. 240 // Used on STS and MediaStreamGraph threads. 241 // Not used outside initialization in MediaPipelineTransmit 242 // The m-line index (starting at 0, to match convention) Atomic because 243 // this value is updated from STS, but read on main, and we don't want to 244 // bother with dispatches just to get an int occasionally. 245 Atomic<int> level_; 246 RefPtr<MediaSessionConduit> conduit_; // Our conduit. Written on the main 247 // thread. Read on STS thread. 248 249 // The transport objects. Read/written on STS thread. 250 TransportInfo rtp_; 251 TransportInfo rtcp_; 252 253 // Pointers to the threads we need. Initialized at creation 254 // and used all over the place. 255 nsCOMPtr<nsIEventTarget> main_thread_; 256 nsCOMPtr<nsIEventTarget> sts_thread_; 257 258 // Created on Init. Referenced by the conduit and eventually 259 // destroyed on the STS thread. 260 RefPtr<PipelineTransport> transport_; 261 262 // Only safe to access from STS thread. 263 // Build into TransportInfo? 264 int32_t rtp_packets_sent_; 265 int32_t rtcp_packets_sent_; 266 int32_t rtp_packets_received_; 267 int32_t rtcp_packets_received_; 268 int64_t rtp_bytes_sent_; 269 int64_t rtp_bytes_received_; 270 271 std::vector<uint32_t> ssrcs_received_; 272 273 // Written on Init. Read on STS thread. 274 std::string pc_; 275 std::string description_; 276 277 // Written on Init, all following accesses are on the STS thread. 278 nsAutoPtr<MediaPipelineFilter> filter_; 279 nsAutoPtr<webrtc::RtpHeaderParser> rtp_parser_; 280 281 private: 282 nsresult Init_s(); 283 284 bool IsRtp(const unsigned char *data, size_t len); 285 }; 286 287 class ConduitDeleteEvent: public Runnable 288 { 289 public: 290 explicit ConduitDeleteEvent(already_AddRefed<MediaSessionConduit> aConduit) : 291 mConduit(aConduit) {} 292 293 /* we exist solely to proxy release of the conduit */ 294 NS_IMETHOD Run() override { return NS_OK; } 295 private: 296 RefPtr<MediaSessionConduit> mConduit; 297 }; 298 299 // A specialization of pipeline for reading from an input device 300 // and transmitting to the network. 301 class MediaPipelineTransmit : public MediaPipeline { 302 public: 303 // Set rtcp_transport to nullptr to use rtcp-mux 304 MediaPipelineTransmit(const std::string& pc, 305 nsCOMPtr<nsIEventTarget> main_thread, 306 nsCOMPtr<nsIEventTarget> sts_thread, 307 dom::MediaStreamTrack* domtrack, 308 const std::string& track_id, 309 int level, 310 RefPtr<MediaSessionConduit> conduit, 311 RefPtr<TransportFlow> rtp_transport, 312 RefPtr<TransportFlow> rtcp_transport, 313 nsAutoPtr<MediaPipelineFilter> filter); 314 315 // Initialize (stuff here may fail) 316 nsresult Init() override; 317 318 virtual void AttachToTrack(const std::string& track_id); 319 320 // written and used from MainThread 321 bool IsVideo() const override; 322 323 #if !defined(MOZILLA_EXTERNAL_LINKAGE) 324 // When the principal of the domtrack changes, it calls through to here 325 // so that we can determine whether to enable track transmission. 326 // `track` has to be null or equal `domtrack_` for us to apply the update. 327 virtual void UpdateSinkIdentity_m(dom::MediaStreamTrack* track, 328 nsIPrincipal* principal, 329 const PeerIdentity* sinkIdentity); 330 #endif 331 332 // Called on the main thread. 333 void DetachMedia() override; 334 335 // Override MediaPipeline::TransportReady. 336 nsresult TransportReady_s(TransportInfo &info) override; 337 338 // Replace a track with a different one 339 // In non-compliance with the likely final spec, allow the new 340 // track to be part of a different stream (since we don't support 341 // multiple tracks of a type in a stream yet). bug 1056650 342 virtual nsresult ReplaceTrack(dom::MediaStreamTrack& domtrack); 343 344 // Separate classes to allow ref counting 345 class PipelineListener; 346 class VideoFrameFeeder; 347 class PipelineVideoSink; 348 349 protected: 350 ~MediaPipelineTransmit(); 351 352 private: 353 RefPtr<PipelineListener> listener_; 354 RefPtr<AudioProxyThread> audio_processing_; 355 #if !defined(MOZILLA_EXTERNAL_LINKAGE) 356 RefPtr<VideoFrameFeeder> feeder_; 357 RefPtr<VideoFrameConverter> converter_; 358 #endif 359 RefPtr<PipelineVideoSink> video_sink_; 360 dom::MediaStreamTrack* domtrack_; 361 }; 362 363 364 // A specialization of pipeline for reading from the network and 365 // rendering video. 366 class MediaPipelineReceive : public MediaPipeline { 367 public: 368 // Set rtcp_transport to nullptr to use rtcp-mux 369 MediaPipelineReceive(const std::string& pc, 370 nsCOMPtr<nsIEventTarget> main_thread, 371 nsCOMPtr<nsIEventTarget> sts_thread, 372 SourceMediaStream *stream, 373 const std::string& track_id, 374 int level, 375 RefPtr<MediaSessionConduit> conduit, 376 RefPtr<TransportFlow> rtp_transport, 377 RefPtr<TransportFlow> rtcp_transport, 378 nsAutoPtr<MediaPipelineFilter> filter); 379 380 int segments_added() const { return segments_added_; } 381 382 #ifndef USE_FAKE_MEDIA_STREAMS 383 // Sets the PrincipalHandle we set on the media chunks produced by this 384 // pipeline. Must be called on the main thread. 385 virtual void SetPrincipalHandle_m(const PrincipalHandle& principal_handle) = 0; 386 #endif // USE_FAKE_MEDIA_STREAMS 387 protected: 388 ~MediaPipelineReceive(); 389 390 RefPtr<SourceMediaStream> stream_; 391 int segments_added_; 392 393 private: 394 }; 395 396 397 // A specialization of pipeline for reading from the network and 398 // rendering audio. 399 class MediaPipelineReceiveAudio : public MediaPipelineReceive { 400 public: 401 MediaPipelineReceiveAudio(const std::string& pc, 402 nsCOMPtr<nsIEventTarget> main_thread, 403 nsCOMPtr<nsIEventTarget> sts_thread, 404 SourceMediaStream* stream, 405 // This comes from an msid attribute. Everywhere 406 // but MediaStreamGraph uses this. 407 const std::string& media_stream_track_id, 408 // This is an integer identifier that is only 409 // unique within a single DOMMediaStream, which is 410 // used by MediaStreamGraph 411 TrackID numeric_track_id, 412 int level, 413 RefPtr<AudioSessionConduit> conduit, 414 RefPtr<TransportFlow> rtp_transport, 415 RefPtr<TransportFlow> rtcp_transport, 416 nsAutoPtr<MediaPipelineFilter> filter); 417 418 void DetachMedia() override; 419 420 nsresult Init() override; 421 bool IsVideo() const override { return false; } 422 423 #ifndef USE_FAKE_MEDIA_STREAMS 424 void SetPrincipalHandle_m(const PrincipalHandle& principal_handle) override; 425 #endif // USE_FAKE_MEDIA_STREAMS 426 427 private: 428 // Separate class to allow ref counting 429 class PipelineListener; 430 431 RefPtr<PipelineListener> listener_; 432 }; 433 434 435 // A specialization of pipeline for reading from the network and 436 // rendering video. 437 class MediaPipelineReceiveVideo : public MediaPipelineReceive { 438 public: 439 MediaPipelineReceiveVideo(const std::string& pc, 440 nsCOMPtr<nsIEventTarget> main_thread, 441 nsCOMPtr<nsIEventTarget> sts_thread, 442 SourceMediaStream *stream, 443 // This comes from an msid attribute. Everywhere 444 // but MediaStreamGraph uses this. 445 const std::string& media_stream_track_id, 446 // This is an integer identifier that is only 447 // unique within a single DOMMediaStream, which is 448 // used by MediaStreamGraph 449 TrackID numeric_track_id, 450 int level, 451 RefPtr<VideoSessionConduit> conduit, 452 RefPtr<TransportFlow> rtp_transport, 453 RefPtr<TransportFlow> rtcp_transport, 454 nsAutoPtr<MediaPipelineFilter> filter); 455 456 // Called on the main thread. 457 void DetachMedia() override; 458 459 nsresult Init() override; 460 bool IsVideo() const override { return true; } 461 462 #ifndef USE_FAKE_MEDIA_STREAMS 463 void SetPrincipalHandle_m(const PrincipalHandle& principal_handle) override; 464 #endif // USE_FAKE_MEDIA_STREAMS 465 466 private: 467 class PipelineRenderer; 468 friend class PipelineRenderer; 469 470 // Separate class to allow ref counting 471 class PipelineListener; 472 473 RefPtr<PipelineRenderer> renderer_; 474 RefPtr<PipelineListener> listener_; 475 }; 476 477 478 } // namespace mozilla 479 #endif 480