1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
3 * You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5 #ifndef _PEER_CONNECTION_MEDIA_H_
6 #define _PEER_CONNECTION_MEDIA_H_
7
8 #include <string>
9 #include <vector>
10 #include <map>
11
12 #include "nspr.h"
13 #include "prlock.h"
14
15 #include "mozilla/RefPtr.h"
16 #include "mozilla/UniquePtr.h"
17 #include "nsComponentManagerUtils.h"
18 #include "nsIProtocolProxyCallback.h"
19
20 #include "signaling/src/jsep/JsepSession.h"
21 #include "AudioSegment.h"
22
23 #if !defined(MOZILLA_EXTERNAL_LINKAGE)
24 #include "Layers.h"
25 #include "VideoUtils.h"
26 #include "ImageLayers.h"
27 #include "VideoSegment.h"
28 #include "MediaStreamTrack.h"
29 #endif
30
xmalloc(size_t n)31 class nsIPrincipal;
32
33 namespace mozilla {
34 class DataChannel;
35 class PeerIdentity;
36 class MediaPipelineFactory;
37 namespace dom {
38 struct RTCInboundRTPStreamStats;
39 struct RTCOutboundRTPStreamStats;
40 }
41 }
42
43 #include "nricectxhandler.h"
44 #include "nriceresolver.h"
45 #include "nricemediastream.h"
46 #include "MediaPipeline.h"
47
xmallocarray(size_t nmemb,size_t size)48 namespace mozilla {
49
50 class PeerConnectionImpl;
51 class PeerConnectionMedia;
52 class PCUuidGenerator;
53
54 class SourceStreamInfo {
55 public:
56 SourceStreamInfo(DOMMediaStream* aMediaStream,
57 PeerConnectionMedia *aParent,
58 const std::string& aId)
59 : mMediaStream(aMediaStream),
60 mParent(aParent),
61 mId(aId) {
62 MOZ_ASSERT(mMediaStream);
63 }
64
65 SourceStreamInfo(already_AddRefed<DOMMediaStream>& aMediaStream,
66 PeerConnectionMedia *aParent,
67 const std::string& aId)
68 : mMediaStream(aMediaStream),
69 mParent(aParent),
70 mId(aId) {
71 MOZ_ASSERT(mMediaStream);
72 }
73
74 virtual ~SourceStreamInfo() {}
75
76 DOMMediaStream* GetMediaStream() const {
77 return mMediaStream;
78 }
79
80 nsresult StorePipeline(const std::string& trackId,
81 const RefPtr<MediaPipeline>& aPipeline);
82
83 virtual void AddTrack(const std::string& trackId,
84 const RefPtr<dom::MediaStreamTrack>& aTrack)
85 {
86 mTracks.insert(std::make_pair(trackId, aTrack));
87 }
88 virtual void RemoveTrack(const std::string& trackId);
89 bool HasTrack(const std::string& trackId) const
90 {
91 return !!mTracks.count(trackId);
92 }
93 size_t GetTrackCount() const { return mTracks.size(); }
94
95 // This method exists for stats and the unittests.
96 // It allows visibility into the pipelines and flows.
97 const std::map<std::string, RefPtr<MediaPipeline>>&
98 GetPipelines() const { return mPipelines; }
99 RefPtr<MediaPipeline> GetPipelineByTrackId_m(const std::string& trackId);
100 // This is needed so PeerConnectionImpl can unregister itself as
101 // PrincipalChangeObserver from each track.
102 const std::map<std::string, RefPtr<dom::MediaStreamTrack>>&
103 GetMediaStreamTracks() const { return mTracks; }
104 dom::MediaStreamTrack* GetTrackById(const std::string& trackId) const
105 {
106 auto it = mTracks.find(trackId);
107 if (it == mTracks.end()) {
108 return nullptr;
109 }
110
111 return it->second;
112 }
113 const std::string& GetId() const { return mId; }
114
115 void DetachTransport_s();
116 virtual void DetachMedia_m();
117 bool AnyCodecHasPluginID(uint64_t aPluginID);
118 protected:
119 void EndTrack(MediaStream* stream, dom::MediaStreamTrack* track);
120 RefPtr<DOMMediaStream> mMediaStream;
121 PeerConnectionMedia *mParent;
122 const std::string mId;
123 // These get set up before we generate our local description, the pipelines
124 // and conduits are set up once offer/answer completes.
125 std::map<std::string, RefPtr<dom::MediaStreamTrack>> mTracks;
126 std::map<std::string, RefPtr<MediaPipeline>> mPipelines;
127 };
128
129 // TODO(ekr@rtfm.com): Refactor {Local,Remote}SourceStreamInfo
130 // bug 837539.
131 class LocalSourceStreamInfo : public SourceStreamInfo {
132 ~LocalSourceStreamInfo() {
133 mMediaStream = nullptr;
134 }
135 public:
136 LocalSourceStreamInfo(DOMMediaStream *aMediaStream,
137 PeerConnectionMedia *aParent,
138 const std::string& aId)
139 : SourceStreamInfo(aMediaStream, aParent, aId) {}
140
141 nsresult TakePipelineFrom(RefPtr<LocalSourceStreamInfo>& info,
142 const std::string& oldTrackId,
143 dom::MediaStreamTrack& aNewTrack,
144 const std::string& newTrackId);
145
146 #if !defined(MOZILLA_EXTERNAL_LINKAGE)
147 void UpdateSinkIdentity_m(dom::MediaStreamTrack* aTrack,
148 nsIPrincipal* aPrincipal,
149 const PeerIdentity* aSinkIdentity);
150 #endif
151
152 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(LocalSourceStreamInfo)
153
154 private:
155 already_AddRefed<MediaPipeline> ForgetPipelineByTrackId_m(
156 const std::string& trackId);
157 };
158
159 #if !defined(MOZILLA_EXTERNAL_LINKAGE)
160 class RemoteTrackSource : public dom::MediaStreamTrackSource
161 {
162 public:
163 explicit RemoteTrackSource(nsIPrincipal* aPrincipal, const nsString& aLabel)
164 : dom::MediaStreamTrackSource(aPrincipal, aLabel) {}
165
166 dom::MediaSourceEnum GetMediaSource() const override
167 {
168 return dom::MediaSourceEnum::Other;
169 }
170
171 already_AddRefed<PledgeVoid>
172 ApplyConstraints(nsPIDOMWindowInner* aWindow,
173 const dom::MediaTrackConstraints& aConstraints) override;
174
175 void Stop() override
176 {
177 // XXX (Bug 1314270): Implement rejection logic if necessary when we have
178 // clarity in the spec.
179 }
180
181 void SetPrincipal(nsIPrincipal* aPrincipal)
182 {
183 mPrincipal = aPrincipal;
184 PrincipalChanged();
185 }
186
187 protected:
188 virtual ~RemoteTrackSource() {}
189 };
190 #endif
191
192 class RemoteSourceStreamInfo : public SourceStreamInfo {
193 ~RemoteSourceStreamInfo() {}
194 public:
195 RemoteSourceStreamInfo(already_AddRefed<DOMMediaStream> aMediaStream,
196 PeerConnectionMedia *aParent,
197 const std::string& aId)
198 : SourceStreamInfo(aMediaStream, aParent, aId),
199 mReceiving(false)
200 {
201 }
202
203 void DetachMedia_m() override;
204 void RemoveTrack(const std::string& trackId) override;
205 void SyncPipeline(RefPtr<MediaPipelineReceive> aPipeline);
206
207 #if !defined(MOZILLA_EXTERNAL_LINKAGE)
208 void UpdatePrincipal_m(nsIPrincipal* aPrincipal);
209 #endif
210
211 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(RemoteSourceStreamInfo)
212
213 void AddTrack(const std::string& trackId,
214 const RefPtr<dom::MediaStreamTrack>& aTrack) override
215 {
216 SourceStreamInfo::AddTrack(trackId, aTrack);
217 }
218
219 TrackID GetNumericTrackId(const std::string& trackId) const
220 {
221 dom::MediaStreamTrack* track = GetTrackById(trackId);
222 if (!track) {
223 return TRACK_INVALID;
224 }
225 return track->mTrackID;
226 }
227
228 void StartReceiving();
229
230 private:
231 // True iff SetPullEnabled(true) has been called on the DOMMediaStream. This
232 // happens when offer/answer concludes.
233 bool mReceiving;
234 };
235
236 class PeerConnectionMedia : public sigslot::has_slots<> {
237 ~PeerConnectionMedia()
238 {
239 MOZ_RELEASE_ASSERT(!mMainThread);
240 }
241
242 public:
243 explicit PeerConnectionMedia(PeerConnectionImpl *parent);
244
245 enum IceRestartState { ICE_RESTART_NONE,
246 ICE_RESTART_PROVISIONAL,
247 ICE_RESTART_COMMITTED
248 };
249
250 PeerConnectionImpl* GetPC() { return mParent; }
251 nsresult Init(const std::vector<NrIceStunServer>& stun_servers,
252 const std::vector<NrIceTurnServer>& turn_servers,
253 NrIceCtx::Policy policy);
254 // WARNING: This destroys the object!
255 void SelfDestruct();
256
257 RefPtr<NrIceCtxHandler> ice_ctx_hdlr() const { return mIceCtxHdlr; }
258 RefPtr<NrIceCtx> ice_ctx() const { return mIceCtxHdlr->ctx(); }
259
260 RefPtr<NrIceMediaStream> ice_media_stream(size_t i) const {
261 return mIceCtxHdlr->ctx()->GetStream(i);
262 }
263
264 size_t num_ice_media_streams() const {
265 return mIceCtxHdlr->ctx()->GetStreamCount();
266 }
267
268 // Ensure ICE transports exist that we might need when offer/answer concludes
269 void EnsureTransports(const JsepSession& aSession);
270
271 // Activate or remove ICE transports at the conclusion of offer/answer,
272 // or when rollback occurs.
273 void ActivateOrRemoveTransports(const JsepSession& aSession);
274
275 // Start ICE checks.
276 void StartIceChecks(const JsepSession& session);
277
278 bool IsIceRestarting() const;
279 IceRestartState GetIceRestartState() const;
280
281 // Begin ICE restart
282 void BeginIceRestart(const std::string& ufrag,
283 const std::string& pwd);
284 // Commit ICE Restart - offer/answer complete, no rollback possible
285 void CommitIceRestart();
286 // Finalize ICE restart
287 void FinalizeIceRestart();
288 // Abort ICE restart
289 void RollbackIceRestart();
290
291 // Process a trickle ICE candidate.
292 void AddIceCandidate(const std::string& candidate, const std::string& mid,
293 uint32_t aMLine);
294
295 // Handle complete media pipelines.
296 nsresult UpdateMediaPipelines(const JsepSession& session);
297
298 // Add a track (main thread only)
299 nsresult AddTrack(DOMMediaStream& aMediaStream,
300 const std::string& streamId,
301 dom::MediaStreamTrack& aTrack,
302 const std::string& trackId);
303
304 nsresult RemoveLocalTrack(const std::string& streamId,
305 const std::string& trackId);
306 nsresult RemoveRemoteTrack(const std::string& streamId,
307 const std::string& trackId);
308
309 // Get a specific local stream
310 uint32_t LocalStreamsLength()
311 {
312 return mLocalSourceStreams.Length();
313 }
314 LocalSourceStreamInfo* GetLocalStreamByIndex(int index);
315 LocalSourceStreamInfo* GetLocalStreamById(const std::string& id);
316 LocalSourceStreamInfo* GetLocalStreamByTrackId(const std::string& id);
317
318 // Get a specific remote stream
319 uint32_t RemoteStreamsLength()
320 {
321 return mRemoteSourceStreams.Length();
322 }
323
324 RemoteSourceStreamInfo* GetRemoteStreamByIndex(size_t index);
325 RemoteSourceStreamInfo* GetRemoteStreamById(const std::string& id);
326 RemoteSourceStreamInfo* GetRemoteStreamByTrackId(const std::string& id);
327
328 // Add a remote stream.
329 nsresult AddRemoteStream(RefPtr<RemoteSourceStreamInfo> aInfo);
330
331 nsresult ReplaceTrack(const std::string& aOldStreamId,
332 const std::string& aOldTrackId,
333 dom::MediaStreamTrack& aNewTrack,
334 const std::string& aNewStreamId,
335 const std::string& aNewTrackId);
336
337 #if !defined(MOZILLA_EXTERNAL_LINKAGE)
338 // In cases where the peer isn't yet identified, we disable the pipeline (not
339 // the stream, that would potentially affect others), so that it sends
340 // black/silence. Once the peer is identified, re-enable those streams.
341 // aTrack will be set if this update came from a principal change on aTrack.
342 void UpdateSinkIdentity_m(dom::MediaStreamTrack* aTrack,
343 nsIPrincipal* aPrincipal,
344 const PeerIdentity* aSinkIdentity);
345 // this determines if any track is peerIdentity constrained
346 bool AnyLocalTrackHasPeerIdentity() const;
347 // When we finally learn who is on the other end, we need to change the ownership
348 // on streams
349 void UpdateRemoteStreamPrincipals_m(nsIPrincipal* aPrincipal);
350 #endif
351
352 bool AnyCodecHasPluginID(uint64_t aPluginID);
353
354 const nsCOMPtr<nsIThread>& GetMainThread() const { return mMainThread; }
355 const nsCOMPtr<nsIEventTarget>& GetSTSThread() const { return mSTSThread; }
356
357 static size_t GetTransportFlowIndex(int aStreamIndex, bool aRtcp)
358 {
359 return aStreamIndex * 2 + (aRtcp ? 1 : 0);
360 }
361
362 // Get a transport flow either RTP/RTCP for a particular stream
363 // A stream can be of audio/video/datachannel/budled(?) types
364 RefPtr<TransportFlow> GetTransportFlow(int aStreamIndex, bool aIsRtcp) {
365 int index_inner = GetTransportFlowIndex(aStreamIndex, aIsRtcp);
366
367 if (mTransportFlows.find(index_inner) == mTransportFlows.end())
368 return nullptr;
369
370 return mTransportFlows[index_inner];
371 }
372
373 // Add a transport flow
374 void AddTransportFlow(int aIndex, bool aRtcp,
375 const RefPtr<TransportFlow> &aFlow);
376 void RemoveTransportFlow(int aIndex, bool aRtcp);
377 void ConnectDtlsListener_s(const RefPtr<TransportFlow>& aFlow);
378 void DtlsConnected_s(TransportLayer* aFlow,
379 TransportLayer::State state);
380 static void DtlsConnected_m(const std::string& aParentHandle,
381 bool aPrivacyRequested);
382
383 RefPtr<AudioSessionConduit> GetAudioConduit(size_t level) {
384 auto it = mConduits.find(level);
385 if (it == mConduits.end()) {
386 return nullptr;
387 }
388
389 if (it->second.first) {
390 MOZ_ASSERT(false, "In GetAudioConduit, we found a video conduit!");
391 return nullptr;
392 }
393
394 return RefPtr<AudioSessionConduit>(
395 static_cast<AudioSessionConduit*>(it->second.second.get()));
396 }
397
398 RefPtr<VideoSessionConduit> GetVideoConduit(size_t level) {
399 auto it = mConduits.find(level);
400 if (it == mConduits.end()) {
401 return nullptr;
402 }
403
404 if (!it->second.first) {
405 MOZ_ASSERT(false, "In GetVideoConduit, we found an audio conduit!");
406 return nullptr;
407 }
408
409 return RefPtr<VideoSessionConduit>(
410 static_cast<VideoSessionConduit*>(it->second.second.get()));
411 }
412
413 // Add a conduit
414 void AddAudioConduit(size_t level, const RefPtr<AudioSessionConduit> &aConduit) {
415 mConduits[level] = std::make_pair(false, aConduit);
416 }
417
418 void AddVideoConduit(size_t level, const RefPtr<VideoSessionConduit> &aConduit) {
419 mConduits[level] = std::make_pair(true, aConduit);
420 }
421
422 // ICE state signals
423 sigslot::signal2<NrIceCtx*, NrIceCtx::GatheringState>
424 SignalIceGatheringStateChange;
425 sigslot::signal2<NrIceCtx*, NrIceCtx::ConnectionState>
426 SignalIceConnectionStateChange;
427 // This passes a candidate:... attribute and level
428 sigslot::signal2<const std::string&, uint16_t> SignalCandidate;
429 // This passes address, port, level of the default candidate.
430 sigslot::signal5<const std::string&, uint16_t,
431 const std::string&, uint16_t, uint16_t>
432 SignalUpdateDefaultCandidate;
433 sigslot::signal1<uint16_t>
434 SignalEndOfLocalCandidates;
435
436 private:
437 nsresult InitProxy();
438 class ProtocolProxyQueryHandler : public nsIProtocolProxyCallback {
439 public:
440 explicit ProtocolProxyQueryHandler(PeerConnectionMedia *pcm) :
441 pcm_(pcm) {}
442
443 NS_IMETHOD OnProxyAvailable(nsICancelable *request,
444 nsIChannel *aChannel,
445 nsIProxyInfo *proxyinfo,
446 nsresult result) override;
447 NS_DECL_ISUPPORTS
448
449 private:
450 void SetProxyOnPcm(nsIProxyInfo& proxyinfo);
451 RefPtr<PeerConnectionMedia> pcm_;
452 virtual ~ProtocolProxyQueryHandler() {}
453 };
454
455 // Shutdown media transport. Must be called on STS thread.
456 void ShutdownMediaTransport_s();
457
458 // Final destruction of the media stream. Must be called on the main
459 // thread.
460 void SelfDestruct_m();
461
462 // Manage ICE transports.
463 void EnsureTransport_s(size_t aLevel, size_t aComponentCount);
464 void ActivateOrRemoveTransport_s(
465 size_t aMLine,
466 size_t aComponentCount,
467 const std::string& aUfrag,
468 const std::string& aPassword,
469 const std::vector<std::string>& aCandidateList);
470 void RemoveTransportsAtOrAfter_s(size_t aMLine);
471
472 void GatherIfReady();
473 void FlushIceCtxOperationQueueIfReady();
474 void PerformOrEnqueueIceCtxOperation(nsIRunnable* runnable);
475 void EnsureIceGathering_s(bool aDefaultRouteOnly, bool aProxyOnly);
476 void StartIceChecks_s(bool aIsControlling,
477 bool aIsIceLite,
478 const std::vector<std::string>& aIceOptionsList);
479
480 void BeginIceRestart_s(RefPtr<NrIceCtx> new_ctx);
481 void FinalizeIceRestart_s();
482 void RollbackIceRestart_s();
483 bool GetPrefDefaultAddressOnly() const;
484 bool GetPrefProxyOnly() const;
485
486 void ConnectSignals(NrIceCtx *aCtx, NrIceCtx *aOldCtx=nullptr);
487
488 // Process a trickle ICE candidate.
489 void AddIceCandidate_s(const std::string& aCandidate, const std::string& aMid,
490 uint32_t aMLine);
491
492
493 // ICE events
494 void IceGatheringStateChange_s(NrIceCtx* ctx,
495 NrIceCtx::GatheringState state);
496 void IceConnectionStateChange_s(NrIceCtx* ctx,
497 NrIceCtx::ConnectionState state);
498 void IceStreamReady_s(NrIceMediaStream *aStream);
499 void OnCandidateFound_s(NrIceMediaStream *aStream,
500 const std::string& aCandidate);
501 void EndOfLocalCandidates(const std::string& aDefaultAddr,
502 uint16_t aDefaultPort,
503 const std::string& aDefaultRtcpAddr,
504 uint16_t aDefaultRtcpPort,
505 uint16_t aMLine);
506 void GetDefaultCandidates(const NrIceMediaStream& aStream,
507 NrIceCandidate* aCandidate,
508 NrIceCandidate* aRtcpCandidate);
509
510 void IceGatheringStateChange_m(NrIceCtx* ctx,
511 NrIceCtx::GatheringState state);
512 void IceConnectionStateChange_m(NrIceCtx* ctx,
513 NrIceCtx::ConnectionState state);
514 void OnCandidateFound_m(const std::string& aCandidateLine,
515 const std::string& aDefaultAddr,
516 uint16_t aDefaultPort,
517 const std::string& aDefaultRtcpAddr,
518 uint16_t aDefaultRtcpPort,
519 uint16_t aMLine);
520 void EndOfLocalCandidates_m(const std::string& aDefaultAddr,
521 uint16_t aDefaultPort,
522 const std::string& aDefaultRtcpAddr,
523 uint16_t aDefaultRtcpPort,
524 uint16_t aMLine);
525 bool IsIceCtxReady() const {
526 return mProxyResolveCompleted;
527 }
528
529 // The parent PC
530 PeerConnectionImpl *mParent;
531 // and a loose handle on it for event driven stuff
532 std::string mParentHandle;
533 std::string mParentName;
534
535 // A list of streams returned from GetUserMedia
536 // This is only accessed on the main thread (with one special exception)
537 nsTArray<RefPtr<LocalSourceStreamInfo> > mLocalSourceStreams;
538
539 // A list of streams provided by the other side
540 // This is only accessed on the main thread (with one special exception)
541 nsTArray<RefPtr<RemoteSourceStreamInfo> > mRemoteSourceStreams;
542
543 std::map<size_t, std::pair<bool, RefPtr<MediaSessionConduit>>> mConduits;
544
545 // ICE objects
546 RefPtr<NrIceCtxHandler> mIceCtxHdlr;
547
548 // DNS
549 RefPtr<NrIceResolver> mDNSResolver;
550
551 // Transport flows: even is RTP, odd is RTCP
552 std::map<int, RefPtr<TransportFlow> > mTransportFlows;
553
554 // UUID Generator
555 UniquePtr<PCUuidGenerator> mUuidGen;
556
557 // The main thread.
558 nsCOMPtr<nsIThread> mMainThread;
559
560 // The STS thread.
561 nsCOMPtr<nsIEventTarget> mSTSThread;
562
563 // Used whenever we need to dispatch a runnable to STS to tweak something
564 // on our ICE ctx, but are not ready to do so at the moment (eg; we are
565 // waiting to get a callback with our http proxy config before we start
566 // gathering or start checking)
567 std::vector<nsCOMPtr<nsIRunnable>> mQueuedIceCtxOperations;
568
569 // Used to cancel any ongoing proxy request.
570 nsCOMPtr<nsICancelable> mProxyRequest;
571
572 // Used to track the state of the request.
573 bool mProxyResolveCompleted;
574
575 // Used to store the result of the request.
576 UniquePtr<NrIceProxyServer> mProxyServer;
577
578 // Used to track the state of ice restart
579 IceRestartState mIceRestartState;
580
581 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(PeerConnectionMedia)
582 };
583
584 } // namespace mozilla
585
586 #endif
587