1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #ifdef XP_WIN
8 # include "objbase.h"
9 #endif
10
11 #include "mozilla/dom/HTMLMediaElement.h"
12 #include "AudioDeviceInfo.h"
13 #include "AudioStreamTrack.h"
14 #include "AutoplayPolicy.h"
15 #include "ChannelMediaDecoder.h"
16 #include "DOMMediaStream.h"
17 #include "DecoderDoctorDiagnostics.h"
18 #include "DecoderDoctorLogger.h"
19 #include "DecoderTraits.h"
20 #include "FrameStatistics.h"
21 #include "GMPCrashHelper.h"
22 #include "GVAutoplayPermissionRequest.h"
23 #ifdef MOZ_ANDROID_HLS_SUPPORT
24 # include "HLSDecoder.h"
25 #endif
26 #include "HTMLMediaElement.h"
27 #include "ImageContainer.h"
28 #include "Layers.h"
29 #include "MP4Decoder.h"
30 #include "MediaContainerType.h"
31 #include "MediaError.h"
32 #include "MediaManager.h"
33 #include "MediaMetadataManager.h"
34 #include "MediaResource.h"
35 #include "MediaShutdownManager.h"
36 #include "MediaSourceDecoder.h"
37 #include "MediaStreamError.h"
38 #include "MediaTrackGraphImpl.h"
39 #include "MediaTrackListener.h"
40 #include "MediaStreamWindowCapturer.h"
41 #include "MediaTrack.h"
42 #include "MediaTrackList.h"
43 #include "SVGObserverUtils.h"
44 #include "TimeRanges.h"
45 #include "VideoFrameContainer.h"
46 #include "VideoOutput.h"
47 #include "VideoStreamTrack.h"
48 #include "base/basictypes.h"
49 #include "jsapi.h"
50 #include "mozilla/ArrayUtils.h"
51 #include "mozilla/AsyncEventDispatcher.h"
52 #include "mozilla/EMEUtils.h"
53 #include "mozilla/EventDispatcher.h"
54 #include "mozilla/FloatingPoint.h"
55 #include "mozilla/MathAlgorithms.h"
56 #include "mozilla/NotNull.h"
57 #include "mozilla/Preferences.h"
58 #include "mozilla/PresShell.h"
59 #include "mozilla/Sprintf.h"
60 #include "mozilla/StaticPrefs_media.h"
61 #include "mozilla/Telemetry.h"
62 #include "mozilla/dom/AudioTrack.h"
63 #include "mozilla/dom/AudioTrackList.h"
64 #include "mozilla/dom/BlobURLProtocolHandler.h"
65 #include "mozilla/dom/ContentMediaController.h"
66 #include "mozilla/dom/ElementInlines.h"
67 #include "mozilla/dom/HTMLAudioElement.h"
68 #include "mozilla/dom/HTMLInputElement.h"
69 #include "mozilla/dom/HTMLMediaElementBinding.h"
70 #include "mozilla/dom/HTMLSourceElement.h"
71 #include "mozilla/dom/HTMLVideoElement.h"
72 #include "mozilla/dom/MediaControlUtils.h"
73 #include "mozilla/dom/MediaEncryptedEvent.h"
74 #include "mozilla/dom/MediaErrorBinding.h"
75 #include "mozilla/dom/MediaSource.h"
76 #include "mozilla/dom/PlayPromise.h"
77 #include "mozilla/dom/Promise.h"
78 #include "mozilla/dom/TextTrack.h"
79 #include "mozilla/dom/UserActivation.h"
80 #include "mozilla/dom/VideoPlaybackQuality.h"
81 #include "mozilla/dom/VideoTrack.h"
82 #include "mozilla/dom/VideoTrackList.h"
83 #include "mozilla/dom/WakeLock.h"
84 #include "mozilla/dom/power/PowerManagerService.h"
85 #include "mozilla/net/UrlClassifierFeatureFactory.h"
86 #include "nsAttrValueInlines.h"
87 #include "nsContentPolicyUtils.h"
88 #include "nsContentUtils.h"
89 #include "nsCycleCollectionParticipant.h"
90 #include "nsDisplayList.h"
91 #include "nsDocShell.h"
92 #include "nsError.h"
93 #include "nsGenericHTMLElement.h"
94 #include "nsGkAtoms.h"
95 #include "nsIAsyncVerifyRedirectCallback.h"
96 #include "nsICachingChannel.h"
97 #include "nsIClassOfService.h"
98 #include "nsIContentPolicy.h"
99 #include "nsIDocShell.h"
100 #include "mozilla/dom/Document.h"
101 #include "nsIFrame.h"
102 #include "nsIObserverService.h"
103 #include "nsIRequest.h"
104 #include "nsIScriptError.h"
105 #include "nsISupportsPrimitives.h"
106 #include "nsITimer.h"
107 #include "nsJSUtils.h"
108 #include "nsLayoutUtils.h"
109 #include "nsMediaFragmentURIParser.h"
110 #include "nsMimeTypes.h"
111 #include "nsNetUtil.h"
112 #include "nsNodeInfoManager.h"
113 #include "nsPresContext.h"
114 #include "nsQueryObject.h"
115 #include "nsRange.h"
116 #include "nsSize.h"
117 #include "nsThreadUtils.h"
118 #include "nsURIHashKey.h"
119 #include "nsVideoFrame.h"
120 #include "ReferrerInfo.h"
121 #include "xpcpublic.h"
122 #include <algorithm>
123 #include <cmath>
124 #include <limits>
125 #include <type_traits>
126
127 mozilla::LazyLogModule gMediaElementLog("nsMediaElement");
128 static mozilla::LazyLogModule gMediaElementEventsLog("nsMediaElementEvents");
129
130 extern mozilla::LazyLogModule gAutoplayPermissionLog;
131 #define AUTOPLAY_LOG(msg, ...) \
132 MOZ_LOG(gAutoplayPermissionLog, LogLevel::Debug, (msg, ##__VA_ARGS__))
133
134 // avoid redefined macro in unified build
135 #undef MEDIACONTROL_LOG
136 #define MEDIACONTROL_LOG(msg, ...) \
137 MOZ_LOG(gMediaControlLog, LogLevel::Debug, \
138 ("HTMLMediaElement=%p, " msg, this, ##__VA_ARGS__))
139
140 #undef CONTROLLER_TIMER_LOG
141 #define CONTROLLER_TIMER_LOG(element, msg, ...) \
142 MOZ_LOG(gMediaControlLog, LogLevel::Debug, \
143 ("HTMLMediaElement=%p, " msg, element, ##__VA_ARGS__))
144
145 #define LOG(type, msg) MOZ_LOG(gMediaElementLog, type, msg)
146 #define LOG_EVENT(type, msg) MOZ_LOG(gMediaElementEventsLog, type, msg)
147
148 using namespace mozilla::layers;
149 using mozilla::net::nsMediaFragmentURIParser;
150 using namespace mozilla::dom::HTMLMediaElement_Binding;
151
152 namespace mozilla {
153 namespace dom {
154
155 using AudibleState = AudioChannelService::AudibleState;
156
157 // Number of milliseconds between progress events as defined by spec
158 static const uint32_t PROGRESS_MS = 350;
159
160 // Number of milliseconds of no data before a stall event is fired as defined by
161 // spec
162 static const uint32_t STALL_MS = 3000;
163
164 // Used by AudioChannel for suppresssing the volume to this ratio.
165 #define FADED_VOLUME_RATIO 0.25
166
167 // These constants are arbitrary
168 // Minimum playbackRate for a media
169 static const double MIN_PLAYBACKRATE = 1.0 / 16;
170 // Maximum playbackRate for a media
171 static const double MAX_PLAYBACKRATE = 16.0;
172 // These are the limits beyonds which SoundTouch does not perform too well and
173 // when speech is hard to understand anyway. Threshold above which audio is
174 // muted
175 static const double THRESHOLD_HIGH_PLAYBACKRATE_AUDIO = 4.0;
176 // Threshold under which audio is muted
177 static const double THRESHOLD_LOW_PLAYBACKRATE_AUDIO = 0.25;
178
ClampPlaybackRate(double aPlaybackRate)179 static double ClampPlaybackRate(double aPlaybackRate) {
180 MOZ_ASSERT(aPlaybackRate >= 0.0);
181
182 if (aPlaybackRate == 0.0) {
183 return aPlaybackRate;
184 }
185 if (aPlaybackRate < MIN_PLAYBACKRATE) {
186 return MIN_PLAYBACKRATE;
187 }
188 if (aPlaybackRate > MAX_PLAYBACKRATE) {
189 return MAX_PLAYBACKRATE;
190 }
191 return aPlaybackRate;
192 }
193
194 // Media error values. These need to match the ones in MediaError.webidl.
195 static const unsigned short MEDIA_ERR_ABORTED = 1;
196 static const unsigned short MEDIA_ERR_NETWORK = 2;
197 static const unsigned short MEDIA_ERR_DECODE = 3;
198 static const unsigned short MEDIA_ERR_SRC_NOT_SUPPORTED = 4;
199
ResolvePromisesWithUndefined(const nsTArray<RefPtr<PlayPromise>> & aPromises)200 static void ResolvePromisesWithUndefined(
201 const nsTArray<RefPtr<PlayPromise>>& aPromises) {
202 for (auto& promise : aPromises) {
203 promise->MaybeResolveWithUndefined();
204 }
205 }
206
RejectPromises(const nsTArray<RefPtr<PlayPromise>> & aPromises,nsresult aError)207 static void RejectPromises(const nsTArray<RefPtr<PlayPromise>>& aPromises,
208 nsresult aError) {
209 for (auto& promise : aPromises) {
210 promise->MaybeReject(aError);
211 }
212 }
213
214 // Under certain conditions there may be no-one holding references to
215 // a media element from script, DOM parent, etc, but the element may still
216 // fire meaningful events in the future so we can't destroy it yet:
217 // 1) If the element is delaying the load event (or would be, if it were
218 // in a document), then events up to loadeddata or error could be fired,
219 // so we need to stay alive.
220 // 2) If the element is not paused and playback has not ended, then
221 // we will (or might) play, sending timeupdate and ended events and possibly
222 // audio output, so we need to stay alive.
223 // 3) if the element is seeking then we will fire seeking events and possibly
224 // start playing afterward, so we need to stay alive.
225 // 4) If autoplay could start playback in this element (if we got enough data),
226 // then we need to stay alive.
227 // 5) if the element is currently loading, not suspended, and its source is
228 // not a MediaSource, then script might be waiting for progress events or a
229 // 'stalled' or 'suspend' event, so we need to stay alive.
230 // If we're already suspended then (all other conditions being met),
231 // it's OK to just disappear without firing any more events,
232 // since we have the freedom to remain suspended indefinitely. Note
233 // that we could use this 'suspended' loophole to garbage-collect a suspended
234 // element in case 4 even if it had 'autoplay' set, but we choose not to.
235 // If someone throws away all references to a loading 'autoplay' element
236 // sound should still eventually play.
237 // 6) If the source is a MediaSource, most loading events will not fire unless
238 // appendBuffer() is called on a SourceBuffer, in which case something is
239 // already referencing the SourceBuffer, which keeps the associated media
240 // element alive. Further, a MediaSource will never time out the resource
241 // fetch, and so should not keep the media element alive if it is
242 // unreferenced. A pending 'stalled' event keeps the media element alive.
243 //
244 // Media elements owned by inactive documents (i.e. documents not contained in
245 // any document viewer) should never hold a self-reference because none of the
246 // above conditions are allowed: the element will stop loading and playing
247 // and never resume loading or playing unless its owner document changes to
248 // an active document (which can only happen if there is an external reference
249 // to the element).
250 // Media elements with no owner doc should be able to hold a self-reference.
251 // Something native must have created the element and may expect it to
252 // stay alive to play.
253
254 // It's very important that any change in state which could change the value of
255 // needSelfReference in AddRemoveSelfReference be followed by a call to
256 // AddRemoveSelfReference before this element could die!
257 // It's especially important if needSelfReference would change to 'true',
258 // since if we neglect to add a self-reference, this element might be
259 // garbage collected while there are still event listeners that should
260 // receive events. If we neglect to remove the self-reference then the element
261 // just lives longer than it needs to.
262
263 class nsMediaEvent : public Runnable {
264 public:
nsMediaEvent(const char * aName,HTMLMediaElement * aElement)265 explicit nsMediaEvent(const char* aName, HTMLMediaElement* aElement)
266 : Runnable(aName),
267 mElement(aElement),
268 mLoadID(mElement->GetCurrentLoadID()) {}
269 ~nsMediaEvent() = default;
270
271 NS_IMETHOD Run() override = 0;
272
273 protected:
IsCancelled()274 bool IsCancelled() { return mElement->GetCurrentLoadID() != mLoadID; }
275
276 RefPtr<HTMLMediaElement> mElement;
277 uint32_t mLoadID;
278 };
279
280 class HTMLMediaElement::nsAsyncEventRunner : public nsMediaEvent {
281 private:
282 nsString mName;
283
284 public:
nsAsyncEventRunner(const nsAString & aName,HTMLMediaElement * aElement)285 nsAsyncEventRunner(const nsAString& aName, HTMLMediaElement* aElement)
286 : nsMediaEvent("HTMLMediaElement::nsAsyncEventRunner", aElement),
287 mName(aName) {}
288
Run()289 NS_IMETHOD Run() override {
290 // Silently cancel if our load has been cancelled.
291 if (IsCancelled()) return NS_OK;
292
293 return mElement->DispatchEvent(mName);
294 }
295 };
296
297 /*
298 * If no error is passed while constructing an instance, the instance will
299 * resolve the passed promises with undefined; otherwise, the instance will
300 * reject the passed promises with the passed error.
301 *
302 * The constructor appends the constructed instance into the passed media
303 * element's mPendingPlayPromisesRunners member and once the the runner is run
304 * (whether fulfilled or canceled), it removes itself from
305 * mPendingPlayPromisesRunners.
306 */
307 class HTMLMediaElement::nsResolveOrRejectPendingPlayPromisesRunner
308 : public nsMediaEvent {
309 nsTArray<RefPtr<PlayPromise>> mPromises;
310 nsresult mError;
311
312 public:
nsResolveOrRejectPendingPlayPromisesRunner(HTMLMediaElement * aElement,nsTArray<RefPtr<PlayPromise>> && aPromises,nsresult aError=NS_OK)313 nsResolveOrRejectPendingPlayPromisesRunner(
314 HTMLMediaElement* aElement, nsTArray<RefPtr<PlayPromise>>&& aPromises,
315 nsresult aError = NS_OK)
316 : nsMediaEvent(
317 "HTMLMediaElement::nsResolveOrRejectPendingPlayPromisesRunner",
318 aElement),
319 mPromises(std::move(aPromises)),
320 mError(aError) {
321 mElement->mPendingPlayPromisesRunners.AppendElement(this);
322 }
323
ResolveOrReject()324 void ResolveOrReject() {
325 if (NS_SUCCEEDED(mError)) {
326 ResolvePromisesWithUndefined(mPromises);
327 } else {
328 RejectPromises(mPromises, mError);
329 }
330 }
331
Run()332 NS_IMETHOD Run() override {
333 if (!IsCancelled()) {
334 ResolveOrReject();
335 }
336
337 mElement->mPendingPlayPromisesRunners.RemoveElement(this);
338 return NS_OK;
339 }
340 };
341
342 class HTMLMediaElement::nsNotifyAboutPlayingRunner
343 : public nsResolveOrRejectPendingPlayPromisesRunner {
344 public:
nsNotifyAboutPlayingRunner(HTMLMediaElement * aElement,nsTArray<RefPtr<PlayPromise>> && aPendingPlayPromises)345 nsNotifyAboutPlayingRunner(
346 HTMLMediaElement* aElement,
347 nsTArray<RefPtr<PlayPromise>>&& aPendingPlayPromises)
348 : nsResolveOrRejectPendingPlayPromisesRunner(
349 aElement, std::move(aPendingPlayPromises)) {}
350
Run()351 NS_IMETHOD Run() override {
352 if (IsCancelled()) {
353 mElement->mPendingPlayPromisesRunners.RemoveElement(this);
354 return NS_OK;
355 }
356
357 mElement->DispatchEvent(NS_LITERAL_STRING("playing"));
358 return nsResolveOrRejectPendingPlayPromisesRunner::Run();
359 }
360 };
361
362 class nsSourceErrorEventRunner : public nsMediaEvent {
363 private:
364 nsCOMPtr<nsIContent> mSource;
365
366 public:
nsSourceErrorEventRunner(HTMLMediaElement * aElement,nsIContent * aSource)367 nsSourceErrorEventRunner(HTMLMediaElement* aElement, nsIContent* aSource)
368 : nsMediaEvent("dom::nsSourceErrorEventRunner", aElement),
369 mSource(aSource) {}
370
Run()371 NS_IMETHOD Run() override {
372 // Silently cancel if our load has been cancelled.
373 if (IsCancelled()) return NS_OK;
374 LOG_EVENT(LogLevel::Debug,
375 ("%p Dispatching simple event source error", mElement.get()));
376 return nsContentUtils::DispatchTrustedEvent(
377 mElement->OwnerDoc(), mSource, NS_LITERAL_STRING("error"),
378 CanBubble::eNo, Cancelable::eNo);
379 }
380 };
381
382 /**
383 * We use MediaControlEventListener to listen MediaControlKeysEvent in order to
384 * play and pause media element when user press media control keys and update
385 * media's playback and audible state to the media controller.
386 *
387 * Use `Start()` to start listening event and use `Stop()` to stop listening
388 * event. In addition, notifying any change to media controller MUST be done
389 * after successfully calling `Start()`.
390 */
391 class HTMLMediaElement::MediaControlEventListener final
392 : public ContentControlKeyEventReceiver {
393 public:
NS_INLINE_DECL_REFCOUNTING(MediaControlEventListener,override)394 NS_INLINE_DECL_REFCOUNTING(MediaControlEventListener, override)
395
396 MOZ_INIT_OUTSIDE_CTOR explicit MediaControlEventListener(
397 HTMLMediaElement* aElement)
398 : mElement(aElement) {
399 MOZ_ASSERT(NS_IsMainThread());
400 MOZ_ASSERT(aElement);
401 }
402
403 // Return false if the listener can't be started. Otherwise, return true.
Start()404 bool Start() {
405 MOZ_ASSERT(NS_IsMainThread());
406 if (IsStarted()) {
407 // We have already been started, do not notify start twice.
408 return true;
409 }
410
411 // Fail to init media agent, we are not able to notify the media controller
412 // any update and also are not able to receive media control key events.
413 if (!InitMediaAgent()) {
414 MEDIACONTROL_LOG("Fail to init content media agent!");
415 return false;
416 }
417
418 NotifyPlaybackStateChanged(MediaPlaybackState::eStarted);
419 return true;
420 }
421
Stop()422 void Stop() {
423 MOZ_ASSERT(NS_IsMainThread());
424 if (!IsStarted()) {
425 // We have already been stopped, do not notify stop twice.
426 return;
427 }
428 NotifyMediaStoppedPlaying();
429 NotifyPlaybackStateChanged(MediaPlaybackState::eStopped);
430
431 // Remove ourselves from media agent, which would stop receiving event.
432 mControlAgent->RemoveReceiver(this);
433 mControlAgent = nullptr;
434 }
435
IsStarted() const436 bool IsStarted() const { return mState != MediaPlaybackState::eStopped; }
437
438 /**
439 * Following methods should only be used after starting listener.
440 */
NotifyMediaStartedPlaying()441 void NotifyMediaStartedPlaying() {
442 MOZ_ASSERT(NS_IsMainThread());
443 MOZ_ASSERT(IsStarted());
444 if (mState == MediaPlaybackState::eStarted ||
445 mState == MediaPlaybackState::ePaused) {
446 NotifyPlaybackStateChanged(MediaPlaybackState::ePlayed);
447 // If media is `inaudible` in the beginning, then we don't need to notify
448 // the state, because notifying `inaudible` should always come after
449 // notifying `audible`.
450 if (mIsOwnerAudible) {
451 NotifyAudibleStateChanged(MediaAudibleState::eAudible);
452 }
453 }
454 }
455
NotifyMediaStoppedPlaying()456 void NotifyMediaStoppedPlaying() {
457 MOZ_ASSERT(NS_IsMainThread());
458 MOZ_ASSERT(IsStarted());
459 if (mState == MediaPlaybackState::ePlayed) {
460 NotifyPlaybackStateChanged(MediaPlaybackState::ePaused);
461 // As media are going to be paused, so no sound is possible to be heard.
462 if (mIsOwnerAudible) {
463 NotifyAudibleStateChanged(MediaAudibleState::eInaudible);
464 }
465 }
466 }
467
UpdateMediaAudibleState(bool aIsOwnerAudible)468 void UpdateMediaAudibleState(bool aIsOwnerAudible) {
469 MOZ_ASSERT(NS_IsMainThread());
470 MOZ_ASSERT(IsStarted());
471 if (mIsOwnerAudible == aIsOwnerAudible) {
472 return;
473 }
474 mIsOwnerAudible = aIsOwnerAudible;
475 MEDIACONTROL_LOG("Media becomes %s",
476 mIsOwnerAudible ? "audible" : "inaudible");
477 // If media hasn't started playing, it doesn't make sense to update media
478 // audible state. Therefore, in that case we would noitfy the audible state
479 // when media starts playing.
480 if (mState == MediaPlaybackState::ePlayed) {
481 NotifyAudibleStateChanged(mIsOwnerAudible
482 ? MediaAudibleState::eAudible
483 : MediaAudibleState::eInaudible);
484 }
485 }
486
SetPictureInPictureModeEnabled(bool aIsEnabled)487 void SetPictureInPictureModeEnabled(bool aIsEnabled) {
488 MOZ_ASSERT(NS_IsMainThread());
489 MOZ_ASSERT(IsStarted());
490 if (mIsPictureInPictureEnabled == aIsEnabled) {
491 return;
492 }
493 mIsPictureInPictureEnabled = aIsEnabled;
494 mControlAgent->NotifyPictureInPictureModeChanged(
495 this, mIsPictureInPictureEnabled);
496 }
497
HandleEvent(MediaControlKeysEvent aEvent)498 void HandleEvent(MediaControlKeysEvent aEvent) override {
499 MOZ_ASSERT(NS_IsMainThread());
500 MOZ_ASSERT(IsStarted());
501 MEDIACONTROL_LOG("HandleEvent '%s'", ToMediaControlKeysEventStr(aEvent));
502 if (aEvent == MediaControlKeysEvent::ePlay && Owner()->Paused()) {
503 Owner()->Play();
504 } else if ((aEvent == MediaControlKeysEvent::ePause ||
505 aEvent == MediaControlKeysEvent::eStop) &&
506 !Owner()->Paused()) {
507 Owner()->Pause();
508 }
509 }
510
UpdateOwnerBrowsingContextIfNeeded()511 void UpdateOwnerBrowsingContextIfNeeded() {
512 // Has not notified any information about the owner context yet.
513 if (!IsStarted()) {
514 return;
515 }
516
517 BrowsingContext* currentBC = GetCurrentBrowsingContext();
518 MOZ_ASSERT(currentBC && mOwnerBrowsingContext);
519 // Still in the same browsing context, no need to update.
520 if (currentBC == mOwnerBrowsingContext) {
521 return;
522 }
523 MEDIACONTROL_LOG("Change browsing context from %" PRIu64 " to %" PRIu64,
524 mOwnerBrowsingContext->Id(), currentBC->Id());
525 // This situation would happen when we start a media in an original browsing
526 // context, then we move it to another browsing context, such as an iframe,
527 // so its owner browsing context would be changed. Therefore, we should
528 // reset the media status for the previous browsing context by calling
529 // `Stop()`, in which the listener would notify `ePaused` (if it's playing)
530 // and `eStop`. Then calls `Start()`, in which the listener would notify
531 // `eStart` to the new browsing context. If the media was playing before,
532 // we would also notify `ePlayed`.
533 bool wasInPlayingState = mState == MediaPlaybackState::ePlayed;
534 Stop();
535 Unused << Start();
536 if (wasInPlayingState) {
537 NotifyMediaStartedPlaying();
538 }
539 }
540
GetBrowsingContext() const541 BrowsingContext* GetBrowsingContext() const override {
542 return mOwnerBrowsingContext;
543 }
544
545 private:
546 ~MediaControlEventListener() = default;
547
548 // The media can be moved around different browsing context, so this context
549 // might be different from `mOwnerBrowsingContext` that we use to initialize
550 // the `ContentMediaAgent`.
GetCurrentBrowsingContext() const551 BrowsingContext* GetCurrentBrowsingContext() const {
552 nsPIDOMWindowInner* window = Owner()->OwnerDoc()->GetInnerWindow();
553 return window ? window->GetBrowsingContext() : nullptr;
554 }
555
InitMediaAgent()556 bool InitMediaAgent() {
557 MOZ_ASSERT(NS_IsMainThread());
558 BrowsingContext* currentBC = GetCurrentBrowsingContext();
559 mControlAgent = ContentMediaAgent::Get(currentBC);
560 if (!mControlAgent) {
561 return false;
562 }
563 mOwnerBrowsingContext = currentBC;
564 MOZ_ASSERT(mOwnerBrowsingContext);
565 MEDIACONTROL_LOG("Init agent in browsing context %" PRIu64,
566 mOwnerBrowsingContext->Id());
567 mControlAgent->AddReceiver(this);
568 return true;
569 }
570
Owner() const571 HTMLMediaElement* Owner() const {
572 MOZ_ASSERT(mElement);
573 return mElement.get();
574 }
575
NotifyPlaybackStateChanged(MediaPlaybackState aState)576 void NotifyPlaybackStateChanged(MediaPlaybackState aState) {
577 MOZ_ASSERT(NS_IsMainThread());
578 MOZ_ASSERT(mControlAgent);
579 MEDIACONTROL_LOG("NotifyMediaState from state='%s' to state='%s'",
580 ToMediaPlaybackStateStr(mState),
581 ToMediaPlaybackStateStr(aState));
582 MOZ_ASSERT(mState != aState, "Should not notify same state again!");
583 mState = aState;
584 mControlAgent->NotifyPlaybackStateChanged(this, mState);
585 }
586
NotifyAudibleStateChanged(MediaAudibleState aState)587 void NotifyAudibleStateChanged(MediaAudibleState aState) {
588 MOZ_ASSERT(NS_IsMainThread());
589 MOZ_ASSERT(IsStarted());
590 mControlAgent->NotifyAudibleStateChanged(this, aState);
591 }
592
593 MediaPlaybackState mState = MediaPlaybackState::eStopped;
594 WeakPtr<HTMLMediaElement> mElement;
595 RefPtr<ContentMediaAgent> mControlAgent;
596 bool mIsPictureInPictureEnabled = false;
597 bool mIsOwnerAudible = false;
598 BrowsingContext* MOZ_NON_OWNING_REF mOwnerBrowsingContext = nullptr;
599 };
600
601 class HTMLMediaElement::MediaStreamTrackListener
602 : public DOMMediaStream::TrackListener {
603 public:
MediaStreamTrackListener(HTMLMediaElement * aElement)604 explicit MediaStreamTrackListener(HTMLMediaElement* aElement)
605 : mElement(aElement) {}
606
NotifyTrackAdded(const RefPtr<MediaStreamTrack> & aTrack)607 void NotifyTrackAdded(const RefPtr<MediaStreamTrack>& aTrack) override {
608 if (!mElement) {
609 return;
610 }
611 mElement->NotifyMediaStreamTrackAdded(aTrack);
612 }
613
NotifyTrackRemoved(const RefPtr<MediaStreamTrack> & aTrack)614 void NotifyTrackRemoved(const RefPtr<MediaStreamTrack>& aTrack) override {
615 if (!mElement) {
616 return;
617 }
618 mElement->NotifyMediaStreamTrackRemoved(aTrack);
619 }
620
OnActive()621 void OnActive() {
622 MOZ_ASSERT(mElement);
623
624 // mediacapture-main says:
625 // Note that once ended equals true the HTMLVideoElement will not play media
626 // even if new MediaStreamTracks are added to the MediaStream (causing it to
627 // return to the active state) unless autoplay is true or the web
628 // application restarts the element, e.g., by calling play().
629 //
630 // This is vague on exactly how to go from becoming active to playing, when
631 // autoplaying. However, per the media element spec, to play an autoplaying
632 // media element, we must load the source and reach readyState
633 // HAVE_ENOUGH_DATA [1]. Hence, a MediaStream being assigned to a media
634 // element and becoming active runs the load algorithm, so that it can
635 // eventually be played.
636 //
637 // [1]
638 // https://html.spec.whatwg.org/multipage/media.html#ready-states:event-media-play
639
640 LOG(LogLevel::Debug, ("%p, mSrcStream %p became active, checking if we "
641 "need to run the load algorithm",
642 mElement.get(), mElement->mSrcStream.get()));
643 if (!mElement->IsPlaybackEnded()) {
644 return;
645 }
646 if (!mElement->Autoplay()) {
647 return;
648 }
649 LOG(LogLevel::Info, ("%p, mSrcStream %p became active on autoplaying, "
650 "ended element. Reloading.",
651 mElement.get(), mElement->mSrcStream.get()));
652 mElement->DoLoad();
653 }
654
NotifyActive()655 void NotifyActive() override {
656 if (!mElement) {
657 return;
658 }
659
660 if (!mElement->IsVideo()) {
661 // Audio elements use NotifyAudible().
662 return;
663 }
664
665 OnActive();
666 }
667
NotifyAudible()668 void NotifyAudible() override {
669 if (!mElement) {
670 return;
671 }
672
673 if (mElement->IsVideo()) {
674 // Video elements use NotifyActive().
675 return;
676 }
677
678 OnActive();
679 }
680
OnInactive()681 void OnInactive() {
682 MOZ_ASSERT(mElement);
683
684 if (mElement->IsPlaybackEnded()) {
685 return;
686 }
687 LOG(LogLevel::Debug, ("%p, mSrcStream %p became inactive", mElement.get(),
688 mElement->mSrcStream.get()));
689
690 mElement->PlaybackEnded();
691 }
692
NotifyInactive()693 void NotifyInactive() override {
694 if (!mElement) {
695 return;
696 }
697
698 if (!mElement->IsVideo()) {
699 // Audio elements use NotifyInaudible().
700 return;
701 }
702
703 OnInactive();
704 }
705
NotifyInaudible()706 void NotifyInaudible() override {
707 if (!mElement) {
708 return;
709 }
710
711 if (mElement->IsVideo()) {
712 // Video elements use NotifyInactive().
713 return;
714 }
715
716 OnInactive();
717 }
718
719 protected:
720 const WeakPtr<HTMLMediaElement> mElement;
721 };
722
723 /**
724 * This listener observes the first video frame to arrive with a non-empty size,
725 * and renders it to its VideoFrameContainer.
726 */
727 class HTMLMediaElement::FirstFrameListener : public VideoOutput {
728 public:
FirstFrameListener(VideoFrameContainer * aContainer,AbstractThread * aMainThread)729 FirstFrameListener(VideoFrameContainer* aContainer,
730 AbstractThread* aMainThread)
731 : VideoOutput(aContainer, aMainThread) {
732 MOZ_ASSERT(NS_IsMainThread());
733 }
734
735 // NB that this overrides VideoOutput::NotifyRealtimeTrackData, so we can
736 // filter out all frames but the first one with a real size. This allows us to
737 // later re-use the logic in VideoOutput for rendering that frame.
NotifyRealtimeTrackData(MediaTrackGraph * aGraph,TrackTime aTrackOffset,const MediaSegment & aMedia)738 void NotifyRealtimeTrackData(MediaTrackGraph* aGraph, TrackTime aTrackOffset,
739 const MediaSegment& aMedia) override {
740 MOZ_ASSERT(aMedia.GetType() == MediaSegment::VIDEO);
741
742 if (mInitialSizeFound) {
743 return;
744 }
745
746 const VideoSegment& video = static_cast<const VideoSegment&>(aMedia);
747 for (VideoSegment::ConstChunkIterator c(video); !c.IsEnded(); c.Next()) {
748 if (c->mFrame.GetIntrinsicSize() != gfx::IntSize(0, 0)) {
749 mInitialSizeFound = true;
750
751 // Pick the first frame and run it through the rendering code.
752 VideoSegment segment;
753 segment.AppendFrame(do_AddRef(c->mFrame.GetImage()),
754 c->mFrame.GetIntrinsicSize(),
755 c->mFrame.GetPrincipalHandle(),
756 c->mFrame.GetForceBlack(), c->mTimeStamp);
757 VideoOutput::NotifyRealtimeTrackData(aGraph, aTrackOffset, segment);
758 return;
759 }
760 }
761 }
762
763 private:
764 // Whether a frame with a concrete size has been received. May only be
765 // accessed on the MTG's appending thread. (this is a direct listener so we
766 // get called by whoever is producing this track's data)
767 bool mInitialSizeFound = false;
768 };
769
770 /**
771 * Helper class that manages audio and video outputs for all enabled tracks in a
772 * media element. It also manages calculating the current time when playing a
773 * MediaStream.
774 */
775 class HTMLMediaElement::MediaStreamRenderer
776 : public DOMMediaStream::TrackListener {
777 public:
778 NS_INLINE_DECL_REFCOUNTING(MediaStreamRenderer)
779
MediaStreamRenderer(AbstractThread * aMainThread,VideoFrameContainer * aVideoContainer,void * aAudioOutputKey)780 MediaStreamRenderer(AbstractThread* aMainThread,
781 VideoFrameContainer* aVideoContainer,
782 void* aAudioOutputKey)
783 : mVideoContainer(aVideoContainer),
784 mAudioOutputKey(aAudioOutputKey),
785 mWatchManager(this, aMainThread) {}
786
Shutdown()787 void Shutdown() {
788 for (const auto& t : mAudioTracks.Clone()) {
789 if (t) {
790 RemoveTrack(t->AsAudioStreamTrack());
791 }
792 }
793 if (mVideoTrack) {
794 RemoveTrack(mVideoTrack->AsVideoStreamTrack());
795 }
796 mWatchManager.Shutdown();
797 }
798
UpdateGraphTime()799 void UpdateGraphTime() {
800 mGraphTime =
801 mGraphTimeDummy->mTrack->Graph()->CurrentTime() - *mGraphTimeOffset;
802 }
803
SetProgressingCurrentTime(bool aProgress)804 void SetProgressingCurrentTime(bool aProgress) {
805 if (aProgress == mProgressingCurrentTime) {
806 return;
807 }
808
809 MOZ_DIAGNOSTIC_ASSERT(mGraphTimeDummy);
810 mProgressingCurrentTime = aProgress;
811 MediaTrackGraph* graph = mGraphTimeDummy->mTrack->Graph();
812 if (mProgressingCurrentTime) {
813 mGraphTimeOffset = Some(graph->CurrentTime().Ref() - mGraphTime);
814 mWatchManager.Watch(graph->CurrentTime(),
815 &MediaStreamRenderer::UpdateGraphTime);
816 } else {
817 mWatchManager.Unwatch(graph->CurrentTime(),
818 &MediaStreamRenderer::UpdateGraphTime);
819 }
820 }
821
Start()822 void Start() {
823 if (mRendering) {
824 return;
825 }
826
827 mRendering = true;
828
829 if (!mGraphTimeDummy) {
830 return;
831 }
832
833 for (const auto& t : mAudioTracks) {
834 if (t) {
835 t->AsAudioStreamTrack()->AddAudioOutput(mAudioOutputKey);
836 t->AsAudioStreamTrack()->SetAudioOutputVolume(mAudioOutputKey,
837 mAudioOutputVolume);
838 }
839 }
840
841 if (mVideoTrack) {
842 mVideoTrack->AsVideoStreamTrack()->AddVideoOutput(mVideoContainer);
843 }
844 }
845
Stop()846 void Stop() {
847 if (!mRendering) {
848 return;
849 }
850
851 mRendering = false;
852
853 if (!mGraphTimeDummy) {
854 return;
855 }
856
857 for (const auto& t : mAudioTracks) {
858 if (t) {
859 t->AsAudioStreamTrack()->RemoveAudioOutput(mAudioOutputKey);
860 }
861 }
862
863 if (mVideoTrack) {
864 mVideoTrack->AsVideoStreamTrack()->RemoveVideoOutput(mVideoContainer);
865 }
866 }
867
SetAudioOutputVolume(float aVolume)868 void SetAudioOutputVolume(float aVolume) {
869 if (mAudioOutputVolume == aVolume) {
870 return;
871 }
872 mAudioOutputVolume = aVolume;
873 if (!mRendering) {
874 return;
875 }
876 for (const auto& t : mAudioTracks) {
877 if (t) {
878 t->AsAudioStreamTrack()->SetAudioOutputVolume(mAudioOutputKey,
879 mAudioOutputVolume);
880 }
881 }
882 }
883
AddTrack(AudioStreamTrack * aTrack)884 void AddTrack(AudioStreamTrack* aTrack) {
885 MOZ_DIAGNOSTIC_ASSERT(!mAudioTracks.Contains(aTrack));
886 mAudioTracks.AppendElement(aTrack);
887 EnsureGraphTimeDummy();
888 if (mRendering) {
889 aTrack->AddAudioOutput(mAudioOutputKey);
890 aTrack->SetAudioOutputVolume(mAudioOutputKey, mAudioOutputVolume);
891 }
892 }
AddTrack(VideoStreamTrack * aTrack)893 void AddTrack(VideoStreamTrack* aTrack) {
894 MOZ_DIAGNOSTIC_ASSERT(!mVideoTrack);
895 if (!mVideoContainer) {
896 return;
897 }
898 mVideoTrack = aTrack;
899 EnsureGraphTimeDummy();
900 if (mRendering) {
901 aTrack->AddVideoOutput(mVideoContainer);
902 }
903 }
904
RemoveTrack(AudioStreamTrack * aTrack)905 void RemoveTrack(AudioStreamTrack* aTrack) {
906 MOZ_DIAGNOSTIC_ASSERT(mAudioTracks.Contains(aTrack));
907 if (mRendering) {
908 aTrack->RemoveAudioOutput(mAudioOutputKey);
909 }
910 mAudioTracks.RemoveElement(aTrack);
911 }
RemoveTrack(VideoStreamTrack * aTrack)912 void RemoveTrack(VideoStreamTrack* aTrack) {
913 MOZ_DIAGNOSTIC_ASSERT(mVideoTrack == aTrack);
914 if (!mVideoContainer) {
915 return;
916 }
917 if (mRendering) {
918 aTrack->RemoveVideoOutput(mVideoContainer);
919 }
920 mVideoTrack = nullptr;
921 }
922
CurrentTime() const923 double CurrentTime() const {
924 if (!mGraphTimeDummy) {
925 return 0.0;
926 }
927
928 return mGraphTimeDummy->mTrack->GraphImpl()->MediaTimeToSeconds(mGraphTime);
929 }
930
CurrentGraphTime()931 Watchable<GraphTime>& CurrentGraphTime() { return mGraphTime; }
932
933 // Set if we're rendering video.
934 const RefPtr<VideoFrameContainer> mVideoContainer;
935
936 // Set if we're rendering audio, nullptr otherwise.
937 void* const mAudioOutputKey;
938
939 private:
~MediaStreamRenderer()940 ~MediaStreamRenderer() { Shutdown(); }
941
EnsureGraphTimeDummy()942 void EnsureGraphTimeDummy() {
943 if (mGraphTimeDummy) {
944 return;
945 }
946
947 MediaTrackGraph* graph = nullptr;
948 for (const auto& t : mAudioTracks) {
949 if (t && !t->Ended()) {
950 graph = t->Graph();
951 break;
952 }
953 }
954
955 if (!graph && mVideoTrack && !mVideoTrack->Ended()) {
956 graph = mVideoTrack->Graph();
957 }
958
959 if (!graph) {
960 return;
961 }
962
963 // This dummy keeps `graph` alive and ensures access to it.
964 mGraphTimeDummy = MakeRefPtr<SharedDummyTrack>(
965 graph->CreateSourceTrack(MediaSegment::AUDIO));
966 }
967
968 // True when all tracks are being rendered, i.e., when the media element is
969 // playing.
970 bool mRendering = false;
971
972 // True while we're progressing mGraphTime. False otherwise.
973 bool mProgressingCurrentTime = false;
974
975 // The audio output volume for all audio tracks.
976 float mAudioOutputVolume = 1.0f;
977
978 // WatchManager for mGraphTime.
979 WatchManager<MediaStreamRenderer> mWatchManager;
980
981 // A dummy MediaTrack to guarantee a MediaTrackGraph is kept alive while
982 // we're actively rendering, so we can track the graph's current time. Set
983 // when the first track is added, never unset.
984 RefPtr<SharedDummyTrack> mGraphTimeDummy;
985
986 // Watchable that relays the graph's currentTime updates to the media element
987 // only while we're rendering. This is the current time of the rendering in
988 // GraphTime units.
989 Watchable<GraphTime> mGraphTime = {0, "MediaStreamRenderer::mGraphTime"};
990
991 // Nothing until a track has been added. Then, the current GraphTime at the
992 // time when we were last Start()ed.
993 Maybe<GraphTime> mGraphTimeOffset;
994
995 // Currently enabled (and rendered) audio tracks.
996 nsTArray<WeakPtr<MediaStreamTrack>> mAudioTracks;
997
998 // Currently selected (and rendered) video track.
999 WeakPtr<MediaStreamTrack> mVideoTrack;
1000 };
1001
1002 class HTMLMediaElement::MediaElementTrackSource
1003 : public MediaStreamTrackSource,
1004 public MediaStreamTrackSource::Sink,
1005 public MediaStreamTrackConsumer {
1006 public:
1007 NS_DECL_ISUPPORTS_INHERITED
NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaElementTrackSource,MediaStreamTrackSource)1008 NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaElementTrackSource,
1009 MediaStreamTrackSource)
1010
1011 /* MediaDecoder track source */
1012 MediaElementTrackSource(nsISerialEventTarget* aMainThreadEventTarget,
1013 ProcessedMediaTrack* aTrack, nsIPrincipal* aPrincipal,
1014 OutputMuteState aMuteState)
1015 : MediaStreamTrackSource(aPrincipal, nsString()),
1016 mMainThreadEventTarget(aMainThreadEventTarget),
1017 mTrack(aTrack),
1018 mIntendedElementMuteState(aMuteState),
1019 mElementMuteState(aMuteState) {
1020 MOZ_ASSERT(mTrack);
1021 }
1022
1023 /* MediaStream track source */
MediaElementTrackSource(nsISerialEventTarget * aMainThreadEventTarget,MediaStreamTrack * aCapturedTrack,MediaStreamTrackSource * aCapturedTrackSource,ProcessedMediaTrack * aTrack,MediaInputPort * aPort,OutputMuteState aMuteState)1024 MediaElementTrackSource(nsISerialEventTarget* aMainThreadEventTarget,
1025 MediaStreamTrack* aCapturedTrack,
1026 MediaStreamTrackSource* aCapturedTrackSource,
1027 ProcessedMediaTrack* aTrack, MediaInputPort* aPort,
1028 OutputMuteState aMuteState)
1029 : MediaStreamTrackSource(aCapturedTrackSource->GetPrincipal(),
1030 nsString()),
1031 mMainThreadEventTarget(aMainThreadEventTarget),
1032 mCapturedTrack(aCapturedTrack),
1033 mCapturedTrackSource(aCapturedTrackSource),
1034 mTrack(aTrack),
1035 mPort(aPort),
1036 mIntendedElementMuteState(aMuteState),
1037 mElementMuteState(aMuteState) {
1038 MOZ_ASSERT(mTrack);
1039 MOZ_ASSERT(mCapturedTrack);
1040 MOZ_ASSERT(mCapturedTrackSource);
1041 MOZ_ASSERT(mPort);
1042
1043 mCapturedTrack->AddConsumer(this);
1044 mCapturedTrackSource->RegisterSink(this);
1045 }
1046
SetEnabled(bool aEnabled)1047 void SetEnabled(bool aEnabled) {
1048 if (!mTrack) {
1049 return;
1050 }
1051 mTrack->SetEnabled(aEnabled ? DisabledTrackMode::ENABLED
1052 : DisabledTrackMode::SILENCE_FREEZE);
1053 }
1054
SetPrincipal(RefPtr<nsIPrincipal> aPrincipal)1055 void SetPrincipal(RefPtr<nsIPrincipal> aPrincipal) {
1056 mPrincipal = std::move(aPrincipal);
1057 MediaStreamTrackSource::PrincipalChanged();
1058 }
1059
SetMutedByElement(OutputMuteState aMuteState)1060 void SetMutedByElement(OutputMuteState aMuteState) {
1061 if (mIntendedElementMuteState == aMuteState) {
1062 return;
1063 }
1064 mIntendedElementMuteState = aMuteState;
1065 mMainThreadEventTarget->Dispatch(NS_NewRunnableFunction(
1066 "MediaElementTrackSource::SetMutedByElement",
1067 [self = RefPtr<MediaElementTrackSource>(this), this, aMuteState] {
1068 mElementMuteState = aMuteState;
1069 MediaStreamTrackSource::MutedChanged(Muted());
1070 }));
1071 }
1072
Destroy()1073 void Destroy() override {
1074 if (mCapturedTrack) {
1075 mCapturedTrack->RemoveConsumer(this);
1076 mCapturedTrack = nullptr;
1077 }
1078 if (mCapturedTrackSource) {
1079 mCapturedTrackSource->UnregisterSink(this);
1080 mCapturedTrackSource = nullptr;
1081 }
1082 if (mTrack && !mTrack->IsDestroyed()) {
1083 mTrack->Destroy();
1084 }
1085 if (mPort) {
1086 mPort->Destroy();
1087 mPort = nullptr;
1088 }
1089 }
1090
GetMediaSource() const1091 MediaSourceEnum GetMediaSource() const override {
1092 return MediaSourceEnum::Other;
1093 }
1094
Stop()1095 void Stop() override {
1096 // Do nothing. There may appear new output streams
1097 // that need tracks sourced from this source, so we
1098 // cannot destroy things yet.
1099 }
1100
1101 /**
1102 * Do not keep the track source alive. The source lifetime is controlled by
1103 * its associated tracks.
1104 */
KeepsSourceAlive() const1105 bool KeepsSourceAlive() const override { return false; }
1106
1107 /**
1108 * Do not keep the track source on. It is controlled by its associated tracks.
1109 */
Enabled() const1110 bool Enabled() const override { return false; }
1111
Disable()1112 void Disable() override {}
1113
Enable()1114 void Enable() override {}
1115
PrincipalChanged()1116 void PrincipalChanged() override {
1117 if (!mCapturedTrackSource) {
1118 // This could happen during shutdown.
1119 return;
1120 }
1121
1122 SetPrincipal(mCapturedTrackSource->GetPrincipal());
1123 }
1124
MutedChanged(bool aNewState)1125 void MutedChanged(bool aNewState) override {
1126 MediaStreamTrackSource::MutedChanged(Muted());
1127 }
1128
OverrideEnded()1129 void OverrideEnded() override {
1130 Destroy();
1131 MediaStreamTrackSource::OverrideEnded();
1132 }
1133
NotifyEnabledChanged(MediaStreamTrack * aTrack,bool aEnabled)1134 void NotifyEnabledChanged(MediaStreamTrack* aTrack, bool aEnabled) override {
1135 MediaStreamTrackSource::MutedChanged(Muted());
1136 }
1137
Muted() const1138 bool Muted() const {
1139 return mElementMuteState == OutputMuteState::Muted ||
1140 (mCapturedTrack &&
1141 (mCapturedTrack->Muted() || !mCapturedTrack->Enabled()));
1142 }
1143
Track() const1144 ProcessedMediaTrack* Track() const { return mTrack; }
1145
1146 private:
~MediaElementTrackSource()1147 virtual ~MediaElementTrackSource() { Destroy(); };
1148
1149 const RefPtr<nsISerialEventTarget> mMainThreadEventTarget;
1150 RefPtr<MediaStreamTrack> mCapturedTrack;
1151 RefPtr<MediaStreamTrackSource> mCapturedTrackSource;
1152 const RefPtr<ProcessedMediaTrack> mTrack;
1153 RefPtr<MediaInputPort> mPort;
1154 // The mute state as intended by the media element.
1155 OutputMuteState mIntendedElementMuteState;
1156 // The mute state as applied to this track source. It is applied async, so
1157 // needs to be tracked separately from the intended state.
1158 OutputMuteState mElementMuteState;
1159 };
1160
OutputMediaStream(RefPtr<DOMMediaStream> aStream,bool aCapturingAudioOnly,bool aFinishWhenEnded)1161 HTMLMediaElement::OutputMediaStream::OutputMediaStream(
1162 RefPtr<DOMMediaStream> aStream, bool aCapturingAudioOnly,
1163 bool aFinishWhenEnded)
1164 : mStream(std::move(aStream)),
1165 mCapturingAudioOnly(aCapturingAudioOnly),
1166 mFinishWhenEnded(aFinishWhenEnded) {}
1167 HTMLMediaElement::OutputMediaStream::~OutputMediaStream() = default;
1168
ImplCycleCollectionTraverse(nsCycleCollectionTraversalCallback & aCallback,HTMLMediaElement::OutputMediaStream & aField,const char * aName,uint32_t aFlags)1169 void ImplCycleCollectionTraverse(nsCycleCollectionTraversalCallback& aCallback,
1170 HTMLMediaElement::OutputMediaStream& aField,
1171 const char* aName, uint32_t aFlags) {
1172 ImplCycleCollectionTraverse(aCallback, aField.mStream, "mStream", aFlags);
1173 ImplCycleCollectionTraverse(aCallback, aField.mLiveTracks, "mLiveTracks",
1174 aFlags);
1175 ImplCycleCollectionTraverse(aCallback, aField.mFinishWhenEndedLoadingSrc,
1176 "mFinishWhenEndedLoadingSrc", aFlags);
1177 ImplCycleCollectionTraverse(aCallback, aField.mFinishWhenEndedAttrStream,
1178 "mFinishWhenEndedAttrStream", aFlags);
1179 }
1180
ImplCycleCollectionUnlink(HTMLMediaElement::OutputMediaStream & aField)1181 void ImplCycleCollectionUnlink(HTMLMediaElement::OutputMediaStream& aField) {
1182 ImplCycleCollectionUnlink(aField.mStream);
1183 ImplCycleCollectionUnlink(aField.mLiveTracks);
1184 ImplCycleCollectionUnlink(aField.mFinishWhenEndedLoadingSrc);
1185 ImplCycleCollectionUnlink(aField.mFinishWhenEndedAttrStream);
1186 }
1187
1188 NS_IMPL_ADDREF_INHERITED(HTMLMediaElement::MediaElementTrackSource,
1189 MediaStreamTrackSource)
1190 NS_IMPL_RELEASE_INHERITED(HTMLMediaElement::MediaElementTrackSource,
1191 MediaStreamTrackSource)
1192 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(
1193 HTMLMediaElement::MediaElementTrackSource)
1194 NS_INTERFACE_MAP_END_INHERITING(MediaStreamTrackSource)
1195 NS_IMPL_CYCLE_COLLECTION_CLASS(HTMLMediaElement::MediaElementTrackSource)
1196 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(
1197 HTMLMediaElement::MediaElementTrackSource, MediaStreamTrackSource)
1198 tmp->Destroy();
1199 NS_IMPL_CYCLE_COLLECTION_UNLINK(mCapturedTrack)
1200 NS_IMPL_CYCLE_COLLECTION_UNLINK(mCapturedTrackSource)
1201 NS_IMPL_CYCLE_COLLECTION_UNLINK_END
1202 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(
1203 HTMLMediaElement::MediaElementTrackSource, MediaStreamTrackSource)
1204 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mCapturedTrack)
1205 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mCapturedTrackSource)
1206 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
1207
1208 /**
1209 * There is a reference cycle involving this class: MediaLoadListener
1210 * holds a reference to the HTMLMediaElement, which holds a reference
1211 * to an nsIChannel, which holds a reference to this listener.
1212 * We break the reference cycle in OnStartRequest by clearing mElement.
1213 */
1214 class HTMLMediaElement::MediaLoadListener final
1215 : public nsIStreamListener,
1216 public nsIChannelEventSink,
1217 public nsIInterfaceRequestor,
1218 public nsIObserver,
1219 public nsIThreadRetargetableStreamListener {
1220 ~MediaLoadListener() = default;
1221
1222 NS_DECL_THREADSAFE_ISUPPORTS
1223 NS_DECL_NSIREQUESTOBSERVER
1224 NS_DECL_NSISTREAMLISTENER
1225 NS_DECL_NSICHANNELEVENTSINK
1226 NS_DECL_NSIOBSERVER
1227 NS_DECL_NSIINTERFACEREQUESTOR
1228 NS_DECL_NSITHREADRETARGETABLESTREAMLISTENER
1229
1230 public:
MediaLoadListener(HTMLMediaElement * aElement)1231 explicit MediaLoadListener(HTMLMediaElement* aElement)
1232 : mElement(aElement), mLoadID(aElement->GetCurrentLoadID()) {
1233 MOZ_ASSERT(mElement, "Must pass an element to call back");
1234 }
1235
1236 private:
1237 RefPtr<HTMLMediaElement> mElement;
1238 nsCOMPtr<nsIStreamListener> mNextListener;
1239 const uint32_t mLoadID;
1240 };
1241
NS_IMPL_ISUPPORTS(HTMLMediaElement::MediaLoadListener,nsIRequestObserver,nsIStreamListener,nsIChannelEventSink,nsIInterfaceRequestor,nsIObserver,nsIThreadRetargetableStreamListener)1242 NS_IMPL_ISUPPORTS(HTMLMediaElement::MediaLoadListener, nsIRequestObserver,
1243 nsIStreamListener, nsIChannelEventSink, nsIInterfaceRequestor,
1244 nsIObserver, nsIThreadRetargetableStreamListener)
1245
1246 NS_IMETHODIMP
1247 HTMLMediaElement::MediaLoadListener::Observe(nsISupports* aSubject,
1248 const char* aTopic,
1249 const char16_t* aData) {
1250 nsContentUtils::UnregisterShutdownObserver(this);
1251
1252 // Clear mElement to break cycle so we don't leak on shutdown
1253 mElement = nullptr;
1254 return NS_OK;
1255 }
1256
1257 NS_IMETHODIMP
OnStartRequest(nsIRequest * aRequest)1258 HTMLMediaElement::MediaLoadListener::OnStartRequest(nsIRequest* aRequest) {
1259 nsContentUtils::UnregisterShutdownObserver(this);
1260
1261 if (!mElement) {
1262 // We've been notified by the shutdown observer, and are shutting down.
1263 return NS_BINDING_ABORTED;
1264 }
1265
1266 // The element is only needed until we've had a chance to call
1267 // InitializeDecoderForChannel. So make sure mElement is cleared here.
1268 RefPtr<HTMLMediaElement> element;
1269 element.swap(mElement);
1270
1271 if (mLoadID != element->GetCurrentLoadID()) {
1272 // The channel has been cancelled before we had a chance to create
1273 // a decoder. Abort, don't dispatch an "error" event, as the new load
1274 // may not be in an error state.
1275 return NS_BINDING_ABORTED;
1276 }
1277
1278 // Don't continue to load if the request failed or has been canceled.
1279 nsresult status;
1280 nsresult rv = aRequest->GetStatus(&status);
1281 NS_ENSURE_SUCCESS(rv, rv);
1282 if (NS_FAILED(status)) {
1283 if (element) {
1284 // Handle media not loading error because source was a tracking URL (or
1285 // fingerprinting, cryptomining, etc).
1286 // We make a note of this media node by including it in a dedicated
1287 // array of blocked tracking nodes under its parent document.
1288 if (net::UrlClassifierFeatureFactory::IsClassifierBlockingErrorCode(
1289 status)) {
1290 element->OwnerDoc()->AddBlockedNodeByClassifier(element);
1291 }
1292 element->NotifyLoadError(
1293 nsPrintfCString("%u: %s", uint32_t(status), "Request failed"));
1294 }
1295 return status;
1296 }
1297
1298 nsCOMPtr<nsIHttpChannel> hc = do_QueryInterface(aRequest);
1299 bool succeeded;
1300 if (hc && NS_SUCCEEDED(hc->GetRequestSucceeded(&succeeded)) && !succeeded) {
1301 uint32_t responseStatus = 0;
1302 Unused << hc->GetResponseStatus(&responseStatus);
1303 nsAutoCString statusText;
1304 Unused << hc->GetResponseStatusText(statusText);
1305 element->NotifyLoadError(
1306 nsPrintfCString("%u: %s", responseStatus, statusText.get()));
1307
1308 nsAutoString code;
1309 code.AppendInt(responseStatus);
1310 nsAutoString src;
1311 element->GetCurrentSrc(src);
1312 AutoTArray<nsString, 2> params = {code, src};
1313 element->ReportLoadError("MediaLoadHttpError", params);
1314 return NS_BINDING_ABORTED;
1315 }
1316
1317 nsCOMPtr<nsIChannel> channel = do_QueryInterface(aRequest);
1318 if (channel &&
1319 NS_SUCCEEDED(rv = element->InitializeDecoderForChannel(
1320 channel, getter_AddRefs(mNextListener))) &&
1321 mNextListener) {
1322 rv = mNextListener->OnStartRequest(aRequest);
1323 } else {
1324 // If InitializeDecoderForChannel() returned an error, fire a network error.
1325 if (NS_FAILED(rv) && !mNextListener) {
1326 // Load failed, attempt to load the next candidate resource. If there
1327 // are none, this will trigger a MEDIA_ERR_SRC_NOT_SUPPORTED error.
1328 element->NotifyLoadError(NS_LITERAL_CSTRING("Failed to init decoder"));
1329 }
1330 // If InitializeDecoderForChannel did not return a listener (but may
1331 // have otherwise succeeded), we abort the connection since we aren't
1332 // interested in keeping the channel alive ourselves.
1333 rv = NS_BINDING_ABORTED;
1334 }
1335
1336 return rv;
1337 }
1338
1339 NS_IMETHODIMP
OnStopRequest(nsIRequest * aRequest,nsresult aStatus)1340 HTMLMediaElement::MediaLoadListener::OnStopRequest(nsIRequest* aRequest,
1341 nsresult aStatus) {
1342 if (mNextListener) {
1343 return mNextListener->OnStopRequest(aRequest, aStatus);
1344 }
1345 return NS_OK;
1346 }
1347
1348 NS_IMETHODIMP
OnDataAvailable(nsIRequest * aRequest,nsIInputStream * aStream,uint64_t aOffset,uint32_t aCount)1349 HTMLMediaElement::MediaLoadListener::OnDataAvailable(nsIRequest* aRequest,
1350 nsIInputStream* aStream,
1351 uint64_t aOffset,
1352 uint32_t aCount) {
1353 if (!mNextListener) {
1354 NS_ERROR(
1355 "Must have a chained listener; OnStartRequest should have "
1356 "canceled this request");
1357 return NS_BINDING_ABORTED;
1358 }
1359 return mNextListener->OnDataAvailable(aRequest, aStream, aOffset, aCount);
1360 }
1361
1362 NS_IMETHODIMP
AsyncOnChannelRedirect(nsIChannel * aOldChannel,nsIChannel * aNewChannel,uint32_t aFlags,nsIAsyncVerifyRedirectCallback * cb)1363 HTMLMediaElement::MediaLoadListener::AsyncOnChannelRedirect(
1364 nsIChannel* aOldChannel, nsIChannel* aNewChannel, uint32_t aFlags,
1365 nsIAsyncVerifyRedirectCallback* cb) {
1366 // TODO is this really correct?? See bug #579329.
1367 if (mElement) {
1368 mElement->OnChannelRedirect(aOldChannel, aNewChannel, aFlags);
1369 }
1370 nsCOMPtr<nsIChannelEventSink> sink = do_QueryInterface(mNextListener);
1371 if (sink) {
1372 return sink->AsyncOnChannelRedirect(aOldChannel, aNewChannel, aFlags, cb);
1373 }
1374 cb->OnRedirectVerifyCallback(NS_OK);
1375 return NS_OK;
1376 }
1377
1378 NS_IMETHODIMP
CheckListenerChain()1379 HTMLMediaElement::MediaLoadListener::CheckListenerChain() {
1380 MOZ_ASSERT(mNextListener);
1381 nsCOMPtr<nsIThreadRetargetableStreamListener> retargetable =
1382 do_QueryInterface(mNextListener);
1383 if (retargetable) {
1384 return retargetable->CheckListenerChain();
1385 }
1386 return NS_ERROR_NO_INTERFACE;
1387 }
1388
1389 NS_IMETHODIMP
GetInterface(const nsIID & aIID,void ** aResult)1390 HTMLMediaElement::MediaLoadListener::GetInterface(const nsIID& aIID,
1391 void** aResult) {
1392 return QueryInterface(aIID, aResult);
1393 }
1394
ReportLoadError(const char * aMsg,const nsTArray<nsString> & aParams)1395 void HTMLMediaElement::ReportLoadError(const char* aMsg,
1396 const nsTArray<nsString>& aParams) {
1397 ReportToConsole(nsIScriptError::warningFlag, aMsg, aParams);
1398 }
1399
ReportToConsole(uint32_t aErrorFlags,const char * aMsg,const nsTArray<nsString> & aParams) const1400 void HTMLMediaElement::ReportToConsole(
1401 uint32_t aErrorFlags, const char* aMsg,
1402 const nsTArray<nsString>& aParams) const {
1403 nsContentUtils::ReportToConsole(aErrorFlags, NS_LITERAL_CSTRING("Media"),
1404 OwnerDoc(), nsContentUtils::eDOM_PROPERTIES,
1405 aMsg, aParams);
1406 }
1407
1408 class HTMLMediaElement::AudioChannelAgentCallback final
1409 : public nsIAudioChannelAgentCallback {
1410 public:
1411 NS_DECL_CYCLE_COLLECTING_ISUPPORTS
NS_DECL_CYCLE_COLLECTION_CLASS(AudioChannelAgentCallback)1412 NS_DECL_CYCLE_COLLECTION_CLASS(AudioChannelAgentCallback)
1413
1414 explicit AudioChannelAgentCallback(HTMLMediaElement* aOwner)
1415 : mOwner(aOwner),
1416 mAudioChannelVolume(1.0),
1417 mPlayingThroughTheAudioChannel(false),
1418 mIsOwnerAudible(IsOwnerAudible()),
1419 mIsShutDown(false) {
1420 MOZ_ASSERT(mOwner);
1421 MaybeCreateAudioChannelAgent();
1422 }
1423
UpdateAudioChannelPlayingState()1424 void UpdateAudioChannelPlayingState() {
1425 MOZ_ASSERT(!mIsShutDown);
1426 bool playingThroughTheAudioChannel = IsPlayingThroughTheAudioChannel();
1427
1428 if (playingThroughTheAudioChannel != mPlayingThroughTheAudioChannel) {
1429 if (!MaybeCreateAudioChannelAgent()) {
1430 return;
1431 }
1432
1433 mPlayingThroughTheAudioChannel = playingThroughTheAudioChannel;
1434 if (mPlayingThroughTheAudioChannel) {
1435 StartAudioChannelAgent();
1436 } else {
1437 StopAudioChanelAgent();
1438 }
1439 }
1440 }
1441
NotifyPlayStateChanged()1442 void NotifyPlayStateChanged() {
1443 MOZ_ASSERT(!mIsShutDown);
1444 UpdateAudioChannelPlayingState();
1445 }
1446
WindowVolumeChanged(float aVolume,bool aMuted)1447 NS_IMETHODIMP WindowVolumeChanged(float aVolume, bool aMuted) override {
1448 MOZ_ASSERT(mAudioChannelAgent);
1449
1450 MOZ_LOG(
1451 AudioChannelService::GetAudioChannelLog(), LogLevel::Debug,
1452 ("HTMLMediaElement::AudioChannelAgentCallback, WindowVolumeChanged, "
1453 "this = %p, aVolume = %f, aMuted = %s\n",
1454 this, aVolume, aMuted ? "true" : "false"));
1455
1456 if (mAudioChannelVolume != aVolume) {
1457 mAudioChannelVolume = aVolume;
1458 mOwner->SetVolumeInternal();
1459 }
1460
1461 const uint32_t muted = mOwner->mMuted;
1462 if (aMuted && !mOwner->ComputedMuted()) {
1463 mOwner->SetMutedInternal(muted | MUTED_BY_AUDIO_CHANNEL);
1464 } else if (!aMuted && mOwner->ComputedMuted()) {
1465 mOwner->SetMutedInternal(muted & ~MUTED_BY_AUDIO_CHANNEL);
1466 }
1467
1468 return NS_OK;
1469 }
1470
WindowSuspendChanged(SuspendTypes aSuspend)1471 NS_IMETHODIMP WindowSuspendChanged(SuspendTypes aSuspend) override {
1472 // Currently this method is only be used for delaying autoplay, and we've
1473 // separated related codes to `MediaPlaybackDelayPolicy`.
1474 return NS_OK;
1475 }
1476
WindowAudioCaptureChanged(bool aCapture)1477 NS_IMETHODIMP WindowAudioCaptureChanged(bool aCapture) override {
1478 MOZ_ASSERT(mAudioChannelAgent);
1479 AudioCaptureTrackChangeIfNeeded();
1480 return NS_OK;
1481 }
1482
AudioCaptureTrackChangeIfNeeded()1483 void AudioCaptureTrackChangeIfNeeded() {
1484 MOZ_ASSERT(!mIsShutDown);
1485 if (!IsPlayingStarted()) {
1486 return;
1487 }
1488
1489 MOZ_ASSERT(mAudioChannelAgent);
1490 bool isCapturing = mAudioChannelAgent->IsWindowAudioCapturingEnabled();
1491 mOwner->AudioCaptureTrackChange(isCapturing);
1492 }
1493
NotifyAudioPlaybackChanged(AudibleChangedReasons aReason)1494 void NotifyAudioPlaybackChanged(AudibleChangedReasons aReason) {
1495 MOZ_ASSERT(!mIsShutDown);
1496 if (!IsPlayingStarted()) {
1497 return;
1498 }
1499
1500 AudibleState newAudibleState = IsOwnerAudible();
1501 if (mIsOwnerAudible == newAudibleState) {
1502 return;
1503 }
1504
1505 mIsOwnerAudible = newAudibleState;
1506 mAudioChannelAgent->NotifyStartedAudible(mIsOwnerAudible, aReason);
1507 }
1508
Shutdown()1509 void Shutdown() {
1510 MOZ_ASSERT(!mIsShutDown);
1511 if (mAudioChannelAgent && mAudioChannelAgent->IsPlayingStarted()) {
1512 StopAudioChanelAgent();
1513 }
1514 mAudioChannelAgent = nullptr;
1515 mIsShutDown = true;
1516 }
1517
GetEffectiveVolume() const1518 float GetEffectiveVolume() const {
1519 MOZ_ASSERT(!mIsShutDown);
1520 return mOwner->Volume() * mAudioChannelVolume;
1521 }
1522
1523 private:
~AudioChannelAgentCallback()1524 ~AudioChannelAgentCallback() { MOZ_ASSERT(mIsShutDown); };
1525
MaybeCreateAudioChannelAgent()1526 bool MaybeCreateAudioChannelAgent() {
1527 if (mAudioChannelAgent) {
1528 return true;
1529 }
1530
1531 mAudioChannelAgent = new AudioChannelAgent();
1532 nsresult rv =
1533 mAudioChannelAgent->Init(mOwner->OwnerDoc()->GetInnerWindow(), this);
1534 if (NS_WARN_IF(NS_FAILED(rv))) {
1535 mAudioChannelAgent = nullptr;
1536 MOZ_LOG(
1537 AudioChannelService::GetAudioChannelLog(), LogLevel::Debug,
1538 ("HTMLMediaElement::AudioChannelAgentCallback, Fail to initialize "
1539 "the audio channel agent, this = %p\n",
1540 this));
1541 return false;
1542 }
1543
1544 return true;
1545 }
1546
StartAudioChannelAgent()1547 void StartAudioChannelAgent() {
1548 MOZ_ASSERT(mAudioChannelAgent);
1549 MOZ_ASSERT(!mAudioChannelAgent->IsPlayingStarted());
1550 if (NS_WARN_IF(NS_FAILED(
1551 mAudioChannelAgent->NotifyStartedPlaying(IsOwnerAudible())))) {
1552 return;
1553 }
1554 mAudioChannelAgent->PullInitialUpdate();
1555 }
1556
StopAudioChanelAgent()1557 void StopAudioChanelAgent() {
1558 MOZ_ASSERT(mAudioChannelAgent);
1559 MOZ_ASSERT(mAudioChannelAgent->IsPlayingStarted());
1560 mAudioChannelAgent->NotifyStoppedPlaying();
1561 // If we have started audio capturing before, we have to tell media element
1562 // to clear the output capturing track.
1563 mOwner->AudioCaptureTrackChange(false);
1564 }
1565
IsPlayingStarted()1566 bool IsPlayingStarted() {
1567 if (MaybeCreateAudioChannelAgent()) {
1568 return mAudioChannelAgent->IsPlayingStarted();
1569 }
1570 return false;
1571 }
1572
IsOwnerAudible() const1573 AudibleState IsOwnerAudible() const {
1574 // paused media doesn't produce any sound.
1575 if (mOwner->mPaused) {
1576 return AudibleState::eNotAudible;
1577 }
1578 return mOwner->IsAudible() ? AudibleState::eAudible
1579 : AudibleState::eNotAudible;
1580 }
1581
IsPlayingThroughTheAudioChannel() const1582 bool IsPlayingThroughTheAudioChannel() const {
1583 // If we have an error, we are not playing.
1584 if (mOwner->GetError()) {
1585 return false;
1586 }
1587
1588 // We should consider any bfcached page or inactive document as non-playing.
1589 if (!mOwner->IsActive()) {
1590 return false;
1591 }
1592
1593 // Media is suspended by the docshell.
1594 if (mOwner->ShouldBeSuspendedByInactiveDocShell()) {
1595 return false;
1596 }
1597
1598 // Are we paused
1599 if (mOwner->mPaused) {
1600 return false;
1601 }
1602
1603 // No audio track
1604 if (!mOwner->HasAudio()) {
1605 return false;
1606 }
1607
1608 // A loop always is playing
1609 if (mOwner->HasAttr(kNameSpaceID_None, nsGkAtoms::loop)) {
1610 return true;
1611 }
1612
1613 // If we are actually playing...
1614 if (mOwner->IsCurrentlyPlaying()) {
1615 return true;
1616 }
1617
1618 // If we are playing an external stream.
1619 if (mOwner->mSrcAttrStream) {
1620 return true;
1621 }
1622
1623 return false;
1624 }
1625
1626 RefPtr<AudioChannelAgent> mAudioChannelAgent;
1627 HTMLMediaElement* mOwner;
1628
1629 // The audio channel volume
1630 float mAudioChannelVolume;
1631 // Is this media element playing?
1632 bool mPlayingThroughTheAudioChannel;
1633 // Indicate whether media element is audible for users.
1634 AudibleState mIsOwnerAudible;
1635 bool mIsShutDown;
1636 };
1637
1638 NS_IMPL_CYCLE_COLLECTION_CLASS(HTMLMediaElement::AudioChannelAgentCallback)
1639
1640 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(
1641 HTMLMediaElement::AudioChannelAgentCallback)
1642 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mAudioChannelAgent)
1643 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
1644
1645 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(
1646 HTMLMediaElement::AudioChannelAgentCallback)
1647 NS_IMPL_CYCLE_COLLECTION_UNLINK(mAudioChannelAgent)
1648 NS_IMPL_CYCLE_COLLECTION_UNLINK_END
1649
1650 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(
1651 HTMLMediaElement::AudioChannelAgentCallback)
1652 NS_INTERFACE_MAP_ENTRY(nsIAudioChannelAgentCallback)
1653 NS_INTERFACE_MAP_END
1654
1655 NS_IMPL_CYCLE_COLLECTING_ADDREF(HTMLMediaElement::AudioChannelAgentCallback)
1656 NS_IMPL_CYCLE_COLLECTING_RELEASE(HTMLMediaElement::AudioChannelAgentCallback)
1657
1658 class HTMLMediaElement::ChannelLoader final {
1659 public:
1660 NS_INLINE_DECL_REFCOUNTING(ChannelLoader);
1661
LoadInternal(HTMLMediaElement * aElement)1662 void LoadInternal(HTMLMediaElement* aElement) {
1663 if (mCancelled) {
1664 return;
1665 }
1666
1667 // determine what security checks need to be performed in AsyncOpen().
1668 nsSecurityFlags securityFlags =
1669 aElement->ShouldCheckAllowOrigin()
1670 ? nsILoadInfo::SEC_REQUIRE_CORS_DATA_INHERITS
1671 : nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_DATA_INHERITS;
1672
1673 if (aElement->GetCORSMode() == CORS_USE_CREDENTIALS) {
1674 securityFlags |= nsILoadInfo::SEC_COOKIES_INCLUDE;
1675 }
1676
1677 MOZ_ASSERT(
1678 aElement->IsAnyOfHTMLElements(nsGkAtoms::audio, nsGkAtoms::video));
1679 nsContentPolicyType contentPolicyType =
1680 aElement->IsHTMLElement(nsGkAtoms::audio)
1681 ? nsIContentPolicy::TYPE_INTERNAL_AUDIO
1682 : nsIContentPolicy::TYPE_INTERNAL_VIDEO;
1683
1684 // If aElement has 'triggeringprincipal' attribute, we will use the value as
1685 // triggeringPrincipal for the channel, otherwise it will default to use
1686 // aElement->NodePrincipal().
1687 // This function returns true when aElement has 'triggeringprincipal', so if
1688 // setAttrs is true we will override the origin attributes on the channel
1689 // later.
1690 nsCOMPtr<nsIPrincipal> triggeringPrincipal;
1691 bool setAttrs = nsContentUtils::QueryTriggeringPrincipal(
1692 aElement, aElement->mLoadingSrcTriggeringPrincipal,
1693 getter_AddRefs(triggeringPrincipal));
1694
1695 nsCOMPtr<nsILoadGroup> loadGroup = aElement->GetDocumentLoadGroup();
1696 nsCOMPtr<nsIChannel> channel;
1697 nsresult rv = NS_NewChannelWithTriggeringPrincipal(
1698 getter_AddRefs(channel), aElement->mLoadingSrc,
1699 static_cast<Element*>(aElement), triggeringPrincipal, securityFlags,
1700 contentPolicyType,
1701 nullptr, // aPerformanceStorage
1702 loadGroup,
1703 nullptr, // aCallbacks
1704 nsICachingChannel::LOAD_BYPASS_LOCAL_CACHE_IF_BUSY |
1705 nsIChannel::LOAD_MEDIA_SNIFFER_OVERRIDES_CONTENT_TYPE |
1706 nsIChannel::LOAD_CALL_CONTENT_SNIFFERS);
1707
1708 if (NS_FAILED(rv)) {
1709 // Notify load error so the element will try next resource candidate.
1710 aElement->NotifyLoadError(NS_LITERAL_CSTRING("Fail to create channel"));
1711 return;
1712 }
1713
1714 if (setAttrs) {
1715 nsCOMPtr<nsILoadInfo> loadInfo = channel->LoadInfo();
1716 // The function simply returns NS_OK, so we ignore the return value.
1717 Unused << loadInfo->SetOriginAttributes(
1718 triggeringPrincipal->OriginAttributesRef());
1719 }
1720
1721 nsCOMPtr<nsIClassOfService> cos(do_QueryInterface(channel));
1722 if (cos) {
1723 if (aElement->mUseUrgentStartForChannel) {
1724 cos->AddClassFlags(nsIClassOfService::UrgentStart);
1725
1726 // Reset the flag to avoid loading again without initiated by user
1727 // interaction.
1728 aElement->mUseUrgentStartForChannel = false;
1729 }
1730
1731 // Unconditionally disable throttling since we want the media to fluently
1732 // play even when we switch the tab to background.
1733 cos->AddClassFlags(nsIClassOfService::DontThrottle);
1734 }
1735
1736 // The listener holds a strong reference to us. This creates a
1737 // reference cycle, once we've set mChannel, which is manually broken
1738 // in the listener's OnStartRequest method after it is finished with
1739 // the element. The cycle will also be broken if we get a shutdown
1740 // notification before OnStartRequest fires. Necko guarantees that
1741 // OnStartRequest will eventually fire if we don't shut down first.
1742 RefPtr<MediaLoadListener> loadListener = new MediaLoadListener(aElement);
1743
1744 channel->SetNotificationCallbacks(loadListener);
1745
1746 nsCOMPtr<nsIHttpChannel> hc = do_QueryInterface(channel);
1747 if (hc) {
1748 // Use a byte range request from the start of the resource.
1749 // This enables us to detect if the stream supports byte range
1750 // requests, and therefore seeking, early.
1751 rv = hc->SetRequestHeader(NS_LITERAL_CSTRING("Range"),
1752 NS_LITERAL_CSTRING("bytes=0-"), false);
1753 MOZ_ASSERT(NS_SUCCEEDED(rv));
1754 aElement->SetRequestHeaders(hc);
1755 }
1756
1757 rv = channel->AsyncOpen(loadListener);
1758 if (NS_FAILED(rv)) {
1759 // Notify load error so the element will try next resource candidate.
1760 aElement->NotifyLoadError(NS_LITERAL_CSTRING("Failed to open channel"));
1761 return;
1762 }
1763
1764 // Else the channel must be open and starting to download. If it encounters
1765 // a non-catastrophic failure, it will set a new task to continue loading
1766 // another candidate. It's safe to set it as mChannel now.
1767 mChannel = channel;
1768
1769 // loadListener will be unregistered either on shutdown or when
1770 // OnStartRequest for the channel we just opened fires.
1771 nsContentUtils::RegisterShutdownObserver(loadListener);
1772 }
1773
Load(HTMLMediaElement * aElement)1774 nsresult Load(HTMLMediaElement* aElement) {
1775 MOZ_ASSERT(aElement);
1776 // Per bug 1235183 comment 8, we can't spin the event loop from stable
1777 // state. Defer NS_NewChannel() to a new regular runnable.
1778 return aElement->MainThreadEventTarget()->Dispatch(
1779 NewRunnableMethod<HTMLMediaElement*>("ChannelLoader::LoadInternal",
1780 this, &ChannelLoader::LoadInternal,
1781 aElement));
1782 }
1783
Cancel()1784 void Cancel() {
1785 mCancelled = true;
1786 if (mChannel) {
1787 mChannel->Cancel(NS_BINDING_ABORTED);
1788 mChannel = nullptr;
1789 }
1790 }
1791
Done()1792 void Done() {
1793 MOZ_ASSERT(mChannel);
1794 // Decoder successfully created, the decoder now owns the MediaResource
1795 // which owns the channel.
1796 mChannel = nullptr;
1797 }
1798
Redirect(nsIChannel * aChannel,nsIChannel * aNewChannel,uint32_t aFlags)1799 nsresult Redirect(nsIChannel* aChannel, nsIChannel* aNewChannel,
1800 uint32_t aFlags) {
1801 NS_ASSERTION(aChannel == mChannel, "Channels should match!");
1802 mChannel = aNewChannel;
1803
1804 // Handle forwarding of Range header so that the intial detection
1805 // of seeking support (via result code 206) works across redirects.
1806 nsCOMPtr<nsIHttpChannel> http = do_QueryInterface(aChannel);
1807 NS_ENSURE_STATE(http);
1808
1809 NS_NAMED_LITERAL_CSTRING(rangeHdr, "Range");
1810
1811 nsAutoCString rangeVal;
1812 if (NS_SUCCEEDED(http->GetRequestHeader(rangeHdr, rangeVal))) {
1813 NS_ENSURE_STATE(!rangeVal.IsEmpty());
1814
1815 http = do_QueryInterface(aNewChannel);
1816 NS_ENSURE_STATE(http);
1817
1818 nsresult rv = http->SetRequestHeader(rangeHdr, rangeVal, false);
1819 NS_ENSURE_SUCCESS(rv, rv);
1820 }
1821
1822 return NS_OK;
1823 }
1824
1825 private:
~ChannelLoader()1826 ~ChannelLoader() { MOZ_ASSERT(!mChannel); }
1827 // Holds a reference to the first channel we open to the media resource.
1828 // Once the decoder is created, control over the channel passes to the
1829 // decoder, and we null out this reference. We must store this in case
1830 // we need to cancel the channel before control of it passes to the decoder.
1831 nsCOMPtr<nsIChannel> mChannel;
1832
1833 bool mCancelled = false;
1834 };
1835
1836 class HTMLMediaElement::ErrorSink {
1837 public:
ErrorSink(HTMLMediaElement * aOwner)1838 explicit ErrorSink(HTMLMediaElement* aOwner) : mOwner(aOwner) {
1839 MOZ_ASSERT(mOwner);
1840 }
1841
SetError(uint16_t aErrorCode,const nsACString & aErrorDetails)1842 void SetError(uint16_t aErrorCode, const nsACString& aErrorDetails) {
1843 // Since we have multiple paths calling into DecodeError, e.g.
1844 // MediaKeys::Terminated and EMEH264Decoder::Error. We should take the 1st
1845 // one only in order not to fire multiple 'error' events.
1846 if (mError) {
1847 return;
1848 }
1849
1850 if (!IsValidErrorCode(aErrorCode)) {
1851 NS_ASSERTION(false, "Undefined MediaError codes!");
1852 return;
1853 }
1854
1855 mError = new MediaError(mOwner, aErrorCode, aErrorDetails);
1856 mOwner->DispatchAsyncEvent(NS_LITERAL_STRING("error"));
1857 if (mOwner->ReadyState() == HAVE_NOTHING &&
1858 aErrorCode == MEDIA_ERR_ABORTED) {
1859 // https://html.spec.whatwg.org/multipage/embedded-content.html#media-data-processing-steps-list
1860 // "If the media data fetching process is aborted by the user"
1861 mOwner->DispatchAsyncEvent(NS_LITERAL_STRING("abort"));
1862 mOwner->ChangeNetworkState(NETWORK_EMPTY);
1863 mOwner->DispatchAsyncEvent(NS_LITERAL_STRING("emptied"));
1864 if (mOwner->mDecoder) {
1865 mOwner->ShutdownDecoder();
1866 }
1867 } else if (aErrorCode == MEDIA_ERR_SRC_NOT_SUPPORTED) {
1868 mOwner->ChangeNetworkState(NETWORK_NO_SOURCE);
1869 } else {
1870 mOwner->ChangeNetworkState(NETWORK_IDLE);
1871 }
1872 }
1873
ResetError()1874 void ResetError() { mError = nullptr; }
1875
1876 RefPtr<MediaError> mError;
1877
1878 private:
IsValidErrorCode(const uint16_t & aErrorCode) const1879 bool IsValidErrorCode(const uint16_t& aErrorCode) const {
1880 return (aErrorCode == MEDIA_ERR_DECODE || aErrorCode == MEDIA_ERR_NETWORK ||
1881 aErrorCode == MEDIA_ERR_ABORTED ||
1882 aErrorCode == MEDIA_ERR_SRC_NOT_SUPPORTED);
1883 }
1884
1885 // Media elememt's life cycle would be longer than error sink, so we use the
1886 // raw pointer and this class would only be referenced by media element.
1887 HTMLMediaElement* mOwner;
1888 };
1889
1890 NS_IMPL_CYCLE_COLLECTION_CLASS(HTMLMediaElement)
1891
1892 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(HTMLMediaElement,
1893 nsGenericHTMLElement)
1894 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMediaSource)
1895 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSrcMediaSource)
1896 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSrcStream)
1897 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSrcAttrStream)
1898 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSourcePointer)
1899 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mLoadBlockedDoc)
1900 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSourceLoadCandidate)
1901 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mAudioChannelWrapper)
1902 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mErrorSink->mError)
1903 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOutputStreams)
1904 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOutputTrackSources);
1905 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mPlayed);
1906 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mTextTrackManager)
1907 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mAudioTrackList)
1908 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mVideoTrackList)
1909 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMediaKeys)
1910 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mIncomingMediaKeys)
1911 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSelectedVideoStreamTrack)
1912 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mPendingPlayPromises)
1913 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSeekDOMPromise)
1914 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSetMediaKeysDOMPromise)
1915 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
1916
1917 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(HTMLMediaElement,
1918 nsGenericHTMLElement)
1919 tmp->RemoveMutationObserver(tmp);
1920 if (tmp->mSrcStream) {
1921 // Need to unhook everything that EndSrcMediaStreamPlayback would normally
1922 // do, without creating any new strong references.
1923 if (tmp->mSelectedVideoStreamTrack) {
1924 tmp->mSelectedVideoStreamTrack->RemovePrincipalChangeObserver(tmp);
1925 }
1926 if (tmp->mFirstFrameListener) {
1927 tmp->mSelectedVideoStreamTrack->RemoveVideoOutput(
1928 tmp->mFirstFrameListener);
1929 }
1930 if (tmp->mMediaStreamTrackListener) {
1931 tmp->mSrcStream->UnregisterTrackListener(
1932 tmp->mMediaStreamTrackListener.get());
1933 }
1934 }
1935 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSrcStream)
1936 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSrcAttrStream)
1937 NS_IMPL_CYCLE_COLLECTION_UNLINK(mMediaSource)
1938 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSrcMediaSource)
1939 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSourcePointer)
1940 NS_IMPL_CYCLE_COLLECTION_UNLINK(mLoadBlockedDoc)
1941 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSourceLoadCandidate)
1942 if (tmp->mAudioChannelWrapper) {
1943 tmp->mAudioChannelWrapper->Shutdown();
1944 }
1945 NS_IMPL_CYCLE_COLLECTION_UNLINK(mAudioChannelWrapper)
1946 NS_IMPL_CYCLE_COLLECTION_UNLINK(mErrorSink->mError)
1947 NS_IMPL_CYCLE_COLLECTION_UNLINK(mOutputStreams)
1948 NS_IMPL_CYCLE_COLLECTION_UNLINK(mOutputTrackSources)
1949 NS_IMPL_CYCLE_COLLECTION_UNLINK(mPlayed)
1950 NS_IMPL_CYCLE_COLLECTION_UNLINK(mTextTrackManager)
1951 NS_IMPL_CYCLE_COLLECTION_UNLINK(mAudioTrackList)
1952 NS_IMPL_CYCLE_COLLECTION_UNLINK(mVideoTrackList)
1953 NS_IMPL_CYCLE_COLLECTION_UNLINK(mMediaKeys)
1954 NS_IMPL_CYCLE_COLLECTION_UNLINK(mIncomingMediaKeys)
1955 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSelectedVideoStreamTrack)
1956 NS_IMPL_CYCLE_COLLECTION_UNLINK(mPendingPlayPromises)
1957 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSeekDOMPromise)
1958 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSetMediaKeysDOMPromise)
1959 if (tmp->mMediaControlEventListener) {
1960 tmp->StopListeningMediaControlEventIfNeeded();
1961 tmp->mMediaControlEventListener = nullptr;
1962 }
1963 NS_IMPL_CYCLE_COLLECTION_UNLINK_WEAK_PTR
1964 NS_IMPL_CYCLE_COLLECTION_UNLINK_END
1965
NS_IMPL_ISUPPORTS_CYCLE_COLLECTION_INHERITED_0(HTMLMediaElement,nsGenericHTMLElement)1966 NS_IMPL_ISUPPORTS_CYCLE_COLLECTION_INHERITED_0(HTMLMediaElement,
1967 nsGenericHTMLElement)
1968
1969 void HTMLMediaElement::AddSizeOfExcludingThis(nsWindowSizes& aSizes,
1970 size_t* aNodeSize) const {
1971 nsGenericHTMLElement::AddSizeOfExcludingThis(aSizes, aNodeSize);
1972
1973 // There are many other fields that might be worth reporting, but as seen in
1974 // bug 1595603, mPendingEvents can grow to be very large sometimes, so at
1975 // least report that.
1976 *aNodeSize +=
1977 mPendingEvents.ShallowSizeOfExcludingThis(aSizes.mState.mMallocSizeOf);
1978 for (const nsString& event : mPendingEvents) {
1979 *aNodeSize +=
1980 event.SizeOfExcludingThisIfUnshared(aSizes.mState.mMallocSizeOf);
1981 }
1982 }
1983
ContentRemoved(nsIContent * aChild,nsIContent * aPreviousSibling)1984 void HTMLMediaElement::ContentRemoved(nsIContent* aChild,
1985 nsIContent* aPreviousSibling) {
1986 if (aChild == mSourcePointer) {
1987 mSourcePointer = aPreviousSibling;
1988 }
1989 }
1990
GetMozMediaSourceObject() const1991 already_AddRefed<MediaSource> HTMLMediaElement::GetMozMediaSourceObject()
1992 const {
1993 RefPtr<MediaSource> source = mMediaSource;
1994 return source.forget();
1995 }
1996
MozRequestDebugInfo(ErrorResult & aRv)1997 already_AddRefed<Promise> HTMLMediaElement::MozRequestDebugInfo(
1998 ErrorResult& aRv) {
1999 RefPtr<Promise> promise = CreateDOMPromise(aRv);
2000 if (NS_WARN_IF(aRv.Failed())) {
2001 return nullptr;
2002 }
2003 auto result = MakeUnique<dom::HTMLMediaElementDebugInfo>();
2004 if (mMediaKeys) {
2005 GetEMEInfo(result->mEMEInfo);
2006 }
2007 if (mVideoFrameContainer) {
2008 result->mCompositorDroppedFrames =
2009 mVideoFrameContainer->GetDroppedImageCount();
2010 }
2011 if (mDecoder) {
2012 mDecoder->RequestDebugInfo(result->mDecoder)
2013 ->Then(
2014 mAbstractMainThread, __func__,
2015 [promise, ptr = std::move(result)]() {
2016 promise->MaybeResolve(ptr.get());
2017 },
2018 []() {
2019 MOZ_ASSERT_UNREACHABLE("Unexpected RequestDebugInfo() rejection");
2020 });
2021 } else {
2022 promise->MaybeResolve(result.get());
2023 }
2024 return promise.forget();
2025 }
2026
2027 /* static */
MozEnableDebugLog(const GlobalObject &)2028 void HTMLMediaElement::MozEnableDebugLog(const GlobalObject&) {
2029 DecoderDoctorLogger::EnableLogging();
2030 }
2031
MozRequestDebugLog(ErrorResult & aRv)2032 already_AddRefed<Promise> HTMLMediaElement::MozRequestDebugLog(
2033 ErrorResult& aRv) {
2034 RefPtr<Promise> promise = CreateDOMPromise(aRv);
2035 if (NS_WARN_IF(aRv.Failed())) {
2036 return nullptr;
2037 }
2038
2039 DecoderDoctorLogger::RetrieveMessages(this)->Then(
2040 mAbstractMainThread, __func__,
2041 [promise](const nsACString& aString) {
2042 promise->MaybeResolve(NS_ConvertUTF8toUTF16(aString));
2043 },
2044 [promise](nsresult rv) { promise->MaybeReject(rv); });
2045
2046 return promise.forget();
2047 }
2048
SetVisible(bool aVisible)2049 void HTMLMediaElement::SetVisible(bool aVisible) {
2050 mForcedHidden = !aVisible;
2051 if (mDecoder) {
2052 mDecoder->SetForcedHidden(!aVisible);
2053 }
2054 }
2055
IsVideoDecodingSuspended() const2056 bool HTMLMediaElement::IsVideoDecodingSuspended() const {
2057 return mDecoder && mDecoder->IsVideoDecodingSuspended();
2058 }
2059
IsVisible() const2060 bool HTMLMediaElement::IsVisible() const {
2061 return mVisibilityState == Visibility::ApproximatelyVisible;
2062 }
2063
GetCurrentImage()2064 already_AddRefed<layers::Image> HTMLMediaElement::GetCurrentImage() {
2065 MarkAsTainted();
2066
2067 // TODO: In bug 1345404, handle case when video decoder is already suspended.
2068 ImageContainer* container = GetImageContainer();
2069 if (!container) {
2070 return nullptr;
2071 }
2072
2073 AutoLockImage lockImage(container);
2074 RefPtr<layers::Image> image = lockImage.GetImage(TimeStamp::Now());
2075 return image.forget();
2076 }
2077
HasSuspendTaint() const2078 bool HTMLMediaElement::HasSuspendTaint() const {
2079 MOZ_ASSERT(!mDecoder || (mDecoder->HasSuspendTaint() == mHasSuspendTaint));
2080 return mHasSuspendTaint;
2081 }
2082
GetSrcObject() const2083 already_AddRefed<DOMMediaStream> HTMLMediaElement::GetSrcObject() const {
2084 return do_AddRef(mSrcAttrStream);
2085 }
2086
SetSrcObject(DOMMediaStream & aValue)2087 void HTMLMediaElement::SetSrcObject(DOMMediaStream& aValue) {
2088 SetSrcObject(&aValue);
2089 }
2090
SetSrcObject(DOMMediaStream * aValue)2091 void HTMLMediaElement::SetSrcObject(DOMMediaStream* aValue) {
2092 for (auto& outputStream : mOutputStreams) {
2093 if (aValue == outputStream.mStream) {
2094 ReportToConsole(nsIScriptError::warningFlag,
2095 "MediaElementStreamCaptureCycle");
2096 return;
2097 }
2098 }
2099 mSrcAttrStream = aValue;
2100 UpdateAudioChannelPlayingState();
2101 DoLoad();
2102 }
2103
Ended()2104 bool HTMLMediaElement::Ended() {
2105 return (mDecoder && mDecoder->IsEnded()) ||
2106 (mSrcStream && mSrcStreamReportPlaybackEnded);
2107 }
2108
GetCurrentSrc(nsAString & aCurrentSrc)2109 void HTMLMediaElement::GetCurrentSrc(nsAString& aCurrentSrc) {
2110 nsAutoCString src;
2111 GetCurrentSpec(src);
2112 CopyUTF8toUTF16(src, aCurrentSrc);
2113 }
2114
OnChannelRedirect(nsIChannel * aChannel,nsIChannel * aNewChannel,uint32_t aFlags)2115 nsresult HTMLMediaElement::OnChannelRedirect(nsIChannel* aChannel,
2116 nsIChannel* aNewChannel,
2117 uint32_t aFlags) {
2118 MOZ_ASSERT(mChannelLoader);
2119 return mChannelLoader->Redirect(aChannel, aNewChannel, aFlags);
2120 }
2121
ShutdownDecoder()2122 void HTMLMediaElement::ShutdownDecoder() {
2123 RemoveMediaElementFromURITable();
2124 NS_ASSERTION(mDecoder, "Must have decoder to shut down");
2125
2126 mWaitingForKeyListener.DisconnectIfExists();
2127 if (mMediaSource) {
2128 mMediaSource->CompletePendingTransactions();
2129 }
2130 mDecoder->Shutdown();
2131 DDUNLINKCHILD(mDecoder.get());
2132 mDecoder = nullptr;
2133 ReportAudioTrackSilenceProportionTelemetry();
2134 }
2135
ReportPlayedTimeAfterBlockedTelemetry()2136 void HTMLMediaElement::ReportPlayedTimeAfterBlockedTelemetry() {
2137 if (!mHasPlayEverBeenBlocked) {
2138 return;
2139 }
2140 mHasPlayEverBeenBlocked = false;
2141
2142 const double playTimeThreshold = 7.0;
2143 const double playTimeAfterBlocked = mCurrentLoadPlayTime.Total();
2144 if (playTimeAfterBlocked <= 0.0) {
2145 return;
2146 }
2147
2148 const bool isDurationLessThanTimeThresholdAndMediaPlayedToTheEnd =
2149 Duration() < playTimeThreshold && Ended();
2150 LOG(LogLevel::Debug, ("%p PLAYED_TIME_AFTER_AUTOPLAY_BLOCKED=%f, isVideo=%d",
2151 this, playTimeAfterBlocked, IsVideo()));
2152 if (IsVideo() && playTimeAfterBlocked >= playTimeThreshold) {
2153 AccumulateCategorical(
2154 mozilla::Telemetry::LABELS_MEDIA_PLAYED_TIME_AFTER_AUTOPLAY_BLOCKED::
2155 VPlayedMoreThan7s);
2156 } else if (IsVideo() && playTimeAfterBlocked < playTimeThreshold) {
2157 if (isDurationLessThanTimeThresholdAndMediaPlayedToTheEnd) {
2158 AccumulateCategorical(
2159 mozilla::Telemetry::LABELS_MEDIA_PLAYED_TIME_AFTER_AUTOPLAY_BLOCKED::
2160 VPlayedToTheEnd);
2161 } else {
2162 AccumulateCategorical(
2163 mozilla::Telemetry::LABELS_MEDIA_PLAYED_TIME_AFTER_AUTOPLAY_BLOCKED::
2164 VPlayedLessThan7s);
2165 }
2166 } else if (!IsVideo() && playTimeAfterBlocked >= playTimeThreshold) {
2167 AccumulateCategorical(
2168 mozilla::Telemetry::LABELS_MEDIA_PLAYED_TIME_AFTER_AUTOPLAY_BLOCKED::
2169 APlayedMoreThan7s);
2170 } else if (!IsVideo() && playTimeAfterBlocked < playTimeThreshold) {
2171 if (isDurationLessThanTimeThresholdAndMediaPlayedToTheEnd) {
2172 AccumulateCategorical(
2173 mozilla::Telemetry::LABELS_MEDIA_PLAYED_TIME_AFTER_AUTOPLAY_BLOCKED::
2174 APlayedToTheEnd);
2175 } else {
2176 AccumulateCategorical(
2177 mozilla::Telemetry::LABELS_MEDIA_PLAYED_TIME_AFTER_AUTOPLAY_BLOCKED::
2178 APlayedLessThan7s);
2179 }
2180 }
2181 }
2182
AbortExistingLoads()2183 void HTMLMediaElement::AbortExistingLoads() {
2184 // Abort any already-running instance of the resource selection algorithm.
2185 mLoadWaitStatus = NOT_WAITING;
2186
2187 // Set a new load ID. This will cause events which were enqueued
2188 // with a different load ID to silently be cancelled.
2189 mCurrentLoadID++;
2190
2191 // Immediately reject or resolve the already-dispatched
2192 // nsResolveOrRejectPendingPlayPromisesRunners. These runners won't be
2193 // executed again later since the mCurrentLoadID had been changed.
2194 for (auto& runner : mPendingPlayPromisesRunners) {
2195 runner->ResolveOrReject();
2196 }
2197 mPendingPlayPromisesRunners.Clear();
2198
2199 if (mChannelLoader) {
2200 mChannelLoader->Cancel();
2201 mChannelLoader = nullptr;
2202 }
2203
2204 bool fireTimeUpdate = false;
2205
2206 // We need to remove FirstFrameListener before VideoTracks get emptied.
2207 if (mFirstFrameListener) {
2208 if (mSelectedVideoStreamTrack) {
2209 // Unlink might have removed the selected video track.
2210 mSelectedVideoStreamTrack->RemoveVideoOutput(mFirstFrameListener);
2211 }
2212 mFirstFrameListener = nullptr;
2213 }
2214
2215 if (mDecoder) {
2216 fireTimeUpdate = mDecoder->GetCurrentTime() != 0.0;
2217 ShutdownDecoder();
2218 }
2219 if (mSrcStream) {
2220 EndSrcMediaStreamPlayback();
2221 }
2222
2223 RemoveMediaElementFromURITable();
2224 mLoadingSrc = nullptr;
2225 mLoadingSrcTriggeringPrincipal = nullptr;
2226 DDLOG(DDLogCategory::Property, "loading_src", "");
2227 DDUNLINKCHILD(mMediaSource.get());
2228 mMediaSource = nullptr;
2229
2230 if (mNetworkState == NETWORK_LOADING || mNetworkState == NETWORK_IDLE) {
2231 DispatchAsyncEvent(NS_LITERAL_STRING("abort"));
2232 }
2233
2234 bool hadVideo = HasVideo();
2235 mErrorSink->ResetError();
2236 mCurrentPlayRangeStart = -1.0;
2237 mLoadedDataFired = false;
2238 mAutoplaying = true;
2239 mIsLoadingFromSourceChildren = false;
2240 mSuspendedAfterFirstFrame = false;
2241 mAllowSuspendAfterFirstFrame = true;
2242 mHaveQueuedSelectResource = false;
2243 mSuspendedForPreloadNone = false;
2244 mDownloadSuspendedByCache = false;
2245 mMediaInfo = MediaInfo();
2246 mIsEncrypted = false;
2247 mPendingEncryptedInitData.Reset();
2248 mWaitingForKey = NOT_WAITING_FOR_KEY;
2249 mSourcePointer = nullptr;
2250 mIsBlessed = false;
2251
2252 mTags = nullptr;
2253 mAudioTrackSilenceStartedTime = 0.0;
2254
2255 if (mNetworkState != NETWORK_EMPTY) {
2256 NS_ASSERTION(!mDecoder && !mSrcStream,
2257 "How did someone setup a new stream/decoder already?");
2258 // ChangeNetworkState() will call UpdateAudioChannelPlayingState()
2259 // indirectly which depends on mPaused. So we need to update mPaused first.
2260 if (!mPaused) {
2261 mPaused = true;
2262 RejectPromises(TakePendingPlayPromises(), NS_ERROR_DOM_MEDIA_ABORT_ERR);
2263 }
2264 ChangeNetworkState(NETWORK_EMPTY);
2265 RemoveMediaTracks();
2266 ChangeReadyState(HAVE_NOTHING);
2267
2268 // TODO: Apply the rules for text track cue rendering Bug 865407
2269 if (mTextTrackManager) {
2270 mTextTrackManager->GetTextTracks()->SetCuesInactive();
2271 }
2272
2273 if (fireTimeUpdate) {
2274 // Since we destroyed the decoder above, the current playback position
2275 // will now be reported as 0. The playback position was non-zero when
2276 // we destroyed the decoder, so fire a timeupdate event so that the
2277 // change will be reflected in the controls.
2278 FireTimeUpdate(false);
2279 }
2280 DispatchAsyncEvent(NS_LITERAL_STRING("emptied"));
2281 UpdateAudioChannelPlayingState();
2282 }
2283
2284 if (IsVideo() && hadVideo) {
2285 // Ensure we render transparent black after resetting video resolution.
2286 Maybe<nsIntSize> size = Some(nsIntSize(0, 0));
2287 Invalidate(true, size, false);
2288 }
2289
2290 // As aborting current load would stop current playback, so we have no need to
2291 // resume a paused media element.
2292 ClearResumeDelayedMediaPlaybackAgentIfNeeded();
2293
2294 // We may have changed mPaused, mAutoplaying, and other
2295 // things which can affect AddRemoveSelfReference
2296 AddRemoveSelfReference();
2297
2298 mIsRunningSelectResource = false;
2299
2300 mEventDeliveryPaused = false;
2301 mPendingEvents.Clear();
2302 mCurrentLoadPlayTime.Reset();
2303
2304 AssertReadyStateIsNothing();
2305 }
2306
NoSupportedMediaSourceError(const nsACString & aErrorDetails)2307 void HTMLMediaElement::NoSupportedMediaSourceError(
2308 const nsACString& aErrorDetails) {
2309 if (mDecoder) {
2310 ShutdownDecoder();
2311 }
2312
2313 bool isThirdPartyLoad = false;
2314 nsresult rv = NS_ERROR_NOT_AVAILABLE;
2315 if (mSrcAttrTriggeringPrincipal) {
2316 rv = mSrcAttrTriggeringPrincipal->IsThirdPartyURI(mLoadingSrc,
2317 &isThirdPartyLoad);
2318 }
2319
2320 if (NS_SUCCEEDED(rv) && isThirdPartyLoad) {
2321 // aErrorDetails can include sensitive details like MimeType or HTTP Status
2322 // Code. In case we're loading a 3rd party resource we should not leak this
2323 // and pass a Generic Error Message
2324 mErrorSink->SetError(MEDIA_ERR_SRC_NOT_SUPPORTED,
2325 NS_LITERAL_CSTRING("Failed to open media"));
2326 } else {
2327 mErrorSink->SetError(MEDIA_ERR_SRC_NOT_SUPPORTED, aErrorDetails);
2328 }
2329
2330 RemoveMediaTracks();
2331 ChangeDelayLoadStatus(false);
2332 UpdateAudioChannelPlayingState();
2333 RejectPromises(TakePendingPlayPromises(),
2334 NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR);
2335 }
2336
2337 typedef void (HTMLMediaElement::*SyncSectionFn)();
2338
2339 // Runs a "synchronous section", a function that must run once the event loop
2340 // has reached a "stable state". See:
2341 // http://www.whatwg.org/specs/web-apps/current-work/multipage/webappapis.html#synchronous-section
2342 class nsSyncSection : public nsMediaEvent {
2343 private:
2344 nsCOMPtr<nsIRunnable> mRunnable;
2345
2346 public:
nsSyncSection(HTMLMediaElement * aElement,nsIRunnable * aRunnable)2347 nsSyncSection(HTMLMediaElement* aElement, nsIRunnable* aRunnable)
2348 : nsMediaEvent("dom::nsSyncSection", aElement), mRunnable(aRunnable) {}
2349
Run()2350 NS_IMETHOD Run() override {
2351 // Silently cancel if our load has been cancelled.
2352 if (IsCancelled()) return NS_OK;
2353 mRunnable->Run();
2354 return NS_OK;
2355 }
2356 };
2357
RunInStableState(nsIRunnable * aRunnable)2358 void HTMLMediaElement::RunInStableState(nsIRunnable* aRunnable) {
2359 if (mShuttingDown) {
2360 return;
2361 }
2362
2363 nsCOMPtr<nsIRunnable> event = new nsSyncSection(this, aRunnable);
2364 nsContentUtils::RunInStableState(event.forget());
2365 }
2366
QueueLoadFromSourceTask()2367 void HTMLMediaElement::QueueLoadFromSourceTask() {
2368 if (!mIsLoadingFromSourceChildren || mShuttingDown) {
2369 return;
2370 }
2371
2372 if (mDecoder) {
2373 // Reset readyState to HAVE_NOTHING since we're going to load a new decoder.
2374 ShutdownDecoder();
2375 ChangeReadyState(HAVE_NOTHING);
2376 }
2377
2378 AssertReadyStateIsNothing();
2379
2380 ChangeDelayLoadStatus(true);
2381 ChangeNetworkState(NETWORK_LOADING);
2382 RefPtr<Runnable> r =
2383 NewRunnableMethod("HTMLMediaElement::LoadFromSourceChildren", this,
2384 &HTMLMediaElement::LoadFromSourceChildren);
2385 RunInStableState(r);
2386 }
2387
QueueSelectResourceTask()2388 void HTMLMediaElement::QueueSelectResourceTask() {
2389 // Don't allow multiple async select resource calls to be queued.
2390 if (mHaveQueuedSelectResource) return;
2391 mHaveQueuedSelectResource = true;
2392 ChangeNetworkState(NETWORK_NO_SOURCE);
2393 RefPtr<Runnable> r =
2394 NewRunnableMethod("HTMLMediaElement::SelectResourceWrapper", this,
2395 &HTMLMediaElement::SelectResourceWrapper);
2396 RunInStableState(r);
2397 }
2398
HasSourceChildren(nsIContent * aElement)2399 static bool HasSourceChildren(nsIContent* aElement) {
2400 for (nsIContent* child = aElement->GetFirstChild(); child;
2401 child = child->GetNextSibling()) {
2402 if (child->IsHTMLElement(nsGkAtoms::source)) {
2403 return true;
2404 }
2405 }
2406 return false;
2407 }
2408
DocumentOrigin(Document * aDoc)2409 static nsCString DocumentOrigin(Document* aDoc) {
2410 if (!aDoc) {
2411 return NS_LITERAL_CSTRING("null");
2412 }
2413 nsCOMPtr<nsIPrincipal> principal = aDoc->NodePrincipal();
2414 if (!principal) {
2415 return NS_LITERAL_CSTRING("null");
2416 }
2417 nsCString origin;
2418 if (NS_FAILED(principal->GetOrigin(origin))) {
2419 return NS_LITERAL_CSTRING("null");
2420 }
2421 return origin;
2422 }
2423
Load()2424 void HTMLMediaElement::Load() {
2425 LOG(LogLevel::Debug,
2426 ("%p Load() hasSrcAttrStream=%d hasSrcAttr=%d hasSourceChildren=%d "
2427 "handlingInput=%d hasAutoplayAttr=%d IsAllowedToPlay=%d "
2428 "ownerDoc=%p (%s) ownerDocUserActivated=%d "
2429 "muted=%d volume=%f",
2430 this, !!mSrcAttrStream, HasAttr(kNameSpaceID_None, nsGkAtoms::src),
2431 HasSourceChildren(this), UserActivation::IsHandlingUserInput(),
2432 HasAttr(kNameSpaceID_None, nsGkAtoms::autoplay),
2433 AutoplayPolicy::IsAllowedToPlay(*this), OwnerDoc(),
2434 DocumentOrigin(OwnerDoc()).get(),
2435 OwnerDoc()->HasBeenUserGestureActivated(), mMuted, mVolume));
2436
2437 if (mIsRunningLoadMethod) {
2438 return;
2439 }
2440
2441 mIsDoingExplicitLoad = true;
2442 DoLoad();
2443 }
2444
DoLoad()2445 void HTMLMediaElement::DoLoad() {
2446 // Check if media is allowed for the docshell.
2447 nsCOMPtr<nsIDocShell> docShell = OwnerDoc()->GetDocShell();
2448 if (docShell && !docShell->GetAllowMedia()) {
2449 LOG(LogLevel::Debug, ("%p Media not allowed", this));
2450 return;
2451 }
2452
2453 if (mIsRunningLoadMethod) {
2454 return;
2455 }
2456
2457 if (UserActivation::IsHandlingUserInput()) {
2458 // Detect if user has interacted with element so that play will not be
2459 // blocked when initiated by a script. This enables sites to capture user
2460 // intent to play by calling load() in the click handler of a "catalog
2461 // view" of a gallery of videos.
2462 mIsBlessed = true;
2463 // Mark the channel as urgent-start when autopaly so that it will play the
2464 // media from src after loading enough resource.
2465 if (HasAttr(kNameSpaceID_None, nsGkAtoms::autoplay)) {
2466 mUseUrgentStartForChannel = true;
2467 }
2468 }
2469
2470 SetPlayedOrSeeked(false);
2471 mIsRunningLoadMethod = true;
2472 AbortExistingLoads();
2473 SetPlaybackRate(mDefaultPlaybackRate, IgnoreErrors());
2474 QueueSelectResourceTask();
2475 ResetState();
2476 mIsRunningLoadMethod = false;
2477 }
2478
ResetState()2479 void HTMLMediaElement::ResetState() {
2480 // There might be a pending MediaDecoder::PlaybackPositionChanged() which
2481 // will overwrite |mMediaInfo.mVideo.mDisplay| in UpdateMediaSize() to give
2482 // staled videoWidth and videoHeight. We have to call ForgetElement() here
2483 // such that the staled callbacks won't reach us.
2484 if (mVideoFrameContainer) {
2485 mVideoFrameContainer->ForgetElement();
2486 mVideoFrameContainer = nullptr;
2487 }
2488 if (mMediaStreamRenderer) {
2489 // mMediaStreamRenderer, has a strong reference to mVideoFrameContainer.
2490 mMediaStreamRenderer->Shutdown();
2491 mMediaStreamRenderer = nullptr;
2492 }
2493 }
2494
SelectResourceWrapper()2495 void HTMLMediaElement::SelectResourceWrapper() {
2496 SelectResource();
2497 MaybeBeginCloningVisually();
2498 mIsRunningSelectResource = false;
2499 mHaveQueuedSelectResource = false;
2500 mIsDoingExplicitLoad = false;
2501 }
2502
SelectResource()2503 void HTMLMediaElement::SelectResource() {
2504 if (!mSrcAttrStream && !HasAttr(kNameSpaceID_None, nsGkAtoms::src) &&
2505 !HasSourceChildren(this)) {
2506 // The media element has neither a src attribute nor any source
2507 // element children, abort the load.
2508 ChangeNetworkState(NETWORK_EMPTY);
2509 ChangeDelayLoadStatus(false);
2510 return;
2511 }
2512
2513 ChangeDelayLoadStatus(true);
2514
2515 ChangeNetworkState(NETWORK_LOADING);
2516 DispatchAsyncEvent(NS_LITERAL_STRING("loadstart"));
2517
2518 // Delay setting mIsRunningSeletResource until after UpdatePreloadAction
2519 // so that we don't lose our state change by bailing out of the preload
2520 // state update
2521 UpdatePreloadAction();
2522 mIsRunningSelectResource = true;
2523
2524 // If we have a 'src' attribute, use that exclusively.
2525 nsAutoString src;
2526 if (mSrcAttrStream) {
2527 SetupSrcMediaStreamPlayback(mSrcAttrStream);
2528 } else if (GetAttr(kNameSpaceID_None, nsGkAtoms::src, src)) {
2529 nsCOMPtr<nsIURI> uri;
2530 MediaResult rv = NewURIFromString(src, getter_AddRefs(uri));
2531 if (NS_SUCCEEDED(rv)) {
2532 LOG(LogLevel::Debug, ("%p Trying load from src=%s", this,
2533 NS_ConvertUTF16toUTF8(src).get()));
2534 NS_ASSERTION(
2535 !mIsLoadingFromSourceChildren,
2536 "Should think we're not loading from source children by default");
2537
2538 RemoveMediaElementFromURITable();
2539 mLoadingSrc = uri;
2540 mLoadingSrcTriggeringPrincipal = mSrcAttrTriggeringPrincipal;
2541 DDLOG(DDLogCategory::Property, "loading_src",
2542 nsCString(NS_ConvertUTF16toUTF8(src)));
2543 mMediaSource = mSrcMediaSource;
2544 DDLINKCHILD("mediasource", mMediaSource.get());
2545 UpdatePreloadAction();
2546 if (mPreloadAction == HTMLMediaElement::PRELOAD_NONE && !mMediaSource) {
2547 // preload:none media, suspend the load here before we make any
2548 // network requests.
2549 SuspendLoad();
2550 return;
2551 }
2552
2553 rv = LoadResource();
2554 if (NS_SUCCEEDED(rv)) {
2555 return;
2556 }
2557 } else {
2558 AutoTArray<nsString, 1> params = {src};
2559 ReportLoadError("MediaLoadInvalidURI", params);
2560 rv = MediaResult(rv.Code(), "MediaLoadInvalidURI");
2561 }
2562 // The media element has neither a src attribute nor a source element child:
2563 // set the networkState to NETWORK_EMPTY, and abort these steps; the
2564 // synchronous section ends.
2565 mMainThreadEventTarget->Dispatch(NewRunnableMethod<nsCString>(
2566 "HTMLMediaElement::NoSupportedMediaSourceError", this,
2567 &HTMLMediaElement::NoSupportedMediaSourceError, rv.Description()));
2568 } else {
2569 // Otherwise, the source elements will be used.
2570 mIsLoadingFromSourceChildren = true;
2571 LoadFromSourceChildren();
2572 }
2573 }
2574
NotifyLoadError(const nsACString & aErrorDetails)2575 void HTMLMediaElement::NotifyLoadError(const nsACString& aErrorDetails) {
2576 if (!mIsLoadingFromSourceChildren) {
2577 LOG(LogLevel::Debug, ("NotifyLoadError(), no supported media error"));
2578 NoSupportedMediaSourceError(aErrorDetails);
2579 } else if (mSourceLoadCandidate) {
2580 DispatchAsyncSourceError(mSourceLoadCandidate);
2581 QueueLoadFromSourceTask();
2582 } else {
2583 NS_WARNING("Should know the source we were loading from!");
2584 }
2585 }
2586
NotifyMediaTrackEnabled(dom::MediaTrack * aTrack)2587 void HTMLMediaElement::NotifyMediaTrackEnabled(dom::MediaTrack* aTrack) {
2588 MOZ_ASSERT(aTrack);
2589 if (!aTrack) {
2590 return;
2591 }
2592 #ifdef DEBUG
2593 nsString id;
2594 aTrack->GetId(id);
2595
2596 LOG(LogLevel::Debug, ("MediaElement %p %sTrack with id %s enabled", this,
2597 aTrack->AsAudioTrack() ? "Audio" : "Video",
2598 NS_ConvertUTF16toUTF8(id).get()));
2599 #endif
2600
2601 MOZ_ASSERT((aTrack->AsAudioTrack() && aTrack->AsAudioTrack()->Enabled()) ||
2602 (aTrack->AsVideoTrack() && aTrack->AsVideoTrack()->Selected()));
2603
2604 if (aTrack->AsAudioTrack()) {
2605 SetMutedInternal(mMuted & ~MUTED_BY_AUDIO_TRACK);
2606 } else if (aTrack->AsVideoTrack()) {
2607 if (!IsVideo()) {
2608 MOZ_ASSERT(false);
2609 return;
2610 }
2611 mDisableVideo = false;
2612 } else {
2613 MOZ_ASSERT(false, "Unknown track type");
2614 }
2615
2616 if (mSrcStream) {
2617 if (AudioTrack* t = aTrack->AsAudioTrack()) {
2618 if (mMediaStreamRenderer) {
2619 mMediaStreamRenderer->AddTrack(t->GetAudioStreamTrack());
2620 }
2621 } else if (VideoTrack* t = aTrack->AsVideoTrack()) {
2622 MOZ_ASSERT(!mSelectedVideoStreamTrack);
2623
2624 mSelectedVideoStreamTrack = t->GetVideoStreamTrack();
2625 mSelectedVideoStreamTrack->AddPrincipalChangeObserver(this);
2626 if (mMediaStreamRenderer) {
2627 mMediaStreamRenderer->AddTrack(mSelectedVideoStreamTrack);
2628 }
2629 nsContentUtils::CombineResourcePrincipals(
2630 &mSrcStreamVideoPrincipal, mSelectedVideoStreamTrack->GetPrincipal());
2631 if (VideoFrameContainer* container = GetVideoFrameContainer()) {
2632 HTMLVideoElement* self = static_cast<HTMLVideoElement*>(this);
2633 if (mSrcStreamIsPlaying) {
2634 MaybeBeginCloningVisually();
2635 } else if (self->VideoWidth() <= 1 && self->VideoHeight() <= 1) {
2636 // MediaInfo uses dummy values of 1 for width and height to
2637 // mark video as valid. We need a new first-frame listener
2638 // if size is 0x0 or 1x1.
2639 if (!mFirstFrameListener) {
2640 mFirstFrameListener =
2641 new FirstFrameListener(container, mAbstractMainThread);
2642 }
2643 mSelectedVideoStreamTrack->AddVideoOutput(mFirstFrameListener);
2644 }
2645 }
2646 }
2647 }
2648
2649 // The set of enabled/selected tracks changed.
2650 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
2651 }
2652
NotifyMediaTrackDisabled(dom::MediaTrack * aTrack)2653 void HTMLMediaElement::NotifyMediaTrackDisabled(dom::MediaTrack* aTrack) {
2654 MOZ_ASSERT(aTrack);
2655 if (!aTrack) {
2656 return;
2657 }
2658
2659 nsString id;
2660 aTrack->GetId(id);
2661
2662 LOG(LogLevel::Debug, ("MediaElement %p %sTrack with id %s disabled", this,
2663 aTrack->AsAudioTrack() ? "Audio" : "Video",
2664 NS_ConvertUTF16toUTF8(id).get()));
2665
2666 MOZ_ASSERT((!aTrack->AsAudioTrack() || !aTrack->AsAudioTrack()->Enabled()) &&
2667 (!aTrack->AsVideoTrack() || !aTrack->AsVideoTrack()->Selected()));
2668
2669 if (AudioTrack* t = aTrack->AsAudioTrack()) {
2670 if (mSrcStream) {
2671 if (mMediaStreamRenderer) {
2672 mMediaStreamRenderer->RemoveTrack(t->GetAudioStreamTrack());
2673 }
2674 }
2675 // If we don't have any live tracks, we don't need to mute MediaElement.
2676 MOZ_DIAGNOSTIC_ASSERT(AudioTracks(), "Element can't have been unlinked");
2677 if (AudioTracks()->Length() > 0) {
2678 bool shouldMute = true;
2679 for (uint32_t i = 0; i < AudioTracks()->Length(); ++i) {
2680 if ((*AudioTracks())[i]->Enabled()) {
2681 shouldMute = false;
2682 break;
2683 }
2684 }
2685
2686 if (shouldMute) {
2687 SetMutedInternal(mMuted | MUTED_BY_AUDIO_TRACK);
2688 }
2689 }
2690 } else if (aTrack->AsVideoTrack()) {
2691 if (mSrcStream) {
2692 MOZ_DIAGNOSTIC_ASSERT(mSelectedVideoStreamTrack ==
2693 aTrack->AsVideoTrack()->GetVideoStreamTrack());
2694 if (mFirstFrameListener) {
2695 mSelectedVideoStreamTrack->RemoveVideoOutput(mFirstFrameListener);
2696 mFirstFrameListener = nullptr;
2697 }
2698 if (mMediaStreamRenderer) {
2699 mMediaStreamRenderer->RemoveTrack(mSelectedVideoStreamTrack);
2700 }
2701 mSelectedVideoStreamTrack->RemovePrincipalChangeObserver(this);
2702 mSelectedVideoStreamTrack = nullptr;
2703 }
2704 }
2705
2706 // The set of enabled/selected tracks changed.
2707 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
2708 }
2709
DealWithFailedElement(nsIContent * aSourceElement)2710 void HTMLMediaElement::DealWithFailedElement(nsIContent* aSourceElement) {
2711 if (mShuttingDown) {
2712 return;
2713 }
2714
2715 DispatchAsyncSourceError(aSourceElement);
2716 mMainThreadEventTarget->Dispatch(
2717 NewRunnableMethod("HTMLMediaElement::QueueLoadFromSourceTask", this,
2718 &HTMLMediaElement::QueueLoadFromSourceTask));
2719 }
2720
LoadFromSourceChildren()2721 void HTMLMediaElement::LoadFromSourceChildren() {
2722 NS_ASSERTION(mDelayingLoadEvent,
2723 "Should delay load event (if in document) during load");
2724 NS_ASSERTION(mIsLoadingFromSourceChildren,
2725 "Must remember we're loading from source children");
2726
2727 AddMutationObserverUnlessExists(this);
2728
2729 RemoveMediaTracks();
2730
2731 while (true) {
2732 Element* child = GetNextSource();
2733 if (!child) {
2734 // Exhausted candidates, wait for more candidates to be appended to
2735 // the media element.
2736 mLoadWaitStatus = WAITING_FOR_SOURCE;
2737 ChangeNetworkState(NETWORK_NO_SOURCE);
2738 ChangeDelayLoadStatus(false);
2739 ReportLoadError("MediaLoadExhaustedCandidates");
2740 return;
2741 }
2742
2743 // Must have src attribute.
2744 nsAutoString src;
2745 if (!child->GetAttr(kNameSpaceID_None, nsGkAtoms::src, src)) {
2746 ReportLoadError("MediaLoadSourceMissingSrc");
2747 DealWithFailedElement(child);
2748 return;
2749 }
2750
2751 // If we have a type attribute, it must be a supported type.
2752 nsAutoString type;
2753 if (child->GetAttr(kNameSpaceID_None, nsGkAtoms::type, type) &&
2754 !type.IsEmpty()) {
2755 DecoderDoctorDiagnostics diagnostics;
2756 CanPlayStatus canPlay = GetCanPlay(type, &diagnostics);
2757 diagnostics.StoreFormatDiagnostics(OwnerDoc(), type,
2758 canPlay != CANPLAY_NO, __func__);
2759 if (canPlay == CANPLAY_NO) {
2760 // Check that at least one other source child exists and report that
2761 // we will try to load that one next.
2762 nsIContent* nextChild = mSourcePointer->GetNextSibling();
2763 AutoTArray<nsString, 2> params = {type, src};
2764
2765 while (nextChild) {
2766 if (nextChild && nextChild->IsHTMLElement(nsGkAtoms::source)) {
2767 ReportLoadError("MediaLoadUnsupportedTypeAttributeLoadingNextChild",
2768 params);
2769 break;
2770 }
2771
2772 nextChild = nextChild->GetNextSibling();
2773 };
2774
2775 if (!nextChild) {
2776 ReportLoadError("MediaLoadUnsupportedTypeAttribute", params);
2777 }
2778
2779 DealWithFailedElement(child);
2780 return;
2781 }
2782 }
2783 HTMLSourceElement* childSrc = HTMLSourceElement::FromNode(child);
2784 LOG(LogLevel::Debug,
2785 ("%p Trying load from <source>=%s type=%s", this,
2786 NS_ConvertUTF16toUTF8(src).get(), NS_ConvertUTF16toUTF8(type).get()));
2787
2788 nsCOMPtr<nsIURI> uri;
2789 NewURIFromString(src, getter_AddRefs(uri));
2790 if (!uri) {
2791 AutoTArray<nsString, 1> params = {src};
2792 ReportLoadError("MediaLoadInvalidURI", params);
2793 DealWithFailedElement(child);
2794 return;
2795 }
2796
2797 RemoveMediaElementFromURITable();
2798 mLoadingSrc = uri;
2799 mLoadingSrcTriggeringPrincipal = childSrc->GetSrcTriggeringPrincipal();
2800 DDLOG(DDLogCategory::Property, "loading_src",
2801 nsCString(NS_ConvertUTF16toUTF8(src)));
2802 mMediaSource = childSrc->GetSrcMediaSource();
2803 DDLINKCHILD("mediasource", mMediaSource.get());
2804 NS_ASSERTION(mNetworkState == NETWORK_LOADING,
2805 "Network state should be loading");
2806
2807 if (mPreloadAction == HTMLMediaElement::PRELOAD_NONE && !mMediaSource) {
2808 // preload:none media, suspend the load here before we make any
2809 // network requests.
2810 SuspendLoad();
2811 return;
2812 }
2813
2814 if (NS_SUCCEEDED(LoadResource())) {
2815 return;
2816 }
2817
2818 // If we fail to load, loop back and try loading the next resource.
2819 DispatchAsyncSourceError(child);
2820 }
2821 MOZ_ASSERT_UNREACHABLE("Execution should not reach here!");
2822 }
2823
SuspendLoad()2824 void HTMLMediaElement::SuspendLoad() {
2825 mSuspendedForPreloadNone = true;
2826 ChangeNetworkState(NETWORK_IDLE);
2827 ChangeDelayLoadStatus(false);
2828 }
2829
ResumeLoad(PreloadAction aAction)2830 void HTMLMediaElement::ResumeLoad(PreloadAction aAction) {
2831 NS_ASSERTION(mSuspendedForPreloadNone,
2832 "Must be halted for preload:none to resume from preload:none "
2833 "suspended load.");
2834 mSuspendedForPreloadNone = false;
2835 mPreloadAction = aAction;
2836 ChangeDelayLoadStatus(true);
2837 ChangeNetworkState(NETWORK_LOADING);
2838 if (!mIsLoadingFromSourceChildren) {
2839 // We were loading from the element's src attribute.
2840 MediaResult rv = LoadResource();
2841 if (NS_FAILED(rv)) {
2842 NoSupportedMediaSourceError(rv.Description());
2843 }
2844 } else {
2845 // We were loading from a child <source> element. Try to resume the
2846 // load of that child, and if that fails, try the next child.
2847 if (NS_FAILED(LoadResource())) {
2848 LoadFromSourceChildren();
2849 }
2850 }
2851 }
2852
AllowedToPlay() const2853 bool HTMLMediaElement::AllowedToPlay() const {
2854 return AutoplayPolicy::IsAllowedToPlay(*this);
2855 }
2856
GetPreloadDefault() const2857 uint32_t HTMLMediaElement::GetPreloadDefault() const {
2858 if (mMediaSource) {
2859 return HTMLMediaElement::PRELOAD_ATTR_METADATA;
2860 }
2861 if (OnCellularConnection()) {
2862 return Preferences::GetInt("media.preload.default.cellular",
2863 HTMLMediaElement::PRELOAD_ATTR_NONE);
2864 }
2865 return Preferences::GetInt("media.preload.default",
2866 HTMLMediaElement::PRELOAD_ATTR_METADATA);
2867 }
2868
GetPreloadDefaultAuto() const2869 uint32_t HTMLMediaElement::GetPreloadDefaultAuto() const {
2870 if (OnCellularConnection()) {
2871 return Preferences::GetInt("media.preload.auto.cellular",
2872 HTMLMediaElement::PRELOAD_ATTR_METADATA);
2873 }
2874 return Preferences::GetInt("media.preload.auto",
2875 HTMLMediaElement::PRELOAD_ENOUGH);
2876 }
2877
UpdatePreloadAction()2878 void HTMLMediaElement::UpdatePreloadAction() {
2879 PreloadAction nextAction = PRELOAD_UNDEFINED;
2880 // If autoplay is set, or we're playing, we should always preload data,
2881 // as we'll need it to play.
2882 if ((AutoplayPolicy::IsAllowedToPlay(*this) &&
2883 HasAttr(kNameSpaceID_None, nsGkAtoms::autoplay)) ||
2884 !mPaused) {
2885 nextAction = HTMLMediaElement::PRELOAD_ENOUGH;
2886 } else {
2887 // Find the appropriate preload action by looking at the attribute.
2888 const nsAttrValue* val =
2889 mAttrs.GetAttr(nsGkAtoms::preload, kNameSpaceID_None);
2890 // MSE doesn't work if preload is none, so it ignores the pref when src is
2891 // from MSE.
2892 uint32_t preloadDefault = GetPreloadDefault();
2893 uint32_t preloadAuto = GetPreloadDefaultAuto();
2894 if (!val) {
2895 // Attribute is not set. Use the preload action specified by the
2896 // media.preload.default pref, or just preload metadata if not present.
2897 nextAction = static_cast<PreloadAction>(preloadDefault);
2898 } else if (val->Type() == nsAttrValue::eEnum) {
2899 PreloadAttrValue attr =
2900 static_cast<PreloadAttrValue>(val->GetEnumValue());
2901 if (attr == HTMLMediaElement::PRELOAD_ATTR_EMPTY ||
2902 attr == HTMLMediaElement::PRELOAD_ATTR_AUTO) {
2903 nextAction = static_cast<PreloadAction>(preloadAuto);
2904 } else if (attr == HTMLMediaElement::PRELOAD_ATTR_METADATA) {
2905 nextAction = HTMLMediaElement::PRELOAD_METADATA;
2906 } else if (attr == HTMLMediaElement::PRELOAD_ATTR_NONE) {
2907 nextAction = HTMLMediaElement::PRELOAD_NONE;
2908 }
2909 } else {
2910 // Use the suggested "missing value default" of "metadata", or the value
2911 // specified by the media.preload.default, if present.
2912 nextAction = static_cast<PreloadAction>(preloadDefault);
2913 }
2914 }
2915
2916 if (nextAction == HTMLMediaElement::PRELOAD_NONE && mIsDoingExplicitLoad) {
2917 nextAction = HTMLMediaElement::PRELOAD_METADATA;
2918 }
2919
2920 mPreloadAction = nextAction;
2921
2922 if (nextAction == HTMLMediaElement::PRELOAD_ENOUGH) {
2923 if (mSuspendedForPreloadNone) {
2924 // Our load was previouly suspended due to the media having preload
2925 // value "none". The preload value has changed to preload:auto, so
2926 // resume the load.
2927 ResumeLoad(PRELOAD_ENOUGH);
2928 } else {
2929 // Preload as much of the video as we can, i.e. don't suspend after
2930 // the first frame.
2931 StopSuspendingAfterFirstFrame();
2932 }
2933
2934 } else if (nextAction == HTMLMediaElement::PRELOAD_METADATA) {
2935 // Ensure that the video can be suspended after first frame.
2936 mAllowSuspendAfterFirstFrame = true;
2937 if (mSuspendedForPreloadNone) {
2938 // Our load was previouly suspended due to the media having preload
2939 // value "none". The preload value has changed to preload:metadata, so
2940 // resume the load. We'll pause the load again after we've read the
2941 // metadata.
2942 ResumeLoad(PRELOAD_METADATA);
2943 }
2944 }
2945 }
2946
LoadResource()2947 MediaResult HTMLMediaElement::LoadResource() {
2948 NS_ASSERTION(mDelayingLoadEvent,
2949 "Should delay load event (if in document) during load");
2950
2951 if (mChannelLoader) {
2952 mChannelLoader->Cancel();
2953 mChannelLoader = nullptr;
2954 }
2955
2956 // Set the media element's CORS mode only when loading a resource
2957 mCORSMode = AttrValueToCORSMode(GetParsedAttr(nsGkAtoms::crossorigin));
2958
2959 HTMLMediaElement* other = LookupMediaElementURITable(mLoadingSrc);
2960 if (other && other->mDecoder) {
2961 // Clone it.
2962 // TODO: remove the cast by storing ChannelMediaDecoder in the URI table.
2963 nsresult rv = InitializeDecoderAsClone(
2964 static_cast<ChannelMediaDecoder*>(other->mDecoder.get()));
2965 if (NS_SUCCEEDED(rv)) return rv;
2966 }
2967
2968 if (mMediaSource) {
2969 MediaDecoderInit decoderInit(
2970 this, mMuted ? 0.0 : mVolume, mPreservesPitch,
2971 ClampPlaybackRate(mPlaybackRate),
2972 mPreloadAction == HTMLMediaElement::PRELOAD_METADATA, mHasSuspendTaint,
2973 HasAttr(kNameSpaceID_None, nsGkAtoms::loop),
2974 MediaContainerType(MEDIAMIMETYPE("application/x.mediasource")));
2975
2976 RefPtr<MediaSourceDecoder> decoder = new MediaSourceDecoder(decoderInit);
2977 if (!mMediaSource->Attach(decoder)) {
2978 // TODO: Handle failure: run "If the media data cannot be fetched at
2979 // all, due to network errors, causing the user agent to give up
2980 // trying to fetch the resource" section of resource fetch algorithm.
2981 decoder->Shutdown();
2982 return MediaResult(NS_ERROR_FAILURE, "Failed to attach MediaSource");
2983 }
2984 ChangeDelayLoadStatus(false);
2985 nsresult rv = decoder->Load(mMediaSource->GetPrincipal());
2986 if (NS_FAILED(rv)) {
2987 decoder->Shutdown();
2988 LOG(LogLevel::Debug,
2989 ("%p Failed to load for decoder %p", this, decoder.get()));
2990 return MediaResult(rv, "Fail to load decoder");
2991 }
2992 rv = FinishDecoderSetup(decoder);
2993 return MediaResult(rv, "Failed to set up decoder");
2994 }
2995
2996 AssertReadyStateIsNothing();
2997
2998 RefPtr<ChannelLoader> loader = new ChannelLoader;
2999 nsresult rv = loader->Load(this);
3000 if (NS_SUCCEEDED(rv)) {
3001 mChannelLoader = std::move(loader);
3002 }
3003 return MediaResult(rv, "Failed to load channel");
3004 }
3005
LoadWithChannel(nsIChannel * aChannel,nsIStreamListener ** aListener)3006 nsresult HTMLMediaElement::LoadWithChannel(nsIChannel* aChannel,
3007 nsIStreamListener** aListener) {
3008 NS_ENSURE_ARG_POINTER(aChannel);
3009 NS_ENSURE_ARG_POINTER(aListener);
3010
3011 *aListener = nullptr;
3012
3013 // Make sure we don't reenter during synchronous abort events.
3014 if (mIsRunningLoadMethod) return NS_OK;
3015 mIsRunningLoadMethod = true;
3016 AbortExistingLoads();
3017 mIsRunningLoadMethod = false;
3018
3019 mLoadingSrcTriggeringPrincipal = nullptr;
3020 nsresult rv = aChannel->GetOriginalURI(getter_AddRefs(mLoadingSrc));
3021 NS_ENSURE_SUCCESS(rv, rv);
3022
3023 ChangeDelayLoadStatus(true);
3024 rv = InitializeDecoderForChannel(aChannel, aListener);
3025 if (NS_FAILED(rv)) {
3026 ChangeDelayLoadStatus(false);
3027 return rv;
3028 }
3029
3030 SetPlaybackRate(mDefaultPlaybackRate, IgnoreErrors());
3031 DispatchAsyncEvent(NS_LITERAL_STRING("loadstart"));
3032
3033 return NS_OK;
3034 }
3035
Seeking() const3036 bool HTMLMediaElement::Seeking() const {
3037 return mDecoder && mDecoder->IsSeeking();
3038 }
3039
CurrentTime() const3040 double HTMLMediaElement::CurrentTime() const {
3041 if (mMediaStreamRenderer) {
3042 return mMediaStreamRenderer->CurrentTime();
3043 }
3044
3045 if (mDefaultPlaybackStartPosition == 0.0 && mDecoder) {
3046 return mDecoder->GetCurrentTime();
3047 }
3048
3049 return mDefaultPlaybackStartPosition;
3050 }
3051
FastSeek(double aTime,ErrorResult & aRv)3052 void HTMLMediaElement::FastSeek(double aTime, ErrorResult& aRv) {
3053 LOG(LogLevel::Debug, ("%p FastSeek(%f) called by JS", this, aTime));
3054 Seek(aTime, SeekTarget::PrevSyncPoint, IgnoreErrors());
3055 }
3056
SeekToNextFrame(ErrorResult & aRv)3057 already_AddRefed<Promise> HTMLMediaElement::SeekToNextFrame(ErrorResult& aRv) {
3058 /* This will cause JIT code to be kept around longer, to help performance
3059 * when using SeekToNextFrame to iterate through every frame of a video.
3060 */
3061 nsPIDOMWindowInner* win = OwnerDoc()->GetInnerWindow();
3062
3063 if (win) {
3064 if (JSObject* obj = win->AsGlobal()->GetGlobalJSObject()) {
3065 js::NotifyAnimationActivity(obj);
3066 }
3067 }
3068
3069 Seek(CurrentTime(), SeekTarget::NextFrame, aRv);
3070 if (aRv.Failed()) {
3071 return nullptr;
3072 }
3073
3074 mSeekDOMPromise = CreateDOMPromise(aRv);
3075 if (NS_WARN_IF(aRv.Failed())) {
3076 return nullptr;
3077 }
3078
3079 return do_AddRef(mSeekDOMPromise);
3080 }
3081
SetCurrentTime(double aCurrentTime,ErrorResult & aRv)3082 void HTMLMediaElement::SetCurrentTime(double aCurrentTime, ErrorResult& aRv) {
3083 LOG(LogLevel::Debug,
3084 ("%p SetCurrentTime(%f) called by JS", this, aCurrentTime));
3085 Seek(aCurrentTime, SeekTarget::Accurate, IgnoreErrors());
3086 }
3087
3088 /**
3089 * Check if aValue is inside a range of aRanges, and if so returns true
3090 * and puts the range index in aIntervalIndex. If aValue is not
3091 * inside a range, returns false, and aIntervalIndex
3092 * is set to the index of the range which starts immediately after aValue
3093 * (and can be aRanges.Length() if aValue is after the last range).
3094 */
IsInRanges(TimeRanges & aRanges,double aValue,uint32_t & aIntervalIndex)3095 static bool IsInRanges(TimeRanges& aRanges, double aValue,
3096 uint32_t& aIntervalIndex) {
3097 uint32_t length = aRanges.Length();
3098
3099 for (uint32_t i = 0; i < length; i++) {
3100 double start = aRanges.Start(i);
3101 if (start > aValue) {
3102 aIntervalIndex = i;
3103 return false;
3104 }
3105 double end = aRanges.End(i);
3106 if (aValue <= end) {
3107 aIntervalIndex = i;
3108 return true;
3109 }
3110 }
3111 aIntervalIndex = length;
3112 return false;
3113 }
3114
Seek(double aTime,SeekTarget::Type aSeekType,ErrorResult & aRv)3115 void HTMLMediaElement::Seek(double aTime, SeekTarget::Type aSeekType,
3116 ErrorResult& aRv) {
3117 // Note: Seek is called both by synchronous code that expects errors thrown in
3118 // aRv, as well as asynchronous code that expects a promise. Make sure all
3119 // synchronous errors are returned using aRv, not promise rejections.
3120
3121 // aTime should be non-NaN.
3122 MOZ_ASSERT(!mozilla::IsNaN(aTime));
3123
3124 // Seeking step1, Set the media element's show poster flag to false.
3125 // https://html.spec.whatwg.org/multipage/media.html#dom-media-seek
3126 mShowPoster = false;
3127
3128 // Detect if user has interacted with element by seeking so that
3129 // play will not be blocked when initiated by a script.
3130 if (UserActivation::IsHandlingUserInput()) {
3131 mIsBlessed = true;
3132 }
3133
3134 StopSuspendingAfterFirstFrame();
3135
3136 if (mSrcAttrStream) {
3137 // do nothing since media streams have an empty Seekable range.
3138 aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
3139 return;
3140 }
3141
3142 if (mPlayed && mCurrentPlayRangeStart != -1.0) {
3143 double rangeEndTime = CurrentTime();
3144 LOG(LogLevel::Debug, ("%p Adding \'played\' a range : [%f, %f]", this,
3145 mCurrentPlayRangeStart, rangeEndTime));
3146 // Multiple seek without playing, or seek while playing.
3147 if (mCurrentPlayRangeStart != rangeEndTime) {
3148 mPlayed->Add(mCurrentPlayRangeStart, rangeEndTime);
3149 }
3150 // Reset the current played range start time. We'll re-set it once
3151 // the seek completes.
3152 mCurrentPlayRangeStart = -1.0;
3153 }
3154
3155 if (mReadyState == HAVE_NOTHING) {
3156 mDefaultPlaybackStartPosition = aTime;
3157 aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
3158 return;
3159 }
3160
3161 if (!mDecoder) {
3162 // mDecoder must always be set in order to reach this point.
3163 NS_ASSERTION(mDecoder, "SetCurrentTime failed: no decoder");
3164 aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
3165 return;
3166 }
3167
3168 // Clamp the seek target to inside the seekable ranges.
3169 media::TimeIntervals seekableIntervals = mDecoder->GetSeekable();
3170 if (seekableIntervals.IsInvalid()) {
3171 aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
3172 return;
3173 }
3174 RefPtr<TimeRanges> seekable =
3175 new TimeRanges(ToSupports(OwnerDoc()), seekableIntervals);
3176 uint32_t length = seekable->Length();
3177 if (length == 0) {
3178 aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
3179 return;
3180 }
3181
3182 // If the position we want to seek to is not in a seekable range, we seek
3183 // to the closest position in the seekable ranges instead. If two positions
3184 // are equally close, we seek to the closest position from the currentTime.
3185 // See seeking spec, point 7 :
3186 // http://www.whatwg.org/specs/web-apps/current-work/multipage/the-video-element.html#seeking
3187 uint32_t range = 0;
3188 bool isInRange = IsInRanges(*seekable, aTime, range);
3189 if (!isInRange) {
3190 if (range == 0) {
3191 // aTime is before the first range in |seekable|, the closest point we can
3192 // seek to is the start of the first range.
3193 aTime = seekable->Start(0);
3194 } else if (range == length) {
3195 // Seek target is after the end last range in seekable data.
3196 // Clamp the seek target to the end of the last seekable range.
3197 aTime = seekable->End(length - 1);
3198 } else {
3199 double leftBound = seekable->End(range - 1);
3200 double rightBound = seekable->Start(range);
3201 double distanceLeft = Abs(leftBound - aTime);
3202 double distanceRight = Abs(rightBound - aTime);
3203 if (distanceLeft == distanceRight) {
3204 double currentTime = CurrentTime();
3205 distanceLeft = Abs(leftBound - currentTime);
3206 distanceRight = Abs(rightBound - currentTime);
3207 }
3208 aTime = (distanceLeft < distanceRight) ? leftBound : rightBound;
3209 }
3210 }
3211
3212 // TODO: The spec requires us to update the current time to reflect the
3213 // actual seek target before beginning the synchronous section, but
3214 // that requires changing all MediaDecoderReaders to support telling
3215 // us the fastSeek target, and it's currently not possible to get
3216 // this information as we don't yet control the demuxer for all
3217 // MediaDecoderReaders.
3218
3219 mPlayingBeforeSeek = IsPotentiallyPlaying();
3220
3221 // If the audio track is silent before seeking, we should end current silence
3222 // range and start a new range after seeking. Since seek() could be called
3223 // multiple times before seekEnd() executed, we should only calculate silence
3224 // range when first time seek() called. Calculating on other seek() calls
3225 // would cause a wrong result. In order to get correct time, this checking
3226 // should be called before decoder->seek().
3227 if (IsAudioTrackCurrentlySilent() &&
3228 !mHasAccumulatedSilenceRangeBeforeSeekEnd) {
3229 AccumulateAudioTrackSilence();
3230 mHasAccumulatedSilenceRangeBeforeSeekEnd = true;
3231 }
3232
3233 // The media backend is responsible for dispatching the timeupdate
3234 // event if it changes the playback position as a result of the seek.
3235 LOG(LogLevel::Debug, ("%p SetCurrentTime(%f) starting seek", this, aTime));
3236 mDecoder->Seek(aTime, aSeekType);
3237
3238 // We changed whether we're seeking so we need to AddRemoveSelfReference.
3239 AddRemoveSelfReference();
3240 }
3241
Duration() const3242 double HTMLMediaElement::Duration() const {
3243 if (mSrcStream) {
3244 if (mSrcStreamPlaybackEnded) {
3245 return CurrentTime();
3246 }
3247 return std::numeric_limits<double>::infinity();
3248 }
3249
3250 if (mDecoder) {
3251 return mDecoder->GetDuration();
3252 }
3253
3254 return std::numeric_limits<double>::quiet_NaN();
3255 }
3256
Seekable() const3257 already_AddRefed<TimeRanges> HTMLMediaElement::Seekable() const {
3258 media::TimeIntervals seekable =
3259 mDecoder ? mDecoder->GetSeekable() : media::TimeIntervals();
3260 RefPtr<TimeRanges> ranges = new TimeRanges(ToSupports(OwnerDoc()), seekable);
3261 return ranges.forget();
3262 }
3263
Played()3264 already_AddRefed<TimeRanges> HTMLMediaElement::Played() {
3265 RefPtr<TimeRanges> ranges = new TimeRanges(ToSupports(OwnerDoc()));
3266
3267 uint32_t timeRangeCount = 0;
3268 if (mPlayed) {
3269 timeRangeCount = mPlayed->Length();
3270 }
3271 for (uint32_t i = 0; i < timeRangeCount; i++) {
3272 double begin = mPlayed->Start(i);
3273 double end = mPlayed->End(i);
3274 ranges->Add(begin, end);
3275 }
3276
3277 if (mCurrentPlayRangeStart != -1.0) {
3278 double now = CurrentTime();
3279 if (mCurrentPlayRangeStart != now) {
3280 ranges->Add(mCurrentPlayRangeStart, now);
3281 }
3282 }
3283
3284 ranges->Normalize();
3285 return ranges.forget();
3286 }
3287
Pause(ErrorResult & aRv)3288 void HTMLMediaElement::Pause(ErrorResult& aRv) {
3289 LOG(LogLevel::Debug, ("%p Pause() called by JS", this));
3290 if (mNetworkState == NETWORK_EMPTY) {
3291 LOG(LogLevel::Debug, ("Loading due to Pause()"));
3292 DoLoad();
3293 } else if (mDecoder) {
3294 mDecoder->Pause();
3295 }
3296
3297 bool oldPaused = mPaused;
3298 mPaused = true;
3299 mAutoplaying = false;
3300 // We changed mPaused and mAutoplaying which can affect AddRemoveSelfReference
3301 AddRemoveSelfReference();
3302 UpdateSrcMediaStreamPlaying();
3303 if (mAudioChannelWrapper) {
3304 mAudioChannelWrapper->NotifyPlayStateChanged();
3305 }
3306
3307 // We don't need to resume media which is paused explicitly by user.
3308 ClearResumeDelayedMediaPlaybackAgentIfNeeded();
3309
3310 // Start timer to trigger stopping listening to media control key events.
3311 CreateStopMediaControlTimerIfNeeded();
3312
3313 if (!oldPaused) {
3314 FireTimeUpdate(false);
3315 DispatchAsyncEvent(NS_LITERAL_STRING("pause"));
3316 AsyncRejectPendingPlayPromises(NS_ERROR_DOM_MEDIA_ABORT_ERR);
3317 }
3318 }
3319
SetVolume(double aVolume,ErrorResult & aRv)3320 void HTMLMediaElement::SetVolume(double aVolume, ErrorResult& aRv) {
3321 LOG(LogLevel::Debug, ("%p SetVolume(%f) called by JS", this, aVolume));
3322
3323 if (aVolume < 0.0 || aVolume > 1.0) {
3324 aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR);
3325 return;
3326 }
3327
3328 if (aVolume == mVolume) return;
3329
3330 mVolume = aVolume;
3331
3332 // Here we want just to update the volume.
3333 SetVolumeInternal();
3334
3335 DispatchAsyncEvent(NS_LITERAL_STRING("volumechange"));
3336
3337 // We allow inaudible autoplay. But changing our volume may make this
3338 // media audible. So pause if we are no longer supposed to be autoplaying.
3339 PauseIfShouldNotBePlaying();
3340 }
3341
MozGetMetadata(JSContext * cx,JS::MutableHandle<JSObject * > aRetval,ErrorResult & aRv)3342 void HTMLMediaElement::MozGetMetadata(JSContext* cx,
3343 JS::MutableHandle<JSObject*> aRetval,
3344 ErrorResult& aRv) {
3345 if (mReadyState < HAVE_METADATA) {
3346 aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
3347 return;
3348 }
3349
3350 JS::Rooted<JSObject*> tags(cx, JS_NewPlainObject(cx));
3351 if (!tags) {
3352 aRv.Throw(NS_ERROR_FAILURE);
3353 return;
3354 }
3355 if (mTags) {
3356 for (auto iter = mTags->ConstIter(); !iter.Done(); iter.Next()) {
3357 nsString wideValue;
3358 CopyUTF8toUTF16(iter.UserData(), wideValue);
3359 JS::Rooted<JSString*> string(cx,
3360 JS_NewUCStringCopyZ(cx, wideValue.Data()));
3361 if (!string || !JS_DefineProperty(cx, tags, iter.Key().Data(), string,
3362 JSPROP_ENUMERATE)) {
3363 NS_WARNING("couldn't create metadata object!");
3364 aRv.Throw(NS_ERROR_FAILURE);
3365 return;
3366 }
3367 }
3368 }
3369
3370 aRetval.set(tags);
3371 }
3372
SetMutedInternal(uint32_t aMuted)3373 void HTMLMediaElement::SetMutedInternal(uint32_t aMuted) {
3374 uint32_t oldMuted = mMuted;
3375 mMuted = aMuted;
3376
3377 if (!!aMuted == !!oldMuted) {
3378 return;
3379 }
3380
3381 SetVolumeInternal();
3382 }
3383
PauseIfShouldNotBePlaying()3384 void HTMLMediaElement::PauseIfShouldNotBePlaying() {
3385 if (GetPaused()) {
3386 return;
3387 }
3388 if (!AutoplayPolicy::IsAllowedToPlay(*this)) {
3389 AUTOPLAY_LOG("pause because not allowed to play, element=%p", this);
3390 ErrorResult rv;
3391 Pause(rv);
3392 OwnerDoc()->SetDocTreeHadPlayRevoked();
3393 }
3394 }
3395
SetVolumeInternal()3396 void HTMLMediaElement::SetVolumeInternal() {
3397 float effectiveVolume = ComputedVolume();
3398
3399 if (mDecoder) {
3400 mDecoder->SetVolume(effectiveVolume);
3401 } else if (mMediaStreamRenderer) {
3402 mMediaStreamRenderer->SetAudioOutputVolume(effectiveVolume);
3403 }
3404
3405 NotifyAudioPlaybackChanged(
3406 AudioChannelService::AudibleChangedReasons::eVolumeChanged);
3407 }
3408
SetMuted(bool aMuted)3409 void HTMLMediaElement::SetMuted(bool aMuted) {
3410 LOG(LogLevel::Debug, ("%p SetMuted(%d) called by JS", this, aMuted));
3411 if (aMuted == Muted()) {
3412 return;
3413 }
3414
3415 if (aMuted) {
3416 SetMutedInternal(mMuted | MUTED_BY_CONTENT);
3417 } else {
3418 SetMutedInternal(mMuted & ~MUTED_BY_CONTENT);
3419 }
3420
3421 DispatchAsyncEvent(NS_LITERAL_STRING("volumechange"));
3422
3423 // We allow inaudible autoplay. But changing our mute status may make this
3424 // media audible. So pause if we are no longer supposed to be autoplaying.
3425 PauseIfShouldNotBePlaying();
3426 }
3427
GetAllEnabledMediaTracks(nsTArray<RefPtr<MediaTrack>> & aTracks)3428 void HTMLMediaElement::GetAllEnabledMediaTracks(
3429 nsTArray<RefPtr<MediaTrack>>& aTracks) {
3430 if (AudioTrackList* tracks = AudioTracks()) {
3431 for (size_t i = 0; i < tracks->Length(); ++i) {
3432 AudioTrack* track = (*tracks)[i];
3433 if (track->Enabled()) {
3434 aTracks.AppendElement(track);
3435 }
3436 }
3437 }
3438 if (IsVideo()) {
3439 if (VideoTrackList* tracks = VideoTracks()) {
3440 for (size_t i = 0; i < tracks->Length(); ++i) {
3441 VideoTrack* track = (*tracks)[i];
3442 if (track->Selected()) {
3443 aTracks.AppendElement(track);
3444 }
3445 }
3446 }
3447 }
3448 }
3449
SetCapturedOutputStreamsEnabled(bool aEnabled)3450 void HTMLMediaElement::SetCapturedOutputStreamsEnabled(bool aEnabled) {
3451 for (auto& entry : mOutputTrackSources) {
3452 entry.GetData()->SetEnabled(aEnabled);
3453 }
3454 }
3455
OutputTracksMuted()3456 HTMLMediaElement::OutputMuteState HTMLMediaElement::OutputTracksMuted() {
3457 return mPaused || mReadyState <= HAVE_CURRENT_DATA ? OutputMuteState::Muted
3458 : OutputMuteState::Unmuted;
3459 }
3460
UpdateOutputTracksMuting()3461 void HTMLMediaElement::UpdateOutputTracksMuting() {
3462 for (auto& entry : mOutputTrackSources) {
3463 entry.GetData()->SetMutedByElement(OutputTracksMuted());
3464 }
3465 }
3466
AddOutputTrackSourceToOutputStream(MediaElementTrackSource * aSource,OutputMediaStream & aOutputStream,AddTrackMode aMode)3467 void HTMLMediaElement::AddOutputTrackSourceToOutputStream(
3468 MediaElementTrackSource* aSource, OutputMediaStream& aOutputStream,
3469 AddTrackMode aMode) {
3470 if (aOutputStream.mStream == mSrcStream) {
3471 // Cycle detected. This can happen since tracks are added async.
3472 // We avoid forwarding it to the output here or we'd get into an infloop.
3473 LOG(LogLevel::Warning,
3474 ("NOT adding output track source %p to output stream "
3475 "%p -- cycle detected",
3476 aSource, aOutputStream.mStream.get()));
3477 return;
3478 }
3479
3480 LOG(LogLevel::Debug, ("Adding output track source %p to output stream %p",
3481 aSource, aOutputStream.mStream.get()));
3482
3483 RefPtr<MediaStreamTrack> domTrack;
3484 if (aSource->Track()->mType == MediaSegment::AUDIO) {
3485 domTrack = new AudioStreamTrack(
3486 aOutputStream.mStream->GetParentObject(), aSource->Track(), aSource,
3487 MediaStreamTrackState::Live, aSource->Muted());
3488 } else {
3489 domTrack = new VideoStreamTrack(
3490 aOutputStream.mStream->GetParentObject(), aSource->Track(), aSource,
3491 MediaStreamTrackState::Live, aSource->Muted());
3492 }
3493
3494 aOutputStream.mLiveTracks.AppendElement(domTrack);
3495
3496 switch (aMode) {
3497 case AddTrackMode::ASYNC:
3498 mMainThreadEventTarget->Dispatch(
3499 NewRunnableMethod<StoreRefPtrPassByPtr<MediaStreamTrack>>(
3500 "DOMMediaStream::AddTrackInternal", aOutputStream.mStream,
3501 &DOMMediaStream::AddTrackInternal, domTrack));
3502 break;
3503 case AddTrackMode::SYNC:
3504 aOutputStream.mStream->AddTrackInternal(domTrack);
3505 break;
3506 default:
3507 MOZ_CRASH("Unexpected mode");
3508 }
3509
3510 LOG(LogLevel::Debug,
3511 ("Created capture %s track %p",
3512 domTrack->AsAudioStreamTrack() ? "audio" : "video", domTrack.get()));
3513 }
3514
UpdateOutputTrackSources()3515 void HTMLMediaElement::UpdateOutputTrackSources() {
3516 // This updates the track sources in mOutputTrackSources so they're in sync
3517 // with the tracks being currently played, and state saying whether we should
3518 // be capturing tracks. This method is long so here is a breakdown:
3519 // - Figure out the tracks that should be captured
3520 // - Diff those against currently captured tracks (mOutputTrackSources), into
3521 // tracks-to-add, and tracks-to-remove
3522 // - Remove the tracks in tracks-to-remove and dispatch "removetrack" and
3523 // "ended" events for them
3524 // - If playback has ended, or there is no longer a media provider object,
3525 // remove any OutputMediaStreams that have the finish-when-ended flag set
3526 // - Create track sources for, and add to OutputMediaStreams, the tracks in
3527 // tracks-to-add
3528
3529 const bool shouldHaveTrackSources = mTracksCaptured.Ref() &&
3530 !IsPlaybackEnded() &&
3531 mReadyState >= HAVE_METADATA;
3532
3533 // Add track sources for all enabled/selected MediaTracks.
3534 nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
3535 if (!window) {
3536 return;
3537 }
3538
3539 if (mDecoder) {
3540 mDecoder->SetOutputCaptured(mTracksCaptured.Ref());
3541 }
3542
3543 // Start with all MediaTracks
3544 AutoTArray<RefPtr<MediaTrack>, 4> mediaTracksToAdd;
3545 if (shouldHaveTrackSources) {
3546 GetAllEnabledMediaTracks(mediaTracksToAdd);
3547 }
3548
3549 // ...and all MediaElementTrackSources.
3550 AutoTArray<nsString, 4> trackSourcesToRemove;
3551 for (const auto& entry : mOutputTrackSources) {
3552 trackSourcesToRemove.AppendElement(entry.GetKey());
3553 }
3554
3555 // Then work out the differences.
3556 mediaTracksToAdd.RemoveElementsAt(
3557 std::remove_if(mediaTracksToAdd.begin(), mediaTracksToAdd.end(),
3558 [this, &trackSourcesToRemove](const auto& track) {
3559 const bool remove =
3560 mOutputTrackSources.GetWeak(track->GetId());
3561 if (remove) {
3562 trackSourcesToRemove.RemoveElement(track->GetId());
3563 }
3564 return remove;
3565 }),
3566 mediaTracksToAdd.end());
3567
3568 // First remove stale track sources.
3569 for (const auto& id : trackSourcesToRemove) {
3570 RefPtr<MediaElementTrackSource> source = mOutputTrackSources.GetWeak(id);
3571
3572 LOG(LogLevel::Debug, ("Removing output track source %p for track %s",
3573 source.get(), NS_ConvertUTF16toUTF8(id).get()));
3574
3575 if (mDecoder) {
3576 mDecoder->RemoveOutputTrack(source->Track());
3577 }
3578
3579 // The source of this track just ended. Force-notify that it ended.
3580 // If we bounce it to the MediaTrackGraph it might not be picked up,
3581 // for instance if the MediaInputPort was destroyed in the same
3582 // iteration as it was added.
3583 mMainThreadEventTarget->Dispatch(
3584 NewRunnableMethod("MediaElementTrackSource::OverrideEnded", source,
3585 &MediaElementTrackSource::OverrideEnded));
3586
3587 // Remove the track from the MediaStream after it ended.
3588 for (OutputMediaStream& ms : mOutputStreams) {
3589 if (source->Track()->mType == MediaSegment::VIDEO &&
3590 ms.mCapturingAudioOnly) {
3591 continue;
3592 }
3593 DebugOnly<size_t> length = ms.mLiveTracks.Length();
3594 ms.mLiveTracks.RemoveElementsBy(
3595 [&](const RefPtr<MediaStreamTrack>& aTrack) {
3596 if (&aTrack->GetSource() != source) {
3597 return false;
3598 }
3599 mMainThreadEventTarget->Dispatch(
3600 NewRunnableMethod<RefPtr<MediaStreamTrack>>(
3601 "DOMMediaStream::RemoveTrackInternal", ms.mStream,
3602 &DOMMediaStream::RemoveTrackInternal, aTrack));
3603 return true;
3604 });
3605 MOZ_ASSERT(ms.mLiveTracks.Length() == length - 1);
3606 }
3607
3608 mOutputTrackSources.Remove(id);
3609 }
3610
3611 // Then update finish-when-ended output streams as needed.
3612 for (int32_t i = mOutputStreams.Length() - 1; i >= 0; --i) {
3613 if (!mOutputStreams[i].mFinishWhenEnded) {
3614 continue;
3615 }
3616
3617 if (!mOutputStreams[i].mFinishWhenEndedLoadingSrc &&
3618 !mOutputStreams[i].mFinishWhenEndedAttrStream) {
3619 // This finish-when-ended stream has not seen any source loaded yet.
3620 // Update the loading src if it's time.
3621 if (!IsPlaybackEnded()) {
3622 if (mLoadingSrc) {
3623 mOutputStreams[i].mFinishWhenEndedLoadingSrc = mLoadingSrc;
3624 } else if (mSrcAttrStream) {
3625 mOutputStreams[i].mFinishWhenEndedAttrStream = mSrcAttrStream;
3626 }
3627 }
3628 continue;
3629 }
3630
3631 // Discard finish-when-ended output streams with a loading src set as
3632 // needed.
3633 if (!IsPlaybackEnded() &&
3634 mLoadingSrc == mOutputStreams[i].mFinishWhenEndedLoadingSrc) {
3635 continue;
3636 }
3637 if (!IsPlaybackEnded() &&
3638 mSrcAttrStream == mOutputStreams[i].mFinishWhenEndedAttrStream) {
3639 continue;
3640 }
3641 LOG(LogLevel::Debug,
3642 ("Playback ended or source changed. Discarding stream %p",
3643 mOutputStreams[i].mStream.get()));
3644 mOutputStreams.RemoveElementAt(i);
3645 if (mOutputStreams.IsEmpty()) {
3646 mTracksCaptured = nullptr;
3647 }
3648 }
3649
3650 // Finally add new MediaTracks.
3651 for (const auto& mediaTrack : mediaTracksToAdd) {
3652 nsAutoString id;
3653 mediaTrack->GetId(id);
3654
3655 MediaSegment::Type type;
3656 if (mediaTrack->AsAudioTrack()) {
3657 type = MediaSegment::AUDIO;
3658 } else if (mediaTrack->AsVideoTrack()) {
3659 type = MediaSegment::VIDEO;
3660 } else {
3661 MOZ_CRASH("Unknown track type");
3662 }
3663
3664 RefPtr<ProcessedMediaTrack> track;
3665 RefPtr<MediaElementTrackSource> source;
3666 if (mDecoder) {
3667 track = mTracksCaptured.Ref()->mTrack->Graph()->CreateForwardedInputTrack(
3668 type);
3669 RefPtr<nsIPrincipal> principal = GetCurrentPrincipal();
3670 if (!principal || IsCORSSameOrigin()) {
3671 principal = NodePrincipal();
3672 }
3673 source = MakeAndAddRef<MediaElementTrackSource>(
3674 mMainThreadEventTarget, track, principal, OutputTracksMuted());
3675 mDecoder->AddOutputTrack(track);
3676 } else if (mSrcStream) {
3677 MediaStreamTrack* inputTrack;
3678 if (AudioTrack* t = mediaTrack->AsAudioTrack()) {
3679 inputTrack = t->GetAudioStreamTrack();
3680 } else if (VideoTrack* t = mediaTrack->AsVideoTrack()) {
3681 inputTrack = t->GetVideoStreamTrack();
3682 } else {
3683 MOZ_CRASH("Unknown track type");
3684 }
3685 MOZ_ASSERT(inputTrack);
3686 if (!inputTrack) {
3687 NS_ERROR("Input track not found in source stream");
3688 return;
3689 }
3690 MOZ_DIAGNOSTIC_ASSERT(!inputTrack->Ended());
3691
3692 track = inputTrack->Graph()->CreateForwardedInputTrack(type);
3693 RefPtr<MediaInputPort> port = inputTrack->ForwardTrackContentsTo(track);
3694 source = MakeAndAddRef<MediaElementTrackSource>(
3695 mMainThreadEventTarget, inputTrack, &inputTrack->GetSource(), track,
3696 port, OutputTracksMuted());
3697
3698 // Track is muted initially, so we don't leak data if it's added while
3699 // paused and an MTG iteration passes before the mute comes into effect.
3700 source->SetEnabled(mSrcStreamIsPlaying);
3701 } else {
3702 MOZ_CRASH("Unknown source");
3703 }
3704
3705 LOG(LogLevel::Debug, ("Adding output track source %p for track %s",
3706 source.get(), NS_ConvertUTF16toUTF8(id).get()));
3707
3708 track->QueueSetAutoend(false);
3709 MOZ_DIAGNOSTIC_ASSERT(!mOutputTrackSources.GetWeak(id));
3710 mOutputTrackSources.Put(id, RefPtr{source});
3711
3712 // Add the new track source to any existing output streams
3713 for (OutputMediaStream& ms : mOutputStreams) {
3714 if (source->Track()->mType == MediaSegment::VIDEO &&
3715 ms.mCapturingAudioOnly) {
3716 // If the output stream is for audio only we ignore video sources.
3717 continue;
3718 }
3719 AddOutputTrackSourceToOutputStream(source, ms);
3720 }
3721 }
3722 }
3723
CanBeCaptured(StreamCaptureType aCaptureType)3724 bool HTMLMediaElement::CanBeCaptured(StreamCaptureType aCaptureType) {
3725 // Don't bother capturing when the document has gone away
3726 nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
3727 if (!window) {
3728 return false;
3729 }
3730
3731 // Prevent capturing restricted video
3732 if (aCaptureType == StreamCaptureType::CAPTURE_ALL_TRACKS &&
3733 ContainsRestrictedContent()) {
3734 return false;
3735 }
3736 return true;
3737 }
3738
CaptureStreamInternal(StreamCaptureBehavior aFinishBehavior,StreamCaptureType aStreamCaptureType,MediaTrackGraph * aGraph)3739 already_AddRefed<DOMMediaStream> HTMLMediaElement::CaptureStreamInternal(
3740 StreamCaptureBehavior aFinishBehavior, StreamCaptureType aStreamCaptureType,
3741 MediaTrackGraph* aGraph) {
3742 MOZ_ASSERT(CanBeCaptured(aStreamCaptureType));
3743
3744 MarkAsContentSource(CallerAPI::CAPTURE_STREAM);
3745 MarkAsTainted();
3746
3747 if (mTracksCaptured.Ref() &&
3748 aGraph != mTracksCaptured.Ref()->mTrack->Graph()) {
3749 return nullptr;
3750 }
3751
3752 if (!mTracksCaptured.Ref()) {
3753 // This is the first output stream, or there are no tracks. If the former,
3754 // start capturing all tracks. If the latter, they will be added later.
3755 mTracksCaptured = MakeRefPtr<SharedDummyTrack>(
3756 aGraph->CreateSourceTrack(MediaSegment::AUDIO));
3757 UpdateOutputTrackSources();
3758 }
3759
3760 nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
3761 OutputMediaStream* out = mOutputStreams.EmplaceBack(
3762 MakeRefPtr<DOMMediaStream>(window),
3763 aStreamCaptureType == StreamCaptureType::CAPTURE_AUDIO,
3764 aFinishBehavior == StreamCaptureBehavior::FINISH_WHEN_ENDED);
3765
3766 if (aFinishBehavior == StreamCaptureBehavior::FINISH_WHEN_ENDED &&
3767 !mOutputTrackSources.IsEmpty()) {
3768 // This output stream won't receive any more tracks when playback of the
3769 // current src of this media element ends, or when the src of this media
3770 // element changes. If we're currently playing something (i.e., if there are
3771 // tracks currently captured), set the current src on the output stream so
3772 // this can be tracked. If we're not playing anything,
3773 // UpdateOutputTrackSources will set the current src when it becomes
3774 // available later.
3775 if (mLoadingSrc) {
3776 out->mFinishWhenEndedLoadingSrc = mLoadingSrc;
3777 }
3778 if (mSrcAttrStream) {
3779 out->mFinishWhenEndedAttrStream = mSrcAttrStream;
3780 }
3781 MOZ_ASSERT(out->mFinishWhenEndedLoadingSrc ||
3782 out->mFinishWhenEndedAttrStream);
3783 }
3784
3785 if (aStreamCaptureType == StreamCaptureType::CAPTURE_AUDIO) {
3786 if (mSrcStream) {
3787 // We don't support applying volume and mute to the captured stream, when
3788 // capturing a MediaStream.
3789 ReportToConsole(nsIScriptError::errorFlag,
3790 "MediaElementAudioCaptureOfMediaStreamError");
3791 }
3792
3793 // mAudioCaptured tells the user that the audio played by this media element
3794 // is being routed to the captureStreams *instead* of being played to
3795 // speakers.
3796 mAudioCaptured = true;
3797 }
3798
3799 for (const auto& entry : mOutputTrackSources) {
3800 const RefPtr<MediaElementTrackSource>& source = entry.GetData();
3801 if (source->Track()->mType == MediaSegment::VIDEO) {
3802 // Only add video tracks if we're a video element and the output stream
3803 // wants video.
3804 if (!IsVideo()) {
3805 continue;
3806 }
3807 if (out->mCapturingAudioOnly) {
3808 continue;
3809 }
3810 }
3811 AddOutputTrackSourceToOutputStream(source, *out, AddTrackMode::SYNC);
3812 }
3813
3814 return do_AddRef(out->mStream);
3815 }
3816
CaptureAudio(ErrorResult & aRv,MediaTrackGraph * aGraph)3817 already_AddRefed<DOMMediaStream> HTMLMediaElement::CaptureAudio(
3818 ErrorResult& aRv, MediaTrackGraph* aGraph) {
3819 MOZ_RELEASE_ASSERT(aGraph);
3820
3821 if (!CanBeCaptured(StreamCaptureType::CAPTURE_AUDIO)) {
3822 aRv.Throw(NS_ERROR_FAILURE);
3823 return nullptr;
3824 }
3825
3826 RefPtr<DOMMediaStream> stream =
3827 CaptureStreamInternal(StreamCaptureBehavior::CONTINUE_WHEN_ENDED,
3828 StreamCaptureType::CAPTURE_AUDIO, aGraph);
3829 if (!stream) {
3830 aRv.Throw(NS_ERROR_FAILURE);
3831 return nullptr;
3832 }
3833
3834 return stream.forget();
3835 }
3836
GetAllowedToPlayPromise()3837 RefPtr<GenericNonExclusivePromise> HTMLMediaElement::GetAllowedToPlayPromise() {
3838 MOZ_ASSERT(NS_IsMainThread());
3839 MOZ_ASSERT(!mOutputStreams.IsEmpty(),
3840 "This method should only be called during stream capturing!");
3841 if (AutoplayPolicy::IsAllowedToPlay(*this)) {
3842 AUTOPLAY_LOG("MediaElement %p has allowed to play, resolve promise", this);
3843 return GenericNonExclusivePromise::CreateAndResolve(true, __func__);
3844 }
3845 AUTOPLAY_LOG("create allow-to-play promise for MediaElement %p", this);
3846 return mAllowedToPlayPromise.Ensure(__func__);
3847 }
3848
MozCaptureStream(ErrorResult & aRv)3849 already_AddRefed<DOMMediaStream> HTMLMediaElement::MozCaptureStream(
3850 ErrorResult& aRv) {
3851 MediaTrackGraph::GraphDriverType graphDriverType =
3852 HasAudio() ? MediaTrackGraph::AUDIO_THREAD_DRIVER
3853 : MediaTrackGraph::SYSTEM_THREAD_DRIVER;
3854
3855 nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
3856 if (!window) {
3857 aRv.Throw(NS_ERROR_FAILURE);
3858 return nullptr;
3859 }
3860
3861 if (!CanBeCaptured(StreamCaptureType::CAPTURE_ALL_TRACKS)) {
3862 aRv.Throw(NS_ERROR_FAILURE);
3863 return nullptr;
3864 }
3865
3866 MediaTrackGraph* graph = MediaTrackGraph::GetInstance(
3867 graphDriverType, window, MediaTrackGraph::REQUEST_DEFAULT_SAMPLE_RATE,
3868 MediaTrackGraph::DEFAULT_OUTPUT_DEVICE);
3869
3870 RefPtr<DOMMediaStream> stream =
3871 CaptureStreamInternal(StreamCaptureBehavior::CONTINUE_WHEN_ENDED,
3872 StreamCaptureType::CAPTURE_ALL_TRACKS, graph);
3873 if (!stream) {
3874 aRv.Throw(NS_ERROR_FAILURE);
3875 return nullptr;
3876 }
3877
3878 return stream.forget();
3879 }
3880
MozCaptureStreamUntilEnded(ErrorResult & aRv)3881 already_AddRefed<DOMMediaStream> HTMLMediaElement::MozCaptureStreamUntilEnded(
3882 ErrorResult& aRv) {
3883 MediaTrackGraph::GraphDriverType graphDriverType =
3884 HasAudio() ? MediaTrackGraph::AUDIO_THREAD_DRIVER
3885 : MediaTrackGraph::SYSTEM_THREAD_DRIVER;
3886
3887 nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
3888 if (!window) {
3889 aRv.Throw(NS_ERROR_FAILURE);
3890 return nullptr;
3891 }
3892
3893 if (!CanBeCaptured(StreamCaptureType::CAPTURE_ALL_TRACKS)) {
3894 aRv.Throw(NS_ERROR_FAILURE);
3895 return nullptr;
3896 }
3897
3898 MediaTrackGraph* graph = MediaTrackGraph::GetInstance(
3899 graphDriverType, window, MediaTrackGraph::REQUEST_DEFAULT_SAMPLE_RATE,
3900 MediaTrackGraph::DEFAULT_OUTPUT_DEVICE);
3901
3902 RefPtr<DOMMediaStream> stream =
3903 CaptureStreamInternal(StreamCaptureBehavior::FINISH_WHEN_ENDED,
3904 StreamCaptureType::CAPTURE_ALL_TRACKS, graph);
3905 if (!stream) {
3906 aRv.Throw(NS_ERROR_FAILURE);
3907 return nullptr;
3908 }
3909
3910 return stream.forget();
3911 }
3912
3913 class MediaElementSetForURI : public nsURIHashKey {
3914 public:
MediaElementSetForURI(const nsIURI * aKey)3915 explicit MediaElementSetForURI(const nsIURI* aKey) : nsURIHashKey(aKey) {}
MediaElementSetForURI(MediaElementSetForURI && aOther)3916 MediaElementSetForURI(MediaElementSetForURI&& aOther)
3917 : nsURIHashKey(std::move(aOther)),
3918 mElements(std::move(aOther.mElements)) {}
3919 nsTArray<HTMLMediaElement*> mElements;
3920 };
3921
3922 typedef nsTHashtable<MediaElementSetForURI> MediaElementURITable;
3923 // Elements in this table must have non-null mDecoder and mLoadingSrc, and those
3924 // can't change while the element is in the table. The table is keyed by
3925 // the element's mLoadingSrc. Each entry has a list of all elements with the
3926 // same mLoadingSrc.
3927 static MediaElementURITable* gElementTable;
3928
3929 #ifdef DEBUG
URISafeEquals(nsIURI * a1,nsIURI * a2)3930 static bool URISafeEquals(nsIURI* a1, nsIURI* a2) {
3931 if (!a1 || !a2) {
3932 // Consider two empty URIs *not* equal!
3933 return false;
3934 }
3935 bool equal = false;
3936 nsresult rv = a1->Equals(a2, &equal);
3937 return NS_SUCCEEDED(rv) && equal;
3938 }
3939 // Returns the number of times aElement appears in the media element table
3940 // for aURI. If this returns other than 0 or 1, there's a bug somewhere!
MediaElementTableCount(HTMLMediaElement * aElement,nsIURI * aURI)3941 static unsigned MediaElementTableCount(HTMLMediaElement* aElement,
3942 nsIURI* aURI) {
3943 if (!gElementTable || !aElement) {
3944 return 0;
3945 }
3946 uint32_t uriCount = 0;
3947 uint32_t otherCount = 0;
3948 for (auto it = gElementTable->ConstIter(); !it.Done(); it.Next()) {
3949 MediaElementSetForURI* entry = it.Get();
3950 uint32_t count = 0;
3951 for (const auto& elem : entry->mElements) {
3952 if (elem == aElement) {
3953 count++;
3954 }
3955 }
3956 if (URISafeEquals(aURI, entry->GetKey())) {
3957 uriCount = count;
3958 } else {
3959 otherCount += count;
3960 }
3961 }
3962 NS_ASSERTION(otherCount == 0, "Should not have entries for unknown URIs");
3963 return uriCount;
3964 }
3965 #endif
3966
AddMediaElementToURITable()3967 void HTMLMediaElement::AddMediaElementToURITable() {
3968 NS_ASSERTION(mDecoder, "Call this only with decoder Load called");
3969 NS_ASSERTION(
3970 MediaElementTableCount(this, mLoadingSrc) == 0,
3971 "Should not have entry for element in element table before addition");
3972 if (!gElementTable) {
3973 gElementTable = new MediaElementURITable();
3974 }
3975 MediaElementSetForURI* entry = gElementTable->PutEntry(mLoadingSrc);
3976 entry->mElements.AppendElement(this);
3977 NS_ASSERTION(
3978 MediaElementTableCount(this, mLoadingSrc) == 1,
3979 "Should have a single entry for element in element table after addition");
3980 }
3981
RemoveMediaElementFromURITable()3982 void HTMLMediaElement::RemoveMediaElementFromURITable() {
3983 if (!mDecoder || !mLoadingSrc || !gElementTable) {
3984 return;
3985 }
3986 MediaElementSetForURI* entry = gElementTable->GetEntry(mLoadingSrc);
3987 if (!entry) {
3988 return;
3989 }
3990 entry->mElements.RemoveElement(this);
3991 if (entry->mElements.IsEmpty()) {
3992 gElementTable->RemoveEntry(entry);
3993 if (gElementTable->Count() == 0) {
3994 delete gElementTable;
3995 gElementTable = nullptr;
3996 }
3997 }
3998 NS_ASSERTION(MediaElementTableCount(this, mLoadingSrc) == 0,
3999 "After remove, should no longer have an entry in element table");
4000 }
4001
LookupMediaElementURITable(nsIURI * aURI)4002 HTMLMediaElement* HTMLMediaElement::LookupMediaElementURITable(nsIURI* aURI) {
4003 if (!gElementTable) {
4004 return nullptr;
4005 }
4006 MediaElementSetForURI* entry = gElementTable->GetEntry(aURI);
4007 if (!entry) {
4008 return nullptr;
4009 }
4010 for (uint32_t i = 0; i < entry->mElements.Length(); ++i) {
4011 HTMLMediaElement* elem = entry->mElements[i];
4012 bool equal;
4013 // Look for elements that have the same principal and CORS mode.
4014 // Ditto for anything else that could cause us to send different headers.
4015 if (NS_SUCCEEDED(elem->NodePrincipal()->Equals(NodePrincipal(), &equal)) &&
4016 equal && elem->mCORSMode == mCORSMode) {
4017 // See SetupDecoder() below. We only add a element to the table when
4018 // mDecoder is a ChannelMediaDecoder.
4019 auto decoder = static_cast<ChannelMediaDecoder*>(elem->mDecoder.get());
4020 NS_ASSERTION(decoder, "Decoder gone");
4021 if (decoder->CanClone()) {
4022 return elem;
4023 }
4024 }
4025 }
4026 return nullptr;
4027 }
4028
4029 class HTMLMediaElement::ShutdownObserver : public nsIObserver {
4030 enum class Phase : int8_t { Init, Subscribed, Unsubscribed };
4031
4032 public:
4033 NS_DECL_ISUPPORTS
4034
Observe(nsISupports *,const char * aTopic,const char16_t *)4035 NS_IMETHOD Observe(nsISupports*, const char* aTopic,
4036 const char16_t*) override {
4037 if (mPhase != Phase::Subscribed) {
4038 // Bail out if we are not subscribed for this might be called even after
4039 // |nsContentUtils::UnregisterShutdownObserver(this)|.
4040 return NS_OK;
4041 }
4042 MOZ_DIAGNOSTIC_ASSERT(mWeak);
4043 if (strcmp(aTopic, NS_XPCOM_SHUTDOWN_OBSERVER_ID) == 0) {
4044 mWeak->NotifyShutdownEvent();
4045 }
4046 return NS_OK;
4047 }
Subscribe(HTMLMediaElement * aPtr)4048 void Subscribe(HTMLMediaElement* aPtr) {
4049 MOZ_DIAGNOSTIC_ASSERT(mPhase == Phase::Init);
4050 MOZ_DIAGNOSTIC_ASSERT(!mWeak);
4051 mWeak = aPtr;
4052 nsContentUtils::RegisterShutdownObserver(this);
4053 mPhase = Phase::Subscribed;
4054 }
Unsubscribe()4055 void Unsubscribe() {
4056 MOZ_DIAGNOSTIC_ASSERT(mPhase == Phase::Subscribed);
4057 MOZ_DIAGNOSTIC_ASSERT(mWeak);
4058 MOZ_DIAGNOSTIC_ASSERT(!mAddRefed,
4059 "ReleaseMediaElement should have been called first");
4060 mWeak = nullptr;
4061 nsContentUtils::UnregisterShutdownObserver(this);
4062 mPhase = Phase::Unsubscribed;
4063 }
AddRefMediaElement()4064 void AddRefMediaElement() {
4065 MOZ_DIAGNOSTIC_ASSERT(mWeak);
4066 MOZ_DIAGNOSTIC_ASSERT(!mAddRefed, "Should only ever AddRef once");
4067 mWeak->AddRef();
4068 mAddRefed = true;
4069 }
ReleaseMediaElement()4070 void ReleaseMediaElement() {
4071 MOZ_DIAGNOSTIC_ASSERT(mWeak);
4072 MOZ_DIAGNOSTIC_ASSERT(mAddRefed, "Should only release after AddRef");
4073 mWeak->Release();
4074 mAddRefed = false;
4075 }
4076
4077 private:
~ShutdownObserver()4078 virtual ~ShutdownObserver() {
4079 MOZ_DIAGNOSTIC_ASSERT(mPhase == Phase::Unsubscribed);
4080 MOZ_DIAGNOSTIC_ASSERT(!mWeak);
4081 MOZ_DIAGNOSTIC_ASSERT(!mAddRefed,
4082 "ReleaseMediaElement should have been called first");
4083 }
4084 // Guaranteed to be valid by HTMLMediaElement.
4085 HTMLMediaElement* mWeak = nullptr;
4086 Phase mPhase = Phase::Init;
4087 bool mAddRefed = false;
4088 };
4089
NS_IMPL_ISUPPORTS(HTMLMediaElement::ShutdownObserver,nsIObserver)4090 NS_IMPL_ISUPPORTS(HTMLMediaElement::ShutdownObserver, nsIObserver)
4091
4092 HTMLMediaElement::HTMLMediaElement(
4093 already_AddRefed<mozilla::dom::NodeInfo>&& aNodeInfo)
4094 : nsGenericHTMLElement(std::move(aNodeInfo)),
4095 mWatchManager(this,
4096 OwnerDoc()->AbstractMainThreadFor(TaskCategory::Other)),
4097 mMainThreadEventTarget(OwnerDoc()->EventTargetFor(TaskCategory::Other)),
4098 mAbstractMainThread(
4099 OwnerDoc()->AbstractMainThreadFor(TaskCategory::Other)),
4100 mShutdownObserver(new ShutdownObserver),
4101 mPlayed(new TimeRanges(ToSupports(OwnerDoc()))),
4102 mTracksCaptured(nullptr, "HTMLMediaElement::mTracksCaptured"),
4103 mErrorSink(new ErrorSink(this)),
4104 mAudioChannelWrapper(new AudioChannelAgentCallback(this)),
4105 mSink(std::pair(nsString(), RefPtr<AudioDeviceInfo>())),
4106 mShowPoster(IsVideo()) {
4107 MOZ_ASSERT(mMainThreadEventTarget);
4108 MOZ_ASSERT(mAbstractMainThread);
4109 // Please don't add anything to this constructor or the initialization
4110 // list that can cause AddRef to be called. This prevents subclasses
4111 // from overriding AddRef in a way that works with our refcount
4112 // logging mechanisms. Put these things inside of the ::Init method
4113 // instead.
4114 }
4115
Init()4116 void HTMLMediaElement::Init() {
4117 MOZ_ASSERT(mRefCnt == 0 && !mRefCnt.IsPurple(),
4118 "HTMLMediaElement::Init called when AddRef has been called "
4119 "at least once already, probably in the constructor. Please "
4120 "see the documentation in the HTMLMediaElement constructor.");
4121 MOZ_ASSERT(!mRefCnt.IsPurple());
4122
4123 mAudioTrackList = new AudioTrackList(OwnerDoc()->GetParentObject(), this);
4124 mVideoTrackList = new VideoTrackList(OwnerDoc()->GetParentObject(), this);
4125
4126 DecoderDoctorLogger::LogConstruction(this);
4127
4128 mWatchManager.Watch(mPaused, &HTMLMediaElement::UpdateWakeLock);
4129 mWatchManager.Watch(mPaused, &HTMLMediaElement::UpdateOutputTracksMuting);
4130 mWatchManager.Watch(
4131 mPaused, &HTMLMediaElement::NotifyMediaControlPlaybackStateChanged);
4132 mWatchManager.Watch(mReadyState, &HTMLMediaElement::UpdateOutputTracksMuting);
4133
4134 mWatchManager.Watch(mTracksCaptured,
4135 &HTMLMediaElement::UpdateOutputTrackSources);
4136 mWatchManager.Watch(mReadyState, &HTMLMediaElement::UpdateOutputTrackSources);
4137
4138 mWatchManager.Watch(mDownloadSuspendedByCache,
4139 &HTMLMediaElement::UpdateReadyStateInternal);
4140 mWatchManager.Watch(mFirstFrameLoaded,
4141 &HTMLMediaElement::UpdateReadyStateInternal);
4142 mWatchManager.Watch(mSrcStreamPlaybackEnded,
4143 &HTMLMediaElement::UpdateReadyStateInternal);
4144
4145 ErrorResult rv;
4146
4147 double defaultVolume = Preferences::GetFloat("media.default_volume", 1.0);
4148 SetVolume(defaultVolume, rv);
4149
4150 RegisterActivityObserver();
4151 NotifyOwnerDocumentActivityChanged();
4152
4153 // We initialize the MediaShutdownManager as the HTMLMediaElement is always
4154 // constructed on the main thread, and not during stable state.
4155 // (MediaShutdownManager make use of nsIAsyncShutdownClient which is written
4156 // in JS)
4157 MediaShutdownManager::InitStatics();
4158
4159 #if defined(MOZ_WIDGET_ANDROID)
4160 GVAutoplayPermissionRequestor::AskForPermissionIfNeeded(
4161 OwnerDoc()->GetInnerWindow());
4162 #endif
4163
4164 mShutdownObserver->Subscribe(this);
4165 mInitialized = true;
4166 }
4167
~HTMLMediaElement()4168 HTMLMediaElement::~HTMLMediaElement() {
4169 MOZ_ASSERT(mInitialized,
4170 "HTMLMediaElement must be initialized before it is destroyed.");
4171 NS_ASSERTION(
4172 !mHasSelfReference,
4173 "How can we be destroyed if we're still holding a self reference?");
4174
4175 mWatchManager.Shutdown();
4176
4177 mShutdownObserver->Unsubscribe();
4178
4179 if (mVideoFrameContainer) {
4180 mVideoFrameContainer->ForgetElement();
4181 }
4182 UnregisterActivityObserver();
4183
4184 mSetCDMRequest.DisconnectIfExists();
4185 mAllowedToPlayPromise.RejectIfExists(NS_ERROR_FAILURE, __func__);
4186
4187 if (mDecoder) {
4188 ShutdownDecoder();
4189 }
4190 if (mProgressTimer) {
4191 StopProgress();
4192 }
4193 if (mVideoDecodeSuspendTimer) {
4194 mVideoDecodeSuspendTimer->Cancel();
4195 mVideoDecodeSuspendTimer = nullptr;
4196 }
4197 if (mSrcStream) {
4198 EndSrcMediaStreamPlayback();
4199 }
4200
4201 NS_ASSERTION(MediaElementTableCount(this, mLoadingSrc) == 0,
4202 "Destroyed media element should no longer be in element table");
4203
4204 if (mChannelLoader) {
4205 mChannelLoader->Cancel();
4206 }
4207
4208 if (mAudioChannelWrapper) {
4209 mAudioChannelWrapper->Shutdown();
4210 mAudioChannelWrapper = nullptr;
4211 }
4212
4213 if (mResumeDelayedPlaybackAgent) {
4214 mResumePlaybackRequest.DisconnectIfExists();
4215 mResumeDelayedPlaybackAgent = nullptr;
4216 }
4217
4218 StopListeningMediaControlEventIfNeeded();
4219 mMediaControlEventListener = nullptr;
4220
4221 WakeLockRelease();
4222 ReportPlayedTimeAfterBlockedTelemetry();
4223
4224 DecoderDoctorLogger::LogDestruction(this);
4225 }
4226
StopSuspendingAfterFirstFrame()4227 void HTMLMediaElement::StopSuspendingAfterFirstFrame() {
4228 mAllowSuspendAfterFirstFrame = false;
4229 if (!mSuspendedAfterFirstFrame) return;
4230 mSuspendedAfterFirstFrame = false;
4231 if (mDecoder) {
4232 mDecoder->Resume();
4233 }
4234 }
4235
SetPlayedOrSeeked(bool aValue)4236 void HTMLMediaElement::SetPlayedOrSeeked(bool aValue) {
4237 if (aValue == mHasPlayedOrSeeked) {
4238 return;
4239 }
4240
4241 mHasPlayedOrSeeked = aValue;
4242
4243 // Force a reflow so that the poster frame hides or shows immediately.
4244 nsIFrame* frame = GetPrimaryFrame();
4245 if (!frame) {
4246 return;
4247 }
4248 frame->PresShell()->FrameNeedsReflow(frame, IntrinsicDirty::TreeChange,
4249 NS_FRAME_IS_DIRTY);
4250 }
4251
NotifyXPCOMShutdown()4252 void HTMLMediaElement::NotifyXPCOMShutdown() { ShutdownDecoder(); }
4253
UpdateHadAudibleAutoplayState()4254 void HTMLMediaElement::UpdateHadAudibleAutoplayState() {
4255 // If we're audible, and autoplaying...
4256 if ((Volume() > 0.0 && !Muted()) &&
4257 (!OwnerDoc()->HasBeenUserGestureActivated() || Autoplay())) {
4258 OwnerDoc()->SetDocTreeHadAudibleMedia();
4259 if (AutoplayPolicyTelemetryUtils::WouldBeAllowedToPlayIfAutoplayDisabled(
4260 *this)) {
4261 ScalarAdd(Telemetry::ScalarID::MEDIA_AUTOPLAY_WOULD_BE_ALLOWED_COUNT, 1);
4262 } else {
4263 ScalarAdd(Telemetry::ScalarID::MEDIA_AUTOPLAY_WOULD_NOT_BE_ALLOWED_COUNT,
4264 1);
4265 }
4266 }
4267 }
4268
Play(ErrorResult & aRv)4269 already_AddRefed<Promise> HTMLMediaElement::Play(ErrorResult& aRv) {
4270 LOG(LogLevel::Debug,
4271 ("%p Play() called by JS readyState=%d", this, mReadyState.Ref()));
4272
4273 // 4.8.12.8
4274 // When the play() method on a media element is invoked, the user agent must
4275 // run the following steps.
4276
4277 RefPtr<PlayPromise> promise = CreatePlayPromise(aRv);
4278 if (NS_WARN_IF(aRv.Failed())) {
4279 return nullptr;
4280 }
4281
4282 // 4.8.12.8 - Step 1:
4283 // If the media element is not allowed to play, return a promise rejected
4284 // with a "NotAllowedError" DOMException and abort these steps.
4285 // NOTE: we may require requesting permission from the user, so we do the
4286 // "not allowed" check below.
4287
4288 // 4.8.12.8 - Step 2:
4289 // If the media element's error attribute is not null and its code
4290 // attribute has the value MEDIA_ERR_SRC_NOT_SUPPORTED, return a promise
4291 // rejected with a "NotSupportedError" DOMException and abort these steps.
4292 if (GetError() && GetError()->Code() == MEDIA_ERR_SRC_NOT_SUPPORTED) {
4293 LOG(LogLevel::Debug,
4294 ("%p Play() promise rejected because source not supported.", this));
4295 promise->MaybeReject(NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR);
4296 return promise.forget();
4297 }
4298
4299 // 4.8.12.8 - Step 3:
4300 // Let promise be a new promise and append promise to the list of pending
4301 // play promises.
4302 // Note: Promise appended to list of pending promises as needed below.
4303
4304 if (ShouldBeSuspendedByInactiveDocShell()) {
4305 LOG(LogLevel::Debug, ("%p no allow to play by the docShell for now", this));
4306 mPendingPlayPromises.AppendElement(promise);
4307 return promise.forget();
4308 }
4309
4310 // We may delay starting playback of a media resource for an unvisited tab
4311 // until it's going to foreground or being resumed by the play tab icon.
4312 if (MediaPlaybackDelayPolicy::ShouldDelayPlayback(this)) {
4313 CreateResumeDelayedMediaPlaybackAgentIfNeeded();
4314 LOG(LogLevel::Debug, ("%p delay Play() call", this));
4315 MaybeDoLoad();
4316 // When play is delayed, save a reference to the promise, and return it.
4317 // The promise will be resolved when we resume play by either the tab is
4318 // brought to the foreground, or the audio tab indicator is clicked.
4319 mPendingPlayPromises.AppendElement(promise);
4320 return promise.forget();
4321 }
4322
4323 UpdateHadAudibleAutoplayState();
4324
4325 const bool handlingUserInput = UserActivation::IsHandlingUserInput();
4326 mPendingPlayPromises.AppendElement(promise);
4327
4328 if (AutoplayPolicy::IsAllowedToPlay(*this)) {
4329 AUTOPLAY_LOG("allow MediaElement %p to play", this);
4330 mAllowedToPlayPromise.ResolveIfExists(true, __func__);
4331 PlayInternal(handlingUserInput);
4332 UpdateCustomPolicyAfterPlayed();
4333 } else {
4334 AUTOPLAY_LOG("reject MediaElement %p to play", this);
4335 AsyncRejectPendingPlayPromises(NS_ERROR_DOM_MEDIA_NOT_ALLOWED_ERR);
4336 }
4337 return promise.forget();
4338 }
4339
DispatchEventsWhenPlayWasNotAllowed()4340 void HTMLMediaElement::DispatchEventsWhenPlayWasNotAllowed() {
4341 if (StaticPrefs::media_autoplay_block_event_enabled()) {
4342 DispatchAsyncEvent(NS_LITERAL_STRING("blocked"));
4343 }
4344 #if defined(MOZ_WIDGET_ANDROID)
4345 RefPtr<AsyncEventDispatcher> asyncDispatcher = new AsyncEventDispatcher(
4346 this, NS_LITERAL_STRING("MozAutoplayMediaBlocked"), CanBubble::eYes,
4347 ChromeOnlyDispatch::eYes);
4348 asyncDispatcher->PostDOMEvent();
4349 #endif
4350 MaybeNotifyAutoplayBlocked();
4351 ReportToConsole(nsIScriptError::warningFlag, "BlockAutoplayError");
4352 mHasPlayEverBeenBlocked = true;
4353 mHasEverBeenBlockedForAutoplay = true;
4354 }
4355
MaybeNotifyAutoplayBlocked()4356 void HTMLMediaElement::MaybeNotifyAutoplayBlocked() {
4357 Document* topLevelDoc = OwnerDoc()->GetTopLevelContentDocument();
4358 if (!topLevelDoc) {
4359 return;
4360 }
4361
4362 // This event is used to notify front-end side that we've blocked autoplay,
4363 // so front-end side should show blocking icon as well.
4364 RefPtr<AsyncEventDispatcher> asyncDispatcher = new AsyncEventDispatcher(
4365 topLevelDoc, NS_LITERAL_STRING("GloballyAutoplayBlocked"),
4366 CanBubble::eYes, ChromeOnlyDispatch::eYes);
4367 asyncDispatcher->PostDOMEvent();
4368 }
4369
PlayInternal(bool aHandlingUserInput)4370 void HTMLMediaElement::PlayInternal(bool aHandlingUserInput) {
4371 if (mPreloadAction == HTMLMediaElement::PRELOAD_NONE) {
4372 // The media load algorithm will be initiated by a user interaction.
4373 // We want to boost the channel priority for better responsiveness.
4374 // Note this must be done before UpdatePreloadAction() which will
4375 // update |mPreloadAction|.
4376 mUseUrgentStartForChannel = true;
4377 }
4378
4379 StopSuspendingAfterFirstFrame();
4380 SetPlayedOrSeeked(true);
4381
4382 // 4.8.12.8 - Step 4:
4383 // If the media element's networkState attribute has the value NETWORK_EMPTY,
4384 // invoke the media element's resource selection algorithm.
4385 MaybeDoLoad();
4386 if (mSuspendedForPreloadNone) {
4387 ResumeLoad(PRELOAD_ENOUGH);
4388 }
4389
4390 // 4.8.12.8 - Step 5:
4391 // If the playback has ended and the direction of playback is forwards,
4392 // seek to the earliest possible position of the media resource.
4393
4394 // Even if we just did Load() or ResumeLoad(), we could already have a decoder
4395 // here if we managed to clone an existing decoder.
4396 if (mDecoder) {
4397 if (mDecoder->IsEnded()) {
4398 SetCurrentTime(0);
4399 }
4400 if (!mSuspendedByInactiveDocOrDocshell) {
4401 mDecoder->Play();
4402 }
4403 }
4404
4405 if (mCurrentPlayRangeStart == -1.0) {
4406 mCurrentPlayRangeStart = CurrentTime();
4407 }
4408
4409 const bool oldPaused = mPaused;
4410 mPaused = false;
4411 mAutoplaying = false;
4412
4413 // We changed mPaused and mAutoplaying which can affect AddRemoveSelfReference
4414 // and our preload status.
4415 AddRemoveSelfReference();
4416 UpdatePreloadAction();
4417 UpdateSrcMediaStreamPlaying();
4418
4419 StartListeningMediaControlEventIfNeeded();
4420
4421 // Once play() has been called in a user generated event handler,
4422 // it is allowed to autoplay. Note: we can reach here when not in
4423 // a user generated event handler if our readyState has not yet
4424 // reached HAVE_METADATA.
4425 mIsBlessed |= aHandlingUserInput;
4426
4427 // TODO: If the playback has ended, then the user agent must set
4428 // seek to the effective start.
4429
4430 // 4.8.12.8 - Step 6:
4431 // If the media element's paused attribute is true, run the following steps:
4432 if (oldPaused) {
4433 // 6.1. Change the value of paused to false. (Already done.)
4434 // This step is uplifted because the "block-media-playback" feature needs
4435 // the mPaused to be false before UpdateAudioChannelPlayingState() being
4436 // called.
4437
4438 // 6.2. If the show poster flag is true, set the element's show poster flag
4439 // to false and run the time marches on steps.
4440 if (mShowPoster) {
4441 mShowPoster = false;
4442 if (mTextTrackManager) {
4443 mTextTrackManager->TimeMarchesOn();
4444 }
4445 }
4446
4447 // 6.3. Queue a task to fire a simple event named play at the element.
4448 DispatchAsyncEvent(NS_LITERAL_STRING("play"));
4449
4450 // 6.4. If the media element's readyState attribute has the value
4451 // HAVE_NOTHING, HAVE_METADATA, or HAVE_CURRENT_DATA, queue a task to
4452 // fire a simple event named waiting at the element.
4453 // Otherwise, the media element's readyState attribute has the value
4454 // HAVE_FUTURE_DATA or HAVE_ENOUGH_DATA: notify about playing for the
4455 // element.
4456 switch (mReadyState) {
4457 case HAVE_NOTHING:
4458 DispatchAsyncEvent(NS_LITERAL_STRING("waiting"));
4459 break;
4460 case HAVE_METADATA:
4461 case HAVE_CURRENT_DATA:
4462 DispatchAsyncEvent(NS_LITERAL_STRING("waiting"));
4463 break;
4464 case HAVE_FUTURE_DATA:
4465 case HAVE_ENOUGH_DATA:
4466 NotifyAboutPlaying();
4467 break;
4468 }
4469 } else if (mReadyState >= HAVE_FUTURE_DATA) {
4470 // 7. Otherwise, if the media element's readyState attribute has the value
4471 // HAVE_FUTURE_DATA or HAVE_ENOUGH_DATA, take pending play promises and
4472 // queue a task to resolve pending play promises with the result.
4473 AsyncResolvePendingPlayPromises();
4474 }
4475
4476 // 8. Set the media element's autoplaying flag to false. (Already done.)
4477
4478 // 9. Return promise.
4479 // (Done in caller.)
4480 }
4481
MaybeDoLoad()4482 void HTMLMediaElement::MaybeDoLoad() {
4483 if (mNetworkState == NETWORK_EMPTY) {
4484 DoLoad();
4485 }
4486 }
4487
UpdateWakeLock()4488 void HTMLMediaElement::UpdateWakeLock() {
4489 MOZ_ASSERT(NS_IsMainThread());
4490 // Ensure we have a wake lock if we're playing audibly. This ensures the
4491 // device doesn't sleep while playing.
4492 bool playing = !mPaused;
4493 bool isAudible = Volume() > 0.0 && !mMuted && mIsAudioTrackAudible;
4494 // WakeLock when playing audible media.
4495 if (playing && isAudible) {
4496 CreateAudioWakeLockIfNeeded();
4497 } else {
4498 ReleaseAudioWakeLockIfExists();
4499 }
4500 }
4501
CreateAudioWakeLockIfNeeded()4502 void HTMLMediaElement::CreateAudioWakeLockIfNeeded() {
4503 if (!mWakeLock) {
4504 RefPtr<power::PowerManagerService> pmService =
4505 power::PowerManagerService::GetInstance();
4506 NS_ENSURE_TRUE_VOID(pmService);
4507
4508 ErrorResult rv;
4509 mWakeLock = pmService->NewWakeLock(NS_LITERAL_STRING("audio-playing"),
4510 OwnerDoc()->GetInnerWindow(), rv);
4511 }
4512 }
4513
ReleaseAudioWakeLockIfExists()4514 void HTMLMediaElement::ReleaseAudioWakeLockIfExists() {
4515 if (mWakeLock) {
4516 ErrorResult rv;
4517 mWakeLock->Unlock(rv);
4518 rv.SuppressException();
4519 mWakeLock = nullptr;
4520 }
4521 }
4522
WakeLockRelease()4523 void HTMLMediaElement::WakeLockRelease() { ReleaseAudioWakeLockIfExists(); }
4524
GetEventTargetParent(EventChainPreVisitor & aVisitor)4525 void HTMLMediaElement::GetEventTargetParent(EventChainPreVisitor& aVisitor) {
4526 if (!this->Controls() || !aVisitor.mEvent->mFlags.mIsTrusted) {
4527 nsGenericHTMLElement::GetEventTargetParent(aVisitor);
4528 return;
4529 }
4530
4531 HTMLInputElement* el = nullptr;
4532 nsCOMPtr<nsINode> node;
4533
4534 // We will need to trap pointer, touch, and mouse events within the media
4535 // element, allowing media control exclusive consumption on these events,
4536 // and preventing the content from handling them.
4537 switch (aVisitor.mEvent->mMessage) {
4538 case ePointerDown:
4539 case ePointerUp:
4540 case eTouchEnd:
4541 // Always prevent touchmove captured in video element from being handled by
4542 // content, since we always do that for touchstart.
4543 case eTouchMove:
4544 case eTouchStart:
4545 case eMouseClick:
4546 case eMouseDoubleClick:
4547 case eMouseDown:
4548 case eMouseUp:
4549 aVisitor.mCanHandle = false;
4550 return;
4551
4552 // The *move events however are only comsumed when the range input is being
4553 // dragged.
4554 case ePointerMove:
4555 case eMouseMove:
4556 node = do_QueryInterface(aVisitor.mEvent->mOriginalTarget);
4557 if (node->IsInNativeAnonymousSubtree() || node->IsInUAWidget()) {
4558 if (node->IsHTMLElement(nsGkAtoms::input)) {
4559 // The node is a <input type="range">
4560 el = static_cast<HTMLInputElement*>(node.get());
4561 } else if (node->GetParentNode() &&
4562 node->GetParentNode()->IsHTMLElement(nsGkAtoms::input)) {
4563 // The node is a child of <input type="range">
4564 el = static_cast<HTMLInputElement*>(node->GetParentNode());
4565 }
4566 }
4567 if (el && el->IsDraggingRange()) {
4568 aVisitor.mCanHandle = false;
4569 return;
4570 }
4571 nsGenericHTMLElement::GetEventTargetParent(aVisitor);
4572 return;
4573
4574 default:
4575 nsGenericHTMLElement::GetEventTargetParent(aVisitor);
4576 return;
4577 }
4578 }
4579
ParseAttribute(int32_t aNamespaceID,nsAtom * aAttribute,const nsAString & aValue,nsIPrincipal * aMaybeScriptedPrincipal,nsAttrValue & aResult)4580 bool HTMLMediaElement::ParseAttribute(int32_t aNamespaceID, nsAtom* aAttribute,
4581 const nsAString& aValue,
4582 nsIPrincipal* aMaybeScriptedPrincipal,
4583 nsAttrValue& aResult) {
4584 // Mappings from 'preload' attribute strings to an enumeration.
4585 static const nsAttrValue::EnumTable kPreloadTable[] = {
4586 {"", HTMLMediaElement::PRELOAD_ATTR_EMPTY},
4587 {"none", HTMLMediaElement::PRELOAD_ATTR_NONE},
4588 {"metadata", HTMLMediaElement::PRELOAD_ATTR_METADATA},
4589 {"auto", HTMLMediaElement::PRELOAD_ATTR_AUTO},
4590 {nullptr, 0}};
4591
4592 if (aNamespaceID == kNameSpaceID_None) {
4593 if (aAttribute == nsGkAtoms::crossorigin) {
4594 ParseCORSValue(aValue, aResult);
4595 return true;
4596 }
4597 if (aAttribute == nsGkAtoms::preload) {
4598 return aResult.ParseEnumValue(aValue, kPreloadTable, false);
4599 }
4600 }
4601
4602 return nsGenericHTMLElement::ParseAttribute(aNamespaceID, aAttribute, aValue,
4603 aMaybeScriptedPrincipal, aResult);
4604 }
4605
DoneCreatingElement()4606 void HTMLMediaElement::DoneCreatingElement() {
4607 if (HasAttr(kNameSpaceID_None, nsGkAtoms::muted)) {
4608 mMuted |= MUTED_BY_CONTENT;
4609 }
4610 }
4611
IsHTMLFocusable(bool aWithMouse,bool * aIsFocusable,int32_t * aTabIndex)4612 bool HTMLMediaElement::IsHTMLFocusable(bool aWithMouse, bool* aIsFocusable,
4613 int32_t* aTabIndex) {
4614 if (nsGenericHTMLElement::IsHTMLFocusable(aWithMouse, aIsFocusable,
4615 aTabIndex)) {
4616 return true;
4617 }
4618
4619 *aIsFocusable = true;
4620 return false;
4621 }
4622
TabIndexDefault()4623 int32_t HTMLMediaElement::TabIndexDefault() { return 0; }
4624
AfterSetAttr(int32_t aNameSpaceID,nsAtom * aName,const nsAttrValue * aValue,const nsAttrValue * aOldValue,nsIPrincipal * aMaybeScriptedPrincipal,bool aNotify)4625 nsresult HTMLMediaElement::AfterSetAttr(int32_t aNameSpaceID, nsAtom* aName,
4626 const nsAttrValue* aValue,
4627 const nsAttrValue* aOldValue,
4628 nsIPrincipal* aMaybeScriptedPrincipal,
4629 bool aNotify) {
4630 if (aNameSpaceID == kNameSpaceID_None) {
4631 if (aName == nsGkAtoms::src) {
4632 mSrcMediaSource = nullptr;
4633 mSrcAttrTriggeringPrincipal = nsContentUtils::GetAttrTriggeringPrincipal(
4634 this, aValue ? aValue->GetStringValue() : EmptyString(),
4635 aMaybeScriptedPrincipal);
4636 if (aValue) {
4637 nsString srcStr = aValue->GetStringValue();
4638 nsCOMPtr<nsIURI> uri;
4639 NewURIFromString(srcStr, getter_AddRefs(uri));
4640 if (uri && IsMediaSourceURI(uri)) {
4641 nsresult rv = NS_GetSourceForMediaSourceURI(
4642 uri, getter_AddRefs(mSrcMediaSource));
4643 if (NS_FAILED(rv)) {
4644 nsAutoString spec;
4645 GetCurrentSrc(spec);
4646 AutoTArray<nsString, 1> params = {spec};
4647 ReportLoadError("MediaLoadInvalidURI", params);
4648 }
4649 }
4650 }
4651 } else if (aName == nsGkAtoms::autoplay) {
4652 if (aNotify) {
4653 if (aValue) {
4654 StopSuspendingAfterFirstFrame();
4655 CheckAutoplayDataReady();
4656 }
4657 // This attribute can affect AddRemoveSelfReference
4658 AddRemoveSelfReference();
4659 UpdatePreloadAction();
4660 }
4661 } else if (aName == nsGkAtoms::preload) {
4662 UpdatePreloadAction();
4663 } else if (aName == nsGkAtoms::loop) {
4664 if (mDecoder) {
4665 mDecoder->SetLooping(!!aValue);
4666 }
4667 } else if (aName == nsGkAtoms::controls && IsInComposedDoc()) {
4668 NotifyUAWidgetSetupOrChange();
4669 }
4670 }
4671
4672 // Since AfterMaybeChangeAttr may call DoLoad, make sure that it is called
4673 // *after* any possible changes to mSrcMediaSource.
4674 if (aValue) {
4675 AfterMaybeChangeAttr(aNameSpaceID, aName, aNotify);
4676 }
4677
4678 return nsGenericHTMLElement::AfterSetAttr(
4679 aNameSpaceID, aName, aValue, aOldValue, aMaybeScriptedPrincipal, aNotify);
4680 }
4681
OnAttrSetButNotChanged(int32_t aNamespaceID,nsAtom * aName,const nsAttrValueOrString & aValue,bool aNotify)4682 nsresult HTMLMediaElement::OnAttrSetButNotChanged(
4683 int32_t aNamespaceID, nsAtom* aName, const nsAttrValueOrString& aValue,
4684 bool aNotify) {
4685 AfterMaybeChangeAttr(aNamespaceID, aName, aNotify);
4686
4687 return nsGenericHTMLElement::OnAttrSetButNotChanged(aNamespaceID, aName,
4688 aValue, aNotify);
4689 }
4690
AfterMaybeChangeAttr(int32_t aNamespaceID,nsAtom * aName,bool aNotify)4691 void HTMLMediaElement::AfterMaybeChangeAttr(int32_t aNamespaceID, nsAtom* aName,
4692 bool aNotify) {
4693 if (aNamespaceID == kNameSpaceID_None) {
4694 if (aName == nsGkAtoms::src) {
4695 DoLoad();
4696 }
4697 }
4698 }
4699
BindToTree(BindContext & aContext,nsINode & aParent)4700 nsresult HTMLMediaElement::BindToTree(BindContext& aContext, nsINode& aParent) {
4701 nsresult rv = nsGenericHTMLElement::BindToTree(aContext, aParent);
4702
4703 if (IsInComposedDoc()) {
4704 // Construct Shadow Root so web content can be hidden in the DOM.
4705 AttachAndSetUAShadowRoot();
4706 NotifyUAWidgetSetupOrChange();
4707
4708 // The preload action depends on the value of the autoplay attribute.
4709 // It's value may have changed, so update it.
4710 UpdatePreloadAction();
4711 }
4712
4713 NotifyDecoderActivityChanges();
4714 if (mMediaControlEventListener) {
4715 mMediaControlEventListener->UpdateOwnerBrowsingContextIfNeeded();
4716 }
4717
4718 return rv;
4719 }
4720
4721 /* static */
VideoDecodeSuspendTimerCallback(nsITimer * aTimer,void * aClosure)4722 void HTMLMediaElement::VideoDecodeSuspendTimerCallback(nsITimer* aTimer,
4723 void* aClosure) {
4724 MOZ_ASSERT(NS_IsMainThread());
4725 auto element = static_cast<HTMLMediaElement*>(aClosure);
4726 element->mVideoDecodeSuspendTime.Start();
4727 element->mVideoDecodeSuspendTimer = nullptr;
4728 }
4729
HiddenVideoStart()4730 void HTMLMediaElement::HiddenVideoStart() {
4731 MOZ_ASSERT(NS_IsMainThread());
4732 mHiddenPlayTime.Start();
4733 if (mVideoDecodeSuspendTimer) {
4734 // Already started, just keep it running.
4735 return;
4736 }
4737 NS_NewTimerWithFuncCallback(
4738 getter_AddRefs(mVideoDecodeSuspendTimer), VideoDecodeSuspendTimerCallback,
4739 this, StaticPrefs::media_suspend_bkgnd_video_delay_ms(),
4740 nsITimer::TYPE_ONE_SHOT,
4741 "HTMLMediaElement::VideoDecodeSuspendTimerCallback",
4742 mMainThreadEventTarget);
4743 }
4744
HiddenVideoStop()4745 void HTMLMediaElement::HiddenVideoStop() {
4746 MOZ_ASSERT(NS_IsMainThread());
4747 mHiddenPlayTime.Pause();
4748 mVideoDecodeSuspendTime.Pause();
4749 if (!mVideoDecodeSuspendTimer) {
4750 return;
4751 }
4752 mVideoDecodeSuspendTimer->Cancel();
4753 mVideoDecodeSuspendTimer = nullptr;
4754 }
4755
ReportTelemetry()4756 void HTMLMediaElement::ReportTelemetry() {
4757 FrameStatisticsData data;
4758
4759 if (HTMLVideoElement* vid = HTMLVideoElement::FromNodeOrNull(this)) {
4760 FrameStatistics* stats = vid->GetFrameStatistics();
4761 if (stats) {
4762 data = stats->GetFrameStatisticsData();
4763 uint64_t parsedFrames = stats->GetParsedFrames();
4764 if (parsedFrames) {
4765 uint64_t droppedFrames = stats->GetDroppedFrames();
4766 MOZ_ASSERT(droppedFrames <= parsedFrames);
4767 // Dropped frames <= total frames, so 'percentage' cannot be higher than
4768 // 100 and therefore can fit in a uint32_t (that Telemetry takes).
4769 uint32_t percentage = 100 * droppedFrames / parsedFrames;
4770 LOG(LogLevel::Debug,
4771 ("Reporting telemetry DROPPED_FRAMES_IN_VIDEO_PLAYBACK"));
4772 Telemetry::Accumulate(Telemetry::VIDEO_DROPPED_FRAMES_PROPORTION,
4773 percentage);
4774 }
4775 }
4776 }
4777
4778 if (mMediaInfo.HasVideo() && mMediaInfo.mVideo.mImage.height > 0) {
4779 // We have a valid video.
4780 double playTime = mPlayTime.Total();
4781 double hiddenPlayTime = mHiddenPlayTime.Total();
4782 double videoDecodeSuspendTime = mVideoDecodeSuspendTime.Total();
4783
4784 Telemetry::Accumulate(Telemetry::VIDEO_PLAY_TIME_MS,
4785 SECONDS_TO_MS(playTime));
4786 LOG(LogLevel::Debug, ("%p VIDEO_PLAY_TIME_MS = %f", this, playTime));
4787
4788 Telemetry::Accumulate(Telemetry::VIDEO_HIDDEN_PLAY_TIME_MS,
4789 SECONDS_TO_MS(hiddenPlayTime));
4790 LOG(LogLevel::Debug,
4791 ("%p VIDEO_HIDDEN_PLAY_TIME_MS = %f", this, hiddenPlayTime));
4792
4793 if (playTime > 0.0) {
4794 // We have actually played something -> Report some valid-video telemetry.
4795
4796 // Keyed by audio+video or video alone, and by a resolution range.
4797 nsCString key(mMediaInfo.HasAudio() ? "AV," : "V,");
4798 static const struct {
4799 int32_t mH;
4800 const char* mRes;
4801 } sResolutions[] = {{240, "0<h<=240"}, {480, "240<h<=480"},
4802 {576, "480<h<=576"}, {720, "576<h<=720"},
4803 {1080, "720<h<=1080"}, {2160, "1080<h<=2160"}};
4804 const char* resolution = "h>2160";
4805 int32_t height = mMediaInfo.mVideo.mImage.height;
4806 for (const auto& res : sResolutions) {
4807 if (height <= res.mH) {
4808 resolution = res.mRes;
4809 break;
4810 }
4811 }
4812 key.AppendASCII(resolution);
4813
4814 uint32_t hiddenPercentage =
4815 uint32_t(hiddenPlayTime / playTime * 100.0 + 0.5);
4816 Telemetry::Accumulate(Telemetry::VIDEO_HIDDEN_PLAY_TIME_PERCENTAGE, key,
4817 hiddenPercentage);
4818 // Also accumulate all percentages in an "All" key.
4819 Telemetry::Accumulate(Telemetry::VIDEO_HIDDEN_PLAY_TIME_PERCENTAGE,
4820 NS_LITERAL_CSTRING("All"), hiddenPercentage);
4821 LOG(LogLevel::Debug,
4822 ("%p VIDEO_HIDDEN_PLAY_TIME_PERCENTAGE = %u, keys: '%s' and 'All'",
4823 this, hiddenPercentage, key.get()));
4824
4825 uint32_t videoDecodeSuspendPercentage =
4826 uint32_t(videoDecodeSuspendTime / playTime * 100.0 + 0.5);
4827 Telemetry::Accumulate(Telemetry::VIDEO_INFERRED_DECODE_SUSPEND_PERCENTAGE,
4828 key, videoDecodeSuspendPercentage);
4829 Telemetry::Accumulate(Telemetry::VIDEO_INFERRED_DECODE_SUSPEND_PERCENTAGE,
4830 NS_LITERAL_CSTRING("All"),
4831 videoDecodeSuspendPercentage);
4832 LOG(LogLevel::Debug,
4833 ("%p VIDEO_INFERRED_DECODE_SUSPEND_PERCENTAGE = %u, keys: '%s' and "
4834 "'All'",
4835 this, videoDecodeSuspendPercentage, key.get()));
4836
4837 if (data.mInterKeyframeCount != 0) {
4838 uint32_t average_ms = uint32_t(std::min<uint64_t>(
4839 double(data.mInterKeyframeSum_us) /
4840 double(data.mInterKeyframeCount) / 1000.0 +
4841 0.5,
4842 UINT32_MAX));
4843 Telemetry::Accumulate(Telemetry::VIDEO_INTER_KEYFRAME_AVERAGE_MS, key,
4844 average_ms);
4845 Telemetry::Accumulate(Telemetry::VIDEO_INTER_KEYFRAME_AVERAGE_MS,
4846 NS_LITERAL_CSTRING("All"), average_ms);
4847 LOG(LogLevel::Debug,
4848 ("%p VIDEO_INTER_KEYFRAME_AVERAGE_MS = %u, keys: '%s' and 'All'",
4849 this, average_ms, key.get()));
4850
4851 uint32_t max_ms = uint32_t(std::min<uint64_t>(
4852 (data.mInterKeyFrameMax_us + 500) / 1000, UINT32_MAX));
4853 Telemetry::Accumulate(Telemetry::VIDEO_INTER_KEYFRAME_MAX_MS, key,
4854 max_ms);
4855 Telemetry::Accumulate(Telemetry::VIDEO_INTER_KEYFRAME_MAX_MS,
4856 NS_LITERAL_CSTRING("All"), max_ms);
4857 LOG(LogLevel::Debug,
4858 ("%p VIDEO_INTER_KEYFRAME_MAX_MS = %u, keys: '%s' and 'All'", this,
4859 max_ms, key.get()));
4860 } else {
4861 // Here, we have played *some* of the video, but didn't get more than 1
4862 // keyframe. Report '0' if we have played for longer than the video-
4863 // decode-suspend delay (showing recovery would be difficult).
4864 uint32_t suspendDelay_ms =
4865 StaticPrefs::media_suspend_bkgnd_video_delay_ms();
4866 if (uint32_t(playTime * 1000.0) > suspendDelay_ms) {
4867 Telemetry::Accumulate(Telemetry::VIDEO_INTER_KEYFRAME_MAX_MS, key, 0);
4868 Telemetry::Accumulate(Telemetry::VIDEO_INTER_KEYFRAME_MAX_MS,
4869 NS_LITERAL_CSTRING("All"), 0);
4870 LOG(LogLevel::Debug,
4871 ("%p VIDEO_INTER_KEYFRAME_MAX_MS = 0 (only 1 keyframe), keys: "
4872 "'%s' and 'All'",
4873 this, key.get()));
4874 }
4875 }
4876 }
4877 }
4878 }
4879
UnbindFromTree(bool aNullParent)4880 void HTMLMediaElement::UnbindFromTree(bool aNullParent) {
4881 mVisibilityState = Visibility::Untracked;
4882
4883 if (IsInComposedDoc()) {
4884 NotifyUAWidgetTeardown();
4885 }
4886
4887 nsGenericHTMLElement::UnbindFromTree(aNullParent);
4888
4889 MOZ_ASSERT(IsHidden());
4890 NotifyDecoderActivityChanges();
4891
4892 // https://html.spec.whatwg.org/#playing-the-media-resource:remove-an-element-from-a-document
4893 //
4894 // Dispatch a task to run once we're in a stable state which ensures we're
4895 // paused if we're no longer in a document. Note that we need to dispatch this
4896 // even if there are other tasks in flight for this because these can be
4897 // cancelled if there's a new load.
4898 //
4899 // FIXME(emilio): Per that spec section, we should only do this if we used to
4900 // be connected, though other browsers match our current behavior...
4901 //
4902 // Also, https://github.com/whatwg/html/issues/4928
4903 nsCOMPtr<nsIRunnable> task =
4904 NS_NewRunnableFunction("dom::HTMLMediaElement::UnbindFromTree",
4905 [self = RefPtr<HTMLMediaElement>(this)]() {
4906 if (!self->IsInComposedDoc()) {
4907 self->Pause();
4908 }
4909 });
4910 RunInStableState(task);
4911 }
4912
4913 /* static */
GetCanPlay(const nsAString & aType,DecoderDoctorDiagnostics * aDiagnostics)4914 CanPlayStatus HTMLMediaElement::GetCanPlay(
4915 const nsAString& aType, DecoderDoctorDiagnostics* aDiagnostics) {
4916 Maybe<MediaContainerType> containerType = MakeMediaContainerType(aType);
4917 if (!containerType) {
4918 return CANPLAY_NO;
4919 }
4920 CanPlayStatus status =
4921 DecoderTraits::CanHandleContainerType(*containerType, aDiagnostics);
4922 if (status == CANPLAY_YES &&
4923 (*containerType).ExtendedType().Codecs().IsEmpty()) {
4924 // Per spec: 'Generally, a user agent should never return "probably" for a
4925 // type that allows the `codecs` parameter if that parameter is not
4926 // present.' As all our currently-supported types allow for `codecs`, we can
4927 // do this check here.
4928 // TODO: Instead, missing `codecs` should be checked in each decoder's
4929 // `IsSupportedType` call from `CanHandleCodecsType()`.
4930 // See bug 1399023.
4931 return CANPLAY_MAYBE;
4932 }
4933 return status;
4934 }
4935
CanPlayType(const nsAString & aType,nsAString & aResult)4936 void HTMLMediaElement::CanPlayType(const nsAString& aType, nsAString& aResult) {
4937 DecoderDoctorDiagnostics diagnostics;
4938 CanPlayStatus canPlay = GetCanPlay(aType, &diagnostics);
4939 diagnostics.StoreFormatDiagnostics(OwnerDoc(), aType, canPlay != CANPLAY_NO,
4940 __func__);
4941 switch (canPlay) {
4942 case CANPLAY_NO:
4943 aResult.Truncate();
4944 break;
4945 case CANPLAY_YES:
4946 aResult.AssignLiteral("probably");
4947 break;
4948 case CANPLAY_MAYBE:
4949 aResult.AssignLiteral("maybe");
4950 break;
4951 default:
4952 MOZ_ASSERT_UNREACHABLE("Unexpected case.");
4953 break;
4954 }
4955
4956 LOG(LogLevel::Debug,
4957 ("%p CanPlayType(%s) = \"%s\"", this, NS_ConvertUTF16toUTF8(aType).get(),
4958 NS_ConvertUTF16toUTF8(aResult).get()));
4959 }
4960
AssertReadyStateIsNothing()4961 void HTMLMediaElement::AssertReadyStateIsNothing() {
4962 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
4963 if (mReadyState != HAVE_NOTHING) {
4964 char buf[1024];
4965 SprintfLiteral(buf,
4966 "readyState=%d networkState=%d mLoadWaitStatus=%d "
4967 "mSourceLoadCandidate=%d "
4968 "mIsLoadingFromSourceChildren=%d mPreloadAction=%d "
4969 "mSuspendedForPreloadNone=%d error=%d",
4970 int(mReadyState), int(mNetworkState), int(mLoadWaitStatus),
4971 !!mSourceLoadCandidate, mIsLoadingFromSourceChildren,
4972 int(mPreloadAction), mSuspendedForPreloadNone,
4973 GetError() ? GetError()->Code() : 0);
4974 MOZ_CRASH_UNSAFE_PRINTF("ReadyState should be HAVE_NOTHING! %s", buf);
4975 }
4976 #endif
4977 }
4978
InitializeDecoderAsClone(ChannelMediaDecoder * aOriginal)4979 nsresult HTMLMediaElement::InitializeDecoderAsClone(
4980 ChannelMediaDecoder* aOriginal) {
4981 NS_ASSERTION(mLoadingSrc, "mLoadingSrc must already be set");
4982 NS_ASSERTION(mDecoder == nullptr, "Shouldn't have a decoder");
4983 AssertReadyStateIsNothing();
4984
4985 MediaDecoderInit decoderInit(
4986 this, mMuted ? 0.0 : mVolume, mPreservesPitch,
4987 ClampPlaybackRate(mPlaybackRate),
4988 mPreloadAction == HTMLMediaElement::PRELOAD_METADATA, mHasSuspendTaint,
4989 HasAttr(kNameSpaceID_None, nsGkAtoms::loop), aOriginal->ContainerType());
4990
4991 RefPtr<ChannelMediaDecoder> decoder = aOriginal->Clone(decoderInit);
4992 if (!decoder) return NS_ERROR_FAILURE;
4993
4994 LOG(LogLevel::Debug,
4995 ("%p Cloned decoder %p from %p", this, decoder.get(), aOriginal));
4996
4997 return FinishDecoderSetup(decoder);
4998 }
4999
5000 template <typename DecoderType, typename... LoadArgs>
SetupDecoder(DecoderType * aDecoder,LoadArgs &&...aArgs)5001 nsresult HTMLMediaElement::SetupDecoder(DecoderType* aDecoder,
5002 LoadArgs&&... aArgs) {
5003 LOG(LogLevel::Debug, ("%p Created decoder %p for type %s", this, aDecoder,
5004 aDecoder->ContainerType().OriginalString().Data()));
5005
5006 nsresult rv = aDecoder->Load(std::forward<LoadArgs>(aArgs)...);
5007 if (NS_FAILED(rv)) {
5008 aDecoder->Shutdown();
5009 LOG(LogLevel::Debug, ("%p Failed to load for decoder %p", this, aDecoder));
5010 return rv;
5011 }
5012
5013 rv = FinishDecoderSetup(aDecoder);
5014 // Only ChannelMediaDecoder supports resource cloning.
5015 if (std::is_same_v<DecoderType, ChannelMediaDecoder> && NS_SUCCEEDED(rv)) {
5016 AddMediaElementToURITable();
5017 NS_ASSERTION(
5018 MediaElementTableCount(this, mLoadingSrc) == 1,
5019 "Media element should have single table entry if decode initialized");
5020 }
5021
5022 return rv;
5023 }
5024
InitializeDecoderForChannel(nsIChannel * aChannel,nsIStreamListener ** aListener)5025 nsresult HTMLMediaElement::InitializeDecoderForChannel(
5026 nsIChannel* aChannel, nsIStreamListener** aListener) {
5027 NS_ASSERTION(mLoadingSrc, "mLoadingSrc must already be set");
5028 AssertReadyStateIsNothing();
5029
5030 DecoderDoctorDiagnostics diagnostics;
5031
5032 nsAutoCString mimeType;
5033 aChannel->GetContentType(mimeType);
5034 NS_ASSERTION(!mimeType.IsEmpty(), "We should have the Content-Type.");
5035 NS_ConvertUTF8toUTF16 mimeUTF16(mimeType);
5036
5037 RefPtr<HTMLMediaElement> self = this;
5038 auto reportCanPlay = [&, self](bool aCanPlay) {
5039 diagnostics.StoreFormatDiagnostics(self->OwnerDoc(), mimeUTF16, aCanPlay,
5040 __func__);
5041 if (!aCanPlay) {
5042 nsAutoString src;
5043 self->GetCurrentSrc(src);
5044 AutoTArray<nsString, 2> params = {mimeUTF16, src};
5045 self->ReportLoadError("MediaLoadUnsupportedMimeType", params);
5046 }
5047 };
5048
5049 auto onExit = MakeScopeExit([self] {
5050 if (self->mChannelLoader) {
5051 self->mChannelLoader->Done();
5052 self->mChannelLoader = nullptr;
5053 }
5054 });
5055
5056 Maybe<MediaContainerType> containerType = MakeMediaContainerType(mimeType);
5057 if (!containerType) {
5058 reportCanPlay(false);
5059 return NS_ERROR_FAILURE;
5060 }
5061
5062 MediaDecoderInit decoderInit(
5063 this, mMuted ? 0.0 : mVolume, mPreservesPitch,
5064 ClampPlaybackRate(mPlaybackRate),
5065 mPreloadAction == HTMLMediaElement::PRELOAD_METADATA, mHasSuspendTaint,
5066 HasAttr(kNameSpaceID_None, nsGkAtoms::loop), *containerType);
5067
5068 #ifdef MOZ_ANDROID_HLS_SUPPORT
5069 if (HLSDecoder::IsSupportedType(*containerType)) {
5070 RefPtr<HLSDecoder> decoder = HLSDecoder::Create(decoderInit);
5071 if (!decoder) {
5072 reportCanPlay(false);
5073 return NS_ERROR_OUT_OF_MEMORY;
5074 }
5075 reportCanPlay(true);
5076 return SetupDecoder(decoder.get(), aChannel);
5077 }
5078 #endif
5079
5080 RefPtr<ChannelMediaDecoder> decoder =
5081 ChannelMediaDecoder::Create(decoderInit, &diagnostics);
5082 if (!decoder) {
5083 reportCanPlay(false);
5084 return NS_ERROR_FAILURE;
5085 }
5086
5087 reportCanPlay(true);
5088 bool isPrivateBrowsing = NodePrincipal()->GetPrivateBrowsingId() > 0;
5089 return SetupDecoder(decoder.get(), aChannel, isPrivateBrowsing, aListener);
5090 }
5091
FinishDecoderSetup(MediaDecoder * aDecoder)5092 nsresult HTMLMediaElement::FinishDecoderSetup(MediaDecoder* aDecoder) {
5093 ChangeNetworkState(NETWORK_LOADING);
5094
5095 // Set mDecoder now so if methods like GetCurrentSrc get called between
5096 // here and Load(), they work.
5097 SetDecoder(aDecoder);
5098
5099 // Notify the decoder of the initial activity status.
5100 NotifyDecoderActivityChanges();
5101
5102 // Update decoder principal before we start decoding, since it
5103 // can affect how we feed data to MediaStreams
5104 NotifyDecoderPrincipalChanged();
5105
5106 // Set sink device if we have one. Otherwise the default is used.
5107 if (mSink.second) {
5108 mDecoder
5109 ->SetSink(mSink.second)
5110 #ifdef DEBUG
5111 ->Then(mAbstractMainThread, __func__,
5112 [](const GenericPromise::ResolveOrRejectValue& aValue) {
5113 MOZ_ASSERT(aValue.IsResolve() && !aValue.ResolveValue());
5114 });
5115 #else
5116 ;
5117 #endif
5118 }
5119
5120 if (mMediaKeys) {
5121 if (mMediaKeys->GetCDMProxy()) {
5122 mDecoder->SetCDMProxy(mMediaKeys->GetCDMProxy());
5123 } else {
5124 // CDM must have crashed.
5125 ShutdownDecoder();
5126 return NS_ERROR_FAILURE;
5127 }
5128 }
5129
5130 if (mChannelLoader) {
5131 mChannelLoader->Done();
5132 mChannelLoader = nullptr;
5133 }
5134
5135 // We may want to suspend the new stream now.
5136 // This will also do an AddRemoveSelfReference.
5137 NotifyOwnerDocumentActivityChanged();
5138
5139 if (!mDecoder) {
5140 // NotifyOwnerDocumentActivityChanged may shutdown the decoder if the
5141 // owning document is inactive and we're in the EME case. We could try and
5142 // handle this, but at the time of writing it's a pretty niche case, so just
5143 // bail.
5144 return NS_ERROR_FAILURE;
5145 }
5146
5147 if (mSuspendedByInactiveDocOrDocshell) {
5148 mDecoder->Suspend();
5149 }
5150
5151 if (!mPaused) {
5152 SetPlayedOrSeeked(true);
5153 if (!mSuspendedByInactiveDocOrDocshell) {
5154 mDecoder->Play();
5155 }
5156 }
5157
5158 MaybeBeginCloningVisually();
5159
5160 return NS_OK;
5161 }
5162
UpdateSrcMediaStreamPlaying(uint32_t aFlags)5163 void HTMLMediaElement::UpdateSrcMediaStreamPlaying(uint32_t aFlags) {
5164 if (!mSrcStream) {
5165 return;
5166 }
5167
5168 bool shouldPlay = !(aFlags & REMOVING_SRC_STREAM) && !mPaused &&
5169 !mSuspendedByInactiveDocOrDocshell;
5170 if (shouldPlay == mSrcStreamIsPlaying) {
5171 return;
5172 }
5173 mSrcStreamIsPlaying = shouldPlay;
5174
5175 LOG(LogLevel::Debug,
5176 ("MediaElement %p %s playback of DOMMediaStream %p", this,
5177 shouldPlay ? "Setting up" : "Removing", mSrcStream.get()));
5178
5179 if (shouldPlay) {
5180 mSrcStreamPlaybackEnded = false;
5181 mSrcStreamReportPlaybackEnded = false;
5182
5183 if (mMediaStreamRenderer) {
5184 mMediaStreamRenderer->Start();
5185 }
5186
5187 if (mSink.second) {
5188 NS_WARNING(
5189 "setSinkId() when playing a MediaStream is not supported yet and "
5190 "will be ignored");
5191 }
5192
5193 if (mSelectedVideoStreamTrack && GetVideoFrameContainer()) {
5194 MaybeBeginCloningVisually();
5195 }
5196
5197 SetCapturedOutputStreamsEnabled(true); // Unmute
5198 // If the input is a media stream, we don't check its data and always regard
5199 // it as audible when it's playing.
5200 SetAudibleState(true);
5201 } else {
5202 if (mMediaStreamRenderer) {
5203 mMediaStreamRenderer->Stop();
5204 }
5205 VideoFrameContainer* container = GetVideoFrameContainer();
5206 if (mSelectedVideoStreamTrack && container) {
5207 HTMLVideoElement* self = static_cast<HTMLVideoElement*>(this);
5208 if (self->VideoWidth() <= 1 && self->VideoHeight() <= 1) {
5209 // MediaInfo uses dummy values of 1 for width and height to
5210 // mark video as valid. We need a new first-frame listener
5211 // if size is 0x0 or 1x1.
5212 if (!mFirstFrameListener) {
5213 mFirstFrameListener =
5214 new FirstFrameListener(container, mAbstractMainThread);
5215 }
5216 mSelectedVideoStreamTrack->AddVideoOutput(mFirstFrameListener);
5217 }
5218 }
5219
5220 SetCapturedOutputStreamsEnabled(false); // Mute
5221 }
5222 }
5223
UpdateSrcStreamPotentiallyPlaying()5224 void HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying() {
5225 if (!mMediaStreamRenderer) {
5226 // Notifications are async, the renderer could have been cleared.
5227 return;
5228 }
5229
5230 mMediaStreamRenderer->SetProgressingCurrentTime(IsPotentiallyPlaying());
5231 }
5232
UpdateSrcStreamTime()5233 void HTMLMediaElement::UpdateSrcStreamTime() {
5234 MOZ_ASSERT(NS_IsMainThread());
5235
5236 if (mSrcStreamPlaybackEnded) {
5237 // We do a separate FireTimeUpdate() when this is set.
5238 return;
5239 }
5240
5241 FireTimeUpdate(true);
5242 }
5243
SetupSrcMediaStreamPlayback(DOMMediaStream * aStream)5244 void HTMLMediaElement::SetupSrcMediaStreamPlayback(DOMMediaStream* aStream) {
5245 NS_ASSERTION(!mSrcStream && !mFirstFrameListener,
5246 "Should have been ended already");
5247
5248 mSrcStream = aStream;
5249
5250 mMediaStreamRenderer = MakeAndAddRef<MediaStreamRenderer>(
5251 mAbstractMainThread, GetVideoFrameContainer(), this);
5252 mWatchManager.Watch(mPaused,
5253 &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
5254 mWatchManager.Watch(mReadyState,
5255 &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
5256 mWatchManager.Watch(mSrcStreamPlaybackEnded,
5257 &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
5258 mWatchManager.Watch(mSrcStreamPlaybackEnded,
5259 &HTMLMediaElement::UpdateSrcStreamReportPlaybackEnded);
5260 mWatchManager.Watch(mMediaStreamRenderer->CurrentGraphTime(),
5261 &HTMLMediaElement::UpdateSrcStreamTime);
5262 SetVolumeInternal();
5263
5264 UpdateSrcMediaStreamPlaying();
5265 UpdateSrcStreamPotentiallyPlaying();
5266 mSrcStreamVideoPrincipal = NodePrincipal();
5267
5268 // If we pause this media element, track changes in the underlying stream
5269 // will continue to fire events at this element and alter its track list.
5270 // That's simpler than delaying the events, but probably confusing...
5271 nsTArray<RefPtr<MediaStreamTrack>> tracks;
5272 mSrcStream->GetTracks(tracks);
5273 for (const RefPtr<MediaStreamTrack>& track : tracks) {
5274 NotifyMediaStreamTrackAdded(track);
5275 }
5276
5277 mMediaStreamTrackListener = MakeUnique<MediaStreamTrackListener>(this);
5278 mSrcStream->RegisterTrackListener(mMediaStreamTrackListener.get());
5279
5280 ChangeNetworkState(NETWORK_IDLE);
5281 ChangeDelayLoadStatus(false);
5282
5283 // FirstFrameLoaded() will be called when the stream has tracks.
5284 }
5285
EndSrcMediaStreamPlayback()5286 void HTMLMediaElement::EndSrcMediaStreamPlayback() {
5287 MOZ_ASSERT(mSrcStream);
5288
5289 UpdateSrcMediaStreamPlaying(REMOVING_SRC_STREAM);
5290
5291 if (mSelectedVideoStreamTrack) {
5292 mSelectedVideoStreamTrack->RemovePrincipalChangeObserver(this);
5293 }
5294 if (mFirstFrameListener) {
5295 if (mSelectedVideoStreamTrack) {
5296 // Unlink might have removed the selected video track.
5297 mSelectedVideoStreamTrack->RemoveVideoOutput(mFirstFrameListener);
5298 }
5299 }
5300 mSelectedVideoStreamTrack = nullptr;
5301 mFirstFrameListener = nullptr;
5302
5303 if (mMediaStreamRenderer) {
5304 mWatchManager.Unwatch(mPaused,
5305 &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
5306 mWatchManager.Unwatch(mReadyState,
5307 &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
5308 mWatchManager.Unwatch(mSrcStreamPlaybackEnded,
5309 &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
5310 mWatchManager.Unwatch(
5311 mSrcStreamPlaybackEnded,
5312 &HTMLMediaElement::UpdateSrcStreamReportPlaybackEnded);
5313 mWatchManager.Unwatch(mMediaStreamRenderer->CurrentGraphTime(),
5314 &HTMLMediaElement::UpdateSrcStreamTime);
5315 mMediaStreamRenderer->Shutdown();
5316 mMediaStreamRenderer = nullptr;
5317 }
5318
5319 mSrcStream->UnregisterTrackListener(mMediaStreamTrackListener.get());
5320 mMediaStreamTrackListener = nullptr;
5321 mSrcStreamPlaybackEnded = false;
5322 mSrcStreamReportPlaybackEnded = false;
5323 mSrcStreamVideoPrincipal = nullptr;
5324
5325 mSrcStream = nullptr;
5326 }
5327
CreateAudioTrack(AudioStreamTrack * aStreamTrack,nsIGlobalObject * aOwnerGlobal)5328 static already_AddRefed<AudioTrack> CreateAudioTrack(
5329 AudioStreamTrack* aStreamTrack, nsIGlobalObject* aOwnerGlobal) {
5330 nsAutoString id;
5331 nsAutoString label;
5332 aStreamTrack->GetId(id);
5333 aStreamTrack->GetLabel(label, CallerType::System);
5334
5335 return MediaTrackList::CreateAudioTrack(aOwnerGlobal, id,
5336 NS_LITERAL_STRING("main"), label,
5337 EmptyString(), true, aStreamTrack);
5338 }
5339
CreateVideoTrack(VideoStreamTrack * aStreamTrack,nsIGlobalObject * aOwnerGlobal)5340 static already_AddRefed<VideoTrack> CreateVideoTrack(
5341 VideoStreamTrack* aStreamTrack, nsIGlobalObject* aOwnerGlobal) {
5342 nsAutoString id;
5343 nsAutoString label;
5344 aStreamTrack->GetId(id);
5345 aStreamTrack->GetLabel(label, CallerType::System);
5346
5347 return MediaTrackList::CreateVideoTrack(aOwnerGlobal, id,
5348 NS_LITERAL_STRING("main"), label,
5349 EmptyString(), aStreamTrack);
5350 }
5351
NotifyMediaStreamTrackAdded(const RefPtr<MediaStreamTrack> & aTrack)5352 void HTMLMediaElement::NotifyMediaStreamTrackAdded(
5353 const RefPtr<MediaStreamTrack>& aTrack) {
5354 MOZ_ASSERT(aTrack);
5355
5356 if (aTrack->Ended()) {
5357 return;
5358 }
5359
5360 #ifdef DEBUG
5361 nsAutoString id;
5362 aTrack->GetId(id);
5363
5364 LOG(LogLevel::Debug, ("%p, Adding %sTrack with id %s", this,
5365 aTrack->AsAudioStreamTrack() ? "Audio" : "Video",
5366 NS_ConvertUTF16toUTF8(id).get()));
5367 #endif
5368
5369 if (AudioStreamTrack* t = aTrack->AsAudioStreamTrack()) {
5370 MOZ_DIAGNOSTIC_ASSERT(AudioTracks(), "Element can't have been unlinked");
5371 RefPtr<AudioTrack> audioTrack =
5372 CreateAudioTrack(t, AudioTracks()->GetOwnerGlobal());
5373 AudioTracks()->AddTrack(audioTrack);
5374 } else if (VideoStreamTrack* t = aTrack->AsVideoStreamTrack()) {
5375 // TODO: Fix this per the spec on bug 1273443.
5376 if (!IsVideo()) {
5377 return;
5378 }
5379 MOZ_DIAGNOSTIC_ASSERT(VideoTracks(), "Element can't have been unlinked");
5380 RefPtr<VideoTrack> videoTrack =
5381 CreateVideoTrack(t, VideoTracks()->GetOwnerGlobal());
5382 VideoTracks()->AddTrack(videoTrack);
5383 // New MediaStreamTrack added, set the new added video track as selected
5384 // video track when there is no selected track.
5385 if (VideoTracks()->SelectedIndex() == -1) {
5386 MOZ_ASSERT(!mSelectedVideoStreamTrack);
5387 videoTrack->SetEnabledInternal(true, dom::MediaTrack::FIRE_NO_EVENTS);
5388 }
5389 }
5390
5391 // The set of enabled AudioTracks and selected video track might have changed.
5392 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
5393 AbstractThread::DispatchDirectTask(
5394 NewRunnableMethod("HTMLMediaElement::FirstFrameLoaded", this,
5395 &HTMLMediaElement::FirstFrameLoaded));
5396 }
5397
NotifyMediaStreamTrackRemoved(const RefPtr<MediaStreamTrack> & aTrack)5398 void HTMLMediaElement::NotifyMediaStreamTrackRemoved(
5399 const RefPtr<MediaStreamTrack>& aTrack) {
5400 MOZ_ASSERT(aTrack);
5401
5402 nsAutoString id;
5403 aTrack->GetId(id);
5404
5405 LOG(LogLevel::Debug, ("%p, Removing %sTrack with id %s", this,
5406 aTrack->AsAudioStreamTrack() ? "Audio" : "Video",
5407 NS_ConvertUTF16toUTF8(id).get()));
5408
5409 MOZ_DIAGNOSTIC_ASSERT(AudioTracks() && VideoTracks(),
5410 "Element can't have been unlinked");
5411 if (dom::MediaTrack* t = AudioTracks()->GetTrackById(id)) {
5412 AudioTracks()->RemoveTrack(t);
5413 } else if (dom::MediaTrack* t = VideoTracks()->GetTrackById(id)) {
5414 VideoTracks()->RemoveTrack(t);
5415 } else {
5416 NS_ASSERTION(aTrack->AsVideoStreamTrack() && !IsVideo(),
5417 "MediaStreamTrack ended but did not exist in track lists. "
5418 "This is only allowed if a video element ends and we are an "
5419 "audio element.");
5420 return;
5421 }
5422 }
5423
ProcessMediaFragmentURI()5424 void HTMLMediaElement::ProcessMediaFragmentURI() {
5425 nsMediaFragmentURIParser parser(mLoadingSrc);
5426
5427 if (mDecoder && parser.HasEndTime()) {
5428 mFragmentEnd = parser.GetEndTime();
5429 }
5430
5431 if (parser.HasStartTime()) {
5432 SetCurrentTime(parser.GetStartTime());
5433 mFragmentStart = parser.GetStartTime();
5434 }
5435 }
5436
MetadataLoaded(const MediaInfo * aInfo,UniquePtr<const MetadataTags> aTags)5437 void HTMLMediaElement::MetadataLoaded(const MediaInfo* aInfo,
5438 UniquePtr<const MetadataTags> aTags) {
5439 MOZ_ASSERT(NS_IsMainThread());
5440
5441 if (mDecoder) {
5442 ConstructMediaTracks(aInfo);
5443 }
5444
5445 SetMediaInfo(*aInfo);
5446
5447 mIsEncrypted =
5448 aInfo->IsEncrypted() || mPendingEncryptedInitData.IsEncrypted();
5449 mTags = std::move(aTags);
5450 mLoadedDataFired = false;
5451 ChangeReadyState(HAVE_METADATA);
5452
5453 // Add output tracks synchronously now to be sure they're available in
5454 // "loadedmetadata" event handlers.
5455 UpdateOutputTrackSources();
5456
5457 DispatchAsyncEvent(NS_LITERAL_STRING("durationchange"));
5458 if (IsVideo() && HasVideo()) {
5459 DispatchAsyncEvent(NS_LITERAL_STRING("resize"));
5460 }
5461 NS_ASSERTION(!HasVideo() || (mMediaInfo.mVideo.mDisplay.width > 0 &&
5462 mMediaInfo.mVideo.mDisplay.height > 0),
5463 "Video resolution must be known on 'loadedmetadata'");
5464 DispatchAsyncEvent(NS_LITERAL_STRING("loadedmetadata"));
5465
5466 if (mDecoder && mDecoder->IsTransportSeekable() &&
5467 mDecoder->IsMediaSeekable()) {
5468 ProcessMediaFragmentURI();
5469 mDecoder->SetFragmentEndTime(mFragmentEnd);
5470 }
5471 if (mIsEncrypted) {
5472 // We only support playback of encrypted content via MSE by default.
5473 if (!mMediaSource && Preferences::GetBool("media.eme.mse-only", true)) {
5474 DecodeError(
5475 MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
5476 "Encrypted content not supported outside of MSE"));
5477 return;
5478 }
5479
5480 // Dispatch a distinct 'encrypted' event for each initData we have.
5481 for (const auto& initData : mPendingEncryptedInitData.mInitDatas) {
5482 DispatchEncrypted(initData.mInitData, initData.mType);
5483 }
5484 mPendingEncryptedInitData.Reset();
5485 }
5486
5487 if (IsVideo() && aInfo->HasVideo()) {
5488 // We are a video element playing video so update the screen wakelock
5489 NotifyOwnerDocumentActivityChanged();
5490 }
5491
5492 if (mDefaultPlaybackStartPosition != 0.0) {
5493 SetCurrentTime(mDefaultPlaybackStartPosition);
5494 mDefaultPlaybackStartPosition = 0.0;
5495 }
5496
5497 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
5498 }
5499
FirstFrameLoaded()5500 void HTMLMediaElement::FirstFrameLoaded() {
5501 LOG(LogLevel::Debug,
5502 ("%p, FirstFrameLoaded() mFirstFrameLoaded=%d mWaitingForKey=%d", this,
5503 mFirstFrameLoaded.Ref(), mWaitingForKey));
5504
5505 NS_ASSERTION(!mSuspendedAfterFirstFrame, "Should not have already suspended");
5506
5507 if (!mFirstFrameLoaded) {
5508 mFirstFrameLoaded = true;
5509 }
5510
5511 ChangeDelayLoadStatus(false);
5512
5513 if (mDecoder && mAllowSuspendAfterFirstFrame && mPaused &&
5514 !HasAttr(kNameSpaceID_None, nsGkAtoms::autoplay) &&
5515 mPreloadAction == HTMLMediaElement::PRELOAD_METADATA) {
5516 mSuspendedAfterFirstFrame = true;
5517 mDecoder->Suspend();
5518 }
5519 }
5520
NetworkError(const MediaResult & aError)5521 void HTMLMediaElement::NetworkError(const MediaResult& aError) {
5522 if (mReadyState == HAVE_NOTHING) {
5523 NoSupportedMediaSourceError(aError.Description());
5524 } else {
5525 Error(MEDIA_ERR_NETWORK);
5526 }
5527 }
5528
DecodeError(const MediaResult & aError)5529 void HTMLMediaElement::DecodeError(const MediaResult& aError) {
5530 nsAutoString src;
5531 GetCurrentSrc(src);
5532 AutoTArray<nsString, 1> params = {src};
5533 ReportLoadError("MediaLoadDecodeError", params);
5534
5535 DecoderDoctorDiagnostics diagnostics;
5536 diagnostics.StoreDecodeError(OwnerDoc(), aError, src, __func__);
5537
5538 if (mIsLoadingFromSourceChildren) {
5539 mErrorSink->ResetError();
5540 if (mSourceLoadCandidate) {
5541 DispatchAsyncSourceError(mSourceLoadCandidate);
5542 QueueLoadFromSourceTask();
5543 } else {
5544 NS_WARNING("Should know the source we were loading from!");
5545 }
5546 } else if (mReadyState == HAVE_NOTHING) {
5547 NoSupportedMediaSourceError(aError.Description());
5548 } else if (IsCORSSameOrigin()) {
5549 Error(MEDIA_ERR_DECODE, aError.Description());
5550 } else {
5551 Error(MEDIA_ERR_DECODE, "Failed to decode media"_ns);
5552 }
5553 }
5554
DecodeWarning(const MediaResult & aError)5555 void HTMLMediaElement::DecodeWarning(const MediaResult& aError) {
5556 nsAutoString src;
5557 GetCurrentSrc(src);
5558 DecoderDoctorDiagnostics diagnostics;
5559 diagnostics.StoreDecodeWarning(OwnerDoc(), aError, src, __func__);
5560 }
5561
HasError() const5562 bool HTMLMediaElement::HasError() const { return GetError(); }
5563
LoadAborted()5564 void HTMLMediaElement::LoadAborted() { Error(MEDIA_ERR_ABORTED); }
5565
Error(uint16_t aErrorCode,const nsACString & aErrorDetails)5566 void HTMLMediaElement::Error(uint16_t aErrorCode,
5567 const nsACString& aErrorDetails) {
5568 mErrorSink->SetError(aErrorCode, aErrorDetails);
5569 ChangeDelayLoadStatus(false);
5570 UpdateAudioChannelPlayingState();
5571 }
5572
PlaybackEnded()5573 void HTMLMediaElement::PlaybackEnded() {
5574 // We changed state which can affect AddRemoveSelfReference
5575 AddRemoveSelfReference();
5576
5577 NS_ASSERTION(!mDecoder || mDecoder->IsEnded(),
5578 "Decoder fired ended, but not in ended state");
5579
5580 // IsPlaybackEnded() became true.
5581 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
5582
5583 if (mSrcStream) {
5584 LOG(LogLevel::Debug,
5585 ("%p, got duration by reaching the end of the resource", this));
5586 mSrcStreamPlaybackEnded = true;
5587 DispatchAsyncEvent(NS_LITERAL_STRING("durationchange"));
5588 } else {
5589 // mediacapture-main:
5590 // Setting the loop attribute has no effect since a MediaStream has no
5591 // defined end and therefore cannot be looped.
5592 if (HasAttr(kNameSpaceID_None, nsGkAtoms::loop)) {
5593 SetCurrentTime(0);
5594 return;
5595 }
5596 }
5597
5598 FireTimeUpdate(false);
5599
5600 if (!mPaused) {
5601 Pause();
5602 }
5603
5604 if (mSrcStream) {
5605 // A MediaStream that goes from inactive to active shall be eligible for
5606 // autoplay again according to the mediacapture-main spec.
5607 mAutoplaying = true;
5608 }
5609
5610 DispatchAsyncEvent(NS_LITERAL_STRING("ended"));
5611 }
5612
UpdateSrcStreamReportPlaybackEnded()5613 void HTMLMediaElement::UpdateSrcStreamReportPlaybackEnded() {
5614 mSrcStreamReportPlaybackEnded = mSrcStreamPlaybackEnded;
5615 }
5616
SeekStarted()5617 void HTMLMediaElement::SeekStarted() {
5618 DispatchAsyncEvent(NS_LITERAL_STRING("seeking"));
5619 }
5620
SeekCompleted()5621 void HTMLMediaElement::SeekCompleted() {
5622 mPlayingBeforeSeek = false;
5623 SetPlayedOrSeeked(true);
5624 if (mTextTrackManager) {
5625 mTextTrackManager->DidSeek();
5626 }
5627 FireTimeUpdate(false);
5628 DispatchAsyncEvent(NS_LITERAL_STRING("seeked"));
5629 // We changed whether we're seeking so we need to AddRemoveSelfReference
5630 AddRemoveSelfReference();
5631 if (mCurrentPlayRangeStart == -1.0) {
5632 mCurrentPlayRangeStart = CurrentTime();
5633 }
5634
5635 // After seeking completed, if the audio track is silent, start another new
5636 // silence range.
5637 mHasAccumulatedSilenceRangeBeforeSeekEnd = false;
5638 if (IsAudioTrackCurrentlySilent()) {
5639 UpdateAudioTrackSilenceRange(mIsAudioTrackAudible);
5640 }
5641
5642 if (mSeekDOMPromise) {
5643 mAbstractMainThread->Dispatch(NS_NewRunnableFunction(
5644 __func__, [promise = std::move(mSeekDOMPromise)] {
5645 promise->MaybeResolveWithUndefined();
5646 }));
5647 }
5648 MOZ_ASSERT(!mSeekDOMPromise);
5649 }
5650
SeekAborted()5651 void HTMLMediaElement::SeekAborted() {
5652 if (mSeekDOMPromise) {
5653 mAbstractMainThread->Dispatch(NS_NewRunnableFunction(
5654 __func__, [promise = std::move(mSeekDOMPromise)] {
5655 promise->MaybeReject(NS_ERROR_DOM_ABORT_ERR);
5656 }));
5657 }
5658 MOZ_ASSERT(!mSeekDOMPromise);
5659 }
5660
NotifySuspendedByCache(bool aSuspendedByCache)5661 void HTMLMediaElement::NotifySuspendedByCache(bool aSuspendedByCache) {
5662 mDownloadSuspendedByCache = aSuspendedByCache;
5663 }
5664
DownloadSuspended()5665 void HTMLMediaElement::DownloadSuspended() {
5666 if (mNetworkState == NETWORK_LOADING) {
5667 DispatchAsyncEvent(NS_LITERAL_STRING("progress"));
5668 }
5669 ChangeNetworkState(NETWORK_IDLE);
5670 }
5671
DownloadResumed()5672 void HTMLMediaElement::DownloadResumed() {
5673 ChangeNetworkState(NETWORK_LOADING);
5674 }
5675
CheckProgress(bool aHaveNewProgress)5676 void HTMLMediaElement::CheckProgress(bool aHaveNewProgress) {
5677 MOZ_ASSERT(NS_IsMainThread());
5678 MOZ_ASSERT(mNetworkState == NETWORK_LOADING);
5679
5680 TimeStamp now = TimeStamp::NowLoRes();
5681
5682 if (aHaveNewProgress) {
5683 mDataTime = now;
5684 }
5685
5686 // If this is the first progress, or PROGRESS_MS has passed since the last
5687 // progress event fired and more data has arrived since then, fire a
5688 // progress event.
5689 NS_ASSERTION(
5690 (mProgressTime.IsNull() && !aHaveNewProgress) || !mDataTime.IsNull(),
5691 "null TimeStamp mDataTime should not be used in comparison");
5692 if (mProgressTime.IsNull()
5693 ? aHaveNewProgress
5694 : (now - mProgressTime >=
5695 TimeDuration::FromMilliseconds(PROGRESS_MS) &&
5696 mDataTime > mProgressTime)) {
5697 DispatchAsyncEvent(NS_LITERAL_STRING("progress"));
5698 // Resolution() ensures that future data will have now > mProgressTime,
5699 // and so will trigger another event. mDataTime is not reset because it
5700 // is still required to detect stalled; it is similarly offset by
5701 // resolution to indicate the new data has not yet arrived.
5702 mProgressTime = now - TimeDuration::Resolution();
5703 if (mDataTime > mProgressTime) {
5704 mDataTime = mProgressTime;
5705 }
5706 if (!mProgressTimer) {
5707 NS_ASSERTION(aHaveNewProgress,
5708 "timer dispatched when there was no timer");
5709 // Were stalled. Restart timer.
5710 StartProgressTimer();
5711 if (!mLoadedDataFired) {
5712 ChangeDelayLoadStatus(true);
5713 }
5714 }
5715 // Download statistics may have been updated, force a recheck of the
5716 // readyState.
5717 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
5718 }
5719
5720 if (now - mDataTime >= TimeDuration::FromMilliseconds(STALL_MS)) {
5721 if (!mMediaSource) {
5722 DispatchAsyncEvent(NS_LITERAL_STRING("stalled"));
5723 } else {
5724 ChangeDelayLoadStatus(false);
5725 }
5726
5727 NS_ASSERTION(mProgressTimer, "detected stalled without timer");
5728 // Stop timer events, which prevents repeated stalled events until there
5729 // is more progress.
5730 StopProgress();
5731 }
5732
5733 AddRemoveSelfReference();
5734 }
5735
5736 /* static */
ProgressTimerCallback(nsITimer * aTimer,void * aClosure)5737 void HTMLMediaElement::ProgressTimerCallback(nsITimer* aTimer, void* aClosure) {
5738 auto decoder = static_cast<HTMLMediaElement*>(aClosure);
5739 decoder->CheckProgress(false);
5740 }
5741
StartProgressTimer()5742 void HTMLMediaElement::StartProgressTimer() {
5743 MOZ_ASSERT(NS_IsMainThread());
5744 MOZ_ASSERT(mNetworkState == NETWORK_LOADING);
5745 NS_ASSERTION(!mProgressTimer, "Already started progress timer.");
5746
5747 NS_NewTimerWithFuncCallback(
5748 getter_AddRefs(mProgressTimer), ProgressTimerCallback, this, PROGRESS_MS,
5749 nsITimer::TYPE_REPEATING_SLACK, "HTMLMediaElement::ProgressTimerCallback",
5750 mMainThreadEventTarget);
5751 }
5752
StartProgress()5753 void HTMLMediaElement::StartProgress() {
5754 // Record the time now for detecting stalled.
5755 mDataTime = TimeStamp::NowLoRes();
5756 // Reset mProgressTime so that mDataTime is not indicating bytes received
5757 // after the last progress event.
5758 mProgressTime = TimeStamp();
5759 StartProgressTimer();
5760 }
5761
StopProgress()5762 void HTMLMediaElement::StopProgress() {
5763 MOZ_ASSERT(NS_IsMainThread());
5764 if (!mProgressTimer) {
5765 return;
5766 }
5767
5768 mProgressTimer->Cancel();
5769 mProgressTimer = nullptr;
5770 }
5771
DownloadProgressed()5772 void HTMLMediaElement::DownloadProgressed() {
5773 if (mNetworkState != NETWORK_LOADING) {
5774 return;
5775 }
5776 CheckProgress(true);
5777 }
5778
ShouldCheckAllowOrigin()5779 bool HTMLMediaElement::ShouldCheckAllowOrigin() {
5780 return mCORSMode != CORS_NONE;
5781 }
5782
IsCORSSameOrigin()5783 bool HTMLMediaElement::IsCORSSameOrigin() {
5784 bool subsumes;
5785 RefPtr<nsIPrincipal> principal = GetCurrentPrincipal();
5786 return (NS_SUCCEEDED(NodePrincipal()->Subsumes(principal, &subsumes)) &&
5787 subsumes) ||
5788 ShouldCheckAllowOrigin();
5789 }
5790
UpdateReadyStateInternal()5791 void HTMLMediaElement::UpdateReadyStateInternal() {
5792 if (!mDecoder && !mSrcStream) {
5793 // Not initialized - bail out.
5794 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5795 "Not initialized",
5796 this));
5797 return;
5798 }
5799
5800 if (mDecoder && mReadyState < HAVE_METADATA) {
5801 // aNextFrame might have a next frame because the decoder can advance
5802 // on its own thread before MetadataLoaded gets a chance to run.
5803 // The arrival of more data can't change us out of this readyState.
5804 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5805 "Decoder ready state < HAVE_METADATA",
5806 this));
5807 return;
5808 }
5809
5810 if (mDecoder) {
5811 // IsPlaybackEnded() might have become false.
5812 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
5813 }
5814
5815 if (mSrcStream && mReadyState < HAVE_METADATA) {
5816 bool hasAudioTracks = AudioTracks() && !AudioTracks()->IsEmpty();
5817 bool hasVideoTracks = VideoTracks() && !VideoTracks()->IsEmpty();
5818 if (!hasAudioTracks && !hasVideoTracks) {
5819 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5820 "Stream with no tracks",
5821 this));
5822 // Give it one last chance to remove the self reference if needed.
5823 AddRemoveSelfReference();
5824 return;
5825 }
5826
5827 if (IsVideo() && hasVideoTracks && !HasVideo()) {
5828 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5829 "Stream waiting for video",
5830 this));
5831 return;
5832 }
5833
5834 LOG(LogLevel::Debug,
5835 ("MediaElement %p UpdateReadyStateInternal() Stream has "
5836 "metadata; audioTracks=%d, videoTracks=%d, "
5837 "hasVideoFrame=%d",
5838 this, AudioTracks()->Length(), VideoTracks()->Length(), HasVideo()));
5839
5840 // We are playing a stream that has video and a video frame is now set.
5841 // This means we have all metadata needed to change ready state.
5842 MediaInfo mediaInfo = mMediaInfo;
5843 if (hasAudioTracks) {
5844 mediaInfo.EnableAudio();
5845 }
5846 if (hasVideoTracks) {
5847 mediaInfo.EnableVideo();
5848 }
5849 MetadataLoaded(&mediaInfo, nullptr);
5850 }
5851
5852 if (mMediaSource) {
5853 // readyState has changed, assuming it's following the pending mediasource
5854 // operations. Notify the Mediasource that the operations have completed.
5855 mMediaSource->CompletePendingTransactions();
5856 }
5857
5858 enum NextFrameStatus nextFrameStatus = NextFrameStatus();
5859 if (mWaitingForKey == NOT_WAITING_FOR_KEY) {
5860 if (nextFrameStatus == NEXT_FRAME_UNAVAILABLE && mDecoder &&
5861 !mDecoder->IsEnded()) {
5862 nextFrameStatus = mDecoder->NextFrameBufferedStatus();
5863 }
5864 } else if (mWaitingForKey == WAITING_FOR_KEY) {
5865 if (nextFrameStatus == NEXT_FRAME_UNAVAILABLE ||
5866 nextFrameStatus == NEXT_FRAME_UNAVAILABLE_BUFFERING) {
5867 // http://w3c.github.io/encrypted-media/#wait-for-key
5868 // Continuing 7.3.4 Queue a "waitingforkey" Event
5869 // 4. Queue a task to fire a simple event named waitingforkey
5870 // at the media element.
5871 // 5. Set the readyState of media element to HAVE_METADATA.
5872 // NOTE: We'll change to HAVE_CURRENT_DATA or HAVE_METADATA
5873 // depending on whether we've loaded the first frame or not
5874 // below.
5875 // 6. Suspend playback.
5876 // Note: Playback will already be stalled, as the next frame is
5877 // unavailable.
5878 mWaitingForKey = WAITING_FOR_KEY_DISPATCHED;
5879 DispatchAsyncEvent(NS_LITERAL_STRING("waitingforkey"));
5880 }
5881 } else {
5882 MOZ_ASSERT(mWaitingForKey == WAITING_FOR_KEY_DISPATCHED);
5883 if (nextFrameStatus == NEXT_FRAME_AVAILABLE) {
5884 // We have new frames after dispatching "waitingforkey".
5885 // This means we've got the key and can reset mWaitingForKey now.
5886 mWaitingForKey = NOT_WAITING_FOR_KEY;
5887 }
5888 }
5889
5890 if (nextFrameStatus == MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING) {
5891 LOG(LogLevel::Debug,
5892 ("MediaElement %p UpdateReadyStateInternal() "
5893 "NEXT_FRAME_UNAVAILABLE_SEEKING; Forcing HAVE_METADATA",
5894 this));
5895 ChangeReadyState(HAVE_METADATA);
5896 return;
5897 }
5898
5899 if (IsVideo() && VideoTracks() && !VideoTracks()->IsEmpty() &&
5900 !IsPlaybackEnded() && GetImageContainer() &&
5901 !GetImageContainer()->HasCurrentImage()) {
5902 // Don't advance if we are playing video, but don't have a video frame.
5903 // Also, if video became available after advancing to HAVE_CURRENT_DATA
5904 // while we are still playing, we need to revert to HAVE_METADATA until
5905 // a video frame is available.
5906 LOG(LogLevel::Debug,
5907 ("MediaElement %p UpdateReadyStateInternal() "
5908 "Playing video but no video frame; Forcing HAVE_METADATA",
5909 this));
5910 ChangeReadyState(HAVE_METADATA);
5911 return;
5912 }
5913
5914 if (!mFirstFrameLoaded) {
5915 // We haven't yet loaded the first frame, making us unable to determine
5916 // if we have enough valid data at the present stage.
5917 return;
5918 }
5919
5920 if (nextFrameStatus == NEXT_FRAME_UNAVAILABLE_BUFFERING) {
5921 // Force HAVE_CURRENT_DATA when buffering.
5922 ChangeReadyState(HAVE_CURRENT_DATA);
5923 return;
5924 }
5925
5926 // TextTracks must be loaded for the HAVE_ENOUGH_DATA and
5927 // HAVE_FUTURE_DATA.
5928 // So force HAVE_CURRENT_DATA if text tracks not loaded.
5929 if (mTextTrackManager && !mTextTrackManager->IsLoaded()) {
5930 ChangeReadyState(HAVE_CURRENT_DATA);
5931 return;
5932 }
5933
5934 if (mDownloadSuspendedByCache && mDecoder && !mDecoder->IsEnded()) {
5935 // The decoder has signaled that the download has been suspended by the
5936 // media cache. So move readyState into HAVE_ENOUGH_DATA, in case there's
5937 // script waiting for a "canplaythrough" event; without this forced
5938 // transition, we will never fire the "canplaythrough" event if the
5939 // media cache is too small, and scripts are bound to fail. Don't force
5940 // this transition if the decoder is in ended state; the readyState
5941 // should remain at HAVE_CURRENT_DATA in this case.
5942 // Note that this state transition includes the case where we finished
5943 // downloaded the whole data stream.
5944 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5945 "Decoder download suspended by cache",
5946 this));
5947 ChangeReadyState(HAVE_ENOUGH_DATA);
5948 return;
5949 }
5950
5951 if (nextFrameStatus != MediaDecoderOwner::NEXT_FRAME_AVAILABLE) {
5952 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5953 "Next frame not available",
5954 this));
5955 ChangeReadyState(HAVE_CURRENT_DATA);
5956 return;
5957 }
5958
5959 if (mSrcStream) {
5960 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5961 "Stream HAVE_ENOUGH_DATA",
5962 this));
5963 ChangeReadyState(HAVE_ENOUGH_DATA);
5964 return;
5965 }
5966
5967 // Now see if we should set HAVE_ENOUGH_DATA.
5968 // If it's something we don't know the size of, then we can't
5969 // make a real estimate, so we go straight to HAVE_ENOUGH_DATA once
5970 // we've downloaded enough data that our download rate is considered
5971 // reliable. We have to move to HAVE_ENOUGH_DATA at some point or
5972 // autoplay elements for live streams will never play. Otherwise we
5973 // move to HAVE_ENOUGH_DATA if we can play through the entire media
5974 // without stopping to buffer.
5975 if (mDecoder->CanPlayThrough()) {
5976 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5977 "Decoder can play through",
5978 this));
5979 ChangeReadyState(HAVE_ENOUGH_DATA);
5980 return;
5981 }
5982 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5983 "Default; Decoder has future data",
5984 this));
5985 ChangeReadyState(HAVE_FUTURE_DATA);
5986 }
5987
5988 static const char* const gReadyStateToString[] = {
5989 "HAVE_NOTHING", "HAVE_METADATA", "HAVE_CURRENT_DATA", "HAVE_FUTURE_DATA",
5990 "HAVE_ENOUGH_DATA"};
5991
ChangeReadyState(nsMediaReadyState aState)5992 void HTMLMediaElement::ChangeReadyState(nsMediaReadyState aState) {
5993 if (mReadyState == aState) {
5994 return;
5995 }
5996
5997 nsMediaReadyState oldState = mReadyState;
5998 mReadyState = aState;
5999 LOG(LogLevel::Debug,
6000 ("%p Ready state changed to %s", this, gReadyStateToString[aState]));
6001
6002 DDLOG(DDLogCategory::Property, "ready_state", gReadyStateToString[aState]);
6003
6004 // https://html.spec.whatwg.org/multipage/media.html#text-track-cue-active-flag
6005 // The user agent must synchronously unset cues' active flag whenever the
6006 // media element's readyState is changed back to HAVE_NOTHING.
6007 if (mReadyState == HAVE_NOTHING && mTextTrackManager) {
6008 mTextTrackManager->NotifyReset();
6009 }
6010
6011 if (mNetworkState == NETWORK_EMPTY) {
6012 return;
6013 }
6014
6015 UpdateAudioChannelPlayingState();
6016
6017 // Handle raising of "waiting" event during seek (see 4.8.10.9)
6018 // or
6019 // 4.8.12.7 Ready states:
6020 // "If the previous ready state was HAVE_FUTURE_DATA or more, and the new
6021 // ready state is HAVE_CURRENT_DATA or less
6022 // If the media element was potentially playing before its readyState
6023 // attribute changed to a value lower than HAVE_FUTURE_DATA, and the element
6024 // has not ended playback, and playback has not stopped due to errors,
6025 // paused for user interaction, or paused for in-band content, the user agent
6026 // must queue a task to fire a simple event named timeupdate at the element,
6027 // and queue a task to fire a simple event named waiting at the element."
6028 if (mPlayingBeforeSeek && mReadyState < HAVE_FUTURE_DATA) {
6029 DispatchAsyncEvent(NS_LITERAL_STRING("waiting"));
6030 } else if (oldState >= HAVE_FUTURE_DATA && mReadyState < HAVE_FUTURE_DATA &&
6031 !Paused() && !Ended() && !mErrorSink->mError) {
6032 FireTimeUpdate(false);
6033 DispatchAsyncEvent(NS_LITERAL_STRING("waiting"));
6034 }
6035
6036 if (oldState < HAVE_CURRENT_DATA && mReadyState >= HAVE_CURRENT_DATA &&
6037 !mLoadedDataFired) {
6038 DispatchAsyncEvent(NS_LITERAL_STRING("loadeddata"));
6039 mLoadedDataFired = true;
6040 }
6041
6042 if (oldState < HAVE_FUTURE_DATA && mReadyState >= HAVE_FUTURE_DATA) {
6043 DispatchAsyncEvent(NS_LITERAL_STRING("canplay"));
6044 if (!mPaused) {
6045 if (mDecoder && !mSuspendedByInactiveDocOrDocshell) {
6046 MOZ_ASSERT(AutoplayPolicy::IsAllowedToPlay(*this));
6047 mDecoder->Play();
6048 }
6049 NotifyAboutPlaying();
6050 }
6051 }
6052
6053 CheckAutoplayDataReady();
6054
6055 if (oldState < HAVE_ENOUGH_DATA && mReadyState >= HAVE_ENOUGH_DATA) {
6056 DispatchAsyncEvent(NS_LITERAL_STRING("canplaythrough"));
6057 }
6058 }
6059
6060 static const char* const gNetworkStateToString[] = {"EMPTY", "IDLE", "LOADING",
6061 "NO_SOURCE"};
6062
ChangeNetworkState(nsMediaNetworkState aState)6063 void HTMLMediaElement::ChangeNetworkState(nsMediaNetworkState aState) {
6064 if (mNetworkState == aState) {
6065 return;
6066 }
6067
6068 nsMediaNetworkState oldState = mNetworkState;
6069 mNetworkState = aState;
6070 LOG(LogLevel::Debug,
6071 ("%p Network state changed to %s", this, gNetworkStateToString[aState]));
6072 DDLOG(DDLogCategory::Property, "network_state",
6073 gNetworkStateToString[aState]);
6074
6075 if (oldState == NETWORK_LOADING) {
6076 // Stop progress notification when exiting NETWORK_LOADING.
6077 StopProgress();
6078 }
6079
6080 if (mNetworkState == NETWORK_LOADING) {
6081 // Start progress notification when entering NETWORK_LOADING.
6082 StartProgress();
6083 } else if (mNetworkState == NETWORK_IDLE && !mErrorSink->mError) {
6084 // Fire 'suspend' event when entering NETWORK_IDLE and no error presented.
6085 DispatchAsyncEvent(NS_LITERAL_STRING("suspend"));
6086 }
6087
6088 // According to the resource selection (step2, step9-18), dedicated media
6089 // source failure step (step4) and aborting existing load (step4), set show
6090 // poster flag to true. https://html.spec.whatwg.org/multipage/media.html
6091 if (mNetworkState == NETWORK_NO_SOURCE || mNetworkState == NETWORK_EMPTY) {
6092 mShowPoster = true;
6093 }
6094
6095 // Changing mNetworkState affects AddRemoveSelfReference().
6096 AddRemoveSelfReference();
6097 }
6098
CanActivateAutoplay()6099 bool HTMLMediaElement::CanActivateAutoplay() {
6100 // We also activate autoplay when playing a media source since the data
6101 // download is controlled by the script and there is no way to evaluate
6102 // MediaDecoder::CanPlayThrough().
6103
6104 if (!HasAttr(kNameSpaceID_None, nsGkAtoms::autoplay)) {
6105 return false;
6106 }
6107
6108 if (!mAutoplaying) {
6109 return false;
6110 }
6111
6112 if (IsEditable()) {
6113 return false;
6114 }
6115
6116 if (!mPaused) {
6117 return false;
6118 }
6119
6120 if (mSuspendedByInactiveDocOrDocshell) {
6121 return false;
6122 }
6123
6124 // Static document is used for print preview and printing, should not be
6125 // autoplay
6126 if (OwnerDoc()->IsStaticDocument()) {
6127 return false;
6128 }
6129
6130 if (ShouldBeSuspendedByInactiveDocShell()) {
6131 LOG(LogLevel::Debug, ("%p prohibiting autoplay by the docShell", this));
6132 return false;
6133 }
6134
6135 if (MediaPlaybackDelayPolicy::ShouldDelayPlayback(this)) {
6136 CreateResumeDelayedMediaPlaybackAgentIfNeeded();
6137 LOG(LogLevel::Debug, ("%p delay playing from autoplay", this));
6138 return false;
6139 }
6140
6141 return mReadyState >= HAVE_ENOUGH_DATA;
6142 }
6143
CheckAutoplayDataReady()6144 void HTMLMediaElement::CheckAutoplayDataReady() {
6145 if (!CanActivateAutoplay()) {
6146 return;
6147 }
6148
6149 UpdateHadAudibleAutoplayState();
6150 if (!AutoplayPolicy::IsAllowedToPlay(*this)) {
6151 DispatchEventsWhenPlayWasNotAllowed();
6152 return;
6153 }
6154
6155 mAllowedToPlayPromise.ResolveIfExists(true, __func__);
6156 mPaused = false;
6157 // We changed mPaused which can affect AddRemoveSelfReference
6158 AddRemoveSelfReference();
6159 UpdateSrcMediaStreamPlaying();
6160 UpdateAudioChannelPlayingState();
6161
6162 StartListeningMediaControlEventIfNeeded();
6163
6164 if (mDecoder) {
6165 SetPlayedOrSeeked(true);
6166 if (mCurrentPlayRangeStart == -1.0) {
6167 mCurrentPlayRangeStart = CurrentTime();
6168 }
6169 MOZ_ASSERT(!mSuspendedByInactiveDocOrDocshell);
6170 mDecoder->Play();
6171 } else if (mSrcStream) {
6172 SetPlayedOrSeeked(true);
6173 }
6174
6175 // https://html.spec.whatwg.org/multipage/media.html#ready-states:show-poster-flag
6176 if (mShowPoster) {
6177 mShowPoster = false;
6178 if (mTextTrackManager) {
6179 mTextTrackManager->TimeMarchesOn();
6180 }
6181 }
6182
6183 // For blocked media, the event would be pending until it is resumed.
6184 DispatchAsyncEvent(NS_LITERAL_STRING("play"));
6185
6186 DispatchAsyncEvent(NS_LITERAL_STRING("playing"));
6187 }
6188
IsActive() const6189 bool HTMLMediaElement::IsActive() const {
6190 Document* ownerDoc = OwnerDoc();
6191 return ownerDoc->IsActive() && ownerDoc->IsVisible();
6192 }
6193
IsHidden() const6194 bool HTMLMediaElement::IsHidden() const {
6195 return !IsInComposedDoc() || OwnerDoc()->Hidden();
6196 }
6197
GetVideoFrameContainer()6198 VideoFrameContainer* HTMLMediaElement::GetVideoFrameContainer() {
6199 if (mShuttingDown) {
6200 return nullptr;
6201 }
6202
6203 if (mVideoFrameContainer) return mVideoFrameContainer;
6204
6205 // Only video frames need an image container.
6206 if (!IsVideo()) {
6207 return nullptr;
6208 }
6209
6210 mVideoFrameContainer = new VideoFrameContainer(
6211 this, LayerManager::CreateImageContainer(ImageContainer::ASYNCHRONOUS));
6212
6213 return mVideoFrameContainer;
6214 }
6215
PrincipalChanged(MediaStreamTrack * aTrack)6216 void HTMLMediaElement::PrincipalChanged(MediaStreamTrack* aTrack) {
6217 if (aTrack != mSelectedVideoStreamTrack) {
6218 return;
6219 }
6220
6221 nsContentUtils::CombineResourcePrincipals(&mSrcStreamVideoPrincipal,
6222 aTrack->GetPrincipal());
6223
6224 LOG(LogLevel::Debug,
6225 ("HTMLMediaElement %p video track principal changed to %p (combined "
6226 "into %p). Waiting for it to reach VideoFrameContainer before setting.",
6227 this, aTrack->GetPrincipal(), mSrcStreamVideoPrincipal.get()));
6228
6229 if (mVideoFrameContainer) {
6230 UpdateSrcStreamVideoPrincipal(
6231 mVideoFrameContainer->GetLastPrincipalHandle());
6232 }
6233 }
6234
UpdateSrcStreamVideoPrincipal(const PrincipalHandle & aPrincipalHandle)6235 void HTMLMediaElement::UpdateSrcStreamVideoPrincipal(
6236 const PrincipalHandle& aPrincipalHandle) {
6237 nsTArray<RefPtr<VideoStreamTrack>> videoTracks;
6238 mSrcStream->GetVideoTracks(videoTracks);
6239
6240 PrincipalHandle handle(aPrincipalHandle);
6241 for (const RefPtr<VideoStreamTrack>& track : videoTracks) {
6242 if (PrincipalHandleMatches(handle, track->GetPrincipal()) &&
6243 !track->Ended()) {
6244 // When the PrincipalHandle for the VideoFrameContainer changes to that of
6245 // a live track in mSrcStream we know that a removed track was displayed
6246 // but is no longer so.
6247 LOG(LogLevel::Debug, ("HTMLMediaElement %p VideoFrameContainer's "
6248 "PrincipalHandle matches track %p. That's all we "
6249 "need.",
6250 this, track.get()));
6251 mSrcStreamVideoPrincipal = track->GetPrincipal();
6252 break;
6253 }
6254 }
6255 }
6256
PrincipalHandleChangedForVideoFrameContainer(VideoFrameContainer * aContainer,const PrincipalHandle & aNewPrincipalHandle)6257 void HTMLMediaElement::PrincipalHandleChangedForVideoFrameContainer(
6258 VideoFrameContainer* aContainer,
6259 const PrincipalHandle& aNewPrincipalHandle) {
6260 MOZ_ASSERT(NS_IsMainThread());
6261
6262 if (!mSrcStream) {
6263 return;
6264 }
6265
6266 LOG(LogLevel::Debug, ("HTMLMediaElement %p PrincipalHandle changed in "
6267 "VideoFrameContainer.",
6268 this));
6269
6270 UpdateSrcStreamVideoPrincipal(aNewPrincipalHandle);
6271 }
6272
DispatchEvent(const nsAString & aName)6273 nsresult HTMLMediaElement::DispatchEvent(const nsAString& aName) {
6274 LOG_EVENT(LogLevel::Debug, ("%p Dispatching event %s", this,
6275 NS_ConvertUTF16toUTF8(aName).get()));
6276
6277 // Save events that occur while in the bfcache. These will be dispatched
6278 // if the page comes out of the bfcache.
6279 if (mEventDeliveryPaused) {
6280 mPendingEvents.AppendElement(aName);
6281 return NS_OK;
6282 }
6283
6284 return nsContentUtils::DispatchTrustedEvent(
6285 OwnerDoc(), static_cast<nsIContent*>(this), aName, CanBubble::eNo,
6286 Cancelable::eNo);
6287 }
6288
DispatchAsyncEvent(const nsAString & aName)6289 void HTMLMediaElement::DispatchAsyncEvent(const nsAString& aName) {
6290 LOG_EVENT(LogLevel::Debug,
6291 ("%p Queuing event %s", this, NS_ConvertUTF16toUTF8(aName).get()));
6292 DDLOG(DDLogCategory::Event, "HTMLMediaElement",
6293 nsCString(NS_ConvertUTF16toUTF8(aName)));
6294
6295 // Save events that occur while in the bfcache. These will be dispatched
6296 // if the page comes out of the bfcache.
6297 if (mEventDeliveryPaused) {
6298 mPendingEvents.AppendElement(aName);
6299 return;
6300 }
6301
6302 nsCOMPtr<nsIRunnable> event;
6303
6304 if (aName.EqualsLiteral("playing")) {
6305 event = new nsNotifyAboutPlayingRunner(this, TakePendingPlayPromises());
6306 } else {
6307 event = new nsAsyncEventRunner(aName, this);
6308 }
6309
6310 mMainThreadEventTarget->Dispatch(event.forget());
6311
6312 if ((aName.EqualsLiteral("play") || aName.EqualsLiteral("playing"))) {
6313 mPlayTime.Start();
6314 mCurrentLoadPlayTime.Start();
6315 if (IsHidden()) {
6316 HiddenVideoStart();
6317 }
6318 } else if (aName.EqualsLiteral("waiting")) {
6319 mPlayTime.Pause();
6320 mCurrentLoadPlayTime.Pause();
6321 HiddenVideoStop();
6322 } else if (aName.EqualsLiteral("pause")) {
6323 mPlayTime.Pause();
6324 mCurrentLoadPlayTime.Pause();
6325 HiddenVideoStop();
6326 }
6327
6328 // It would happen when (1) media aborts current load (2) media pauses (3)
6329 // media end (4) media unbind from tree (because we would pause it)
6330 if (aName.EqualsLiteral("pause")) {
6331 ReportPlayedTimeAfterBlockedTelemetry();
6332 }
6333 }
6334
DispatchPendingMediaEvents()6335 nsresult HTMLMediaElement::DispatchPendingMediaEvents() {
6336 NS_ASSERTION(!mEventDeliveryPaused,
6337 "Must not be in bfcache when dispatching pending media events");
6338
6339 uint32_t count = mPendingEvents.Length();
6340 for (uint32_t i = 0; i < count; ++i) {
6341 DispatchAsyncEvent(mPendingEvents[i]);
6342 }
6343 mPendingEvents.Clear();
6344
6345 return NS_OK;
6346 }
6347
IsPotentiallyPlaying() const6348 bool HTMLMediaElement::IsPotentiallyPlaying() const {
6349 // TODO:
6350 // playback has not stopped due to errors,
6351 // and the element has not paused for user interaction
6352 return !mPaused &&
6353 (mReadyState == HAVE_ENOUGH_DATA || mReadyState == HAVE_FUTURE_DATA) &&
6354 !IsPlaybackEnded();
6355 }
6356
IsPlaybackEnded() const6357 bool HTMLMediaElement::IsPlaybackEnded() const {
6358 // TODO:
6359 // the current playback position is equal to the effective end of the media
6360 // resource. See bug 449157.
6361 if (mDecoder) {
6362 return mReadyState >= HAVE_METADATA && mDecoder->IsEnded();
6363 } else if (mSrcStream) {
6364 return mReadyState >= HAVE_METADATA && mSrcStreamPlaybackEnded;
6365 } else {
6366 return false;
6367 }
6368 }
6369
GetCurrentPrincipal()6370 already_AddRefed<nsIPrincipal> HTMLMediaElement::GetCurrentPrincipal() {
6371 if (mDecoder) {
6372 return mDecoder->GetCurrentPrincipal();
6373 }
6374 if (mSrcStream) {
6375 nsTArray<RefPtr<MediaStreamTrack>> tracks;
6376 mSrcStream->GetTracks(tracks);
6377 nsCOMPtr<nsIPrincipal> principal = mSrcStream->GetPrincipal();
6378 return principal.forget();
6379 }
6380 return nullptr;
6381 }
6382
HadCrossOriginRedirects()6383 bool HTMLMediaElement::HadCrossOriginRedirects() {
6384 if (mDecoder) {
6385 return mDecoder->HadCrossOriginRedirects();
6386 }
6387 return false;
6388 }
6389
GetCurrentVideoPrincipal()6390 already_AddRefed<nsIPrincipal> HTMLMediaElement::GetCurrentVideoPrincipal() {
6391 if (mDecoder) {
6392 return mDecoder->GetCurrentPrincipal();
6393 }
6394 if (mSrcStream) {
6395 nsCOMPtr<nsIPrincipal> principal = mSrcStreamVideoPrincipal;
6396 return principal.forget();
6397 }
6398 return nullptr;
6399 }
6400
NotifyDecoderPrincipalChanged()6401 void HTMLMediaElement::NotifyDecoderPrincipalChanged() {
6402 RefPtr<nsIPrincipal> principal = GetCurrentPrincipal();
6403 bool isSameOrigin = !principal || IsCORSSameOrigin();
6404 mDecoder->UpdateSameOriginStatus(isSameOrigin);
6405
6406 if (isSameOrigin) {
6407 principal = NodePrincipal();
6408 }
6409 for (const auto& entry : mOutputTrackSources) {
6410 entry.GetData()->SetPrincipal(principal);
6411 }
6412 mDecoder->SetOutputTracksPrincipal(principal);
6413 }
6414
Invalidate(bool aImageSizeChanged,Maybe<nsIntSize> & aNewIntrinsicSize,bool aForceInvalidate)6415 void HTMLMediaElement::Invalidate(bool aImageSizeChanged,
6416 Maybe<nsIntSize>& aNewIntrinsicSize,
6417 bool aForceInvalidate) {
6418 nsIFrame* frame = GetPrimaryFrame();
6419 if (aNewIntrinsicSize) {
6420 UpdateMediaSize(aNewIntrinsicSize.value());
6421 if (frame) {
6422 nsPresContext* presContext = frame->PresContext();
6423 PresShell* presShell = presContext->PresShell();
6424 presShell->FrameNeedsReflow(frame, IntrinsicDirty::StyleChange,
6425 NS_FRAME_IS_DIRTY);
6426 }
6427 }
6428
6429 RefPtr<ImageContainer> imageContainer = GetImageContainer();
6430 bool asyncInvalidate =
6431 imageContainer && imageContainer->IsAsync() && !aForceInvalidate;
6432 if (frame) {
6433 if (aImageSizeChanged) {
6434 frame->InvalidateFrame();
6435 } else {
6436 frame->InvalidateLayer(DisplayItemType::TYPE_VIDEO, nullptr, nullptr,
6437 asyncInvalidate ? nsIFrame::UPDATE_IS_ASYNC : 0);
6438 }
6439 }
6440
6441 SVGObserverUtils::InvalidateDirectRenderingObservers(this);
6442 }
6443
UpdateMediaSize(const nsIntSize & aSize)6444 void HTMLMediaElement::UpdateMediaSize(const nsIntSize& aSize) {
6445 MOZ_ASSERT(NS_IsMainThread());
6446
6447 if (IsVideo() && mReadyState != HAVE_NOTHING &&
6448 mMediaInfo.mVideo.mDisplay != aSize) {
6449 DispatchAsyncEvent(NS_LITERAL_STRING("resize"));
6450 }
6451
6452 mMediaInfo.mVideo.mDisplay = aSize;
6453 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
6454
6455 if (mFirstFrameListener) {
6456 if (mSelectedVideoStreamTrack) {
6457 // Unlink might have removed the selected video track.
6458 mSelectedVideoStreamTrack->RemoveVideoOutput(mFirstFrameListener);
6459 }
6460 // The first-frame listener won't be needed again for this stream.
6461 mFirstFrameListener = nullptr;
6462 }
6463 }
6464
SuspendOrResumeElement(bool aSuspendElement)6465 void HTMLMediaElement::SuspendOrResumeElement(bool aSuspendElement) {
6466 LOG(LogLevel::Debug, ("%p SuspendOrResumeElement(suspend=%d) hidden=%d", this,
6467 aSuspendElement, OwnerDoc()->Hidden()));
6468 if (aSuspendElement == mSuspendedByInactiveDocOrDocshell) {
6469 return;
6470 }
6471
6472 mSuspendedByInactiveDocOrDocshell = aSuspendElement;
6473 UpdateSrcMediaStreamPlaying();
6474 UpdateAudioChannelPlayingState();
6475
6476 if (aSuspendElement) {
6477 mCurrentLoadPlayTime.Pause();
6478 ReportTelemetry();
6479
6480 // For EME content, we may force destruction of the CDM client (and CDM
6481 // instance if this is the last client for that CDM instance) and
6482 // the CDM's decoder. This ensures the CDM gets reliable and prompt
6483 // shutdown notifications, as it may have book-keeping it needs
6484 // to do on shutdown.
6485 if (mMediaKeys) {
6486 nsAutoString keySystem;
6487 mMediaKeys->GetKeySystem(keySystem);
6488 }
6489 if (mDecoder) {
6490 mDecoder->Pause();
6491 mDecoder->Suspend();
6492 mDecoder->SetDelaySeekMode(true);
6493 }
6494 mEventDeliveryPaused = true;
6495 // We won't want to resume media element from the bfcache.
6496 ClearResumeDelayedMediaPlaybackAgentIfNeeded();
6497 StopListeningMediaControlEventIfNeeded();
6498 } else {
6499 if (!mPaused) {
6500 mCurrentLoadPlayTime.Start();
6501 }
6502 if (mDecoder) {
6503 mDecoder->Resume();
6504 if (!mPaused && !mDecoder->IsEnded()) {
6505 mDecoder->Play();
6506 }
6507 mDecoder->SetDelaySeekMode(false);
6508 }
6509 if (mEventDeliveryPaused) {
6510 mEventDeliveryPaused = false;
6511 DispatchPendingMediaEvents();
6512 }
6513 // If the media element has been blocked and isn't still allowed to play
6514 // when it comes back from the bfcache, we would notify front end to show
6515 // the blocking icon in order to inform user that the site is still being
6516 // blocked.
6517 if (mHasEverBeenBlockedForAutoplay &&
6518 !AutoplayPolicy::IsAllowedToPlay(*this)) {
6519 MaybeNotifyAutoplayBlocked();
6520 }
6521 // If we stopped listening to the event when we suspended media element,
6522 // then we should restart to listen to the event if we haven't done so yet.
6523 if (mMediaControlEventListener &&
6524 !mMediaControlEventListener->IsStarted()) {
6525 StartListeningMediaControlEventIfNeeded();
6526 }
6527 }
6528 if (StaticPrefs::media_testing_only_events()) {
6529 auto dispatcher = MakeRefPtr<AsyncEventDispatcher>(
6530 this, NS_LITERAL_STRING("MozMediaSuspendChanged"), CanBubble::eYes,
6531 ChromeOnlyDispatch::eYes);
6532 dispatcher->PostDOMEvent();
6533 }
6534 }
6535
IsBeingDestroyed()6536 bool HTMLMediaElement::IsBeingDestroyed() {
6537 nsIDocShell* docShell = OwnerDoc()->GetDocShell();
6538 bool isBeingDestroyed = false;
6539 if (docShell) {
6540 docShell->IsBeingDestroyed(&isBeingDestroyed);
6541 }
6542 return isBeingDestroyed;
6543 }
6544
ShouldBeSuspendedByInactiveDocShell() const6545 bool HTMLMediaElement::ShouldBeSuspendedByInactiveDocShell() const {
6546 nsIDocShell* docShell = OwnerDoc()->GetDocShell();
6547 if (!docShell) {
6548 return false;
6549 }
6550 bool isDocShellActive = false;
6551 docShell->GetIsActive(&isDocShellActive);
6552 return !isDocShellActive && docShell->GetSuspendMediaWhenInactive();
6553 }
6554
NotifyOwnerDocumentActivityChanged()6555 void HTMLMediaElement::NotifyOwnerDocumentActivityChanged() {
6556 bool visible = !IsHidden();
6557 if (visible) {
6558 // Visible -> Just pause hidden play time (no-op if already paused).
6559 HiddenVideoStop();
6560 } else if (mPlayTime.IsStarted()) {
6561 // Not visible, play time is running -> Start hidden play time if needed.
6562 HiddenVideoStart();
6563 }
6564
6565 if (mDecoder && !IsBeingDestroyed()) {
6566 NotifyDecoderActivityChanges();
6567 }
6568
6569 // We would suspend media when the document is inactive, or its docshell has
6570 // been set to hidden and explicitly wants to suspend media. In those cases,
6571 // the media would be not visible and we don't want them to continue playing.
6572 bool shouldSuspend = !IsActive() || ShouldBeSuspendedByInactiveDocShell();
6573 SuspendOrResumeElement(shouldSuspend);
6574
6575 // If the owning document has become inactive we should shutdown the CDM.
6576 if (!OwnerDoc()->IsCurrentActiveDocument() && mMediaKeys) {
6577 // We don't shutdown MediaKeys here because it also listens for document
6578 // activity and will take care of shutting down itself.
6579 DDUNLINKCHILD(mMediaKeys.get());
6580 mMediaKeys = nullptr;
6581 if (mDecoder) {
6582 ShutdownDecoder();
6583 }
6584 }
6585
6586 AddRemoveSelfReference();
6587 }
6588
AddRemoveSelfReference()6589 void HTMLMediaElement::AddRemoveSelfReference() {
6590 // XXX we could release earlier here in many situations if we examined
6591 // which event listeners are attached. Right now we assume there is a
6592 // potential listener for every event. We would also have to keep the
6593 // element alive if it was playing and producing audio output --- right now
6594 // that's covered by the !mPaused check.
6595 Document* ownerDoc = OwnerDoc();
6596
6597 // See the comment at the top of this file for the explanation of this
6598 // boolean expression.
6599 bool needSelfReference =
6600 !mShuttingDown && ownerDoc->IsActive() &&
6601 (mDelayingLoadEvent || (!mPaused && !Ended()) ||
6602 (mDecoder && mDecoder->IsSeeking()) || CanActivateAutoplay() ||
6603 (mMediaSource ? mProgressTimer : mNetworkState == NETWORK_LOADING));
6604
6605 if (needSelfReference != mHasSelfReference) {
6606 mHasSelfReference = needSelfReference;
6607 RefPtr<HTMLMediaElement> self = this;
6608 if (needSelfReference) {
6609 // The shutdown observer will hold a strong reference to us. This
6610 // will do to keep us alive. We need to know about shutdown so that
6611 // we can release our self-reference.
6612 mMainThreadEventTarget->Dispatch(NS_NewRunnableFunction(
6613 "dom::HTMLMediaElement::AddSelfReference",
6614 [self]() { self->mShutdownObserver->AddRefMediaElement(); }));
6615 } else {
6616 // Dispatch Release asynchronously so that we don't destroy this object
6617 // inside a call stack of method calls on this object
6618 mMainThreadEventTarget->Dispatch(NS_NewRunnableFunction(
6619 "dom::HTMLMediaElement::AddSelfReference",
6620 [self]() { self->mShutdownObserver->ReleaseMediaElement(); }));
6621 }
6622 }
6623 }
6624
NotifyShutdownEvent()6625 void HTMLMediaElement::NotifyShutdownEvent() {
6626 mShuttingDown = true;
6627 ResetState();
6628 AddRemoveSelfReference();
6629 }
6630
DispatchAsyncSourceError(nsIContent * aSourceElement)6631 void HTMLMediaElement::DispatchAsyncSourceError(nsIContent* aSourceElement) {
6632 LOG_EVENT(LogLevel::Debug, ("%p Queuing simple source error event", this));
6633
6634 nsCOMPtr<nsIRunnable> event =
6635 new nsSourceErrorEventRunner(this, aSourceElement);
6636 mMainThreadEventTarget->Dispatch(event.forget());
6637 }
6638
NotifyAddedSource()6639 void HTMLMediaElement::NotifyAddedSource() {
6640 // If a source element is inserted as a child of a media element
6641 // that has no src attribute and whose networkState has the value
6642 // NETWORK_EMPTY, the user agent must invoke the media element's
6643 // resource selection algorithm.
6644 if (!HasAttr(kNameSpaceID_None, nsGkAtoms::src) &&
6645 mNetworkState == NETWORK_EMPTY) {
6646 AssertReadyStateIsNothing();
6647 QueueSelectResourceTask();
6648 }
6649
6650 // A load was paused in the resource selection algorithm, waiting for
6651 // a new source child to be added, resume the resource selection algorithm.
6652 if (mLoadWaitStatus == WAITING_FOR_SOURCE) {
6653 // Rest the flag so we don't queue multiple LoadFromSourceTask() when
6654 // multiple <source> are attached in an event loop.
6655 mLoadWaitStatus = NOT_WAITING;
6656 QueueLoadFromSourceTask();
6657 }
6658 }
6659
GetNextSource()6660 Element* HTMLMediaElement::GetNextSource() {
6661 mSourceLoadCandidate = nullptr;
6662
6663 while (true) {
6664 if (mSourcePointer == nsINode::GetLastChild()) {
6665 return nullptr; // no more children
6666 }
6667
6668 if (!mSourcePointer) {
6669 mSourcePointer = nsINode::GetFirstChild();
6670 } else {
6671 mSourcePointer = mSourcePointer->GetNextSibling();
6672 }
6673 nsIContent* child = mSourcePointer;
6674
6675 // If child is a <source> element, it is the next candidate.
6676 if (child && child->IsHTMLElement(nsGkAtoms::source)) {
6677 mSourceLoadCandidate = child;
6678 return child->AsElement();
6679 }
6680 }
6681 MOZ_ASSERT_UNREACHABLE("Execution should not reach here!");
6682 return nullptr;
6683 }
6684
ChangeDelayLoadStatus(bool aDelay)6685 void HTMLMediaElement::ChangeDelayLoadStatus(bool aDelay) {
6686 if (mDelayingLoadEvent == aDelay) return;
6687
6688 mDelayingLoadEvent = aDelay;
6689
6690 LOG(LogLevel::Debug, ("%p ChangeDelayLoadStatus(%d) doc=0x%p", this, aDelay,
6691 mLoadBlockedDoc.get()));
6692 if (mDecoder) {
6693 mDecoder->SetLoadInBackground(!aDelay);
6694 }
6695 if (aDelay) {
6696 mLoadBlockedDoc = OwnerDoc();
6697 mLoadBlockedDoc->BlockOnload();
6698 } else {
6699 // mLoadBlockedDoc might be null due to GC unlinking
6700 if (mLoadBlockedDoc) {
6701 mLoadBlockedDoc->UnblockOnload(false);
6702 mLoadBlockedDoc = nullptr;
6703 }
6704 }
6705
6706 // We changed mDelayingLoadEvent which can affect AddRemoveSelfReference
6707 AddRemoveSelfReference();
6708 }
6709
GetDocumentLoadGroup()6710 already_AddRefed<nsILoadGroup> HTMLMediaElement::GetDocumentLoadGroup() {
6711 if (!OwnerDoc()->IsActive()) {
6712 NS_WARNING("Load group requested for media element in inactive document.");
6713 }
6714 return OwnerDoc()->GetDocumentLoadGroup();
6715 }
6716
CopyInnerTo(Element * aDest)6717 nsresult HTMLMediaElement::CopyInnerTo(Element* aDest) {
6718 nsresult rv = nsGenericHTMLElement::CopyInnerTo(aDest);
6719 NS_ENSURE_SUCCESS(rv, rv);
6720 if (aDest->OwnerDoc()->IsStaticDocument()) {
6721 HTMLMediaElement* dest = static_cast<HTMLMediaElement*>(aDest);
6722 dest->SetMediaInfo(mMediaInfo);
6723 }
6724 return rv;
6725 }
6726
Buffered() const6727 already_AddRefed<TimeRanges> HTMLMediaElement::Buffered() const {
6728 media::TimeIntervals buffered =
6729 mDecoder ? mDecoder->GetBuffered() : media::TimeIntervals();
6730 RefPtr<TimeRanges> ranges = new TimeRanges(ToSupports(OwnerDoc()), buffered);
6731 return ranges.forget();
6732 }
6733
SetRequestHeaders(nsIHttpChannel * aChannel)6734 void HTMLMediaElement::SetRequestHeaders(nsIHttpChannel* aChannel) {
6735 // Send Accept header for video and audio types only (Bug 489071)
6736 SetAcceptHeader(aChannel);
6737
6738 // Apache doesn't send Content-Length when gzip transfer encoding is used,
6739 // which prevents us from estimating the video length (if explicit
6740 // Content-Duration and a length spec in the container are not present either)
6741 // and from seeking. So, disable the standard "Accept-Encoding: gzip,deflate"
6742 // that we usually send. See bug 614760.
6743 DebugOnly<nsresult> rv = aChannel->SetRequestHeader(
6744 NS_LITERAL_CSTRING("Accept-Encoding"), EmptyCString(), false);
6745 MOZ_ASSERT(NS_SUCCEEDED(rv));
6746
6747 // Set the Referrer header
6748 //
6749 // FIXME: Shouldn't this use the Element constructor? Though I guess it
6750 // doesn't matter as no HTMLMediaElement supports the referrerinfo attribute.
6751 auto referrerInfo = MakeRefPtr<ReferrerInfo>(*OwnerDoc());
6752 rv = aChannel->SetReferrerInfoWithoutClone(referrerInfo);
6753 MOZ_ASSERT(NS_SUCCEEDED(rv));
6754 }
6755
FireTimeUpdate(bool aPeriodic)6756 void HTMLMediaElement::FireTimeUpdate(bool aPeriodic) {
6757 NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
6758
6759 TimeStamp now = TimeStamp::Now();
6760 double time = CurrentTime();
6761
6762 // Fire a timeupdate event if this is not a periodic update (i.e. it's a
6763 // timeupdate event mandated by the spec), or if it's a periodic update
6764 // and TIMEUPDATE_MS has passed since the last timeupdate event fired and
6765 // the time has changed.
6766 if (!aPeriodic || (mLastCurrentTime != time &&
6767 (mTimeUpdateTime.IsNull() ||
6768 now - mTimeUpdateTime >=
6769 TimeDuration::FromMilliseconds(TIMEUPDATE_MS)))) {
6770 DispatchAsyncEvent(NS_LITERAL_STRING("timeupdate"));
6771 mTimeUpdateTime = now;
6772 mLastCurrentTime = time;
6773 }
6774 if (mFragmentEnd >= 0.0 && time >= mFragmentEnd) {
6775 Pause();
6776 mFragmentEnd = -1.0;
6777 mFragmentStart = -1.0;
6778 mDecoder->SetFragmentEndTime(mFragmentEnd);
6779 }
6780
6781 // Update the cues displaying on the video.
6782 // Here mTextTrackManager can be null if the cycle collector has unlinked
6783 // us before our parent. In that case UnbindFromTree will call us
6784 // when our parent is unlinked.
6785 if (mTextTrackManager) {
6786 mTextTrackManager->TimeMarchesOn();
6787 }
6788 }
6789
GetError() const6790 MediaError* HTMLMediaElement::GetError() const { return mErrorSink->mError; }
6791
GetCurrentSpec(nsCString & aString)6792 void HTMLMediaElement::GetCurrentSpec(nsCString& aString) {
6793 if (mLoadingSrc) {
6794 mLoadingSrc->GetSpec(aString);
6795 } else {
6796 aString.Truncate();
6797 }
6798 }
6799
MozFragmentEnd()6800 double HTMLMediaElement::MozFragmentEnd() {
6801 double duration = Duration();
6802
6803 // If there is no end fragment, or the fragment end is greater than the
6804 // duration, return the duration.
6805 return (mFragmentEnd < 0.0 || mFragmentEnd > duration) ? duration
6806 : mFragmentEnd;
6807 }
6808
SetDefaultPlaybackRate(double aDefaultPlaybackRate,ErrorResult & aRv)6809 void HTMLMediaElement::SetDefaultPlaybackRate(double aDefaultPlaybackRate,
6810 ErrorResult& aRv) {
6811 if (mSrcAttrStream) {
6812 return;
6813 }
6814
6815 if (aDefaultPlaybackRate < 0) {
6816 aRv.Throw(NS_ERROR_NOT_IMPLEMENTED);
6817 return;
6818 }
6819
6820 double defaultPlaybackRate = ClampPlaybackRate(aDefaultPlaybackRate);
6821
6822 if (mDefaultPlaybackRate == defaultPlaybackRate) {
6823 return;
6824 }
6825
6826 mDefaultPlaybackRate = defaultPlaybackRate;
6827 DispatchAsyncEvent(NS_LITERAL_STRING("ratechange"));
6828 }
6829
SetPlaybackRate(double aPlaybackRate,ErrorResult & aRv)6830 void HTMLMediaElement::SetPlaybackRate(double aPlaybackRate, ErrorResult& aRv) {
6831 if (mSrcAttrStream) {
6832 return;
6833 }
6834
6835 // Changing the playback rate of a media that has more than two channels is
6836 // not supported.
6837 if (aPlaybackRate < 0) {
6838 aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
6839 return;
6840 }
6841
6842 if (mPlaybackRate == aPlaybackRate) {
6843 return;
6844 }
6845
6846 mPlaybackRate = aPlaybackRate;
6847
6848 if (mPlaybackRate != 0.0 &&
6849 (mPlaybackRate > THRESHOLD_HIGH_PLAYBACKRATE_AUDIO ||
6850 mPlaybackRate < THRESHOLD_LOW_PLAYBACKRATE_AUDIO)) {
6851 SetMutedInternal(mMuted | MUTED_BY_INVALID_PLAYBACK_RATE);
6852 } else {
6853 SetMutedInternal(mMuted & ~MUTED_BY_INVALID_PLAYBACK_RATE);
6854 }
6855
6856 if (mDecoder) {
6857 mDecoder->SetPlaybackRate(ClampPlaybackRate(mPlaybackRate));
6858 }
6859 DispatchAsyncEvent(NS_LITERAL_STRING("ratechange"));
6860 }
6861
SetMozPreservesPitch(bool aPreservesPitch)6862 void HTMLMediaElement::SetMozPreservesPitch(bool aPreservesPitch) {
6863 mPreservesPitch = aPreservesPitch;
6864 if (mDecoder) {
6865 mDecoder->SetPreservesPitch(mPreservesPitch);
6866 }
6867 }
6868
GetImageContainer()6869 ImageContainer* HTMLMediaElement::GetImageContainer() {
6870 VideoFrameContainer* container = GetVideoFrameContainer();
6871 return container ? container->GetImageContainer() : nullptr;
6872 }
6873
UpdateAudioChannelPlayingState()6874 void HTMLMediaElement::UpdateAudioChannelPlayingState() {
6875 if (mAudioChannelWrapper) {
6876 mAudioChannelWrapper->UpdateAudioChannelPlayingState();
6877 }
6878 }
6879
VisibilityString(Visibility aVisibility)6880 static const char* VisibilityString(Visibility aVisibility) {
6881 switch (aVisibility) {
6882 case Visibility::Untracked: {
6883 return "Untracked";
6884 }
6885 case Visibility::ApproximatelyNonVisible: {
6886 return "ApproximatelyNonVisible";
6887 }
6888 case Visibility::ApproximatelyVisible: {
6889 return "ApproximatelyVisible";
6890 }
6891 }
6892
6893 return "NAN";
6894 }
6895
OnVisibilityChange(Visibility aNewVisibility)6896 void HTMLMediaElement::OnVisibilityChange(Visibility aNewVisibility) {
6897 LOG(LogLevel::Debug,
6898 ("OnVisibilityChange(): %s\n", VisibilityString(aNewVisibility)));
6899
6900 mVisibilityState = aNewVisibility;
6901 if (StaticPrefs::media_test_video_suspend()) {
6902 DispatchAsyncEvent(NS_LITERAL_STRING("visibilitychanged"));
6903 }
6904
6905 if (!mDecoder) {
6906 return;
6907 }
6908
6909 switch (aNewVisibility) {
6910 case Visibility::Untracked: {
6911 MOZ_ASSERT_UNREACHABLE("Shouldn't notify for untracked visibility");
6912 return;
6913 }
6914 case Visibility::ApproximatelyNonVisible: {
6915 if (mPlayTime.IsStarted()) {
6916 // Not visible, play time is running -> Start hidden play time if
6917 // needed.
6918 HiddenVideoStart();
6919 }
6920 break;
6921 }
6922 case Visibility::ApproximatelyVisible: {
6923 // Visible -> Just pause hidden play time (no-op if already paused).
6924 HiddenVideoStop();
6925 break;
6926 }
6927 }
6928
6929 NotifyDecoderActivityChanges();
6930 }
6931
GetMediaKeys() const6932 MediaKeys* HTMLMediaElement::GetMediaKeys() const { return mMediaKeys; }
6933
ContainsRestrictedContent()6934 bool HTMLMediaElement::ContainsRestrictedContent() {
6935 return GetMediaKeys() != nullptr;
6936 }
6937
SetCDMProxyFailure(const MediaResult & aResult)6938 void HTMLMediaElement::SetCDMProxyFailure(const MediaResult& aResult) {
6939 LOG(LogLevel::Debug, ("%s", __func__));
6940 MOZ_ASSERT(mSetMediaKeysDOMPromise);
6941
6942 ResetSetMediaKeysTempVariables();
6943
6944 mSetMediaKeysDOMPromise->MaybeReject(aResult.Code(), aResult.Message());
6945 }
6946
RemoveMediaKeys()6947 void HTMLMediaElement::RemoveMediaKeys() {
6948 LOG(LogLevel::Debug, ("%s", __func__));
6949 // 5.2.3 Stop using the CDM instance represented by the mediaKeys attribute
6950 // to decrypt media data and remove the association with the media element.
6951 if (mMediaKeys) {
6952 mMediaKeys->Unbind();
6953 }
6954 mMediaKeys = nullptr;
6955 }
6956
TryRemoveMediaKeysAssociation()6957 bool HTMLMediaElement::TryRemoveMediaKeysAssociation() {
6958 MOZ_ASSERT(mMediaKeys);
6959 LOG(LogLevel::Debug, ("%s", __func__));
6960 // 5.2.1 If the user agent or CDM do not support removing the association,
6961 // let this object's attaching media keys value be false and reject promise
6962 // with a new DOMException whose name is NotSupportedError.
6963 // 5.2.2 If the association cannot currently be removed, let this object's
6964 // attaching media keys value be false and reject promise with a new
6965 // DOMException whose name is InvalidStateError.
6966 if (mDecoder) {
6967 RefPtr<HTMLMediaElement> self = this;
6968 mDecoder->SetCDMProxy(nullptr)
6969 ->Then(
6970 mAbstractMainThread, __func__,
6971 [self]() {
6972 self->mSetCDMRequest.Complete();
6973
6974 self->RemoveMediaKeys();
6975 if (self->AttachNewMediaKeys()) {
6976 // No incoming MediaKeys object or MediaDecoder is not
6977 // created yet.
6978 self->MakeAssociationWithCDMResolved();
6979 }
6980 },
6981 [self](const MediaResult& aResult) {
6982 self->mSetCDMRequest.Complete();
6983 // 5.2.4 If the preceding step failed, let this object's
6984 // attaching media keys value be false and reject promise with
6985 // a new DOMException whose name is the appropriate error name.
6986 self->SetCDMProxyFailure(aResult);
6987 })
6988 ->Track(mSetCDMRequest);
6989 return false;
6990 }
6991
6992 RemoveMediaKeys();
6993 return true;
6994 }
6995
DetachExistingMediaKeys()6996 bool HTMLMediaElement::DetachExistingMediaKeys() {
6997 LOG(LogLevel::Debug, ("%s", __func__));
6998 MOZ_ASSERT(mSetMediaKeysDOMPromise);
6999 // 5.1 If mediaKeys is not null, CDM instance represented by mediaKeys is
7000 // already in use by another media element, and the user agent is unable
7001 // to use it with this element, let this object's attaching media keys
7002 // value be false and reject promise with a new DOMException whose name
7003 // is QuotaExceededError.
7004 if (mIncomingMediaKeys && mIncomingMediaKeys->IsBoundToMediaElement()) {
7005 SetCDMProxyFailure(MediaResult(
7006 NS_ERROR_DOM_QUOTA_EXCEEDED_ERR,
7007 "MediaKeys object is already bound to another HTMLMediaElement"));
7008 return false;
7009 }
7010
7011 // 5.2 If the mediaKeys attribute is not null, run the following steps:
7012 if (mMediaKeys) {
7013 return TryRemoveMediaKeysAssociation();
7014 }
7015 return true;
7016 }
7017
MakeAssociationWithCDMResolved()7018 void HTMLMediaElement::MakeAssociationWithCDMResolved() {
7019 LOG(LogLevel::Debug, ("%s", __func__));
7020 MOZ_ASSERT(mSetMediaKeysDOMPromise);
7021
7022 // 5.4 Set the mediaKeys attribute to mediaKeys.
7023 mMediaKeys = mIncomingMediaKeys;
7024 // 5.5 Let this object's attaching media keys value be false.
7025 ResetSetMediaKeysTempVariables();
7026 // 5.6 Resolve promise.
7027 mSetMediaKeysDOMPromise->MaybeResolveWithUndefined();
7028 mSetMediaKeysDOMPromise = nullptr;
7029 }
7030
TryMakeAssociationWithCDM(CDMProxy * aProxy)7031 bool HTMLMediaElement::TryMakeAssociationWithCDM(CDMProxy* aProxy) {
7032 LOG(LogLevel::Debug, ("%s", __func__));
7033 MOZ_ASSERT(aProxy);
7034
7035 // 5.3.3 Queue a task to run the "Attempt to Resume Playback If Necessary"
7036 // algorithm on the media element.
7037 // Note: Setting the CDMProxy on the MediaDecoder will unblock playback.
7038 if (mDecoder) {
7039 // CDMProxy is set asynchronously in MediaFormatReader, once it's done,
7040 // HTMLMediaElement should resolve or reject the DOM promise.
7041 RefPtr<HTMLMediaElement> self = this;
7042 mDecoder->SetCDMProxy(aProxy)
7043 ->Then(
7044 mAbstractMainThread, __func__,
7045 [self]() {
7046 self->mSetCDMRequest.Complete();
7047 self->MakeAssociationWithCDMResolved();
7048 },
7049 [self](const MediaResult& aResult) {
7050 self->mSetCDMRequest.Complete();
7051 self->SetCDMProxyFailure(aResult);
7052 })
7053 ->Track(mSetCDMRequest);
7054 return false;
7055 }
7056 return true;
7057 }
7058
AttachNewMediaKeys()7059 bool HTMLMediaElement::AttachNewMediaKeys() {
7060 LOG(LogLevel::Debug,
7061 ("%s incoming MediaKeys(%p)", __func__, mIncomingMediaKeys.get()));
7062 MOZ_ASSERT(mSetMediaKeysDOMPromise);
7063
7064 // 5.3. If mediaKeys is not null, run the following steps:
7065 if (mIncomingMediaKeys) {
7066 auto cdmProxy = mIncomingMediaKeys->GetCDMProxy();
7067 if (!cdmProxy) {
7068 SetCDMProxyFailure(MediaResult(
7069 NS_ERROR_DOM_INVALID_STATE_ERR,
7070 "CDM crashed before binding MediaKeys object to HTMLMediaElement"));
7071 return false;
7072 }
7073
7074 // 5.3.1 Associate the CDM instance represented by mediaKeys with the
7075 // media element for decrypting media data.
7076 if (NS_FAILED(mIncomingMediaKeys->Bind(this))) {
7077 // 5.3.2 If the preceding step failed, run the following steps:
7078
7079 // 5.3.2.1 Set the mediaKeys attribute to null.
7080 mMediaKeys = nullptr;
7081 // 5.3.2.2 Let this object's attaching media keys value be false.
7082 // 5.3.2.3 Reject promise with a new DOMException whose name is
7083 // the appropriate error name.
7084 SetCDMProxyFailure(
7085 MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,
7086 "Failed to bind MediaKeys object to HTMLMediaElement"));
7087 return false;
7088 }
7089 return TryMakeAssociationWithCDM(cdmProxy);
7090 }
7091 return true;
7092 }
7093
ResetSetMediaKeysTempVariables()7094 void HTMLMediaElement::ResetSetMediaKeysTempVariables() {
7095 mAttachingMediaKey = false;
7096 mIncomingMediaKeys = nullptr;
7097 }
7098
SetMediaKeys(mozilla::dom::MediaKeys * aMediaKeys,ErrorResult & aRv)7099 already_AddRefed<Promise> HTMLMediaElement::SetMediaKeys(
7100 mozilla::dom::MediaKeys* aMediaKeys, ErrorResult& aRv) {
7101 LOG(LogLevel::Debug, ("%p SetMediaKeys(%p) mMediaKeys=%p mDecoder=%p", this,
7102 aMediaKeys, mMediaKeys.get(), mDecoder.get()));
7103
7104 if (MozAudioCaptured()) {
7105 aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
7106 return nullptr;
7107 }
7108
7109 nsPIDOMWindowInner* win = OwnerDoc()->GetInnerWindow();
7110 if (!win) {
7111 aRv.Throw(NS_ERROR_UNEXPECTED);
7112 return nullptr;
7113 }
7114 RefPtr<DetailedPromise> promise = DetailedPromise::Create(
7115 win->AsGlobal(), aRv,
7116 NS_LITERAL_CSTRING("HTMLMediaElement.setMediaKeys"));
7117 if (aRv.Failed()) {
7118 return nullptr;
7119 }
7120
7121 // 1. If mediaKeys and the mediaKeys attribute are the same object,
7122 // return a resolved promise.
7123 if (mMediaKeys == aMediaKeys) {
7124 promise->MaybeResolveWithUndefined();
7125 return promise.forget();
7126 }
7127
7128 // 2. If this object's attaching media keys value is true, return a
7129 // promise rejected with a new DOMException whose name is InvalidStateError.
7130 if (mAttachingMediaKey) {
7131 promise->MaybeRejectWithInvalidStateError(
7132 "A MediaKeys object is in attaching operation.");
7133 return promise.forget();
7134 }
7135
7136 // 3. Let this object's attaching media keys value be true.
7137 mAttachingMediaKey = true;
7138 mIncomingMediaKeys = aMediaKeys;
7139
7140 // 4. Let promise be a new promise.
7141 mSetMediaKeysDOMPromise = promise;
7142
7143 // 5. Run the following steps in parallel:
7144
7145 // 5.1 & 5.2 & 5.3
7146 if (!DetachExistingMediaKeys() || !AttachNewMediaKeys()) {
7147 return promise.forget();
7148 }
7149
7150 // 5.4, 5.5, 5.6
7151 MakeAssociationWithCDMResolved();
7152
7153 // 6. Return promise.
7154 return promise.forget();
7155 }
7156
GetOnencrypted()7157 EventHandlerNonNull* HTMLMediaElement::GetOnencrypted() {
7158 return EventTarget::GetEventHandler(nsGkAtoms::onencrypted);
7159 }
7160
SetOnencrypted(EventHandlerNonNull * aCallback)7161 void HTMLMediaElement::SetOnencrypted(EventHandlerNonNull* aCallback) {
7162 EventTarget::SetEventHandler(nsGkAtoms::onencrypted, aCallback);
7163 }
7164
GetOnwaitingforkey()7165 EventHandlerNonNull* HTMLMediaElement::GetOnwaitingforkey() {
7166 return EventTarget::GetEventHandler(nsGkAtoms::onwaitingforkey);
7167 }
7168
SetOnwaitingforkey(EventHandlerNonNull * aCallback)7169 void HTMLMediaElement::SetOnwaitingforkey(EventHandlerNonNull* aCallback) {
7170 EventTarget::SetEventHandler(nsGkAtoms::onwaitingforkey, aCallback);
7171 }
7172
DispatchEncrypted(const nsTArray<uint8_t> & aInitData,const nsAString & aInitDataType)7173 void HTMLMediaElement::DispatchEncrypted(const nsTArray<uint8_t>& aInitData,
7174 const nsAString& aInitDataType) {
7175 LOG(LogLevel::Debug, ("%p DispatchEncrypted initDataType='%s'", this,
7176 NS_ConvertUTF16toUTF8(aInitDataType).get()));
7177
7178 if (mReadyState == HAVE_NOTHING) {
7179 // Ready state not HAVE_METADATA (yet), don't dispatch encrypted now.
7180 // Queueing for later dispatch in MetadataLoaded.
7181 mPendingEncryptedInitData.AddInitData(aInitDataType, aInitData);
7182 return;
7183 }
7184
7185 RefPtr<MediaEncryptedEvent> event;
7186 if (IsCORSSameOrigin()) {
7187 event = MediaEncryptedEvent::Constructor(this, aInitDataType, aInitData);
7188 } else {
7189 event = MediaEncryptedEvent::Constructor(this);
7190 }
7191
7192 RefPtr<AsyncEventDispatcher> asyncDispatcher =
7193 new AsyncEventDispatcher(this, event);
7194 asyncDispatcher->PostDOMEvent();
7195 }
7196
IsEventAttributeNameInternal(nsAtom * aName)7197 bool HTMLMediaElement::IsEventAttributeNameInternal(nsAtom* aName) {
7198 return aName == nsGkAtoms::onencrypted ||
7199 nsGenericHTMLElement::IsEventAttributeNameInternal(aName);
7200 }
7201
NotifyWaitingForKey()7202 void HTMLMediaElement::NotifyWaitingForKey() {
7203 LOG(LogLevel::Debug, ("%p, NotifyWaitingForKey()", this));
7204
7205 // http://w3c.github.io/encrypted-media/#wait-for-key
7206 // 7.3.4 Queue a "waitingforkey" Event
7207 // 1. Let the media element be the specified HTMLMediaElement object.
7208 // 2. If the media element's waiting for key value is true, abort these steps.
7209 if (mWaitingForKey == NOT_WAITING_FOR_KEY) {
7210 // 3. Set the media element's waiting for key value to true.
7211 // Note: algorithm continues in UpdateReadyStateInternal() when all decoded
7212 // data enqueued in the MDSM is consumed.
7213 mWaitingForKey = WAITING_FOR_KEY;
7214 // mWaitingForKey changed outside of UpdateReadyStateInternal. This may
7215 // affect mReadyState.
7216 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
7217 }
7218 }
7219
AudioTracks()7220 AudioTrackList* HTMLMediaElement::AudioTracks() { return mAudioTrackList; }
7221
VideoTracks()7222 VideoTrackList* HTMLMediaElement::VideoTracks() { return mVideoTrackList; }
7223
GetTextTracks()7224 TextTrackList* HTMLMediaElement::GetTextTracks() {
7225 return GetOrCreateTextTrackManager()->GetTextTracks();
7226 }
7227
AddTextTrack(TextTrackKind aKind,const nsAString & aLabel,const nsAString & aLanguage)7228 already_AddRefed<TextTrack> HTMLMediaElement::AddTextTrack(
7229 TextTrackKind aKind, const nsAString& aLabel, const nsAString& aLanguage) {
7230 return GetOrCreateTextTrackManager()->AddTextTrack(
7231 aKind, aLabel, aLanguage, TextTrackMode::Hidden,
7232 TextTrackReadyState::Loaded, TextTrackSource::AddTextTrack);
7233 }
7234
PopulatePendingTextTrackList()7235 void HTMLMediaElement::PopulatePendingTextTrackList() {
7236 if (mTextTrackManager) {
7237 mTextTrackManager->PopulatePendingList();
7238 }
7239 }
7240
GetOrCreateTextTrackManager()7241 TextTrackManager* HTMLMediaElement::GetOrCreateTextTrackManager() {
7242 if (!mTextTrackManager) {
7243 mTextTrackManager = new TextTrackManager(this);
7244 mTextTrackManager->AddListeners();
7245 }
7246 return mTextTrackManager;
7247 }
7248
NextFrameStatus()7249 MediaDecoderOwner::NextFrameStatus HTMLMediaElement::NextFrameStatus() {
7250 if (mDecoder) {
7251 return mDecoder->NextFrameStatus();
7252 }
7253 if (mSrcStream) {
7254 AutoTArray<RefPtr<MediaTrack>, 4> tracks;
7255 GetAllEnabledMediaTracks(tracks);
7256 if (!tracks.IsEmpty() && !mSrcStreamPlaybackEnded) {
7257 return NEXT_FRAME_AVAILABLE;
7258 }
7259 return NEXT_FRAME_UNAVAILABLE;
7260 }
7261 return NEXT_FRAME_UNINITIALIZED;
7262 }
7263
SetDecoder(MediaDecoder * aDecoder)7264 void HTMLMediaElement::SetDecoder(MediaDecoder* aDecoder) {
7265 MOZ_ASSERT(aDecoder); // Use ShutdownDecoder() to clear.
7266 if (mDecoder) {
7267 ShutdownDecoder();
7268 }
7269 mDecoder = aDecoder;
7270 DDLINKCHILD("decoder", mDecoder.get());
7271 if (mDecoder && mForcedHidden) {
7272 mDecoder->SetForcedHidden(mForcedHidden);
7273 }
7274 }
7275
ComputedVolume() const7276 float HTMLMediaElement::ComputedVolume() const {
7277 return mMuted
7278 ? 0.0f
7279 : mAudioChannelWrapper ? mAudioChannelWrapper->GetEffectiveVolume()
7280 : mVolume;
7281 }
7282
ComputedMuted() const7283 bool HTMLMediaElement::ComputedMuted() const {
7284 return (mMuted & MUTED_BY_AUDIO_CHANNEL);
7285 }
7286
IsSuspendedByInactiveDocOrDocShell() const7287 bool HTMLMediaElement::IsSuspendedByInactiveDocOrDocShell() const {
7288 return mSuspendedByInactiveDocOrDocshell;
7289 }
7290
IsCurrentlyPlaying() const7291 bool HTMLMediaElement::IsCurrentlyPlaying() const {
7292 // We have playable data, but we still need to check whether data is "real"
7293 // current data.
7294 return mReadyState >= HAVE_CURRENT_DATA && !IsPlaybackEnded();
7295 }
7296
SetAudibleState(bool aAudible)7297 void HTMLMediaElement::SetAudibleState(bool aAudible) {
7298 if (mIsAudioTrackAudible != aAudible) {
7299 UpdateAudioTrackSilenceRange(aAudible);
7300 mIsAudioTrackAudible = aAudible;
7301 NotifyAudioPlaybackChanged(
7302 AudioChannelService::AudibleChangedReasons::eDataAudibleChanged);
7303 }
7304 }
7305
IsAudioTrackCurrentlySilent() const7306 bool HTMLMediaElement::IsAudioTrackCurrentlySilent() const {
7307 return HasAudio() && !mIsAudioTrackAudible;
7308 }
7309
UpdateAudioTrackSilenceRange(bool aAudible)7310 void HTMLMediaElement::UpdateAudioTrackSilenceRange(bool aAudible) {
7311 if (!HasAudio()) {
7312 return;
7313 }
7314
7315 if (!aAudible) {
7316 mAudioTrackSilenceStartedTime = CurrentTime();
7317 return;
7318 }
7319
7320 AccumulateAudioTrackSilence();
7321 }
7322
AccumulateAudioTrackSilence()7323 void HTMLMediaElement::AccumulateAudioTrackSilence() {
7324 MOZ_ASSERT(HasAudio());
7325 const double current = CurrentTime();
7326 if (current < mAudioTrackSilenceStartedTime) {
7327 return;
7328 }
7329 const auto start =
7330 media::TimeUnit::FromSeconds(mAudioTrackSilenceStartedTime);
7331 const auto end = media::TimeUnit::FromSeconds(current);
7332 mSilenceTimeRanges += media::TimeInterval(start, end);
7333 }
7334
ReportAudioTrackSilenceProportionTelemetry()7335 void HTMLMediaElement::ReportAudioTrackSilenceProportionTelemetry() {
7336 if (!HasAudio()) {
7337 return;
7338 }
7339
7340 // Add last silence range to our ranges set.
7341 if (!mIsAudioTrackAudible) {
7342 AccumulateAudioTrackSilence();
7343 }
7344
7345 RefPtr<TimeRanges> ranges = Played();
7346 const uint32_t lengthPlayedRange = ranges->Length();
7347 const uint32_t lengthSilenceRange = mSilenceTimeRanges.Length();
7348 if (!lengthPlayedRange || !lengthSilenceRange) {
7349 return;
7350 }
7351
7352 double playedTime = 0.0, silenceTime = 0.0;
7353 for (uint32_t idx = 0; idx < lengthPlayedRange; idx++) {
7354 playedTime += ranges->End(idx) - ranges->Start(idx);
7355 }
7356
7357 for (uint32_t idx = 0; idx < lengthSilenceRange; idx++) {
7358 silenceTime += mSilenceTimeRanges.End(idx).ToSeconds() -
7359 mSilenceTimeRanges.Start(idx).ToSeconds();
7360 }
7361
7362 double silenceProportion = (silenceTime / playedTime) * 100;
7363 // silenceProportion should be in the range [0, 100]
7364 silenceProportion = std::min(100.0, std::max(silenceProportion, 0.0));
7365 Telemetry::Accumulate(Telemetry::AUDIO_TRACK_SILENCE_PROPORTION,
7366 silenceProportion);
7367 }
7368
NotifyAudioPlaybackChanged(AudibleChangedReasons aReason)7369 void HTMLMediaElement::NotifyAudioPlaybackChanged(
7370 AudibleChangedReasons aReason) {
7371 if (mAudioChannelWrapper) {
7372 mAudioChannelWrapper->NotifyAudioPlaybackChanged(aReason);
7373 }
7374 if (mMediaControlEventListener && mMediaControlEventListener->IsStarted()) {
7375 mMediaControlEventListener->UpdateMediaAudibleState(IsAudible());
7376 }
7377 // only request wake lock for audible media.
7378 UpdateWakeLock();
7379 }
7380
SetMediaInfo(const MediaInfo & aInfo)7381 void HTMLMediaElement::SetMediaInfo(const MediaInfo& aInfo) {
7382 const bool oldHasAudio = mMediaInfo.HasAudio();
7383 mMediaInfo = aInfo;
7384 if ((aInfo.HasAudio() != oldHasAudio) && mResumeDelayedPlaybackAgent) {
7385 mResumeDelayedPlaybackAgent->UpdateAudibleState(this, IsAudible());
7386 }
7387 if (mAudioChannelWrapper) {
7388 mAudioChannelWrapper->AudioCaptureTrackChangeIfNeeded();
7389 }
7390 UpdateWakeLock();
7391 }
7392
AudioCaptureTrackChange(bool aCapture)7393 void HTMLMediaElement::AudioCaptureTrackChange(bool aCapture) {
7394 // No need to capture a silent media element.
7395 if (!HasAudio()) {
7396 return;
7397 }
7398
7399 if (aCapture && !mStreamWindowCapturer) {
7400 nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
7401 if (!window) {
7402 return;
7403 }
7404
7405 MediaTrackGraph* mtg = MediaTrackGraph::GetInstance(
7406 MediaTrackGraph::AUDIO_THREAD_DRIVER, window,
7407 MediaTrackGraph::REQUEST_DEFAULT_SAMPLE_RATE,
7408 MediaTrackGraph::DEFAULT_OUTPUT_DEVICE);
7409 RefPtr<DOMMediaStream> stream =
7410 CaptureStreamInternal(StreamCaptureBehavior::CONTINUE_WHEN_ENDED,
7411 StreamCaptureType::CAPTURE_AUDIO, mtg);
7412 mStreamWindowCapturer =
7413 MakeUnique<MediaStreamWindowCapturer>(stream, window->WindowID());
7414 } else if (!aCapture && mStreamWindowCapturer) {
7415 for (size_t i = 0; i < mOutputStreams.Length(); i++) {
7416 if (mOutputStreams[i].mStream == mStreamWindowCapturer->mStream) {
7417 // We own this MediaStream, it is not exposed to JS.
7418 AutoTArray<RefPtr<MediaStreamTrack>, 2> tracks;
7419 mStreamWindowCapturer->mStream->GetTracks(tracks);
7420 for (auto& track : tracks) {
7421 track->Stop();
7422 }
7423 mOutputStreams.RemoveElementAt(i);
7424 break;
7425 }
7426 }
7427 mStreamWindowCapturer = nullptr;
7428 if (mOutputStreams.IsEmpty()) {
7429 mTracksCaptured = nullptr;
7430 }
7431 }
7432 }
7433
NotifyCueDisplayStatesChanged()7434 void HTMLMediaElement::NotifyCueDisplayStatesChanged() {
7435 if (!mTextTrackManager) {
7436 return;
7437 }
7438
7439 mTextTrackManager->DispatchUpdateCueDisplay();
7440 }
7441
MarkAsContentSource(CallerAPI aAPI)7442 void HTMLMediaElement::MarkAsContentSource(CallerAPI aAPI) {
7443 const bool isVisible = mVisibilityState == Visibility::ApproximatelyVisible;
7444
7445 LOG(LogLevel::Debug,
7446 ("%p Log VIDEO_AS_CONTENT_SOURCE: visibility = %u, API: '%d' and 'All'",
7447 this, isVisible, static_cast<int>(aAPI)));
7448
7449 if (!isVisible) {
7450 LOG(LogLevel::Debug,
7451 ("%p Log VIDEO_AS_CONTENT_SOURCE_IN_TREE_OR_NOT: inTree = %u, API: "
7452 "'%d' and 'All'",
7453 this, IsInComposedDoc(), static_cast<int>(aAPI)));
7454 }
7455 }
7456
UpdateCustomPolicyAfterPlayed()7457 void HTMLMediaElement::UpdateCustomPolicyAfterPlayed() {
7458 if (mAudioChannelWrapper) {
7459 mAudioChannelWrapper->NotifyPlayStateChanged();
7460 }
7461 }
7462
AbstractMainThread() const7463 AbstractThread* HTMLMediaElement::AbstractMainThread() const {
7464 MOZ_ASSERT(mAbstractMainThread);
7465
7466 return mAbstractMainThread;
7467 }
7468
TakePendingPlayPromises()7469 nsTArray<RefPtr<PlayPromise>> HTMLMediaElement::TakePendingPlayPromises() {
7470 return std::move(mPendingPlayPromises);
7471 }
7472
NotifyAboutPlaying()7473 void HTMLMediaElement::NotifyAboutPlaying() {
7474 // Stick to the DispatchAsyncEvent() call path for now because we want to
7475 // trigger some telemetry-related codes in the DispatchAsyncEvent() method.
7476 DispatchAsyncEvent(NS_LITERAL_STRING("playing"));
7477 }
7478
CreatePlayPromise(ErrorResult & aRv) const7479 already_AddRefed<PlayPromise> HTMLMediaElement::CreatePlayPromise(
7480 ErrorResult& aRv) const {
7481 nsPIDOMWindowInner* win = OwnerDoc()->GetInnerWindow();
7482
7483 if (!win) {
7484 aRv.Throw(NS_ERROR_UNEXPECTED);
7485 return nullptr;
7486 }
7487
7488 RefPtr<PlayPromise> promise = PlayPromise::Create(win->AsGlobal(), aRv);
7489 LOG(LogLevel::Debug, ("%p created PlayPromise %p", this, promise.get()));
7490
7491 return promise.forget();
7492 }
7493
CreateDOMPromise(ErrorResult & aRv) const7494 already_AddRefed<Promise> HTMLMediaElement::CreateDOMPromise(
7495 ErrorResult& aRv) const {
7496 nsPIDOMWindowInner* win = OwnerDoc()->GetInnerWindow();
7497
7498 if (!win) {
7499 aRv.Throw(NS_ERROR_UNEXPECTED);
7500 return nullptr;
7501 }
7502
7503 return Promise::Create(win->AsGlobal(), aRv);
7504 }
7505
AsyncResolvePendingPlayPromises()7506 void HTMLMediaElement::AsyncResolvePendingPlayPromises() {
7507 if (mShuttingDown) {
7508 return;
7509 }
7510
7511 nsCOMPtr<nsIRunnable> event = new nsResolveOrRejectPendingPlayPromisesRunner(
7512 this, TakePendingPlayPromises());
7513
7514 mMainThreadEventTarget->Dispatch(event.forget());
7515 }
7516
AsyncRejectPendingPlayPromises(nsresult aError)7517 void HTMLMediaElement::AsyncRejectPendingPlayPromises(nsresult aError) {
7518 if (!mPaused) {
7519 mPaused = true;
7520 DispatchAsyncEvent(NS_LITERAL_STRING("pause"));
7521 }
7522
7523 if (mShuttingDown) {
7524 return;
7525 }
7526
7527 if (aError == NS_ERROR_DOM_MEDIA_NOT_ALLOWED_ERR) {
7528 DispatchEventsWhenPlayWasNotAllowed();
7529 }
7530
7531 nsCOMPtr<nsIRunnable> event = new nsResolveOrRejectPendingPlayPromisesRunner(
7532 this, TakePendingPlayPromises(), aError);
7533
7534 mMainThreadEventTarget->Dispatch(event.forget());
7535 }
7536
GetEMEInfo(dom::EMEDebugInfo & aInfo)7537 void HTMLMediaElement::GetEMEInfo(dom::EMEDebugInfo& aInfo) {
7538 if (!mMediaKeys) {
7539 return;
7540 }
7541 mMediaKeys->GetKeySystem(aInfo.mKeySystem);
7542 mMediaKeys->GetSessionsInfo(aInfo.mSessionsInfo);
7543 }
7544
NotifyDecoderActivityChanges() const7545 void HTMLMediaElement::NotifyDecoderActivityChanges() const {
7546 if (mDecoder) {
7547 mDecoder->NotifyOwnerActivityChanged(!IsHidden(), mVisibilityState,
7548 IsInComposedDoc());
7549 }
7550 }
7551
GetDocument() const7552 Document* HTMLMediaElement::GetDocument() const { return OwnerDoc(); }
7553
IsAudible() const7554 bool HTMLMediaElement::IsAudible() const {
7555 // No audio track.
7556 if (!HasAudio()) {
7557 return false;
7558 }
7559
7560 // Muted or the volume should not be ~0
7561 if (mMuted || (std::fabs(Volume()) <= 1e-7)) {
7562 return false;
7563 }
7564
7565 return mIsAudioTrackAudible;
7566 }
7567
ConstructMediaTracks(const MediaInfo * aInfo)7568 void HTMLMediaElement::ConstructMediaTracks(const MediaInfo* aInfo) {
7569 if (!aInfo) {
7570 return;
7571 }
7572
7573 AudioTrackList* audioList = AudioTracks();
7574 if (audioList && aInfo->HasAudio()) {
7575 const TrackInfo& info = aInfo->mAudio;
7576 RefPtr<AudioTrack> track = MediaTrackList::CreateAudioTrack(
7577 audioList->GetOwnerGlobal(), info.mId, info.mKind, info.mLabel,
7578 info.mLanguage, info.mEnabled);
7579
7580 audioList->AddTrack(track);
7581 }
7582
7583 VideoTrackList* videoList = VideoTracks();
7584 if (videoList && aInfo->HasVideo()) {
7585 const TrackInfo& info = aInfo->mVideo;
7586 RefPtr<VideoTrack> track = MediaTrackList::CreateVideoTrack(
7587 videoList->GetOwnerGlobal(), info.mId, info.mKind, info.mLabel,
7588 info.mLanguage);
7589
7590 videoList->AddTrack(track);
7591 track->SetEnabledInternal(info.mEnabled, MediaTrack::FIRE_NO_EVENTS);
7592 }
7593 }
7594
RemoveMediaTracks()7595 void HTMLMediaElement::RemoveMediaTracks() {
7596 if (mAudioTrackList) {
7597 mAudioTrackList->RemoveTracks();
7598 }
7599 if (mVideoTrackList) {
7600 mVideoTrackList->RemoveTracks();
7601 }
7602 }
7603
7604 class MediaElementGMPCrashHelper : public GMPCrashHelper {
7605 public:
MediaElementGMPCrashHelper(HTMLMediaElement * aElement)7606 explicit MediaElementGMPCrashHelper(HTMLMediaElement* aElement)
7607 : mElement(aElement) {
7608 MOZ_ASSERT(NS_IsMainThread()); // WeakPtr isn't thread safe.
7609 }
GetPluginCrashedEventTarget()7610 already_AddRefed<nsPIDOMWindowInner> GetPluginCrashedEventTarget() override {
7611 MOZ_ASSERT(NS_IsMainThread()); // WeakPtr isn't thread safe.
7612 if (!mElement) {
7613 return nullptr;
7614 }
7615 return do_AddRef(mElement->OwnerDoc()->GetInnerWindow());
7616 }
7617
7618 private:
7619 WeakPtr<HTMLMediaElement> mElement;
7620 };
7621
CreateGMPCrashHelper()7622 already_AddRefed<GMPCrashHelper> HTMLMediaElement::CreateGMPCrashHelper() {
7623 return MakeAndAddRef<MediaElementGMPCrashHelper>(this);
7624 }
7625
MarkAsTainted()7626 void HTMLMediaElement::MarkAsTainted() {
7627 mHasSuspendTaint = true;
7628
7629 if (mDecoder) {
7630 mDecoder->SetSuspendTaint(true);
7631 }
7632 }
7633
HasDebuggerOrTabsPrivilege(JSContext * aCx,JSObject * aObj)7634 bool HasDebuggerOrTabsPrivilege(JSContext* aCx, JSObject* aObj) {
7635 return nsContentUtils::CallerHasPermission(aCx, nsGkAtoms::debugger) ||
7636 nsContentUtils::CallerHasPermission(aCx, nsGkAtoms::tabs);
7637 }
7638
ReportCanPlayTelemetry()7639 void HTMLMediaElement::ReportCanPlayTelemetry() {
7640 LOG(LogLevel::Debug, ("%s", __func__));
7641
7642 RefPtr<nsIThread> thread;
7643 nsresult rv = NS_NewNamedThread("MediaTelemetry", getter_AddRefs(thread));
7644 if (NS_WARN_IF(NS_FAILED(rv))) {
7645 return;
7646 }
7647
7648 RefPtr<AbstractThread> abstractThread = mAbstractMainThread;
7649
7650 thread->Dispatch(
7651 NS_NewRunnableFunction(
7652 "dom::HTMLMediaElement::ReportCanPlayTelemetry",
7653 [thread, abstractThread]() {
7654 #if XP_WIN
7655 // Windows Media Foundation requires MSCOM to be inited.
7656 DebugOnly<HRESULT> hr = CoInitializeEx(0, COINIT_MULTITHREADED);
7657 MOZ_ASSERT(hr == S_OK);
7658 #endif
7659 bool aac = MP4Decoder::IsSupportedType(
7660 MediaContainerType(MEDIAMIMETYPE(AUDIO_MP4)), nullptr);
7661 bool h264 = MP4Decoder::IsSupportedType(
7662 MediaContainerType(MEDIAMIMETYPE(VIDEO_MP4)), nullptr);
7663 #if XP_WIN
7664 CoUninitialize();
7665 #endif
7666 abstractThread->Dispatch(NS_NewRunnableFunction(
7667 "dom::HTMLMediaElement::ReportCanPlayTelemetry",
7668 [thread, aac, h264]() {
7669 LOG(LogLevel::Debug,
7670 ("MediaTelemetry aac=%d h264=%d", aac, h264));
7671 thread->AsyncShutdown();
7672 }));
7673 }),
7674 NS_DISPATCH_NORMAL);
7675 }
7676
SetSinkId(const nsAString & aSinkId,ErrorResult & aRv)7677 already_AddRefed<Promise> HTMLMediaElement::SetSinkId(const nsAString& aSinkId,
7678 ErrorResult& aRv) {
7679 nsCOMPtr<nsPIDOMWindowInner> win = OwnerDoc()->GetInnerWindow();
7680 if (!win) {
7681 aRv.Throw(NS_ERROR_UNEXPECTED);
7682 return nullptr;
7683 }
7684
7685 RefPtr<Promise> promise = Promise::Create(win->AsGlobal(), aRv);
7686 if (aRv.Failed()) {
7687 return nullptr;
7688 }
7689
7690 if (mSink.first.Equals(aSinkId)) {
7691 promise->MaybeResolveWithUndefined();
7692 return promise.forget();
7693 }
7694
7695 nsString sinkId(aSinkId);
7696 MediaManager::Get()
7697 ->GetSinkDevice(win, sinkId)
7698 ->Then(
7699 mAbstractMainThread, __func__,
7700 [self = RefPtr<HTMLMediaElement>(this)](
7701 RefPtr<AudioDeviceInfo>&& aInfo) {
7702 // Sink found switch output device.
7703 MOZ_ASSERT(aInfo);
7704 if (self->mDecoder) {
7705 RefPtr<SinkInfoPromise> p = self->mDecoder->SetSink(aInfo)->Then(
7706 self->mAbstractMainThread, __func__,
7707 [aInfo](const GenericPromise::ResolveOrRejectValue& aValue) {
7708 if (aValue.IsResolve()) {
7709 return SinkInfoPromise::CreateAndResolve(aInfo, __func__);
7710 }
7711 return SinkInfoPromise::CreateAndReject(
7712 aValue.RejectValue(), __func__);
7713 });
7714 return p;
7715 }
7716 if (self->mSrcAttrStream) {
7717 // Set Sink Id through MTG is not supported yet.
7718 return SinkInfoPromise::CreateAndReject(NS_ERROR_ABORT, __func__);
7719 }
7720 // No media attached to the element save it for later.
7721 return SinkInfoPromise::CreateAndResolve(aInfo, __func__);
7722 },
7723 [](nsresult res) {
7724 // Promise is rejected, sink not found.
7725 return SinkInfoPromise::CreateAndReject(res, __func__);
7726 })
7727 ->Then(mAbstractMainThread, __func__,
7728 [promise, self = RefPtr<HTMLMediaElement>(this),
7729 sinkId](const SinkInfoPromise::ResolveOrRejectValue& aValue) {
7730 if (aValue.IsResolve()) {
7731 self->mSink = std::pair(sinkId, aValue.ResolveValue());
7732 promise->MaybeResolveWithUndefined();
7733 } else {
7734 switch (aValue.RejectValue()) {
7735 case NS_ERROR_ABORT:
7736 promise->MaybeReject(NS_ERROR_DOM_ABORT_ERR);
7737 break;
7738 case NS_ERROR_NOT_AVAILABLE: {
7739 promise->MaybeRejectWithNotFoundError(
7740 "The object can not be found here.");
7741 break;
7742 }
7743 case NS_ERROR_DOM_MEDIA_NOT_ALLOWED_ERR:
7744 promise->MaybeReject(NS_ERROR_DOM_NOT_ALLOWED_ERR);
7745 break;
7746 default:
7747 MOZ_ASSERT_UNREACHABLE("Invalid error.");
7748 }
7749 }
7750 });
7751
7752 aRv = NS_OK;
7753 return promise.forget();
7754 }
7755
NotifyTextTrackModeChanged()7756 void HTMLMediaElement::NotifyTextTrackModeChanged() {
7757 if (mPendingTextTrackChanged) {
7758 return;
7759 }
7760 mPendingTextTrackChanged = true;
7761 mAbstractMainThread->Dispatch(
7762 NS_NewRunnableFunction("HTMLMediaElement::NotifyTextTrackModeChanged",
7763 [this, self = RefPtr<HTMLMediaElement>(this)]() {
7764 mPendingTextTrackChanged = false;
7765 if (!mTextTrackManager) {
7766 return;
7767 }
7768 GetTextTracks()->CreateAndDispatchChangeEvent();
7769 // https://html.spec.whatwg.org/multipage/media.html#text-track-model:show-poster-flag
7770 if (!mShowPoster) {
7771 mTextTrackManager->TimeMarchesOn();
7772 }
7773 }));
7774 }
7775
CreateResumeDelayedMediaPlaybackAgentIfNeeded()7776 void HTMLMediaElement::CreateResumeDelayedMediaPlaybackAgentIfNeeded() {
7777 if (mResumeDelayedPlaybackAgent) {
7778 return;
7779 }
7780 mResumeDelayedPlaybackAgent =
7781 MediaPlaybackDelayPolicy::CreateResumeDelayedPlaybackAgent(this,
7782 IsAudible());
7783 if (!mResumeDelayedPlaybackAgent) {
7784 LOG(LogLevel::Debug,
7785 ("%p Failed to create a delayed playback agant", this));
7786 return;
7787 }
7788 mResumeDelayedPlaybackAgent->GetResumePromise()
7789 ->Then(
7790 mAbstractMainThread, __func__,
7791 [self = RefPtr<HTMLMediaElement>(this)]() {
7792 LOG(LogLevel::Debug, ("%p Resume delayed Play() call", self.get()));
7793 self->mResumePlaybackRequest.Complete();
7794 self->mResumeDelayedPlaybackAgent = nullptr;
7795 IgnoredErrorResult dummy;
7796 RefPtr<Promise> toBeIgnored = self->Play(dummy);
7797 },
7798 [self = RefPtr<HTMLMediaElement>(this)]() {
7799 LOG(LogLevel::Debug,
7800 ("%p Can not resume delayed Play() call", self.get()));
7801 self->mResumePlaybackRequest.Complete();
7802 self->mResumeDelayedPlaybackAgent = nullptr;
7803 })
7804 ->Track(mResumePlaybackRequest);
7805 }
7806
ClearResumeDelayedMediaPlaybackAgentIfNeeded()7807 void HTMLMediaElement::ClearResumeDelayedMediaPlaybackAgentIfNeeded() {
7808 if (mResumeDelayedPlaybackAgent) {
7809 mResumePlaybackRequest.DisconnectIfExists();
7810 mResumeDelayedPlaybackAgent = nullptr;
7811 }
7812 }
7813
NotifyMediaControlPlaybackStateChanged()7814 void HTMLMediaElement::NotifyMediaControlPlaybackStateChanged() {
7815 if (!mMediaControlEventListener || !mMediaControlEventListener->IsStarted()) {
7816 return;
7817 }
7818 if (mPaused) {
7819 mMediaControlEventListener->NotifyMediaStoppedPlaying();
7820 } else {
7821 mMediaControlEventListener->NotifyMediaStartedPlaying();
7822 }
7823 }
7824
StartListeningMediaControlEventIfNeeded()7825 void HTMLMediaElement::StartListeningMediaControlEventIfNeeded() {
7826 if (mPaused) {
7827 MEDIACONTROL_LOG("Not listening because media is paused");
7828 return;
7829 }
7830
7831 // In order to filter out notification-ish sound, we use this pref to set the
7832 // eligible media duration to prevent showing media control for those short
7833 // sound.
7834 if (Duration() <
7835 StaticPrefs::media_mediacontrol_eligible_media_duration_s()) {
7836 MEDIACONTROL_LOG("Not listening because media's duration %f is too short.",
7837 Duration());
7838 return;
7839 }
7840
7841 // As we would like to start listening to media control event again so we
7842 // should clear the timer, which is used to stop listening to the event.
7843 ClearStopMediaControlTimerIfNeeded();
7844
7845 if (!mMediaControlEventListener) {
7846 mMediaControlEventListener = new MediaControlEventListener(this);
7847 }
7848
7849 if (mMediaControlEventListener->IsStarted() ||
7850 !mMediaControlEventListener->Start()) {
7851 return;
7852 }
7853
7854 // If `mPaused` was being changed at the time the listener didn't start, then
7855 // this method won't be called. Eg. if the media becomes audible after it has
7856 // been playing for a while. Therefore, we have to manually update playback
7857 // state after starting listener.
7858 NotifyMediaControlPlaybackStateChanged();
7859
7860 // `UpdateMediaAudibleState()` could only be used after we start the listener,
7861 // but the audible state update could happen before that. Therefore, we have
7862 // to manually update media's audible state as well.
7863 mMediaControlEventListener->UpdateMediaAudibleState(IsAudible());
7864
7865 // Picture-in-Picture mode can be enabled before we start the listener so we
7866 // manually update the status here in case not to forgot to propagate that.
7867 mMediaControlEventListener->SetPictureInPictureModeEnabled(
7868 IsBeingUsedInPictureInPictureMode());
7869 }
7870
StopListeningMediaControlEventIfNeeded()7871 void HTMLMediaElement::StopListeningMediaControlEventIfNeeded() {
7872 if (mMediaControlEventListener && mMediaControlEventListener->IsStarted()) {
7873 mMediaControlEventListener->Stop();
7874 }
7875 }
7876
7877 /* static */
StopMediaControlTimerCallback(nsITimer * aTimer,void * aClosure)7878 void HTMLMediaElement::StopMediaControlTimerCallback(nsITimer* aTimer,
7879 void* aClosure) {
7880 MOZ_ASSERT(NS_IsMainThread());
7881 auto element = static_cast<HTMLMediaElement*>(aClosure);
7882 CONTROLLER_TIMER_LOG(element,
7883 "Runnning stop media control timmer callback function");
7884 element->StopListeningMediaControlEventIfNeeded();
7885 element->mStopMediaControlTimer = nullptr;
7886 }
7887
CreateStopMediaControlTimerIfNeeded()7888 void HTMLMediaElement::CreateStopMediaControlTimerIfNeeded() {
7889 MOZ_ASSERT(NS_IsMainThread());
7890 // We would create timer only when `mMediaControlEventListener` exists and has
7891 // been started.
7892 if (mStopMediaControlTimer || !mMediaControlEventListener ||
7893 !mMediaControlEventListener->IsStarted()) {
7894 return;
7895 }
7896
7897 // As the media element being used in the PIP mode would always display on the
7898 // screen, users would have high chance to interact with it again, so we don't
7899 // want to stop media control.
7900 if (IsBeingUsedInPictureInPictureMode()) {
7901 MEDIACONTROL_LOG("No need to create a timer for PIP video.");
7902 return;
7903 }
7904
7905 if (!Paused()) {
7906 MEDIACONTROL_LOG("No need to create a timer for playing media.");
7907 return;
7908 }
7909
7910 MEDIACONTROL_LOG("Start stop media control timer");
7911 NS_NewTimerWithFuncCallback(
7912 getter_AddRefs(mStopMediaControlTimer), StopMediaControlTimerCallback,
7913 this, StaticPrefs::media_mediacontrol_stopcontrol_timer_ms(),
7914 nsITimer::TYPE_ONE_SHOT,
7915 "HTMLMediaElement::StopMediaControlTimerCallback",
7916 mMainThreadEventTarget);
7917 }
7918
ClearStopMediaControlTimerIfNeeded()7919 void HTMLMediaElement::ClearStopMediaControlTimerIfNeeded() {
7920 if (mStopMediaControlTimer) {
7921 MEDIACONTROL_LOG("Cancel stop media control timer");
7922 mStopMediaControlTimer->Cancel();
7923 mStopMediaControlTimer = nullptr;
7924 }
7925 }
7926
UpdateMediaControlAfterPictureInPictureModeChanged()7927 void HTMLMediaElement::UpdateMediaControlAfterPictureInPictureModeChanged() {
7928 // Hasn't started to connect with media control, no need to update anything.
7929 if (!mMediaControlEventListener || !mMediaControlEventListener->IsStarted()) {
7930 return;
7931 }
7932 if (IsBeingUsedInPictureInPictureMode()) {
7933 mMediaControlEventListener->SetPictureInPictureModeEnabled(true);
7934 } else {
7935 mMediaControlEventListener->SetPictureInPictureModeEnabled(false);
7936 CreateStopMediaControlTimerIfNeeded();
7937 }
7938 }
7939
IsBeingUsedInPictureInPictureMode() const7940 bool HTMLMediaElement::IsBeingUsedInPictureInPictureMode() const {
7941 if (!IsVideo()) {
7942 return false;
7943 }
7944 return static_cast<const HTMLVideoElement*>(this)->IsCloningElementVisually();
7945 }
7946
7947 } // namespace dom
7948 } // namespace mozilla
7949
7950 #undef LOG
7951 #undef LOG_EVENT
7952