1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #ifdef XP_WIN
8 # include "objbase.h"
9 #endif
10
11 #include "mozilla/dom/HTMLMediaElement.h"
12
13 #include <unordered_map>
14
15 #include "AudioDeviceInfo.h"
16 #include "AudioStreamTrack.h"
17 #include "AutoplayPolicy.h"
18 #include "ChannelMediaDecoder.h"
19 #include "CrossGraphPort.h"
20 #include "DOMMediaStream.h"
21 #include "DecoderDoctorDiagnostics.h"
22 #include "DecoderDoctorLogger.h"
23 #include "DecoderTraits.h"
24 #include "FrameStatistics.h"
25 #include "GMPCrashHelper.h"
26 #include "GVAutoplayPermissionRequest.h"
27 #ifdef MOZ_ANDROID_HLS_SUPPORT
28 # include "HLSDecoder.h"
29 #endif
30 #include "HTMLMediaElement.h"
31 #include "ImageContainer.h"
32 #include "Layers.h"
33 #include "MP4Decoder.h"
34 #include "MediaContainerType.h"
35 #include "MediaError.h"
36 #include "MediaManager.h"
37 #include "MediaMetadataManager.h"
38 #include "MediaResource.h"
39 #include "MediaShutdownManager.h"
40 #include "MediaSourceDecoder.h"
41 #include "MediaStreamError.h"
42 #include "MediaTrackGraphImpl.h"
43 #include "MediaTrackListener.h"
44 #include "MediaStreamWindowCapturer.h"
45 #include "MediaTrack.h"
46 #include "MediaTrackList.h"
47 #include "Navigator.h"
48 #include "TimeRanges.h"
49 #include "VideoFrameContainer.h"
50 #include "VideoOutput.h"
51 #include "VideoStreamTrack.h"
52 #include "base/basictypes.h"
53 #include "jsapi.h"
54 #include "mozilla/ArrayUtils.h"
55 #include "mozilla/AsyncEventDispatcher.h"
56 #include "mozilla/EMEUtils.h"
57 #include "mozilla/EventDispatcher.h"
58 #include "mozilla/FloatingPoint.h"
59 #include "mozilla/MathAlgorithms.h"
60 #include "mozilla/NotNull.h"
61 #include "mozilla/Preferences.h"
62 #include "mozilla/PresShell.h"
63 #include "mozilla/ScopeExit.h"
64 #include "mozilla/Sprintf.h"
65 #include "mozilla/StaticPrefs_media.h"
66 #include "mozilla/SVGObserverUtils.h"
67 #include "mozilla/Telemetry.h"
68 #include "mozilla/dom/AudioTrack.h"
69 #include "mozilla/dom/AudioTrackList.h"
70 #include "mozilla/dom/BlobURLProtocolHandler.h"
71 #include "mozilla/dom/ContentMediaController.h"
72 #include "mozilla/dom/ElementInlines.h"
73 #include "mozilla/dom/HTMLAudioElement.h"
74 #include "mozilla/dom/HTMLInputElement.h"
75 #include "mozilla/dom/HTMLMediaElementBinding.h"
76 #include "mozilla/dom/HTMLSourceElement.h"
77 #include "mozilla/dom/HTMLVideoElement.h"
78 #include "mozilla/dom/MediaControlUtils.h"
79 #include "mozilla/dom/MediaDevices.h"
80 #include "mozilla/dom/MediaEncryptedEvent.h"
81 #include "mozilla/dom/MediaErrorBinding.h"
82 #include "mozilla/dom/MediaSource.h"
83 #include "mozilla/dom/PlayPromise.h"
84 #include "mozilla/dom/Promise.h"
85 #include "mozilla/dom/TextTrack.h"
86 #include "mozilla/dom/UserActivation.h"
87 #include "mozilla/dom/VideoPlaybackQuality.h"
88 #include "mozilla/dom/VideoTrack.h"
89 #include "mozilla/dom/VideoTrackList.h"
90 #include "mozilla/dom/WakeLock.h"
91 #include "mozilla/dom/WindowGlobalChild.h"
92 #include "mozilla/dom/power/PowerManagerService.h"
93 #include "mozilla/net/UrlClassifierFeatureFactory.h"
94 #include "nsAttrValueInlines.h"
95 #include "nsContentPolicyUtils.h"
96 #include "nsContentUtils.h"
97 #include "nsCycleCollectionParticipant.h"
98 #include "nsDisplayList.h"
99 #include "nsDocShell.h"
100 #include "nsError.h"
101 #include "nsGenericHTMLElement.h"
102 #include "nsGkAtoms.h"
103 #include "nsIAsyncVerifyRedirectCallback.h"
104 #include "nsICachingChannel.h"
105 #include "nsIClassOfService.h"
106 #include "nsIContentPolicy.h"
107 #include "nsIDocShell.h"
108 #include "mozilla/dom/Document.h"
109 #include "nsIFrame.h"
110 #include "nsIHttpChannel.h"
111 #include "nsIObserverService.h"
112 #include "nsIRequest.h"
113 #include "nsIScriptError.h"
114 #include "nsISupportsPrimitives.h"
115 #include "nsIThreadRetargetableStreamListener.h"
116 #include "nsITimer.h"
117 #include "nsJSUtils.h"
118 #include "nsLayoutUtils.h"
119 #include "nsMediaFragmentURIParser.h"
120 #include "nsMimeTypes.h"
121 #include "nsNetUtil.h"
122 #include "nsNodeInfoManager.h"
123 #include "nsPresContext.h"
124 #include "nsQueryObject.h"
125 #include "nsRange.h"
126 #include "nsSize.h"
127 #include "nsThreadUtils.h"
128 #include "nsURIHashKey.h"
129 #include "nsVideoFrame.h"
130 #include "ReferrerInfo.h"
131 #include "xpcpublic.h"
132 #include <algorithm>
133 #include <cmath>
134 #include <limits>
135 #include <type_traits>
136
137 mozilla::LazyLogModule gMediaElementLog("nsMediaElement");
138 mozilla::LazyLogModule gMediaElementEventsLog("nsMediaElementEvents");
139
140 extern mozilla::LazyLogModule gAutoplayPermissionLog;
141 #define AUTOPLAY_LOG(msg, ...) \
142 MOZ_LOG(gAutoplayPermissionLog, LogLevel::Debug, (msg, ##__VA_ARGS__))
143
144 // avoid redefined macro in unified build
145 #undef MEDIACONTROL_LOG
146 #define MEDIACONTROL_LOG(msg, ...) \
147 MOZ_LOG(gMediaControlLog, LogLevel::Debug, \
148 ("HTMLMediaElement=%p, " msg, this, ##__VA_ARGS__))
149
150 #undef CONTROLLER_TIMER_LOG
151 #define CONTROLLER_TIMER_LOG(element, msg, ...) \
152 MOZ_LOG(gMediaControlLog, LogLevel::Debug, \
153 ("HTMLMediaElement=%p, " msg, element, ##__VA_ARGS__))
154
155 #define LOG(type, msg) MOZ_LOG(gMediaElementLog, type, msg)
156 #define LOG_EVENT(type, msg) MOZ_LOG(gMediaElementEventsLog, type, msg)
157
158 using namespace mozilla::layers;
159 using mozilla::net::nsMediaFragmentURIParser;
160 using namespace mozilla::dom::HTMLMediaElement_Binding;
161
162 namespace mozilla::dom {
163
164 using AudibleState = AudioChannelService::AudibleState;
165 using SinkInfoPromise = MediaDevices::SinkInfoPromise;
166
167 // Number of milliseconds between progress events as defined by spec
168 static const uint32_t PROGRESS_MS = 350;
169
170 // Number of milliseconds of no data before a stall event is fired as defined by
171 // spec
172 static const uint32_t STALL_MS = 3000;
173
174 // Used by AudioChannel for suppresssing the volume to this ratio.
175 #define FADED_VOLUME_RATIO 0.25
176
177 // These constants are arbitrary
178 // Minimum playbackRate for a media
179 static const double MIN_PLAYBACKRATE = 1.0 / 16;
180 // Maximum playbackRate for a media
181 static const double MAX_PLAYBACKRATE = 16.0;
182 // These are the limits beyonds which SoundTouch does not perform too well and
183 // when speech is hard to understand anyway. Threshold above which audio is
184 // muted
185 static const double THRESHOLD_HIGH_PLAYBACKRATE_AUDIO = 4.0;
186 // Threshold under which audio is muted
187 static const double THRESHOLD_LOW_PLAYBACKRATE_AUDIO = 0.25;
188
ClampPlaybackRate(double aPlaybackRate)189 static double ClampPlaybackRate(double aPlaybackRate) {
190 MOZ_ASSERT(aPlaybackRate >= 0.0);
191
192 if (aPlaybackRate == 0.0) {
193 return aPlaybackRate;
194 }
195 if (aPlaybackRate < MIN_PLAYBACKRATE) {
196 return MIN_PLAYBACKRATE;
197 }
198 if (aPlaybackRate > MAX_PLAYBACKRATE) {
199 return MAX_PLAYBACKRATE;
200 }
201 return aPlaybackRate;
202 }
203
204 // Media error values. These need to match the ones in MediaError.webidl.
205 static const unsigned short MEDIA_ERR_ABORTED = 1;
206 static const unsigned short MEDIA_ERR_NETWORK = 2;
207 static const unsigned short MEDIA_ERR_DECODE = 3;
208 static const unsigned short MEDIA_ERR_SRC_NOT_SUPPORTED = 4;
209
210 /**
211 * EventBlocker helps media element to postpone the event delivery by storing
212 * the event runner, and execute them once media element decides not to postpone
213 * the event delivery. If media element never resumes the event delivery, then
214 * those runner would be cancelled.
215 * For example, we postpone the event delivery when media element entering to
216 * the bf-cache.
217 */
218 class HTMLMediaElement::EventBlocker final : public nsISupports {
219 public:
220 NS_DECL_CYCLE_COLLECTING_ISUPPORTS_FINAL
NS_DECL_CYCLE_COLLECTION_CLASS(EventBlocker)221 NS_DECL_CYCLE_COLLECTION_CLASS(EventBlocker)
222
223 explicit EventBlocker(HTMLMediaElement* aElement) : mElement(aElement) {}
224
SetBlockEventDelivery(bool aShouldBlock)225 void SetBlockEventDelivery(bool aShouldBlock) {
226 MOZ_ASSERT(NS_IsMainThread());
227 if (mShouldBlockEventDelivery == aShouldBlock) {
228 return;
229 }
230 LOG_EVENT(LogLevel::Debug,
231 ("%p %s event delivery", mElement.get(),
232 mShouldBlockEventDelivery ? "block" : "unblock"));
233 mShouldBlockEventDelivery = aShouldBlock;
234 if (!mShouldBlockEventDelivery) {
235 DispatchPendingMediaEvents();
236 }
237 }
238
PostponeEvent(nsMediaEventRunner * aRunner)239 void PostponeEvent(nsMediaEventRunner* aRunner) {
240 MOZ_ASSERT(NS_IsMainThread());
241 // Element has been CCed, which would break the weak pointer.
242 if (!mElement) {
243 return;
244 }
245 MOZ_ASSERT(mShouldBlockEventDelivery);
246 MOZ_ASSERT(mElement);
247 LOG_EVENT(LogLevel::Debug,
248 ("%p postpone runner %s for %s", mElement.get(),
249 NS_ConvertUTF16toUTF8(aRunner->Name()).get(),
250 NS_ConvertUTF16toUTF8(aRunner->EventName()).get()));
251 mPendingEventRunners.AppendElement(aRunner);
252 }
253
Shutdown()254 void Shutdown() {
255 MOZ_ASSERT(NS_IsMainThread());
256 for (auto& runner : mPendingEventRunners) {
257 runner->Cancel();
258 }
259 mPendingEventRunners.Clear();
260 }
261
ShouldBlockEventDelivery() const262 bool ShouldBlockEventDelivery() const {
263 MOZ_ASSERT(NS_IsMainThread());
264 return mShouldBlockEventDelivery;
265 }
266
SizeOfExcludingThis(MallocSizeOf & aMallocSizeOf) const267 size_t SizeOfExcludingThis(MallocSizeOf& aMallocSizeOf) const {
268 MOZ_ASSERT(NS_IsMainThread());
269 size_t total = 0;
270 for (const auto& runner : mPendingEventRunners) {
271 total += aMallocSizeOf(runner);
272 }
273 return total;
274 }
275
276 private:
277 ~EventBlocker() = default;
278
DispatchPendingMediaEvents()279 void DispatchPendingMediaEvents() {
280 MOZ_ASSERT(mElement);
281 for (auto& runner : mPendingEventRunners) {
282 LOG_EVENT(LogLevel::Debug,
283 ("%p execute runner %s for %s", mElement.get(),
284 NS_ConvertUTF16toUTF8(runner->Name()).get(),
285 NS_ConvertUTF16toUTF8(runner->EventName()).get()));
286 GetMainThreadSerialEventTarget()->Dispatch(runner.forget());
287 }
288 mPendingEventRunners.Clear();
289 }
290
291 WeakPtr<HTMLMediaElement> mElement;
292 bool mShouldBlockEventDelivery = false;
293 // Contains event runners which should not be run for now because we want
294 // to block all events delivery. They would be dispatched once media element
295 // decides unblocking them.
296 nsTArray<RefPtr<nsMediaEventRunner>> mPendingEventRunners;
297 };
298
299 NS_IMPL_CYCLE_COLLECTION(HTMLMediaElement::EventBlocker, mPendingEventRunners)
300 NS_IMPL_CYCLE_COLLECTING_ADDREF(HTMLMediaElement::EventBlocker)
301 NS_IMPL_CYCLE_COLLECTING_RELEASE(HTMLMediaElement::EventBlocker)
302 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(HTMLMediaElement::EventBlocker)
303 NS_INTERFACE_MAP_ENTRY(nsISupports)
304 NS_INTERFACE_MAP_END
305
306 /**
307 * We use MediaControlKeyListener to listen to media control key in order to
308 * play and pause media element when user press media control keys and update
309 * media's playback and audible state to the media controller.
310 *
311 * Use `Start()` to start listening event and use `Stop()` to stop listening
312 * event. In addition, notifying any change to media controller MUST be done
313 * after successfully calling `Start()`.
314 */
315 class HTMLMediaElement::MediaControlKeyListener final
316 : public ContentMediaControlKeyReceiver {
317 public:
NS_INLINE_DECL_REFCOUNTING(MediaControlKeyListener,override)318 NS_INLINE_DECL_REFCOUNTING(MediaControlKeyListener, override)
319
320 MOZ_INIT_OUTSIDE_CTOR explicit MediaControlKeyListener(
321 HTMLMediaElement* aElement)
322 : mElement(aElement) {
323 MOZ_ASSERT(NS_IsMainThread());
324 MOZ_ASSERT(aElement);
325 }
326
327 /**
328 * Start listening to the media control keys which would make media being able
329 * to be controlled via pressing media control keys.
330 */
Start()331 void Start() {
332 MOZ_ASSERT(NS_IsMainThread());
333 if (IsStarted()) {
334 // We have already been started, do not notify start twice.
335 return;
336 }
337
338 // Fail to init media agent, we are not able to notify the media controller
339 // any update and also are not able to receive media control key events.
340 if (!InitMediaAgent()) {
341 MEDIACONTROL_LOG("Failed to start due to not able to init media agent!");
342 return;
343 }
344
345 NotifyPlaybackStateChanged(MediaPlaybackState::eStarted);
346 // If owner has started playing before the listener starts, we should update
347 // the playing state as well. Eg. media starts inaudily and becomes audible
348 // later.
349 if (!Owner()->Paused()) {
350 NotifyMediaStartedPlaying();
351 }
352 if (StaticPrefs::media_mediacontrol_testingevents_enabled()) {
353 auto dispatcher = MakeRefPtr<AsyncEventDispatcher>(
354 Owner(), u"MozStartMediaControl"_ns, CanBubble::eYes,
355 ChromeOnlyDispatch::eYes);
356 dispatcher->PostDOMEvent();
357 }
358 }
359
360 /**
361 * Stop listening to the media control keys which would make media not be able
362 * to be controlled via pressing media control keys. If we haven't started
363 * listening to the media control keys, then nothing would happen.
364 */
StopIfNeeded()365 void StopIfNeeded() {
366 MOZ_ASSERT(NS_IsMainThread());
367 if (!IsStarted()) {
368 // We have already been stopped, do not notify stop twice.
369 return;
370 }
371 NotifyMediaStoppedPlaying();
372 NotifyPlaybackStateChanged(MediaPlaybackState::eStopped);
373
374 // Remove ourselves from media agent, which would stop receiving event.
375 mControlAgent->RemoveReceiver(this);
376 mControlAgent = nullptr;
377 }
378
IsStarted() const379 bool IsStarted() const { return mState != MediaPlaybackState::eStopped; }
380
IsPlaying() const381 bool IsPlaying() const override {
382 return Owner() ? !Owner()->Paused() : false;
383 }
384
385 /**
386 * Following methods should only be used after starting listener.
387 */
NotifyMediaStartedPlaying()388 void NotifyMediaStartedPlaying() {
389 MOZ_ASSERT(NS_IsMainThread());
390 MOZ_ASSERT(IsStarted());
391 if (mState == MediaPlaybackState::eStarted ||
392 mState == MediaPlaybackState::ePaused) {
393 NotifyPlaybackStateChanged(MediaPlaybackState::ePlayed);
394 // If media is `inaudible` in the beginning, then we don't need to notify
395 // the state, because notifying `inaudible` should always come after
396 // notifying `audible`.
397 if (mIsOwnerAudible) {
398 NotifyAudibleStateChanged(MediaAudibleState::eAudible);
399 }
400 }
401 }
402
NotifyMediaStoppedPlaying()403 void NotifyMediaStoppedPlaying() {
404 MOZ_ASSERT(NS_IsMainThread());
405 MOZ_ASSERT(IsStarted());
406 if (mState == MediaPlaybackState::ePlayed) {
407 NotifyPlaybackStateChanged(MediaPlaybackState::ePaused);
408 // As media are going to be paused, so no sound is possible to be heard.
409 if (mIsOwnerAudible) {
410 NotifyAudibleStateChanged(MediaAudibleState::eInaudible);
411 }
412 }
413 }
414
415 // This method can be called before the listener starts, which would cache
416 // the audible state and update after the listener starts.
UpdateMediaAudibleState(bool aIsOwnerAudible)417 void UpdateMediaAudibleState(bool aIsOwnerAudible) {
418 MOZ_ASSERT(NS_IsMainThread());
419 if (mIsOwnerAudible == aIsOwnerAudible) {
420 return;
421 }
422 mIsOwnerAudible = aIsOwnerAudible;
423 MEDIACONTROL_LOG("Media becomes %s",
424 mIsOwnerAudible ? "audible" : "inaudible");
425 // If media hasn't started playing, it doesn't make sense to update media
426 // audible state. Therefore, in that case we would noitfy the audible state
427 // when media starts playing.
428 if (mState == MediaPlaybackState::ePlayed) {
429 NotifyAudibleStateChanged(mIsOwnerAudible
430 ? MediaAudibleState::eAudible
431 : MediaAudibleState::eInaudible);
432 }
433 }
434
SetPictureInPictureModeEnabled(bool aIsEnabled)435 void SetPictureInPictureModeEnabled(bool aIsEnabled) {
436 MOZ_ASSERT(NS_IsMainThread());
437 if (mIsPictureInPictureEnabled == aIsEnabled) {
438 return;
439 }
440 // PIP state changes might happen before the listener starts or stops where
441 // we haven't call `InitMediaAgent()` yet. Eg. Reset the PIP video's src,
442 // then cancel the PIP. In addition, not like playback and audible state
443 // which should be restricted to update via the same agent in order to keep
444 // those states correct in each `ContextMediaInfo`, PIP state can be updated
445 // through any browsing context, so we would use `ContentMediaAgent::Get()`
446 // directly to update PIP state.
447 mIsPictureInPictureEnabled = aIsEnabled;
448 if (RefPtr<IMediaInfoUpdater> updater =
449 ContentMediaAgent::Get(GetCurrentBrowsingContext())) {
450 updater->SetIsInPictureInPictureMode(mOwnerBrowsingContextId,
451 mIsPictureInPictureEnabled);
452 }
453 }
454
HandleMediaKey(MediaControlKey aKey)455 void HandleMediaKey(MediaControlKey aKey) override {
456 MOZ_ASSERT(NS_IsMainThread());
457 MOZ_ASSERT(IsStarted());
458 MEDIACONTROL_LOG("HandleEvent '%s'", ToMediaControlKeyStr(aKey));
459 if (aKey == MediaControlKey::Play) {
460 Owner()->Play();
461 } else if (aKey == MediaControlKey::Pause) {
462 Owner()->Pause();
463 } else {
464 MOZ_ASSERT(aKey == MediaControlKey::Stop,
465 "Not supported key for media element!");
466 Owner()->Pause();
467 StopIfNeeded();
468 }
469 }
470
UpdateOwnerBrowsingContextIfNeeded()471 void UpdateOwnerBrowsingContextIfNeeded() {
472 // Has not notified any information about the owner context yet.
473 if (!IsStarted()) {
474 return;
475 }
476
477 BrowsingContext* currentBC = GetCurrentBrowsingContext();
478 MOZ_ASSERT(currentBC);
479 // Still in the same browsing context, no need to update.
480 if (currentBC->Id() == mOwnerBrowsingContextId) {
481 return;
482 }
483 MEDIACONTROL_LOG("Change browsing context from %" PRIu64 " to %" PRIu64,
484 mOwnerBrowsingContextId, currentBC->Id());
485 // This situation would happen when we start a media in an original browsing
486 // context, then we move it to another browsing context, such as an iframe,
487 // so its owner browsing context would be changed. Therefore, we should
488 // reset the media status for the previous browsing context by calling
489 // `Stop()`, in which the listener would notify `ePaused` (if it's playing)
490 // and `eStop`. Then calls `Start()`, in which the listener would notify
491 // `eStart` to the new browsing context. If the media was playing before,
492 // we would also notify `ePlayed`.
493 bool wasInPlayingState = mState == MediaPlaybackState::ePlayed;
494 StopIfNeeded();
495 Start();
496 if (wasInPlayingState) {
497 NotifyMediaStartedPlaying();
498 }
499 }
500
501 private:
502 ~MediaControlKeyListener() = default;
503
504 // The media can be moved around different browsing contexts, so this context
505 // might be different from the one that we used to initialize
506 // `ContentMediaAgent`.
GetCurrentBrowsingContext() const507 BrowsingContext* GetCurrentBrowsingContext() const {
508 // Owner has been CCed, which would break the link of the weaker pointer.
509 if (!Owner()) {
510 return nullptr;
511 }
512 nsPIDOMWindowInner* window = Owner()->OwnerDoc()->GetInnerWindow();
513 return window ? window->GetBrowsingContext() : nullptr;
514 }
515
InitMediaAgent()516 bool InitMediaAgent() {
517 MOZ_ASSERT(NS_IsMainThread());
518 BrowsingContext* currentBC = GetCurrentBrowsingContext();
519 mControlAgent = ContentMediaAgent::Get(currentBC);
520 if (!mControlAgent) {
521 return false;
522 }
523 MOZ_ASSERT(currentBC);
524 mOwnerBrowsingContextId = currentBC->Id();
525 MEDIACONTROL_LOG("Init agent in browsing context %" PRIu64,
526 mOwnerBrowsingContextId);
527 mControlAgent->AddReceiver(this);
528 return true;
529 }
530
Owner() const531 HTMLMediaElement* Owner() const {
532 // `mElement` would be clear during CC unlinked, but it would only happen
533 // after stopping the listener.
534 MOZ_ASSERT(mElement || !IsStarted());
535 return mElement.get();
536 }
537
NotifyPlaybackStateChanged(MediaPlaybackState aState)538 void NotifyPlaybackStateChanged(MediaPlaybackState aState) {
539 MOZ_ASSERT(NS_IsMainThread());
540 MOZ_ASSERT(mControlAgent);
541 MEDIACONTROL_LOG("NotifyMediaState from state='%s' to state='%s'",
542 ToMediaPlaybackStateStr(mState),
543 ToMediaPlaybackStateStr(aState));
544 MOZ_ASSERT(mState != aState, "Should not notify same state again!");
545 mState = aState;
546 mControlAgent->NotifyMediaPlaybackChanged(mOwnerBrowsingContextId, mState);
547 }
548
NotifyAudibleStateChanged(MediaAudibleState aState)549 void NotifyAudibleStateChanged(MediaAudibleState aState) {
550 MOZ_ASSERT(NS_IsMainThread());
551 MOZ_ASSERT(IsStarted());
552 mControlAgent->NotifyMediaAudibleChanged(mOwnerBrowsingContextId, aState);
553 }
554
555 MediaPlaybackState mState = MediaPlaybackState::eStopped;
556 WeakPtr<HTMLMediaElement> mElement;
557 RefPtr<ContentMediaAgent> mControlAgent;
558 bool mIsPictureInPictureEnabled = false;
559 bool mIsOwnerAudible = false;
560 uint64_t mOwnerBrowsingContextId;
561 };
562
563 class HTMLMediaElement::MediaStreamTrackListener
564 : public DOMMediaStream::TrackListener {
565 public:
MediaStreamTrackListener(HTMLMediaElement * aElement)566 explicit MediaStreamTrackListener(HTMLMediaElement* aElement)
567 : mElement(aElement) {}
568
NotifyTrackAdded(const RefPtr<MediaStreamTrack> & aTrack)569 void NotifyTrackAdded(const RefPtr<MediaStreamTrack>& aTrack) override {
570 if (!mElement) {
571 return;
572 }
573 mElement->NotifyMediaStreamTrackAdded(aTrack);
574 }
575
NotifyTrackRemoved(const RefPtr<MediaStreamTrack> & aTrack)576 void NotifyTrackRemoved(const RefPtr<MediaStreamTrack>& aTrack) override {
577 if (!mElement) {
578 return;
579 }
580 mElement->NotifyMediaStreamTrackRemoved(aTrack);
581 }
582
OnActive()583 void OnActive() {
584 MOZ_ASSERT(mElement);
585
586 // mediacapture-main says:
587 // Note that once ended equals true the HTMLVideoElement will not play media
588 // even if new MediaStreamTracks are added to the MediaStream (causing it to
589 // return to the active state) unless autoplay is true or the web
590 // application restarts the element, e.g., by calling play().
591 //
592 // This is vague on exactly how to go from becoming active to playing, when
593 // autoplaying. However, per the media element spec, to play an autoplaying
594 // media element, we must load the source and reach readyState
595 // HAVE_ENOUGH_DATA [1]. Hence, a MediaStream being assigned to a media
596 // element and becoming active runs the load algorithm, so that it can
597 // eventually be played.
598 //
599 // [1]
600 // https://html.spec.whatwg.org/multipage/media.html#ready-states:event-media-play
601
602 LOG(LogLevel::Debug, ("%p, mSrcStream %p became active, checking if we "
603 "need to run the load algorithm",
604 mElement.get(), mElement->mSrcStream.get()));
605 if (!mElement->IsPlaybackEnded()) {
606 return;
607 }
608 if (!mElement->Autoplay()) {
609 return;
610 }
611 LOG(LogLevel::Info, ("%p, mSrcStream %p became active on autoplaying, "
612 "ended element. Reloading.",
613 mElement.get(), mElement->mSrcStream.get()));
614 mElement->DoLoad();
615 }
616
NotifyActive()617 void NotifyActive() override {
618 if (!mElement) {
619 return;
620 }
621
622 if (!mElement->IsVideo()) {
623 // Audio elements use NotifyAudible().
624 return;
625 }
626
627 OnActive();
628 }
629
NotifyAudible()630 void NotifyAudible() override {
631 if (!mElement) {
632 return;
633 }
634
635 if (mElement->IsVideo()) {
636 // Video elements use NotifyActive().
637 return;
638 }
639
640 OnActive();
641 }
642
OnInactive()643 void OnInactive() {
644 MOZ_ASSERT(mElement);
645
646 if (mElement->IsPlaybackEnded()) {
647 return;
648 }
649 LOG(LogLevel::Debug, ("%p, mSrcStream %p became inactive", mElement.get(),
650 mElement->mSrcStream.get()));
651
652 mElement->PlaybackEnded();
653 }
654
NotifyInactive()655 void NotifyInactive() override {
656 if (!mElement) {
657 return;
658 }
659
660 if (!mElement->IsVideo()) {
661 // Audio elements use NotifyInaudible().
662 return;
663 }
664
665 OnInactive();
666 }
667
NotifyInaudible()668 void NotifyInaudible() override {
669 if (!mElement) {
670 return;
671 }
672
673 if (mElement->IsVideo()) {
674 // Video elements use NotifyInactive().
675 return;
676 }
677
678 OnInactive();
679 }
680
681 protected:
682 const WeakPtr<HTMLMediaElement> mElement;
683 };
684
685 /**
686 * Helper class that manages audio and video outputs for all enabled tracks in a
687 * media element. It also manages calculating the current time when playing a
688 * MediaStream.
689 */
690 class HTMLMediaElement::MediaStreamRenderer
691 : public DOMMediaStream::TrackListener {
692 public:
693 NS_INLINE_DECL_REFCOUNTING(MediaStreamRenderer)
694
MediaStreamRenderer(AbstractThread * aMainThread,VideoFrameContainer * aVideoContainer,FirstFrameVideoOutput * aFirstFrameVideoOutput,void * aAudioOutputKey)695 MediaStreamRenderer(AbstractThread* aMainThread,
696 VideoFrameContainer* aVideoContainer,
697 FirstFrameVideoOutput* aFirstFrameVideoOutput,
698 void* aAudioOutputKey)
699 : mVideoContainer(aVideoContainer),
700 mAudioOutputKey(aAudioOutputKey),
701 mWatchManager(this, aMainThread),
702 mFirstFrameVideoOutput(aFirstFrameVideoOutput) {
703 if (mFirstFrameVideoOutput) {
704 mWatchManager.Watch(mFirstFrameVideoOutput->mFirstFrameRendered,
705 &MediaStreamRenderer::SetFirstFrameRendered);
706 }
707 }
708
Shutdown()709 void Shutdown() {
710 for (const auto& t : mAudioTracks.Clone()) {
711 if (t) {
712 RemoveTrack(t->AsAudioStreamTrack());
713 }
714 }
715 if (mVideoTrack) {
716 RemoveTrack(mVideoTrack->AsVideoStreamTrack());
717 }
718 mWatchManager.Shutdown();
719 mFirstFrameVideoOutput = nullptr;
720 }
721
UpdateGraphTime()722 void UpdateGraphTime() {
723 mGraphTime =
724 mGraphTimeDummy->mTrack->Graph()->CurrentTime() - *mGraphTimeOffset;
725 }
726
SetFirstFrameRendered()727 void SetFirstFrameRendered() {
728 if (!mFirstFrameVideoOutput) {
729 return;
730 }
731 if (mVideoTrack) {
732 mVideoTrack->AsVideoStreamTrack()->RemoveVideoOutput(
733 mFirstFrameVideoOutput);
734 }
735 mWatchManager.Unwatch(mFirstFrameVideoOutput->mFirstFrameRendered,
736 &MediaStreamRenderer::SetFirstFrameRendered);
737 mFirstFrameVideoOutput = nullptr;
738 }
739
SetProgressingCurrentTime(bool aProgress)740 void SetProgressingCurrentTime(bool aProgress) {
741 if (aProgress == mProgressingCurrentTime) {
742 return;
743 }
744
745 MOZ_DIAGNOSTIC_ASSERT(mGraphTimeDummy);
746 mProgressingCurrentTime = aProgress;
747 MediaTrackGraph* graph = mGraphTimeDummy->mTrack->Graph();
748 if (mProgressingCurrentTime) {
749 mGraphTimeOffset = Some(graph->CurrentTime().Ref() - mGraphTime);
750 mWatchManager.Watch(graph->CurrentTime(),
751 &MediaStreamRenderer::UpdateGraphTime);
752 } else {
753 mWatchManager.Unwatch(graph->CurrentTime(),
754 &MediaStreamRenderer::UpdateGraphTime);
755 }
756 }
757
Start()758 void Start() {
759 if (mRendering) {
760 return;
761 }
762
763 mRendering = true;
764
765 if (!mGraphTimeDummy) {
766 return;
767 }
768
769 for (const auto& t : mAudioTracks) {
770 if (t) {
771 if (mAudioOutputSink) {
772 t->AsAudioStreamTrack()->SetAudioOutputDevice(mAudioOutputKey,
773 mAudioOutputSink);
774 }
775 t->AsAudioStreamTrack()->AddAudioOutput(mAudioOutputKey);
776 t->AsAudioStreamTrack()->SetAudioOutputVolume(mAudioOutputKey,
777 mAudioOutputVolume);
778 }
779 }
780
781 if (mVideoTrack) {
782 mVideoTrack->AsVideoStreamTrack()->AddVideoOutput(mVideoContainer);
783 }
784 }
785
Stop()786 void Stop() {
787 if (!mRendering) {
788 return;
789 }
790
791 mRendering = false;
792
793 if (!mGraphTimeDummy) {
794 return;
795 }
796
797 for (const auto& t : mAudioTracks) {
798 if (t) {
799 t->AsAudioStreamTrack()->RemoveAudioOutput(mAudioOutputKey);
800 }
801 }
802
803 if (mVideoTrack) {
804 mVideoTrack->AsVideoStreamTrack()->RemoveVideoOutput(mVideoContainer);
805 }
806 }
807
SetAudioOutputVolume(float aVolume)808 void SetAudioOutputVolume(float aVolume) {
809 if (mAudioOutputVolume == aVolume) {
810 return;
811 }
812 mAudioOutputVolume = aVolume;
813 if (!mRendering) {
814 return;
815 }
816 for (const auto& t : mAudioTracks) {
817 if (t) {
818 t->AsAudioStreamTrack()->SetAudioOutputVolume(mAudioOutputKey,
819 mAudioOutputVolume);
820 }
821 }
822 }
823
SetAudioOutputDevice(AudioDeviceInfo * aSink)824 RefPtr<GenericPromise::AllPromiseType> SetAudioOutputDevice(
825 AudioDeviceInfo* aSink) {
826 MOZ_ASSERT(aSink);
827 MOZ_ASSERT(mAudioOutputSink != aSink);
828
829 mAudioOutputSink = aSink;
830
831 if (!mRendering) {
832 return GenericPromise::AllPromiseType::CreateAndResolve(nsTArray<bool>(),
833 __func__);
834 }
835
836 nsTArray<RefPtr<GenericPromise>> promises;
837 for (const auto& t : mAudioTracks) {
838 // SetAudioOutputDevice will create a new output MediaTrack, so the
839 // AudioOutput is removed for the current MediaTrack and re-added after
840 // the new MediaTrack has been created.
841 t->AsAudioStreamTrack()->RemoveAudioOutput(mAudioOutputKey);
842 promises.AppendElement(t->AsAudioStreamTrack()->SetAudioOutputDevice(
843 mAudioOutputKey, mAudioOutputSink));
844 t->AsAudioStreamTrack()->AddAudioOutput(mAudioOutputKey);
845 t->AsAudioStreamTrack()->SetAudioOutputVolume(mAudioOutputKey,
846 mAudioOutputVolume);
847 }
848 if (!promises.Length()) {
849 // Not active track, save it for later
850 return GenericPromise::AllPromiseType::CreateAndResolve(nsTArray<bool>(),
851 __func__);
852 }
853
854 return GenericPromise::All(GetCurrentSerialEventTarget(), promises);
855 }
856
AddTrack(AudioStreamTrack * aTrack)857 void AddTrack(AudioStreamTrack* aTrack) {
858 MOZ_DIAGNOSTIC_ASSERT(!mAudioTracks.Contains(aTrack));
859 mAudioTracks.AppendElement(aTrack);
860 EnsureGraphTimeDummy();
861 if (mRendering) {
862 if (mAudioOutputSink) {
863 aTrack->SetAudioOutputDevice(mAudioOutputKey, mAudioOutputSink);
864 }
865 aTrack->AddAudioOutput(mAudioOutputKey);
866 aTrack->SetAudioOutputVolume(mAudioOutputKey, mAudioOutputVolume);
867 }
868 }
AddTrack(VideoStreamTrack * aTrack)869 void AddTrack(VideoStreamTrack* aTrack) {
870 MOZ_DIAGNOSTIC_ASSERT(!mVideoTrack);
871 if (!mVideoContainer) {
872 return;
873 }
874 mVideoTrack = aTrack;
875 EnsureGraphTimeDummy();
876 if (mFirstFrameVideoOutput) {
877 // Add the first frame output even if we are rendering. It will only
878 // accept one frame. If we are rendering, then the main output will
879 // overwrite that with the same frame (and possibly more frames).
880 aTrack->AddVideoOutput(mFirstFrameVideoOutput);
881 }
882 if (mRendering) {
883 aTrack->AddVideoOutput(mVideoContainer);
884 }
885 }
886
RemoveTrack(AudioStreamTrack * aTrack)887 void RemoveTrack(AudioStreamTrack* aTrack) {
888 MOZ_DIAGNOSTIC_ASSERT(mAudioTracks.Contains(aTrack));
889 if (mRendering) {
890 aTrack->RemoveAudioOutput(mAudioOutputKey);
891 }
892 mAudioTracks.RemoveElement(aTrack);
893 }
RemoveTrack(VideoStreamTrack * aTrack)894 void RemoveTrack(VideoStreamTrack* aTrack) {
895 MOZ_DIAGNOSTIC_ASSERT(mVideoTrack == aTrack);
896 if (!mVideoContainer) {
897 return;
898 }
899 if (mFirstFrameVideoOutput) {
900 aTrack->RemoveVideoOutput(mFirstFrameVideoOutput);
901 }
902 if (mRendering) {
903 aTrack->RemoveVideoOutput(mVideoContainer);
904 }
905 mVideoTrack = nullptr;
906 }
907
CurrentTime() const908 double CurrentTime() const {
909 if (!mGraphTimeDummy) {
910 return 0.0;
911 }
912
913 return mGraphTimeDummy->mTrack->GraphImpl()->MediaTimeToSeconds(mGraphTime);
914 }
915
CurrentGraphTime()916 Watchable<GraphTime>& CurrentGraphTime() { return mGraphTime; }
917
918 // Set if we're rendering video.
919 const RefPtr<VideoFrameContainer> mVideoContainer;
920
921 // Set if we're rendering audio, nullptr otherwise.
922 void* const mAudioOutputKey;
923
924 private:
~MediaStreamRenderer()925 ~MediaStreamRenderer() { Shutdown(); }
926
EnsureGraphTimeDummy()927 void EnsureGraphTimeDummy() {
928 if (mGraphTimeDummy) {
929 return;
930 }
931
932 MediaTrackGraph* graph = nullptr;
933 for (const auto& t : mAudioTracks) {
934 if (t && !t->Ended()) {
935 graph = t->Graph();
936 break;
937 }
938 }
939
940 if (!graph && mVideoTrack && !mVideoTrack->Ended()) {
941 graph = mVideoTrack->Graph();
942 }
943
944 if (!graph) {
945 return;
946 }
947
948 // This dummy keeps `graph` alive and ensures access to it.
949 mGraphTimeDummy = MakeRefPtr<SharedDummyTrack>(
950 graph->CreateSourceTrack(MediaSegment::AUDIO));
951 }
952
953 // True when all tracks are being rendered, i.e., when the media element is
954 // playing.
955 bool mRendering = false;
956
957 // True while we're progressing mGraphTime. False otherwise.
958 bool mProgressingCurrentTime = false;
959
960 // The audio output volume for all audio tracks.
961 float mAudioOutputVolume = 1.0f;
962
963 // The sink device for all audio tracks.
964 RefPtr<AudioDeviceInfo> mAudioOutputSink;
965
966 // WatchManager for mGraphTime.
967 WatchManager<MediaStreamRenderer> mWatchManager;
968
969 // A dummy MediaTrack to guarantee a MediaTrackGraph is kept alive while
970 // we're actively rendering, so we can track the graph's current time. Set
971 // when the first track is added, never unset.
972 RefPtr<SharedDummyTrack> mGraphTimeDummy;
973
974 // Watchable that relays the graph's currentTime updates to the media element
975 // only while we're rendering. This is the current time of the rendering in
976 // GraphTime units.
977 Watchable<GraphTime> mGraphTime = {0, "MediaStreamRenderer::mGraphTime"};
978
979 // Nothing until a track has been added. Then, the current GraphTime at the
980 // time when we were last Start()ed.
981 Maybe<GraphTime> mGraphTimeOffset;
982
983 // Currently enabled (and rendered) audio tracks.
984 nsTArray<WeakPtr<MediaStreamTrack>> mAudioTracks;
985
986 // Currently selected (and rendered) video track.
987 WeakPtr<MediaStreamTrack> mVideoTrack;
988
989 // Holds a reference to the first-frame-getting video output attached to
990 // mVideoTrack. Set by the constructor, unset when the media element tells us
991 // it has rendered the first frame.
992 RefPtr<FirstFrameVideoOutput> mFirstFrameVideoOutput;
993 };
994
995 class HTMLMediaElement::MediaElementTrackSource
996 : public MediaStreamTrackSource,
997 public MediaStreamTrackSource::Sink,
998 public MediaStreamTrackConsumer {
999 public:
1000 NS_DECL_ISUPPORTS_INHERITED
NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaElementTrackSource,MediaStreamTrackSource)1001 NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaElementTrackSource,
1002 MediaStreamTrackSource)
1003
1004 /* MediaDecoder track source */
1005 MediaElementTrackSource(nsISerialEventTarget* aMainThreadEventTarget,
1006 ProcessedMediaTrack* aTrack, nsIPrincipal* aPrincipal,
1007 OutputMuteState aMuteState, bool aHasAlpha)
1008 : MediaStreamTrackSource(aPrincipal, nsString()),
1009 mMainThreadEventTarget(aMainThreadEventTarget),
1010 mTrack(aTrack),
1011 mIntendedElementMuteState(aMuteState),
1012 mElementMuteState(aMuteState),
1013 mMediaDecoderHasAlpha(Some(aHasAlpha)) {
1014 MOZ_ASSERT(mTrack);
1015 }
1016
1017 /* MediaStream track source */
MediaElementTrackSource(nsISerialEventTarget * aMainThreadEventTarget,MediaStreamTrack * aCapturedTrack,MediaStreamTrackSource * aCapturedTrackSource,ProcessedMediaTrack * aTrack,MediaInputPort * aPort,OutputMuteState aMuteState)1018 MediaElementTrackSource(nsISerialEventTarget* aMainThreadEventTarget,
1019 MediaStreamTrack* aCapturedTrack,
1020 MediaStreamTrackSource* aCapturedTrackSource,
1021 ProcessedMediaTrack* aTrack, MediaInputPort* aPort,
1022 OutputMuteState aMuteState)
1023 : MediaStreamTrackSource(aCapturedTrackSource->GetPrincipal(),
1024 nsString()),
1025 mMainThreadEventTarget(aMainThreadEventTarget),
1026 mCapturedTrack(aCapturedTrack),
1027 mCapturedTrackSource(aCapturedTrackSource),
1028 mTrack(aTrack),
1029 mPort(aPort),
1030 mIntendedElementMuteState(aMuteState),
1031 mElementMuteState(aMuteState) {
1032 MOZ_ASSERT(mTrack);
1033 MOZ_ASSERT(mCapturedTrack);
1034 MOZ_ASSERT(mCapturedTrackSource);
1035 MOZ_ASSERT(mPort);
1036
1037 mCapturedTrack->AddConsumer(this);
1038 mCapturedTrackSource->RegisterSink(this);
1039 }
1040
SetEnabled(bool aEnabled)1041 void SetEnabled(bool aEnabled) {
1042 if (!mTrack) {
1043 return;
1044 }
1045 mTrack->SetDisabledTrackMode(aEnabled ? DisabledTrackMode::ENABLED
1046 : DisabledTrackMode::SILENCE_FREEZE);
1047 }
1048
SetPrincipal(RefPtr<nsIPrincipal> aPrincipal)1049 void SetPrincipal(RefPtr<nsIPrincipal> aPrincipal) {
1050 mPrincipal = std::move(aPrincipal);
1051 MediaStreamTrackSource::PrincipalChanged();
1052 }
1053
SetMutedByElement(OutputMuteState aMuteState)1054 void SetMutedByElement(OutputMuteState aMuteState) {
1055 if (mIntendedElementMuteState == aMuteState) {
1056 return;
1057 }
1058 mIntendedElementMuteState = aMuteState;
1059 mMainThreadEventTarget->Dispatch(NS_NewRunnableFunction(
1060 "MediaElementTrackSource::SetMutedByElement",
1061 [self = RefPtr<MediaElementTrackSource>(this), this, aMuteState] {
1062 mElementMuteState = aMuteState;
1063 MediaStreamTrackSource::MutedChanged(Muted());
1064 }));
1065 }
1066
Destroy()1067 void Destroy() override {
1068 if (mCapturedTrack) {
1069 mCapturedTrack->RemoveConsumer(this);
1070 mCapturedTrack = nullptr;
1071 }
1072 if (mCapturedTrackSource) {
1073 mCapturedTrackSource->UnregisterSink(this);
1074 mCapturedTrackSource = nullptr;
1075 }
1076 if (mTrack && !mTrack->IsDestroyed()) {
1077 mTrack->Destroy();
1078 }
1079 if (mPort) {
1080 mPort->Destroy();
1081 mPort = nullptr;
1082 }
1083 }
1084
GetMediaSource() const1085 MediaSourceEnum GetMediaSource() const override {
1086 return MediaSourceEnum::Other;
1087 }
1088
Stop()1089 void Stop() override {
1090 // Do nothing. There may appear new output streams
1091 // that need tracks sourced from this source, so we
1092 // cannot destroy things yet.
1093 }
1094
1095 /**
1096 * Do not keep the track source alive. The source lifetime is controlled by
1097 * its associated tracks.
1098 */
KeepsSourceAlive() const1099 bool KeepsSourceAlive() const override { return false; }
1100
1101 /**
1102 * Do not keep the track source on. It is controlled by its associated tracks.
1103 */
Enabled() const1104 bool Enabled() const override { return false; }
1105
Disable()1106 void Disable() override {}
1107
Enable()1108 void Enable() override {}
1109
PrincipalChanged()1110 void PrincipalChanged() override {
1111 if (!mCapturedTrackSource) {
1112 // This could happen during shutdown.
1113 return;
1114 }
1115
1116 SetPrincipal(mCapturedTrackSource->GetPrincipal());
1117 }
1118
MutedChanged(bool aNewState)1119 void MutedChanged(bool aNewState) override {
1120 MediaStreamTrackSource::MutedChanged(Muted());
1121 }
1122
OverrideEnded()1123 void OverrideEnded() override {
1124 Destroy();
1125 MediaStreamTrackSource::OverrideEnded();
1126 }
1127
NotifyEnabledChanged(MediaStreamTrack * aTrack,bool aEnabled)1128 void NotifyEnabledChanged(MediaStreamTrack* aTrack, bool aEnabled) override {
1129 MediaStreamTrackSource::MutedChanged(Muted());
1130 }
1131
Muted() const1132 bool Muted() const {
1133 return mElementMuteState == OutputMuteState::Muted ||
1134 (mCapturedTrack &&
1135 (mCapturedTrack->Muted() || !mCapturedTrack->Enabled()));
1136 }
1137
HasAlpha() const1138 bool HasAlpha() const override {
1139 if (mCapturedTrack) {
1140 return mCapturedTrack->AsVideoStreamTrack()
1141 ? mCapturedTrack->AsVideoStreamTrack()->HasAlpha()
1142 : false;
1143 }
1144 return mMediaDecoderHasAlpha.valueOr(false);
1145 }
1146
Track() const1147 ProcessedMediaTrack* Track() const { return mTrack; }
1148
1149 private:
~MediaElementTrackSource()1150 virtual ~MediaElementTrackSource() { Destroy(); };
1151
1152 const RefPtr<nsISerialEventTarget> mMainThreadEventTarget;
1153 RefPtr<MediaStreamTrack> mCapturedTrack;
1154 RefPtr<MediaStreamTrackSource> mCapturedTrackSource;
1155 const RefPtr<ProcessedMediaTrack> mTrack;
1156 RefPtr<MediaInputPort> mPort;
1157 // The mute state as intended by the media element.
1158 OutputMuteState mIntendedElementMuteState;
1159 // The mute state as applied to this track source. It is applied async, so
1160 // needs to be tracked separately from the intended state.
1161 OutputMuteState mElementMuteState;
1162 // Some<bool> if this is a MediaDecoder track source.
1163 const Maybe<bool> mMediaDecoderHasAlpha;
1164 };
1165
OutputMediaStream(RefPtr<DOMMediaStream> aStream,bool aCapturingAudioOnly,bool aFinishWhenEnded)1166 HTMLMediaElement::OutputMediaStream::OutputMediaStream(
1167 RefPtr<DOMMediaStream> aStream, bool aCapturingAudioOnly,
1168 bool aFinishWhenEnded)
1169 : mStream(std::move(aStream)),
1170 mCapturingAudioOnly(aCapturingAudioOnly),
1171 mFinishWhenEnded(aFinishWhenEnded) {}
1172 HTMLMediaElement::OutputMediaStream::~OutputMediaStream() = default;
1173
ImplCycleCollectionTraverse(nsCycleCollectionTraversalCallback & aCallback,HTMLMediaElement::OutputMediaStream & aField,const char * aName,uint32_t aFlags)1174 void ImplCycleCollectionTraverse(nsCycleCollectionTraversalCallback& aCallback,
1175 HTMLMediaElement::OutputMediaStream& aField,
1176 const char* aName, uint32_t aFlags) {
1177 ImplCycleCollectionTraverse(aCallback, aField.mStream, "mStream", aFlags);
1178 ImplCycleCollectionTraverse(aCallback, aField.mLiveTracks, "mLiveTracks",
1179 aFlags);
1180 ImplCycleCollectionTraverse(aCallback, aField.mFinishWhenEndedLoadingSrc,
1181 "mFinishWhenEndedLoadingSrc", aFlags);
1182 ImplCycleCollectionTraverse(aCallback, aField.mFinishWhenEndedAttrStream,
1183 "mFinishWhenEndedAttrStream", aFlags);
1184 ImplCycleCollectionTraverse(aCallback, aField.mFinishWhenEndedMediaSource,
1185 "mFinishWhenEndedMediaSource", aFlags);
1186 }
1187
ImplCycleCollectionUnlink(HTMLMediaElement::OutputMediaStream & aField)1188 void ImplCycleCollectionUnlink(HTMLMediaElement::OutputMediaStream& aField) {
1189 ImplCycleCollectionUnlink(aField.mStream);
1190 ImplCycleCollectionUnlink(aField.mLiveTracks);
1191 ImplCycleCollectionUnlink(aField.mFinishWhenEndedLoadingSrc);
1192 ImplCycleCollectionUnlink(aField.mFinishWhenEndedAttrStream);
1193 ImplCycleCollectionUnlink(aField.mFinishWhenEndedMediaSource);
1194 }
1195
1196 NS_IMPL_ADDREF_INHERITED(HTMLMediaElement::MediaElementTrackSource,
1197 MediaStreamTrackSource)
1198 NS_IMPL_RELEASE_INHERITED(HTMLMediaElement::MediaElementTrackSource,
1199 MediaStreamTrackSource)
1200 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(
1201 HTMLMediaElement::MediaElementTrackSource)
1202 NS_INTERFACE_MAP_END_INHERITING(MediaStreamTrackSource)
1203 NS_IMPL_CYCLE_COLLECTION_CLASS(HTMLMediaElement::MediaElementTrackSource)
1204 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(
1205 HTMLMediaElement::MediaElementTrackSource, MediaStreamTrackSource)
1206 tmp->Destroy();
1207 NS_IMPL_CYCLE_COLLECTION_UNLINK(mCapturedTrack)
1208 NS_IMPL_CYCLE_COLLECTION_UNLINK(mCapturedTrackSource)
1209 NS_IMPL_CYCLE_COLLECTION_UNLINK_END
1210 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(
1211 HTMLMediaElement::MediaElementTrackSource, MediaStreamTrackSource)
1212 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mCapturedTrack)
1213 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mCapturedTrackSource)
1214 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
1215
1216 /**
1217 * There is a reference cycle involving this class: MediaLoadListener
1218 * holds a reference to the HTMLMediaElement, which holds a reference
1219 * to an nsIChannel, which holds a reference to this listener.
1220 * We break the reference cycle in OnStartRequest by clearing mElement.
1221 */
1222 class HTMLMediaElement::MediaLoadListener final
1223 : public nsIStreamListener,
1224 public nsIChannelEventSink,
1225 public nsIInterfaceRequestor,
1226 public nsIObserver,
1227 public nsIThreadRetargetableStreamListener {
1228 ~MediaLoadListener() = default;
1229
1230 NS_DECL_THREADSAFE_ISUPPORTS
1231 NS_DECL_NSIREQUESTOBSERVER
1232 NS_DECL_NSISTREAMLISTENER
1233 NS_DECL_NSICHANNELEVENTSINK
1234 NS_DECL_NSIOBSERVER
1235 NS_DECL_NSIINTERFACEREQUESTOR
1236 NS_DECL_NSITHREADRETARGETABLESTREAMLISTENER
1237
1238 public:
MediaLoadListener(HTMLMediaElement * aElement)1239 explicit MediaLoadListener(HTMLMediaElement* aElement)
1240 : mElement(aElement), mLoadID(aElement->GetCurrentLoadID()) {
1241 MOZ_ASSERT(mElement, "Must pass an element to call back");
1242 }
1243
1244 private:
1245 RefPtr<HTMLMediaElement> mElement;
1246 nsCOMPtr<nsIStreamListener> mNextListener;
1247 const uint32_t mLoadID;
1248 };
1249
NS_IMPL_ISUPPORTS(HTMLMediaElement::MediaLoadListener,nsIRequestObserver,nsIStreamListener,nsIChannelEventSink,nsIInterfaceRequestor,nsIObserver,nsIThreadRetargetableStreamListener)1250 NS_IMPL_ISUPPORTS(HTMLMediaElement::MediaLoadListener, nsIRequestObserver,
1251 nsIStreamListener, nsIChannelEventSink, nsIInterfaceRequestor,
1252 nsIObserver, nsIThreadRetargetableStreamListener)
1253
1254 NS_IMETHODIMP
1255 HTMLMediaElement::MediaLoadListener::Observe(nsISupports* aSubject,
1256 const char* aTopic,
1257 const char16_t* aData) {
1258 nsContentUtils::UnregisterShutdownObserver(this);
1259
1260 // Clear mElement to break cycle so we don't leak on shutdown
1261 mElement = nullptr;
1262 return NS_OK;
1263 }
1264
1265 NS_IMETHODIMP
OnStartRequest(nsIRequest * aRequest)1266 HTMLMediaElement::MediaLoadListener::OnStartRequest(nsIRequest* aRequest) {
1267 nsContentUtils::UnregisterShutdownObserver(this);
1268
1269 if (!mElement) {
1270 // We've been notified by the shutdown observer, and are shutting down.
1271 return NS_BINDING_ABORTED;
1272 }
1273
1274 // The element is only needed until we've had a chance to call
1275 // InitializeDecoderForChannel. So make sure mElement is cleared here.
1276 RefPtr<HTMLMediaElement> element;
1277 element.swap(mElement);
1278
1279 if (mLoadID != element->GetCurrentLoadID()) {
1280 // The channel has been cancelled before we had a chance to create
1281 // a decoder. Abort, don't dispatch an "error" event, as the new load
1282 // may not be in an error state.
1283 return NS_BINDING_ABORTED;
1284 }
1285
1286 // Don't continue to load if the request failed or has been canceled.
1287 nsresult status;
1288 nsresult rv = aRequest->GetStatus(&status);
1289 NS_ENSURE_SUCCESS(rv, rv);
1290 if (NS_FAILED(status)) {
1291 if (element) {
1292 // Handle media not loading error because source was a tracking URL (or
1293 // fingerprinting, cryptomining, etc).
1294 // We make a note of this media node by including it in a dedicated
1295 // array of blocked tracking nodes under its parent document.
1296 if (net::UrlClassifierFeatureFactory::IsClassifierBlockingErrorCode(
1297 status)) {
1298 element->OwnerDoc()->AddBlockedNodeByClassifier(element);
1299 }
1300 element->NotifyLoadError(
1301 nsPrintfCString("%u: %s", uint32_t(status), "Request failed"));
1302 }
1303 return status;
1304 }
1305
1306 nsCOMPtr<nsIHttpChannel> hc = do_QueryInterface(aRequest);
1307 bool succeeded;
1308 if (hc && NS_SUCCEEDED(hc->GetRequestSucceeded(&succeeded)) && !succeeded) {
1309 uint32_t responseStatus = 0;
1310 Unused << hc->GetResponseStatus(&responseStatus);
1311 nsAutoCString statusText;
1312 Unused << hc->GetResponseStatusText(statusText);
1313 element->NotifyLoadError(
1314 nsPrintfCString("%u: %s", responseStatus, statusText.get()));
1315
1316 nsAutoString code;
1317 code.AppendInt(responseStatus);
1318 nsAutoString src;
1319 element->GetCurrentSrc(src);
1320 AutoTArray<nsString, 2> params = {code, src};
1321 element->ReportLoadError("MediaLoadHttpError", params);
1322 return NS_BINDING_ABORTED;
1323 }
1324
1325 nsCOMPtr<nsIChannel> channel = do_QueryInterface(aRequest);
1326 if (channel &&
1327 NS_SUCCEEDED(rv = element->InitializeDecoderForChannel(
1328 channel, getter_AddRefs(mNextListener))) &&
1329 mNextListener) {
1330 rv = mNextListener->OnStartRequest(aRequest);
1331 } else {
1332 // If InitializeDecoderForChannel() returned an error, fire a network error.
1333 if (NS_FAILED(rv) && !mNextListener) {
1334 // Load failed, attempt to load the next candidate resource. If there
1335 // are none, this will trigger a MEDIA_ERR_SRC_NOT_SUPPORTED error.
1336 element->NotifyLoadError("Failed to init decoder"_ns);
1337 }
1338 // If InitializeDecoderForChannel did not return a listener (but may
1339 // have otherwise succeeded), we abort the connection since we aren't
1340 // interested in keeping the channel alive ourselves.
1341 rv = NS_BINDING_ABORTED;
1342 }
1343
1344 return rv;
1345 }
1346
1347 NS_IMETHODIMP
OnStopRequest(nsIRequest * aRequest,nsresult aStatus)1348 HTMLMediaElement::MediaLoadListener::OnStopRequest(nsIRequest* aRequest,
1349 nsresult aStatus) {
1350 if (mNextListener) {
1351 return mNextListener->OnStopRequest(aRequest, aStatus);
1352 }
1353 return NS_OK;
1354 }
1355
1356 NS_IMETHODIMP
OnDataAvailable(nsIRequest * aRequest,nsIInputStream * aStream,uint64_t aOffset,uint32_t aCount)1357 HTMLMediaElement::MediaLoadListener::OnDataAvailable(nsIRequest* aRequest,
1358 nsIInputStream* aStream,
1359 uint64_t aOffset,
1360 uint32_t aCount) {
1361 if (!mNextListener) {
1362 NS_ERROR(
1363 "Must have a chained listener; OnStartRequest should have "
1364 "canceled this request");
1365 return NS_BINDING_ABORTED;
1366 }
1367 return mNextListener->OnDataAvailable(aRequest, aStream, aOffset, aCount);
1368 }
1369
1370 NS_IMETHODIMP
AsyncOnChannelRedirect(nsIChannel * aOldChannel,nsIChannel * aNewChannel,uint32_t aFlags,nsIAsyncVerifyRedirectCallback * cb)1371 HTMLMediaElement::MediaLoadListener::AsyncOnChannelRedirect(
1372 nsIChannel* aOldChannel, nsIChannel* aNewChannel, uint32_t aFlags,
1373 nsIAsyncVerifyRedirectCallback* cb) {
1374 // TODO is this really correct?? See bug #579329.
1375 if (mElement) {
1376 mElement->OnChannelRedirect(aOldChannel, aNewChannel, aFlags);
1377 }
1378 nsCOMPtr<nsIChannelEventSink> sink = do_QueryInterface(mNextListener);
1379 if (sink) {
1380 return sink->AsyncOnChannelRedirect(aOldChannel, aNewChannel, aFlags, cb);
1381 }
1382 cb->OnRedirectVerifyCallback(NS_OK);
1383 return NS_OK;
1384 }
1385
1386 NS_IMETHODIMP
CheckListenerChain()1387 HTMLMediaElement::MediaLoadListener::CheckListenerChain() {
1388 MOZ_ASSERT(mNextListener);
1389 nsCOMPtr<nsIThreadRetargetableStreamListener> retargetable =
1390 do_QueryInterface(mNextListener);
1391 if (retargetable) {
1392 return retargetable->CheckListenerChain();
1393 }
1394 return NS_ERROR_NO_INTERFACE;
1395 }
1396
1397 NS_IMETHODIMP
GetInterface(const nsIID & aIID,void ** aResult)1398 HTMLMediaElement::MediaLoadListener::GetInterface(const nsIID& aIID,
1399 void** aResult) {
1400 return QueryInterface(aIID, aResult);
1401 }
1402
ReportLoadError(const char * aMsg,const nsTArray<nsString> & aParams)1403 void HTMLMediaElement::ReportLoadError(const char* aMsg,
1404 const nsTArray<nsString>& aParams) {
1405 ReportToConsole(nsIScriptError::warningFlag, aMsg, aParams);
1406 }
1407
ReportToConsole(uint32_t aErrorFlags,const char * aMsg,const nsTArray<nsString> & aParams) const1408 void HTMLMediaElement::ReportToConsole(
1409 uint32_t aErrorFlags, const char* aMsg,
1410 const nsTArray<nsString>& aParams) const {
1411 nsContentUtils::ReportToConsole(aErrorFlags, "Media"_ns, OwnerDoc(),
1412 nsContentUtils::eDOM_PROPERTIES, aMsg,
1413 aParams);
1414 }
1415
1416 class HTMLMediaElement::AudioChannelAgentCallback final
1417 : public nsIAudioChannelAgentCallback {
1418 public:
1419 NS_DECL_CYCLE_COLLECTING_ISUPPORTS
NS_DECL_CYCLE_COLLECTION_CLASS(AudioChannelAgentCallback)1420 NS_DECL_CYCLE_COLLECTION_CLASS(AudioChannelAgentCallback)
1421
1422 explicit AudioChannelAgentCallback(HTMLMediaElement* aOwner)
1423 : mOwner(aOwner),
1424 mAudioChannelVolume(1.0),
1425 mPlayingThroughTheAudioChannel(false),
1426 mIsOwnerAudible(IsOwnerAudible()),
1427 mIsShutDown(false) {
1428 MOZ_ASSERT(mOwner);
1429 MaybeCreateAudioChannelAgent();
1430 }
1431
UpdateAudioChannelPlayingState()1432 void UpdateAudioChannelPlayingState() {
1433 MOZ_ASSERT(!mIsShutDown);
1434 bool playingThroughTheAudioChannel = IsPlayingThroughTheAudioChannel();
1435
1436 if (playingThroughTheAudioChannel != mPlayingThroughTheAudioChannel) {
1437 if (!MaybeCreateAudioChannelAgent()) {
1438 return;
1439 }
1440
1441 mPlayingThroughTheAudioChannel = playingThroughTheAudioChannel;
1442 if (mPlayingThroughTheAudioChannel) {
1443 StartAudioChannelAgent();
1444 } else {
1445 StopAudioChanelAgent();
1446 }
1447 }
1448 }
1449
NotifyPlayStateChanged()1450 void NotifyPlayStateChanged() {
1451 MOZ_ASSERT(!mIsShutDown);
1452 UpdateAudioChannelPlayingState();
1453 }
1454
WindowVolumeChanged(float aVolume,bool aMuted)1455 NS_IMETHODIMP WindowVolumeChanged(float aVolume, bool aMuted) override {
1456 MOZ_ASSERT(mAudioChannelAgent);
1457
1458 MOZ_LOG(
1459 AudioChannelService::GetAudioChannelLog(), LogLevel::Debug,
1460 ("HTMLMediaElement::AudioChannelAgentCallback, WindowVolumeChanged, "
1461 "this = %p, aVolume = %f, aMuted = %s\n",
1462 this, aVolume, aMuted ? "true" : "false"));
1463
1464 if (mAudioChannelVolume != aVolume) {
1465 mAudioChannelVolume = aVolume;
1466 mOwner->SetVolumeInternal();
1467 }
1468
1469 const uint32_t muted = mOwner->mMuted;
1470 if (aMuted && !mOwner->ComputedMuted()) {
1471 mOwner->SetMutedInternal(muted | MUTED_BY_AUDIO_CHANNEL);
1472 } else if (!aMuted && mOwner->ComputedMuted()) {
1473 mOwner->SetMutedInternal(muted & ~MUTED_BY_AUDIO_CHANNEL);
1474 }
1475
1476 return NS_OK;
1477 }
1478
WindowSuspendChanged(SuspendTypes aSuspend)1479 NS_IMETHODIMP WindowSuspendChanged(SuspendTypes aSuspend) override {
1480 // Currently this method is only be used for delaying autoplay, and we've
1481 // separated related codes to `MediaPlaybackDelayPolicy`.
1482 return NS_OK;
1483 }
1484
WindowAudioCaptureChanged(bool aCapture)1485 NS_IMETHODIMP WindowAudioCaptureChanged(bool aCapture) override {
1486 MOZ_ASSERT(mAudioChannelAgent);
1487 AudioCaptureTrackChangeIfNeeded();
1488 return NS_OK;
1489 }
1490
AudioCaptureTrackChangeIfNeeded()1491 void AudioCaptureTrackChangeIfNeeded() {
1492 MOZ_ASSERT(!mIsShutDown);
1493 if (!IsPlayingStarted()) {
1494 return;
1495 }
1496
1497 MOZ_ASSERT(mAudioChannelAgent);
1498 bool isCapturing = mAudioChannelAgent->IsWindowAudioCapturingEnabled();
1499 mOwner->AudioCaptureTrackChange(isCapturing);
1500 }
1501
NotifyAudioPlaybackChanged(AudibleChangedReasons aReason)1502 void NotifyAudioPlaybackChanged(AudibleChangedReasons aReason) {
1503 MOZ_ASSERT(!mIsShutDown);
1504 AudibleState newAudibleState = IsOwnerAudible();
1505 MOZ_LOG(AudioChannelService::GetAudioChannelLog(), LogLevel::Debug,
1506 ("HTMLMediaElement::AudioChannelAgentCallback, "
1507 "NotifyAudioPlaybackChanged, this=%p, current=%s, new=%s",
1508 this, AudibleStateToStr(mIsOwnerAudible),
1509 AudibleStateToStr(newAudibleState)));
1510 if (mIsOwnerAudible == newAudibleState) {
1511 return;
1512 }
1513
1514 mIsOwnerAudible = newAudibleState;
1515 if (IsPlayingStarted()) {
1516 mAudioChannelAgent->NotifyStartedAudible(mIsOwnerAudible, aReason);
1517 }
1518 }
1519
Shutdown()1520 void Shutdown() {
1521 MOZ_ASSERT(!mIsShutDown);
1522 if (mAudioChannelAgent && mAudioChannelAgent->IsPlayingStarted()) {
1523 StopAudioChanelAgent();
1524 }
1525 mAudioChannelAgent = nullptr;
1526 mIsShutDown = true;
1527 }
1528
GetEffectiveVolume() const1529 float GetEffectiveVolume() const {
1530 MOZ_ASSERT(!mIsShutDown);
1531 return mOwner->Volume() * mAudioChannelVolume;
1532 }
1533
1534 private:
~AudioChannelAgentCallback()1535 ~AudioChannelAgentCallback() { MOZ_ASSERT(mIsShutDown); };
1536
MaybeCreateAudioChannelAgent()1537 bool MaybeCreateAudioChannelAgent() {
1538 if (mAudioChannelAgent) {
1539 return true;
1540 }
1541
1542 mAudioChannelAgent = new AudioChannelAgent();
1543 nsresult rv =
1544 mAudioChannelAgent->Init(mOwner->OwnerDoc()->GetInnerWindow(), this);
1545 if (NS_WARN_IF(NS_FAILED(rv))) {
1546 mAudioChannelAgent = nullptr;
1547 MOZ_LOG(
1548 AudioChannelService::GetAudioChannelLog(), LogLevel::Debug,
1549 ("HTMLMediaElement::AudioChannelAgentCallback, Fail to initialize "
1550 "the audio channel agent, this = %p\n",
1551 this));
1552 return false;
1553 }
1554
1555 return true;
1556 }
1557
StartAudioChannelAgent()1558 void StartAudioChannelAgent() {
1559 MOZ_ASSERT(mAudioChannelAgent);
1560 MOZ_ASSERT(!mAudioChannelAgent->IsPlayingStarted());
1561 if (NS_WARN_IF(NS_FAILED(
1562 mAudioChannelAgent->NotifyStartedPlaying(IsOwnerAudible())))) {
1563 return;
1564 }
1565 mAudioChannelAgent->PullInitialUpdate();
1566 }
1567
StopAudioChanelAgent()1568 void StopAudioChanelAgent() {
1569 MOZ_ASSERT(mAudioChannelAgent);
1570 MOZ_ASSERT(mAudioChannelAgent->IsPlayingStarted());
1571 mAudioChannelAgent->NotifyStoppedPlaying();
1572 // If we have started audio capturing before, we have to tell media element
1573 // to clear the output capturing track.
1574 mOwner->AudioCaptureTrackChange(false);
1575 }
1576
IsPlayingStarted()1577 bool IsPlayingStarted() {
1578 if (MaybeCreateAudioChannelAgent()) {
1579 return mAudioChannelAgent->IsPlayingStarted();
1580 }
1581 return false;
1582 }
1583
IsOwnerAudible() const1584 AudibleState IsOwnerAudible() const {
1585 // paused media doesn't produce any sound.
1586 if (mOwner->mPaused) {
1587 return AudibleState::eNotAudible;
1588 }
1589 return mOwner->IsAudible() ? AudibleState::eAudible
1590 : AudibleState::eNotAudible;
1591 }
1592
IsPlayingThroughTheAudioChannel() const1593 bool IsPlayingThroughTheAudioChannel() const {
1594 // If we have an error, we are not playing.
1595 if (mOwner->GetError()) {
1596 return false;
1597 }
1598
1599 // We should consider any bfcached page or inactive document as non-playing.
1600 if (!mOwner->OwnerDoc()->IsActive()) {
1601 return false;
1602 }
1603
1604 // Media is suspended by the docshell.
1605 if (mOwner->ShouldBeSuspendedByInactiveDocShell()) {
1606 return false;
1607 }
1608
1609 // Are we paused
1610 if (mOwner->mPaused) {
1611 return false;
1612 }
1613
1614 // No audio track
1615 if (!mOwner->HasAudio()) {
1616 return false;
1617 }
1618
1619 // A loop always is playing
1620 if (mOwner->HasAttr(kNameSpaceID_None, nsGkAtoms::loop)) {
1621 return true;
1622 }
1623
1624 // If we are actually playing...
1625 if (mOwner->IsCurrentlyPlaying()) {
1626 return true;
1627 }
1628
1629 // If we are playing an external stream.
1630 if (mOwner->mSrcAttrStream) {
1631 return true;
1632 }
1633
1634 return false;
1635 }
1636
1637 RefPtr<AudioChannelAgent> mAudioChannelAgent;
1638 HTMLMediaElement* mOwner;
1639
1640 // The audio channel volume
1641 float mAudioChannelVolume;
1642 // Is this media element playing?
1643 bool mPlayingThroughTheAudioChannel;
1644 // Indicate whether media element is audible for users.
1645 AudibleState mIsOwnerAudible;
1646 bool mIsShutDown;
1647 };
1648
1649 NS_IMPL_CYCLE_COLLECTION_CLASS(HTMLMediaElement::AudioChannelAgentCallback)
1650
1651 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(
1652 HTMLMediaElement::AudioChannelAgentCallback)
1653 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mAudioChannelAgent)
1654 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
1655
1656 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(
1657 HTMLMediaElement::AudioChannelAgentCallback)
1658 NS_IMPL_CYCLE_COLLECTION_UNLINK(mAudioChannelAgent)
1659 NS_IMPL_CYCLE_COLLECTION_UNLINK_END
1660
1661 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(
1662 HTMLMediaElement::AudioChannelAgentCallback)
1663 NS_INTERFACE_MAP_ENTRY(nsIAudioChannelAgentCallback)
1664 NS_INTERFACE_MAP_END
1665
1666 NS_IMPL_CYCLE_COLLECTING_ADDREF(HTMLMediaElement::AudioChannelAgentCallback)
1667 NS_IMPL_CYCLE_COLLECTING_RELEASE(HTMLMediaElement::AudioChannelAgentCallback)
1668
1669 class HTMLMediaElement::ChannelLoader final {
1670 public:
1671 NS_INLINE_DECL_REFCOUNTING(ChannelLoader);
1672
LoadInternal(HTMLMediaElement * aElement)1673 void LoadInternal(HTMLMediaElement* aElement) {
1674 if (mCancelled) {
1675 return;
1676 }
1677
1678 // determine what security checks need to be performed in AsyncOpen().
1679 nsSecurityFlags securityFlags =
1680 aElement->ShouldCheckAllowOrigin()
1681 ? nsILoadInfo::SEC_REQUIRE_CORS_INHERITS_SEC_CONTEXT
1682 : nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_INHERITS_SEC_CONTEXT;
1683
1684 if (aElement->GetCORSMode() == CORS_USE_CREDENTIALS) {
1685 securityFlags |= nsILoadInfo::SEC_COOKIES_INCLUDE;
1686 }
1687
1688 securityFlags |= nsILoadInfo::SEC_ALLOW_CHROME;
1689
1690 MOZ_ASSERT(
1691 aElement->IsAnyOfHTMLElements(nsGkAtoms::audio, nsGkAtoms::video));
1692 nsContentPolicyType contentPolicyType =
1693 aElement->IsHTMLElement(nsGkAtoms::audio)
1694 ? nsIContentPolicy::TYPE_INTERNAL_AUDIO
1695 : nsIContentPolicy::TYPE_INTERNAL_VIDEO;
1696
1697 // If aElement has 'triggeringprincipal' attribute, we will use the value as
1698 // triggeringPrincipal for the channel, otherwise it will default to use
1699 // aElement->NodePrincipal().
1700 // This function returns true when aElement has 'triggeringprincipal', so if
1701 // setAttrs is true we will override the origin attributes on the channel
1702 // later.
1703 nsCOMPtr<nsIPrincipal> triggeringPrincipal;
1704 bool setAttrs = nsContentUtils::QueryTriggeringPrincipal(
1705 aElement, aElement->mLoadingSrcTriggeringPrincipal,
1706 getter_AddRefs(triggeringPrincipal));
1707
1708 nsCOMPtr<nsILoadGroup> loadGroup = aElement->GetDocumentLoadGroup();
1709 nsCOMPtr<nsIChannel> channel;
1710 nsresult rv = NS_NewChannelWithTriggeringPrincipal(
1711 getter_AddRefs(channel), aElement->mLoadingSrc,
1712 static_cast<Element*>(aElement), triggeringPrincipal, securityFlags,
1713 contentPolicyType,
1714 nullptr, // aPerformanceStorage
1715 loadGroup,
1716 nullptr, // aCallbacks
1717 nsICachingChannel::LOAD_BYPASS_LOCAL_CACHE_IF_BUSY |
1718 nsIChannel::LOAD_MEDIA_SNIFFER_OVERRIDES_CONTENT_TYPE |
1719 nsIChannel::LOAD_CALL_CONTENT_SNIFFERS);
1720
1721 if (NS_FAILED(rv)) {
1722 // Notify load error so the element will try next resource candidate.
1723 aElement->NotifyLoadError("Fail to create channel"_ns);
1724 return;
1725 }
1726
1727 nsCOMPtr<nsILoadInfo> loadInfo = channel->LoadInfo();
1728 if (setAttrs) {
1729 // The function simply returns NS_OK, so we ignore the return value.
1730 Unused << loadInfo->SetOriginAttributes(
1731 triggeringPrincipal->OriginAttributesRef());
1732 }
1733 loadInfo->SetIsMediaRequest(true);
1734 loadInfo->SetIsMediaInitialRequest(true);
1735
1736 nsCOMPtr<nsIClassOfService> cos(do_QueryInterface(channel));
1737 if (cos) {
1738 if (aElement->mUseUrgentStartForChannel) {
1739 cos->AddClassFlags(nsIClassOfService::UrgentStart);
1740
1741 // Reset the flag to avoid loading again without initiated by user
1742 // interaction.
1743 aElement->mUseUrgentStartForChannel = false;
1744 }
1745
1746 // Unconditionally disable throttling since we want the media to fluently
1747 // play even when we switch the tab to background.
1748 cos->AddClassFlags(nsIClassOfService::DontThrottle);
1749 }
1750
1751 // The listener holds a strong reference to us. This creates a
1752 // reference cycle, once we've set mChannel, which is manually broken
1753 // in the listener's OnStartRequest method after it is finished with
1754 // the element. The cycle will also be broken if we get a shutdown
1755 // notification before OnStartRequest fires. Necko guarantees that
1756 // OnStartRequest will eventually fire if we don't shut down first.
1757 RefPtr<MediaLoadListener> loadListener = new MediaLoadListener(aElement);
1758
1759 channel->SetNotificationCallbacks(loadListener);
1760
1761 nsCOMPtr<nsIHttpChannel> hc = do_QueryInterface(channel);
1762 if (hc) {
1763 // Use a byte range request from the start of the resource.
1764 // This enables us to detect if the stream supports byte range
1765 // requests, and therefore seeking, early.
1766 rv = hc->SetRequestHeader("Range"_ns, "bytes=0-"_ns, false);
1767 MOZ_ASSERT(NS_SUCCEEDED(rv));
1768 aElement->SetRequestHeaders(hc);
1769 }
1770
1771 rv = channel->AsyncOpen(loadListener);
1772 if (NS_FAILED(rv)) {
1773 // Notify load error so the element will try next resource candidate.
1774 aElement->NotifyLoadError("Failed to open channel"_ns);
1775 return;
1776 }
1777
1778 // Else the channel must be open and starting to download. If it encounters
1779 // a non-catastrophic failure, it will set a new task to continue loading
1780 // another candidate. It's safe to set it as mChannel now.
1781 mChannel = channel;
1782
1783 // loadListener will be unregistered either on shutdown or when
1784 // OnStartRequest for the channel we just opened fires.
1785 nsContentUtils::RegisterShutdownObserver(loadListener);
1786 }
1787
Load(HTMLMediaElement * aElement)1788 nsresult Load(HTMLMediaElement* aElement) {
1789 MOZ_ASSERT(aElement);
1790 // Per bug 1235183 comment 8, we can't spin the event loop from stable
1791 // state. Defer NS_NewChannel() to a new regular runnable.
1792 return aElement->MainThreadEventTarget()->Dispatch(
1793 NewRunnableMethod<HTMLMediaElement*>("ChannelLoader::LoadInternal",
1794 this, &ChannelLoader::LoadInternal,
1795 aElement));
1796 }
1797
Cancel()1798 void Cancel() {
1799 mCancelled = true;
1800 if (mChannel) {
1801 mChannel->Cancel(NS_BINDING_ABORTED);
1802 mChannel = nullptr;
1803 }
1804 }
1805
Done()1806 void Done() {
1807 MOZ_ASSERT(mChannel);
1808 // Decoder successfully created, the decoder now owns the MediaResource
1809 // which owns the channel.
1810 mChannel = nullptr;
1811 }
1812
Redirect(nsIChannel * aChannel,nsIChannel * aNewChannel,uint32_t aFlags)1813 nsresult Redirect(nsIChannel* aChannel, nsIChannel* aNewChannel,
1814 uint32_t aFlags) {
1815 NS_ASSERTION(aChannel == mChannel, "Channels should match!");
1816 mChannel = aNewChannel;
1817
1818 // Handle forwarding of Range header so that the intial detection
1819 // of seeking support (via result code 206) works across redirects.
1820 nsCOMPtr<nsIHttpChannel> http = do_QueryInterface(aChannel);
1821 NS_ENSURE_STATE(http);
1822
1823 constexpr auto rangeHdr = "Range"_ns;
1824
1825 nsAutoCString rangeVal;
1826 if (NS_SUCCEEDED(http->GetRequestHeader(rangeHdr, rangeVal))) {
1827 NS_ENSURE_STATE(!rangeVal.IsEmpty());
1828
1829 http = do_QueryInterface(aNewChannel);
1830 NS_ENSURE_STATE(http);
1831
1832 nsresult rv = http->SetRequestHeader(rangeHdr, rangeVal, false);
1833 NS_ENSURE_SUCCESS(rv, rv);
1834 }
1835
1836 return NS_OK;
1837 }
1838
1839 private:
~ChannelLoader()1840 ~ChannelLoader() { MOZ_ASSERT(!mChannel); }
1841 // Holds a reference to the first channel we open to the media resource.
1842 // Once the decoder is created, control over the channel passes to the
1843 // decoder, and we null out this reference. We must store this in case
1844 // we need to cancel the channel before control of it passes to the decoder.
1845 nsCOMPtr<nsIChannel> mChannel;
1846
1847 bool mCancelled = false;
1848 };
1849
1850 class HTMLMediaElement::ErrorSink {
1851 public:
ErrorSink(HTMLMediaElement * aOwner)1852 explicit ErrorSink(HTMLMediaElement* aOwner) : mOwner(aOwner) {
1853 MOZ_ASSERT(mOwner);
1854 }
1855
SetError(uint16_t aErrorCode,const nsACString & aErrorDetails)1856 void SetError(uint16_t aErrorCode, const nsACString& aErrorDetails) {
1857 // Since we have multiple paths calling into DecodeError, e.g.
1858 // MediaKeys::Terminated and EMEH264Decoder::Error. We should take the 1st
1859 // one only in order not to fire multiple 'error' events.
1860 if (mError) {
1861 return;
1862 }
1863
1864 if (!IsValidErrorCode(aErrorCode)) {
1865 NS_ASSERTION(false, "Undefined MediaError codes!");
1866 return;
1867 }
1868
1869 mError = new MediaError(mOwner, aErrorCode, aErrorDetails);
1870 mOwner->DispatchAsyncEvent(u"error"_ns);
1871 if (mOwner->ReadyState() == HAVE_NOTHING &&
1872 aErrorCode == MEDIA_ERR_ABORTED) {
1873 // https://html.spec.whatwg.org/multipage/embedded-content.html#media-data-processing-steps-list
1874 // "If the media data fetching process is aborted by the user"
1875 mOwner->DispatchAsyncEvent(u"abort"_ns);
1876 mOwner->ChangeNetworkState(NETWORK_EMPTY);
1877 mOwner->DispatchAsyncEvent(u"emptied"_ns);
1878 if (mOwner->mDecoder) {
1879 mOwner->ShutdownDecoder();
1880 }
1881 } else if (aErrorCode == MEDIA_ERR_SRC_NOT_SUPPORTED) {
1882 mOwner->ChangeNetworkState(NETWORK_NO_SOURCE);
1883 } else {
1884 mOwner->ChangeNetworkState(NETWORK_IDLE);
1885 }
1886 }
1887
ResetError()1888 void ResetError() { mError = nullptr; }
1889
1890 RefPtr<MediaError> mError;
1891
1892 private:
IsValidErrorCode(const uint16_t & aErrorCode) const1893 bool IsValidErrorCode(const uint16_t& aErrorCode) const {
1894 return (aErrorCode == MEDIA_ERR_DECODE || aErrorCode == MEDIA_ERR_NETWORK ||
1895 aErrorCode == MEDIA_ERR_ABORTED ||
1896 aErrorCode == MEDIA_ERR_SRC_NOT_SUPPORTED);
1897 }
1898
1899 // Media elememt's life cycle would be longer than error sink, so we use the
1900 // raw pointer and this class would only be referenced by media element.
1901 HTMLMediaElement* mOwner;
1902 };
1903
1904 NS_IMPL_CYCLE_COLLECTION_CLASS(HTMLMediaElement)
1905
1906 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(HTMLMediaElement,
1907 nsGenericHTMLElement)
1908 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMediaSource)
1909 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSrcMediaSource)
1910 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSrcStream)
1911 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSrcAttrStream)
1912 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSourcePointer)
1913 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mLoadBlockedDoc)
1914 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSourceLoadCandidate)
1915 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mAudioChannelWrapper)
1916 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mErrorSink->mError)
1917 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOutputStreams)
1918 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOutputTrackSources);
1919 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mPlayed);
1920 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mTextTrackManager)
1921 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mAudioTrackList)
1922 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mVideoTrackList)
1923 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMediaKeys)
1924 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mIncomingMediaKeys)
1925 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSelectedVideoStreamTrack)
1926 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mPendingPlayPromises)
1927 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSeekDOMPromise)
1928 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSetMediaKeysDOMPromise)
1929 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mEventBlocker)
1930 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
1931
1932 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(HTMLMediaElement,
1933 nsGenericHTMLElement)
1934 tmp->RemoveMutationObserver(tmp);
1935 if (tmp->mSrcStream) {
1936 // Need to unhook everything that EndSrcMediaStreamPlayback would normally
1937 // do, without creating any new strong references.
1938 if (tmp->mSelectedVideoStreamTrack) {
1939 tmp->mSelectedVideoStreamTrack->RemovePrincipalChangeObserver(tmp);
1940 }
1941 if (tmp->mMediaStreamRenderer) {
1942 tmp->mMediaStreamRenderer->Shutdown();
1943 // We null out mMediaStreamRenderer here since Shutdown() will shut down
1944 // its WatchManager, and UpdateSrcStreamPotentiallyPlaying() contains a
1945 // guard for this.
1946 tmp->mMediaStreamRenderer = nullptr;
1947 }
1948 if (tmp->mSecondaryMediaStreamRenderer) {
1949 tmp->mSecondaryMediaStreamRenderer->Shutdown();
1950 tmp->mSecondaryMediaStreamRenderer = nullptr;
1951 }
1952 if (tmp->mMediaStreamTrackListener) {
1953 tmp->mSrcStream->UnregisterTrackListener(
1954 tmp->mMediaStreamTrackListener.get());
1955 }
1956 }
1957 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSrcStream)
1958 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSrcAttrStream)
1959 NS_IMPL_CYCLE_COLLECTION_UNLINK(mMediaSource)
1960 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSrcMediaSource)
1961 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSourcePointer)
1962 NS_IMPL_CYCLE_COLLECTION_UNLINK(mLoadBlockedDoc)
1963 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSourceLoadCandidate)
1964 if (tmp->mAudioChannelWrapper) {
1965 tmp->mAudioChannelWrapper->Shutdown();
1966 }
1967 NS_IMPL_CYCLE_COLLECTION_UNLINK(mAudioChannelWrapper)
1968 NS_IMPL_CYCLE_COLLECTION_UNLINK(mErrorSink->mError)
1969 NS_IMPL_CYCLE_COLLECTION_UNLINK(mOutputStreams)
1970 NS_IMPL_CYCLE_COLLECTION_UNLINK(mOutputTrackSources)
1971 NS_IMPL_CYCLE_COLLECTION_UNLINK(mPlayed)
1972 NS_IMPL_CYCLE_COLLECTION_UNLINK(mTextTrackManager)
1973 NS_IMPL_CYCLE_COLLECTION_UNLINK(mAudioTrackList)
1974 NS_IMPL_CYCLE_COLLECTION_UNLINK(mVideoTrackList)
1975 NS_IMPL_CYCLE_COLLECTION_UNLINK(mMediaKeys)
1976 NS_IMPL_CYCLE_COLLECTION_UNLINK(mIncomingMediaKeys)
1977 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSelectedVideoStreamTrack)
1978 NS_IMPL_CYCLE_COLLECTION_UNLINK(mPendingPlayPromises)
1979 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSeekDOMPromise)
1980 NS_IMPL_CYCLE_COLLECTION_UNLINK(mSetMediaKeysDOMPromise)
1981 if (tmp->mMediaControlKeyListener) {
1982 tmp->mMediaControlKeyListener->StopIfNeeded();
1983 }
1984 if (tmp->mEventBlocker) {
1985 tmp->mEventBlocker->Shutdown();
1986 }
1987 NS_IMPL_CYCLE_COLLECTION_UNLINK_WEAK_PTR
1988 NS_IMPL_CYCLE_COLLECTION_UNLINK_END
1989
NS_IMPL_ISUPPORTS_CYCLE_COLLECTION_INHERITED_0(HTMLMediaElement,nsGenericHTMLElement)1990 NS_IMPL_ISUPPORTS_CYCLE_COLLECTION_INHERITED_0(HTMLMediaElement,
1991 nsGenericHTMLElement)
1992
1993 void HTMLMediaElement::AddSizeOfExcludingThis(nsWindowSizes& aSizes,
1994 size_t* aNodeSize) const {
1995 nsGenericHTMLElement::AddSizeOfExcludingThis(aSizes, aNodeSize);
1996
1997 // There are many other fields that might be worth reporting, but as seen in
1998 // bug 1595603, the event we postpone to dispatch can grow to be very large
1999 // sometimes, so at least report that.
2000 if (mEventBlocker) {
2001 *aNodeSize +=
2002 mEventBlocker->SizeOfExcludingThis(aSizes.mState.mMallocSizeOf);
2003 }
2004 }
2005
ContentRemoved(nsIContent * aChild,nsIContent * aPreviousSibling)2006 void HTMLMediaElement::ContentRemoved(nsIContent* aChild,
2007 nsIContent* aPreviousSibling) {
2008 if (aChild == mSourcePointer) {
2009 mSourcePointer = aPreviousSibling;
2010 }
2011 }
2012
GetMozMediaSourceObject() const2013 already_AddRefed<MediaSource> HTMLMediaElement::GetMozMediaSourceObject()
2014 const {
2015 RefPtr<MediaSource> source = mMediaSource;
2016 return source.forget();
2017 }
2018
MozRequestDebugInfo(ErrorResult & aRv)2019 already_AddRefed<Promise> HTMLMediaElement::MozRequestDebugInfo(
2020 ErrorResult& aRv) {
2021 RefPtr<Promise> promise = CreateDOMPromise(aRv);
2022 if (NS_WARN_IF(aRv.Failed())) {
2023 return nullptr;
2024 }
2025 auto result = MakeUnique<dom::HTMLMediaElementDebugInfo>();
2026 if (mMediaKeys) {
2027 GetEMEInfo(result->mEMEInfo);
2028 }
2029 if (mVideoFrameContainer) {
2030 result->mCompositorDroppedFrames =
2031 mVideoFrameContainer->GetDroppedImageCount();
2032 }
2033 if (mDecoder) {
2034 mDecoder->RequestDebugInfo(result->mDecoder)
2035 ->Then(
2036 mAbstractMainThread, __func__,
2037 [promise, ptr = std::move(result)]() {
2038 promise->MaybeResolve(ptr.get());
2039 },
2040 []() {
2041 MOZ_ASSERT_UNREACHABLE("Unexpected RequestDebugInfo() rejection");
2042 });
2043 } else {
2044 promise->MaybeResolve(result.get());
2045 }
2046 return promise.forget();
2047 }
2048
2049 /* static */
MozEnableDebugLog(const GlobalObject &)2050 void HTMLMediaElement::MozEnableDebugLog(const GlobalObject&) {
2051 DecoderDoctorLogger::EnableLogging();
2052 }
2053
MozRequestDebugLog(ErrorResult & aRv)2054 already_AddRefed<Promise> HTMLMediaElement::MozRequestDebugLog(
2055 ErrorResult& aRv) {
2056 RefPtr<Promise> promise = CreateDOMPromise(aRv);
2057 if (NS_WARN_IF(aRv.Failed())) {
2058 return nullptr;
2059 }
2060
2061 DecoderDoctorLogger::RetrieveMessages(this)->Then(
2062 mAbstractMainThread, __func__,
2063 [promise](const nsACString& aString) {
2064 promise->MaybeResolve(NS_ConvertUTF8toUTF16(aString));
2065 },
2066 [promise](nsresult rv) { promise->MaybeReject(rv); });
2067
2068 return promise.forget();
2069 }
2070
SetVisible(bool aVisible)2071 void HTMLMediaElement::SetVisible(bool aVisible) {
2072 mForcedHidden = !aVisible;
2073 if (mDecoder) {
2074 mDecoder->SetForcedHidden(!aVisible);
2075 }
2076 }
2077
IsVideoDecodingSuspended() const2078 bool HTMLMediaElement::IsVideoDecodingSuspended() const {
2079 return mDecoder && mDecoder->IsVideoDecodingSuspended();
2080 }
2081
TotalPlayTime() const2082 double HTMLMediaElement::TotalPlayTime() const {
2083 return mDecoder ? mDecoder->GetTotalPlayTimeInSeconds() : -1.0;
2084 }
2085
InvisiblePlayTime() const2086 double HTMLMediaElement::InvisiblePlayTime() const {
2087 return mDecoder ? mDecoder->GetInvisibleVideoPlayTimeInSeconds() : -1.0;
2088 }
2089
VideoDecodeSuspendedTime() const2090 double HTMLMediaElement::VideoDecodeSuspendedTime() const {
2091 return mDecoder ? mDecoder->GetVideoDecodeSuspendedTimeInSeconds() : -1.0;
2092 }
2093
SetFormatDiagnosticsReportForMimeType(const nsAString & aMimeType,DecoderDoctorReportType aType)2094 void HTMLMediaElement::SetFormatDiagnosticsReportForMimeType(
2095 const nsAString& aMimeType, DecoderDoctorReportType aType) {
2096 DecoderDoctorDiagnostics diagnostics;
2097 diagnostics.SetDecoderDoctorReportType(aType);
2098 diagnostics.StoreFormatDiagnostics(OwnerDoc(), aMimeType, false /* can play*/,
2099 __func__);
2100 }
2101
SetDecodeError(const nsAString & aError,ErrorResult & aRv)2102 void HTMLMediaElement::SetDecodeError(const nsAString& aError,
2103 ErrorResult& aRv) {
2104 // The reason we use this map-ish structure is because we can't use
2105 // `CR.NS_ERROR.*` directly in test. In order to use them in test, we have to
2106 // add them into `xpc.msg`. As we won't use `CR.NS_ERROR.*` in the production
2107 // code, adding them to `xpc.msg` seems an overdesign and adding maintenance
2108 // effort (exposing them in CR also needs to add a description, which is
2109 // useless because we won't show them to users)
2110 static struct {
2111 const char* mName;
2112 nsresult mResult;
2113 } kSupportedErrorList[] = {
2114 {"NS_ERROR_DOM_MEDIA_ABORT_ERR", NS_ERROR_DOM_MEDIA_ABORT_ERR},
2115 {"NS_ERROR_DOM_MEDIA_NOT_ALLOWED_ERR",
2116 NS_ERROR_DOM_MEDIA_NOT_ALLOWED_ERR},
2117 {"NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR",
2118 NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR},
2119 {"NS_ERROR_DOM_MEDIA_DECODE_ERR", NS_ERROR_DOM_MEDIA_DECODE_ERR},
2120 {"NS_ERROR_DOM_MEDIA_FATAL_ERR", NS_ERROR_DOM_MEDIA_FATAL_ERR},
2121 {"NS_ERROR_DOM_MEDIA_METADATA_ERR", NS_ERROR_DOM_MEDIA_METADATA_ERR},
2122 {"NS_ERROR_DOM_MEDIA_OVERFLOW_ERR", NS_ERROR_DOM_MEDIA_OVERFLOW_ERR},
2123 {"NS_ERROR_DOM_MEDIA_MEDIASINK_ERR", NS_ERROR_DOM_MEDIA_MEDIASINK_ERR},
2124 {"NS_ERROR_DOM_MEDIA_DEMUXER_ERR", NS_ERROR_DOM_MEDIA_DEMUXER_ERR},
2125 {"NS_ERROR_DOM_MEDIA_CDM_ERR", NS_ERROR_DOM_MEDIA_CDM_ERR},
2126 {"NS_ERROR_DOM_MEDIA_CUBEB_INITIALIZATION_ERR",
2127 NS_ERROR_DOM_MEDIA_CUBEB_INITIALIZATION_ERR}};
2128 for (auto& error : kSupportedErrorList) {
2129 if (strcmp(error.mName, NS_ConvertUTF16toUTF8(aError).get()) == 0) {
2130 DecoderDoctorDiagnostics diagnostics;
2131 diagnostics.StoreDecodeError(OwnerDoc(), error.mResult, u""_ns, __func__);
2132 return;
2133 }
2134 }
2135 aRv.Throw(NS_ERROR_FAILURE);
2136 return;
2137 }
2138
SetAudioSinkFailedStartup()2139 void HTMLMediaElement::SetAudioSinkFailedStartup() {
2140 DecoderDoctorDiagnostics diagnostics;
2141 diagnostics.StoreEvent(OwnerDoc(),
2142 {DecoderDoctorEvent::eAudioSinkStartup,
2143 NS_ERROR_DOM_MEDIA_CUBEB_INITIALIZATION_ERR},
2144 __func__);
2145 }
2146
GetCurrentImage()2147 already_AddRefed<layers::Image> HTMLMediaElement::GetCurrentImage() {
2148 MarkAsTainted();
2149
2150 // TODO: In bug 1345404, handle case when video decoder is already suspended.
2151 ImageContainer* container = GetImageContainer();
2152 if (!container) {
2153 return nullptr;
2154 }
2155
2156 AutoLockImage lockImage(container);
2157 RefPtr<layers::Image> image = lockImage.GetImage(TimeStamp::Now());
2158 return image.forget();
2159 }
2160
HasSuspendTaint() const2161 bool HTMLMediaElement::HasSuspendTaint() const {
2162 MOZ_ASSERT(!mDecoder || (mDecoder->HasSuspendTaint() == mHasSuspendTaint));
2163 return mHasSuspendTaint;
2164 }
2165
GetSrcObject() const2166 already_AddRefed<DOMMediaStream> HTMLMediaElement::GetSrcObject() const {
2167 return do_AddRef(mSrcAttrStream);
2168 }
2169
SetSrcObject(DOMMediaStream & aValue)2170 void HTMLMediaElement::SetSrcObject(DOMMediaStream& aValue) {
2171 SetSrcObject(&aValue);
2172 }
2173
SetSrcObject(DOMMediaStream * aValue)2174 void HTMLMediaElement::SetSrcObject(DOMMediaStream* aValue) {
2175 for (auto& outputStream : mOutputStreams) {
2176 if (aValue == outputStream.mStream) {
2177 ReportToConsole(nsIScriptError::warningFlag,
2178 "MediaElementStreamCaptureCycle");
2179 return;
2180 }
2181 }
2182 mSrcAttrStream = aValue;
2183 UpdateAudioChannelPlayingState();
2184 DoLoad();
2185 }
2186
Ended()2187 bool HTMLMediaElement::Ended() {
2188 return (mDecoder && mDecoder->IsEnded()) ||
2189 (mSrcStream && mSrcStreamReportPlaybackEnded);
2190 }
2191
GetCurrentSrc(nsAString & aCurrentSrc)2192 void HTMLMediaElement::GetCurrentSrc(nsAString& aCurrentSrc) {
2193 nsAutoCString src;
2194 GetCurrentSpec(src);
2195 CopyUTF8toUTF16(src, aCurrentSrc);
2196 }
2197
OnChannelRedirect(nsIChannel * aChannel,nsIChannel * aNewChannel,uint32_t aFlags)2198 nsresult HTMLMediaElement::OnChannelRedirect(nsIChannel* aChannel,
2199 nsIChannel* aNewChannel,
2200 uint32_t aFlags) {
2201 MOZ_ASSERT(mChannelLoader);
2202 return mChannelLoader->Redirect(aChannel, aNewChannel, aFlags);
2203 }
2204
ShutdownDecoder()2205 void HTMLMediaElement::ShutdownDecoder() {
2206 RemoveMediaElementFromURITable();
2207 NS_ASSERTION(mDecoder, "Must have decoder to shut down");
2208
2209 mWaitingForKeyListener.DisconnectIfExists();
2210 if (mMediaSource) {
2211 mMediaSource->CompletePendingTransactions();
2212 }
2213 mDecoder->Shutdown();
2214 DDUNLINKCHILD(mDecoder.get());
2215 mDecoder = nullptr;
2216 }
2217
AbortExistingLoads()2218 void HTMLMediaElement::AbortExistingLoads() {
2219 // Abort any already-running instance of the resource selection algorithm.
2220 mLoadWaitStatus = NOT_WAITING;
2221
2222 // Set a new load ID. This will cause events which were enqueued
2223 // with a different load ID to silently be cancelled.
2224 mCurrentLoadID++;
2225
2226 // Immediately reject or resolve the already-dispatched
2227 // nsResolveOrRejectPendingPlayPromisesRunners. These runners won't be
2228 // executed again later since the mCurrentLoadID had been changed.
2229 for (auto& runner : mPendingPlayPromisesRunners) {
2230 runner->ResolveOrReject();
2231 }
2232 mPendingPlayPromisesRunners.Clear();
2233
2234 if (mChannelLoader) {
2235 mChannelLoader->Cancel();
2236 mChannelLoader = nullptr;
2237 }
2238
2239 bool fireTimeUpdate = false;
2240
2241 if (mDecoder) {
2242 fireTimeUpdate = mDecoder->GetCurrentTime() != 0.0;
2243 ShutdownDecoder();
2244 }
2245 if (mSrcStream) {
2246 EndSrcMediaStreamPlayback();
2247 }
2248
2249 if (mMediaSource) {
2250 OwnerDoc()->RemoveMediaElementWithMSE();
2251 }
2252
2253 RemoveMediaElementFromURITable();
2254 mLoadingSrcTriggeringPrincipal = nullptr;
2255 DDLOG(DDLogCategory::Property, "loading_src", "");
2256 DDUNLINKCHILD(mMediaSource.get());
2257 mMediaSource = nullptr;
2258
2259 if (mNetworkState == NETWORK_LOADING || mNetworkState == NETWORK_IDLE) {
2260 DispatchAsyncEvent(u"abort"_ns);
2261 }
2262
2263 bool hadVideo = HasVideo();
2264 mErrorSink->ResetError();
2265 mCurrentPlayRangeStart = -1.0;
2266 mLoadedDataFired = false;
2267 mAutoplaying = true;
2268 mIsLoadingFromSourceChildren = false;
2269 mSuspendedAfterFirstFrame = false;
2270 mAllowSuspendAfterFirstFrame = true;
2271 mHaveQueuedSelectResource = false;
2272 mSuspendedForPreloadNone = false;
2273 mDownloadSuspendedByCache = false;
2274 mMediaInfo = MediaInfo();
2275 mIsEncrypted = false;
2276 mPendingEncryptedInitData.Reset();
2277 mWaitingForKey = NOT_WAITING_FOR_KEY;
2278 mSourcePointer = nullptr;
2279 mIsBlessed = false;
2280 SetAudibleState(false);
2281
2282 mTags = nullptr;
2283
2284 if (mNetworkState != NETWORK_EMPTY) {
2285 NS_ASSERTION(!mDecoder && !mSrcStream,
2286 "How did someone setup a new stream/decoder already?");
2287 // ChangeNetworkState() will call UpdateAudioChannelPlayingState()
2288 // indirectly which depends on mPaused. So we need to update mPaused first.
2289 if (!mPaused) {
2290 mPaused = true;
2291 PlayPromise::RejectPromises(TakePendingPlayPromises(),
2292 NS_ERROR_DOM_MEDIA_ABORT_ERR);
2293 }
2294 ChangeNetworkState(NETWORK_EMPTY);
2295 RemoveMediaTracks();
2296 UpdateOutputTrackSources();
2297 ChangeReadyState(HAVE_NOTHING);
2298
2299 // TODO: Apply the rules for text track cue rendering Bug 865407
2300 if (mTextTrackManager) {
2301 mTextTrackManager->GetTextTracks()->SetCuesInactive();
2302 }
2303
2304 if (fireTimeUpdate) {
2305 // Since we destroyed the decoder above, the current playback position
2306 // will now be reported as 0. The playback position was non-zero when
2307 // we destroyed the decoder, so fire a timeupdate event so that the
2308 // change will be reflected in the controls.
2309 FireTimeUpdate(TimeupdateType::eMandatory);
2310 }
2311 DispatchAsyncEvent(u"emptied"_ns);
2312 UpdateAudioChannelPlayingState();
2313 }
2314
2315 if (IsVideo() && hadVideo) {
2316 // Ensure we render transparent black after resetting video resolution.
2317 Maybe<nsIntSize> size = Some(nsIntSize(0, 0));
2318 Invalidate(true, size, false);
2319 }
2320
2321 // As aborting current load would stop current playback, so we have no need to
2322 // resume a paused media element.
2323 ClearResumeDelayedMediaPlaybackAgentIfNeeded();
2324
2325 mMediaControlKeyListener->StopIfNeeded();
2326
2327 // We may have changed mPaused, mAutoplaying, and other
2328 // things which can affect AddRemoveSelfReference
2329 AddRemoveSelfReference();
2330
2331 mIsRunningSelectResource = false;
2332
2333 AssertReadyStateIsNothing();
2334 }
2335
NoSupportedMediaSourceError(const nsACString & aErrorDetails)2336 void HTMLMediaElement::NoSupportedMediaSourceError(
2337 const nsACString& aErrorDetails) {
2338 if (mDecoder) {
2339 ShutdownDecoder();
2340 }
2341
2342 bool isThirdPartyLoad = false;
2343 nsresult rv = NS_ERROR_NOT_AVAILABLE;
2344 if (mSrcAttrTriggeringPrincipal && mLoadingSrc) {
2345 rv = mSrcAttrTriggeringPrincipal->IsThirdPartyURI(mLoadingSrc,
2346 &isThirdPartyLoad);
2347 }
2348
2349 if (NS_SUCCEEDED(rv) && isThirdPartyLoad) {
2350 // aErrorDetails can include sensitive details like MimeType or HTTP Status
2351 // Code. In case we're loading a 3rd party resource we should not leak this
2352 // and pass a Generic Error Message
2353 mErrorSink->SetError(MEDIA_ERR_SRC_NOT_SUPPORTED,
2354 "Failed to open media"_ns);
2355 } else {
2356 mErrorSink->SetError(MEDIA_ERR_SRC_NOT_SUPPORTED, aErrorDetails);
2357 }
2358
2359 RemoveMediaTracks();
2360 ChangeDelayLoadStatus(false);
2361 UpdateAudioChannelPlayingState();
2362 PlayPromise::RejectPromises(TakePendingPlayPromises(),
2363 NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR);
2364 }
2365
2366 // Runs a "synchronous section", a function that must run once the event loop
2367 // has reached a "stable state"
2368 // http://www.whatwg.org/specs/web-apps/current-work/multipage/webappapis.html#synchronous-section
RunInStableState(nsIRunnable * aRunnable)2369 void HTMLMediaElement::RunInStableState(nsIRunnable* aRunnable) {
2370 if (mShuttingDown) {
2371 return;
2372 }
2373
2374 nsCOMPtr<nsIRunnable> task = NS_NewRunnableFunction(
2375 "HTMLMediaElement::RunInStableState",
2376 [self = RefPtr<HTMLMediaElement>(this), loadId = GetCurrentLoadID(),
2377 runnable = RefPtr<nsIRunnable>(aRunnable)]() {
2378 if (self->GetCurrentLoadID() != loadId) {
2379 return;
2380 }
2381 runnable->Run();
2382 });
2383 nsContentUtils::RunInStableState(task.forget());
2384 }
2385
QueueLoadFromSourceTask()2386 void HTMLMediaElement::QueueLoadFromSourceTask() {
2387 if (!mIsLoadingFromSourceChildren || mShuttingDown) {
2388 return;
2389 }
2390
2391 if (mDecoder) {
2392 // Reset readyState to HAVE_NOTHING since we're going to load a new decoder.
2393 ShutdownDecoder();
2394 ChangeReadyState(HAVE_NOTHING);
2395 }
2396
2397 AssertReadyStateIsNothing();
2398
2399 ChangeDelayLoadStatus(true);
2400 ChangeNetworkState(NETWORK_LOADING);
2401 RefPtr<Runnable> r =
2402 NewRunnableMethod("HTMLMediaElement::LoadFromSourceChildren", this,
2403 &HTMLMediaElement::LoadFromSourceChildren);
2404 RunInStableState(r);
2405 }
2406
QueueSelectResourceTask()2407 void HTMLMediaElement::QueueSelectResourceTask() {
2408 // Don't allow multiple async select resource calls to be queued.
2409 if (mHaveQueuedSelectResource) return;
2410 mHaveQueuedSelectResource = true;
2411 ChangeNetworkState(NETWORK_NO_SOURCE);
2412 RefPtr<Runnable> r =
2413 NewRunnableMethod("HTMLMediaElement::SelectResourceWrapper", this,
2414 &HTMLMediaElement::SelectResourceWrapper);
2415 RunInStableState(r);
2416 }
2417
HasSourceChildren(nsIContent * aElement)2418 static bool HasSourceChildren(nsIContent* aElement) {
2419 for (nsIContent* child = aElement->GetFirstChild(); child;
2420 child = child->GetNextSibling()) {
2421 if (child->IsHTMLElement(nsGkAtoms::source)) {
2422 return true;
2423 }
2424 }
2425 return false;
2426 }
2427
DocumentOrigin(Document * aDoc)2428 static nsCString DocumentOrigin(Document* aDoc) {
2429 if (!aDoc) {
2430 return "null"_ns;
2431 }
2432 nsCOMPtr<nsIPrincipal> principal = aDoc->NodePrincipal();
2433 if (!principal) {
2434 return "null"_ns;
2435 }
2436 nsCString origin;
2437 if (NS_FAILED(principal->GetOrigin(origin))) {
2438 return "null"_ns;
2439 }
2440 return origin;
2441 }
2442
Load()2443 void HTMLMediaElement::Load() {
2444 LOG(LogLevel::Debug,
2445 ("%p Load() hasSrcAttrStream=%d hasSrcAttr=%d hasSourceChildren=%d "
2446 "handlingInput=%d hasAutoplayAttr=%d IsAllowedToPlay=%d "
2447 "ownerDoc=%p (%s) ownerDocUserActivated=%d "
2448 "muted=%d volume=%f",
2449 this, !!mSrcAttrStream, HasAttr(kNameSpaceID_None, nsGkAtoms::src),
2450 HasSourceChildren(this), UserActivation::IsHandlingUserInput(),
2451 HasAttr(kNameSpaceID_None, nsGkAtoms::autoplay),
2452 AutoplayPolicy::IsAllowedToPlay(*this), OwnerDoc(),
2453 DocumentOrigin(OwnerDoc()).get(),
2454 OwnerDoc()->HasBeenUserGestureActivated(), mMuted, mVolume));
2455
2456 if (mIsRunningLoadMethod) {
2457 return;
2458 }
2459
2460 mIsDoingExplicitLoad = true;
2461 DoLoad();
2462 }
2463
DoLoad()2464 void HTMLMediaElement::DoLoad() {
2465 // Check if media is allowed for the docshell.
2466 nsCOMPtr<nsIDocShell> docShell = OwnerDoc()->GetDocShell();
2467 if (docShell && !docShell->GetAllowMedia()) {
2468 LOG(LogLevel::Debug, ("%p Media not allowed", this));
2469 return;
2470 }
2471
2472 if (mIsRunningLoadMethod) {
2473 return;
2474 }
2475
2476 if (UserActivation::IsHandlingUserInput()) {
2477 // Detect if user has interacted with element so that play will not be
2478 // blocked when initiated by a script. This enables sites to capture user
2479 // intent to play by calling load() in the click handler of a "catalog
2480 // view" of a gallery of videos.
2481 mIsBlessed = true;
2482 // Mark the channel as urgent-start when autopaly so that it will play the
2483 // media from src after loading enough resource.
2484 if (HasAttr(kNameSpaceID_None, nsGkAtoms::autoplay)) {
2485 mUseUrgentStartForChannel = true;
2486 }
2487 }
2488
2489 SetPlayedOrSeeked(false);
2490 mIsRunningLoadMethod = true;
2491 AbortExistingLoads();
2492 SetPlaybackRate(mDefaultPlaybackRate, IgnoreErrors());
2493 QueueSelectResourceTask();
2494 ResetState();
2495 mIsRunningLoadMethod = false;
2496 }
2497
ResetState()2498 void HTMLMediaElement::ResetState() {
2499 // There might be a pending MediaDecoder::PlaybackPositionChanged() which
2500 // will overwrite |mMediaInfo.mVideo.mDisplay| in UpdateMediaSize() to give
2501 // staled videoWidth and videoHeight. We have to call ForgetElement() here
2502 // such that the staled callbacks won't reach us.
2503 if (mVideoFrameContainer) {
2504 mVideoFrameContainer->ForgetElement();
2505 mVideoFrameContainer = nullptr;
2506 }
2507 if (mMediaStreamRenderer) {
2508 // mMediaStreamRenderer, has a strong reference to mVideoFrameContainer.
2509 mMediaStreamRenderer->Shutdown();
2510 mMediaStreamRenderer = nullptr;
2511 }
2512 if (mSecondaryMediaStreamRenderer) {
2513 // mSecondaryMediaStreamRenderer, has a strong reference to
2514 // the secondary VideoFrameContainer.
2515 mSecondaryMediaStreamRenderer->Shutdown();
2516 mSecondaryMediaStreamRenderer = nullptr;
2517 }
2518 }
2519
SelectResourceWrapper()2520 void HTMLMediaElement::SelectResourceWrapper() {
2521 SelectResource();
2522 MaybeBeginCloningVisually();
2523 mIsRunningSelectResource = false;
2524 mHaveQueuedSelectResource = false;
2525 mIsDoingExplicitLoad = false;
2526 }
2527
SelectResource()2528 void HTMLMediaElement::SelectResource() {
2529 if (!mSrcAttrStream && !HasAttr(kNameSpaceID_None, nsGkAtoms::src) &&
2530 !HasSourceChildren(this)) {
2531 // The media element has neither a src attribute nor any source
2532 // element children, abort the load.
2533 ChangeNetworkState(NETWORK_EMPTY);
2534 ChangeDelayLoadStatus(false);
2535 return;
2536 }
2537
2538 ChangeDelayLoadStatus(true);
2539
2540 ChangeNetworkState(NETWORK_LOADING);
2541 DispatchAsyncEvent(u"loadstart"_ns);
2542
2543 // Delay setting mIsRunningSeletResource until after UpdatePreloadAction
2544 // so that we don't lose our state change by bailing out of the preload
2545 // state update
2546 UpdatePreloadAction();
2547 mIsRunningSelectResource = true;
2548
2549 // If we have a 'src' attribute, use that exclusively.
2550 nsAutoString src;
2551 if (mSrcAttrStream) {
2552 SetupSrcMediaStreamPlayback(mSrcAttrStream);
2553 } else if (GetAttr(kNameSpaceID_None, nsGkAtoms::src, src)) {
2554 nsCOMPtr<nsIURI> uri;
2555 MediaResult rv = NewURIFromString(src, getter_AddRefs(uri));
2556 if (NS_SUCCEEDED(rv)) {
2557 LOG(LogLevel::Debug, ("%p Trying load from src=%s", this,
2558 NS_ConvertUTF16toUTF8(src).get()));
2559 NS_ASSERTION(
2560 !mIsLoadingFromSourceChildren,
2561 "Should think we're not loading from source children by default");
2562
2563 if (!mMediaSource) {
2564 OwnerDoc()->AddMediaElementWithMSE();
2565 }
2566
2567 RemoveMediaElementFromURITable();
2568 if (!mSrcMediaSource) {
2569 mLoadingSrc = uri;
2570 } else {
2571 mLoadingSrc = nullptr;
2572 }
2573 mLoadingSrcTriggeringPrincipal = mSrcAttrTriggeringPrincipal;
2574 DDLOG(DDLogCategory::Property, "loading_src",
2575 nsCString(NS_ConvertUTF16toUTF8(src)));
2576 mMediaSource = mSrcMediaSource;
2577 DDLINKCHILD("mediasource", mMediaSource.get());
2578 UpdatePreloadAction();
2579 if (mPreloadAction == HTMLMediaElement::PRELOAD_NONE && !mMediaSource) {
2580 // preload:none media, suspend the load here before we make any
2581 // network requests.
2582 SuspendLoad();
2583 return;
2584 }
2585
2586 rv = LoadResource();
2587 if (NS_SUCCEEDED(rv)) {
2588 return;
2589 }
2590 } else {
2591 AutoTArray<nsString, 1> params = {src};
2592 ReportLoadError("MediaLoadInvalidURI", params);
2593 rv = MediaResult(rv.Code(), "MediaLoadInvalidURI");
2594 }
2595 // The media element has neither a src attribute nor a source element child:
2596 // set the networkState to NETWORK_EMPTY, and abort these steps; the
2597 // synchronous section ends.
2598 mMainThreadEventTarget->Dispatch(NewRunnableMethod<nsCString>(
2599 "HTMLMediaElement::NoSupportedMediaSourceError", this,
2600 &HTMLMediaElement::NoSupportedMediaSourceError, rv.Description()));
2601 } else {
2602 // Otherwise, the source elements will be used.
2603 mIsLoadingFromSourceChildren = true;
2604 LoadFromSourceChildren();
2605 }
2606 }
2607
NotifyLoadError(const nsACString & aErrorDetails)2608 void HTMLMediaElement::NotifyLoadError(const nsACString& aErrorDetails) {
2609 if (!mIsLoadingFromSourceChildren) {
2610 LOG(LogLevel::Debug, ("NotifyLoadError(), no supported media error"));
2611 NoSupportedMediaSourceError(aErrorDetails);
2612 } else if (mSourceLoadCandidate) {
2613 DispatchAsyncSourceError(mSourceLoadCandidate);
2614 QueueLoadFromSourceTask();
2615 } else {
2616 NS_WARNING("Should know the source we were loading from!");
2617 }
2618 }
2619
NotifyMediaTrackAdded(dom::MediaTrack * aTrack)2620 void HTMLMediaElement::NotifyMediaTrackAdded(dom::MediaTrack* aTrack) {
2621 // The set of tracks changed.
2622 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
2623 }
2624
NotifyMediaTrackRemoved(dom::MediaTrack * aTrack)2625 void HTMLMediaElement::NotifyMediaTrackRemoved(dom::MediaTrack* aTrack) {
2626 // The set of tracks changed.
2627 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
2628 }
2629
NotifyMediaTrackEnabled(dom::MediaTrack * aTrack)2630 void HTMLMediaElement::NotifyMediaTrackEnabled(dom::MediaTrack* aTrack) {
2631 MOZ_ASSERT(aTrack);
2632 if (!aTrack) {
2633 return;
2634 }
2635 #ifdef DEBUG
2636 nsString id;
2637 aTrack->GetId(id);
2638
2639 LOG(LogLevel::Debug, ("MediaElement %p %sTrack with id %s enabled", this,
2640 aTrack->AsAudioTrack() ? "Audio" : "Video",
2641 NS_ConvertUTF16toUTF8(id).get()));
2642 #endif
2643
2644 MOZ_ASSERT((aTrack->AsAudioTrack() && aTrack->AsAudioTrack()->Enabled()) ||
2645 (aTrack->AsVideoTrack() && aTrack->AsVideoTrack()->Selected()));
2646
2647 if (aTrack->AsAudioTrack()) {
2648 SetMutedInternal(mMuted & ~MUTED_BY_AUDIO_TRACK);
2649 } else if (aTrack->AsVideoTrack()) {
2650 if (!IsVideo()) {
2651 MOZ_ASSERT(false);
2652 return;
2653 }
2654 mDisableVideo = false;
2655 } else {
2656 MOZ_ASSERT(false, "Unknown track type");
2657 }
2658
2659 if (mSrcStream) {
2660 if (AudioTrack* t = aTrack->AsAudioTrack()) {
2661 if (mMediaStreamRenderer) {
2662 mMediaStreamRenderer->AddTrack(t->GetAudioStreamTrack());
2663 }
2664 } else if (VideoTrack* t = aTrack->AsVideoTrack()) {
2665 MOZ_ASSERT(!mSelectedVideoStreamTrack);
2666
2667 mSelectedVideoStreamTrack = t->GetVideoStreamTrack();
2668 mSelectedVideoStreamTrack->AddPrincipalChangeObserver(this);
2669 if (mMediaStreamRenderer) {
2670 mMediaStreamRenderer->AddTrack(mSelectedVideoStreamTrack);
2671 }
2672 if (mSecondaryMediaStreamRenderer) {
2673 mSecondaryMediaStreamRenderer->AddTrack(mSelectedVideoStreamTrack);
2674 }
2675 if (mMediaInfo.HasVideo()) {
2676 mMediaInfo.mVideo.SetAlpha(mSelectedVideoStreamTrack->HasAlpha());
2677 }
2678 nsContentUtils::CombineResourcePrincipals(
2679 &mSrcStreamVideoPrincipal, mSelectedVideoStreamTrack->GetPrincipal());
2680 }
2681 }
2682
2683 // The set of enabled/selected tracks changed.
2684 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
2685 }
2686
NotifyMediaTrackDisabled(dom::MediaTrack * aTrack)2687 void HTMLMediaElement::NotifyMediaTrackDisabled(dom::MediaTrack* aTrack) {
2688 MOZ_ASSERT(aTrack);
2689 if (!aTrack) {
2690 return;
2691 }
2692
2693 nsString id;
2694 aTrack->GetId(id);
2695
2696 LOG(LogLevel::Debug, ("MediaElement %p %sTrack with id %s disabled", this,
2697 aTrack->AsAudioTrack() ? "Audio" : "Video",
2698 NS_ConvertUTF16toUTF8(id).get()));
2699
2700 MOZ_ASSERT((!aTrack->AsAudioTrack() || !aTrack->AsAudioTrack()->Enabled()) &&
2701 (!aTrack->AsVideoTrack() || !aTrack->AsVideoTrack()->Selected()));
2702
2703 if (AudioTrack* t = aTrack->AsAudioTrack()) {
2704 if (mSrcStream) {
2705 if (mMediaStreamRenderer) {
2706 mMediaStreamRenderer->RemoveTrack(t->GetAudioStreamTrack());
2707 }
2708 }
2709 // If we don't have any live tracks, we don't need to mute MediaElement.
2710 MOZ_DIAGNOSTIC_ASSERT(AudioTracks(), "Element can't have been unlinked");
2711 if (AudioTracks()->Length() > 0) {
2712 bool shouldMute = true;
2713 for (uint32_t i = 0; i < AudioTracks()->Length(); ++i) {
2714 if ((*AudioTracks())[i]->Enabled()) {
2715 shouldMute = false;
2716 break;
2717 }
2718 }
2719
2720 if (shouldMute) {
2721 SetMutedInternal(mMuted | MUTED_BY_AUDIO_TRACK);
2722 }
2723 }
2724 } else if (aTrack->AsVideoTrack()) {
2725 if (mSrcStream) {
2726 MOZ_DIAGNOSTIC_ASSERT(mSelectedVideoStreamTrack ==
2727 aTrack->AsVideoTrack()->GetVideoStreamTrack());
2728 if (mMediaStreamRenderer) {
2729 mMediaStreamRenderer->RemoveTrack(mSelectedVideoStreamTrack);
2730 }
2731 if (mSecondaryMediaStreamRenderer) {
2732 mSecondaryMediaStreamRenderer->RemoveTrack(mSelectedVideoStreamTrack);
2733 }
2734 mSelectedVideoStreamTrack->RemovePrincipalChangeObserver(this);
2735 mSelectedVideoStreamTrack = nullptr;
2736 }
2737 }
2738
2739 // The set of enabled/selected tracks changed.
2740 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
2741 }
2742
DealWithFailedElement(nsIContent * aSourceElement)2743 void HTMLMediaElement::DealWithFailedElement(nsIContent* aSourceElement) {
2744 if (mShuttingDown) {
2745 return;
2746 }
2747
2748 DispatchAsyncSourceError(aSourceElement);
2749 mMainThreadEventTarget->Dispatch(
2750 NewRunnableMethod("HTMLMediaElement::QueueLoadFromSourceTask", this,
2751 &HTMLMediaElement::QueueLoadFromSourceTask));
2752 }
2753
LoadFromSourceChildren()2754 void HTMLMediaElement::LoadFromSourceChildren() {
2755 NS_ASSERTION(mDelayingLoadEvent,
2756 "Should delay load event (if in document) during load");
2757 NS_ASSERTION(mIsLoadingFromSourceChildren,
2758 "Must remember we're loading from source children");
2759
2760 AddMutationObserverUnlessExists(this);
2761
2762 RemoveMediaTracks();
2763
2764 while (true) {
2765 Element* child = GetNextSource();
2766 if (!child) {
2767 // Exhausted candidates, wait for more candidates to be appended to
2768 // the media element.
2769 mLoadWaitStatus = WAITING_FOR_SOURCE;
2770 ChangeNetworkState(NETWORK_NO_SOURCE);
2771 ChangeDelayLoadStatus(false);
2772 ReportLoadError("MediaLoadExhaustedCandidates");
2773 return;
2774 }
2775
2776 // Must have src attribute.
2777 nsAutoString src;
2778 if (!child->GetAttr(kNameSpaceID_None, nsGkAtoms::src, src)) {
2779 ReportLoadError("MediaLoadSourceMissingSrc");
2780 DealWithFailedElement(child);
2781 return;
2782 }
2783
2784 // If we have a type attribute, it must be a supported type.
2785 nsAutoString type;
2786 if (child->GetAttr(kNameSpaceID_None, nsGkAtoms::type, type) &&
2787 !type.IsEmpty()) {
2788 DecoderDoctorDiagnostics diagnostics;
2789 CanPlayStatus canPlay = GetCanPlay(type, &diagnostics);
2790 diagnostics.StoreFormatDiagnostics(OwnerDoc(), type,
2791 canPlay != CANPLAY_NO, __func__);
2792 if (canPlay == CANPLAY_NO) {
2793 // Check that at least one other source child exists and report that
2794 // we will try to load that one next.
2795 nsIContent* nextChild = mSourcePointer->GetNextSibling();
2796 AutoTArray<nsString, 2> params = {type, src};
2797
2798 while (nextChild) {
2799 if (nextChild && nextChild->IsHTMLElement(nsGkAtoms::source)) {
2800 ReportLoadError("MediaLoadUnsupportedTypeAttributeLoadingNextChild",
2801 params);
2802 break;
2803 }
2804
2805 nextChild = nextChild->GetNextSibling();
2806 };
2807
2808 if (!nextChild) {
2809 ReportLoadError("MediaLoadUnsupportedTypeAttribute", params);
2810 }
2811
2812 DealWithFailedElement(child);
2813 return;
2814 }
2815 }
2816 HTMLSourceElement* childSrc = HTMLSourceElement::FromNode(child);
2817 LOG(LogLevel::Debug,
2818 ("%p Trying load from <source>=%s type=%s", this,
2819 NS_ConvertUTF16toUTF8(src).get(), NS_ConvertUTF16toUTF8(type).get()));
2820
2821 nsCOMPtr<nsIURI> uri;
2822 NewURIFromString(src, getter_AddRefs(uri));
2823 if (!uri) {
2824 AutoTArray<nsString, 1> params = {src};
2825 ReportLoadError("MediaLoadInvalidURI", params);
2826 DealWithFailedElement(child);
2827 return;
2828 }
2829
2830 if (!mMediaSource) {
2831 OwnerDoc()->AddMediaElementWithMSE();
2832 }
2833
2834 RemoveMediaElementFromURITable();
2835 mLoadingSrc = uri;
2836 mLoadingSrcTriggeringPrincipal = childSrc->GetSrcTriggeringPrincipal();
2837 DDLOG(DDLogCategory::Property, "loading_src",
2838 nsCString(NS_ConvertUTF16toUTF8(src)));
2839 mMediaSource = childSrc->GetSrcMediaSource();
2840 DDLINKCHILD("mediasource", mMediaSource.get());
2841 NS_ASSERTION(mNetworkState == NETWORK_LOADING,
2842 "Network state should be loading");
2843
2844 if (mPreloadAction == HTMLMediaElement::PRELOAD_NONE && !mMediaSource) {
2845 // preload:none media, suspend the load here before we make any
2846 // network requests.
2847 SuspendLoad();
2848 return;
2849 }
2850
2851 if (NS_SUCCEEDED(LoadResource())) {
2852 return;
2853 }
2854
2855 // If we fail to load, loop back and try loading the next resource.
2856 DispatchAsyncSourceError(child);
2857 }
2858 MOZ_ASSERT_UNREACHABLE("Execution should not reach here!");
2859 }
2860
SuspendLoad()2861 void HTMLMediaElement::SuspendLoad() {
2862 mSuspendedForPreloadNone = true;
2863 ChangeNetworkState(NETWORK_IDLE);
2864 ChangeDelayLoadStatus(false);
2865 }
2866
ResumeLoad(PreloadAction aAction)2867 void HTMLMediaElement::ResumeLoad(PreloadAction aAction) {
2868 NS_ASSERTION(mSuspendedForPreloadNone,
2869 "Must be halted for preload:none to resume from preload:none "
2870 "suspended load.");
2871 mSuspendedForPreloadNone = false;
2872 mPreloadAction = aAction;
2873 ChangeDelayLoadStatus(true);
2874 ChangeNetworkState(NETWORK_LOADING);
2875 if (!mIsLoadingFromSourceChildren) {
2876 // We were loading from the element's src attribute.
2877 MediaResult rv = LoadResource();
2878 if (NS_FAILED(rv)) {
2879 NoSupportedMediaSourceError(rv.Description());
2880 }
2881 } else {
2882 // We were loading from a child <source> element. Try to resume the
2883 // load of that child, and if that fails, try the next child.
2884 if (NS_FAILED(LoadResource())) {
2885 LoadFromSourceChildren();
2886 }
2887 }
2888 }
2889
AllowedToPlay() const2890 bool HTMLMediaElement::AllowedToPlay() const {
2891 return AutoplayPolicy::IsAllowedToPlay(*this);
2892 }
2893
GetPreloadDefault() const2894 uint32_t HTMLMediaElement::GetPreloadDefault() const {
2895 if (mMediaSource) {
2896 return HTMLMediaElement::PRELOAD_ATTR_METADATA;
2897 }
2898 if (OnCellularConnection()) {
2899 return Preferences::GetInt("media.preload.default.cellular",
2900 HTMLMediaElement::PRELOAD_ATTR_NONE);
2901 }
2902 return Preferences::GetInt("media.preload.default",
2903 HTMLMediaElement::PRELOAD_ATTR_METADATA);
2904 }
2905
GetPreloadDefaultAuto() const2906 uint32_t HTMLMediaElement::GetPreloadDefaultAuto() const {
2907 if (OnCellularConnection()) {
2908 return Preferences::GetInt("media.preload.auto.cellular",
2909 HTMLMediaElement::PRELOAD_ATTR_METADATA);
2910 }
2911 return Preferences::GetInt("media.preload.auto",
2912 HTMLMediaElement::PRELOAD_ENOUGH);
2913 }
2914
UpdatePreloadAction()2915 void HTMLMediaElement::UpdatePreloadAction() {
2916 PreloadAction nextAction = PRELOAD_UNDEFINED;
2917 // If autoplay is set, or we're playing, we should always preload data,
2918 // as we'll need it to play.
2919 if ((AutoplayPolicy::IsAllowedToPlay(*this) &&
2920 HasAttr(kNameSpaceID_None, nsGkAtoms::autoplay)) ||
2921 !mPaused) {
2922 nextAction = HTMLMediaElement::PRELOAD_ENOUGH;
2923 } else {
2924 // Find the appropriate preload action by looking at the attribute.
2925 const nsAttrValue* val =
2926 mAttrs.GetAttr(nsGkAtoms::preload, kNameSpaceID_None);
2927 // MSE doesn't work if preload is none, so it ignores the pref when src is
2928 // from MSE.
2929 uint32_t preloadDefault = GetPreloadDefault();
2930 uint32_t preloadAuto = GetPreloadDefaultAuto();
2931 if (!val) {
2932 // Attribute is not set. Use the preload action specified by the
2933 // media.preload.default pref, or just preload metadata if not present.
2934 nextAction = static_cast<PreloadAction>(preloadDefault);
2935 } else if (val->Type() == nsAttrValue::eEnum) {
2936 PreloadAttrValue attr =
2937 static_cast<PreloadAttrValue>(val->GetEnumValue());
2938 if (attr == HTMLMediaElement::PRELOAD_ATTR_EMPTY ||
2939 attr == HTMLMediaElement::PRELOAD_ATTR_AUTO) {
2940 nextAction = static_cast<PreloadAction>(preloadAuto);
2941 } else if (attr == HTMLMediaElement::PRELOAD_ATTR_METADATA) {
2942 nextAction = HTMLMediaElement::PRELOAD_METADATA;
2943 } else if (attr == HTMLMediaElement::PRELOAD_ATTR_NONE) {
2944 nextAction = HTMLMediaElement::PRELOAD_NONE;
2945 }
2946 } else {
2947 // Use the suggested "missing value default" of "metadata", or the value
2948 // specified by the media.preload.default, if present.
2949 nextAction = static_cast<PreloadAction>(preloadDefault);
2950 }
2951 }
2952
2953 if (nextAction == HTMLMediaElement::PRELOAD_NONE && mIsDoingExplicitLoad) {
2954 nextAction = HTMLMediaElement::PRELOAD_METADATA;
2955 }
2956
2957 mPreloadAction = nextAction;
2958
2959 if (nextAction == HTMLMediaElement::PRELOAD_ENOUGH) {
2960 if (mSuspendedForPreloadNone) {
2961 // Our load was previouly suspended due to the media having preload
2962 // value "none". The preload value has changed to preload:auto, so
2963 // resume the load.
2964 ResumeLoad(PRELOAD_ENOUGH);
2965 } else {
2966 // Preload as much of the video as we can, i.e. don't suspend after
2967 // the first frame.
2968 StopSuspendingAfterFirstFrame();
2969 }
2970
2971 } else if (nextAction == HTMLMediaElement::PRELOAD_METADATA) {
2972 // Ensure that the video can be suspended after first frame.
2973 mAllowSuspendAfterFirstFrame = true;
2974 if (mSuspendedForPreloadNone) {
2975 // Our load was previouly suspended due to the media having preload
2976 // value "none". The preload value has changed to preload:metadata, so
2977 // resume the load. We'll pause the load again after we've read the
2978 // metadata.
2979 ResumeLoad(PRELOAD_METADATA);
2980 }
2981 }
2982 }
2983
LoadResource()2984 MediaResult HTMLMediaElement::LoadResource() {
2985 NS_ASSERTION(mDelayingLoadEvent,
2986 "Should delay load event (if in document) during load");
2987
2988 if (mChannelLoader) {
2989 mChannelLoader->Cancel();
2990 mChannelLoader = nullptr;
2991 }
2992
2993 // Set the media element's CORS mode only when loading a resource
2994 mCORSMode = AttrValueToCORSMode(GetParsedAttr(nsGkAtoms::crossorigin));
2995
2996 HTMLMediaElement* other = LookupMediaElementURITable(mLoadingSrc);
2997 if (other && other->mDecoder) {
2998 // Clone it.
2999 // TODO: remove the cast by storing ChannelMediaDecoder in the URI table.
3000 nsresult rv = InitializeDecoderAsClone(
3001 static_cast<ChannelMediaDecoder*>(other->mDecoder.get()));
3002 if (NS_SUCCEEDED(rv)) return rv;
3003 }
3004
3005 if (mMediaSource) {
3006 MediaDecoderInit decoderInit(
3007 this, this, mMuted ? 0.0 : mVolume, mPreservesPitch,
3008 ClampPlaybackRate(mPlaybackRate),
3009 mPreloadAction == HTMLMediaElement::PRELOAD_METADATA, mHasSuspendTaint,
3010 HasAttr(kNameSpaceID_None, nsGkAtoms::loop),
3011 MediaContainerType(MEDIAMIMETYPE("application/x.mediasource")));
3012
3013 RefPtr<MediaSourceDecoder> decoder = new MediaSourceDecoder(decoderInit);
3014 if (!mMediaSource->Attach(decoder)) {
3015 // TODO: Handle failure: run "If the media data cannot be fetched at
3016 // all, due to network errors, causing the user agent to give up
3017 // trying to fetch the resource" section of resource fetch algorithm.
3018 decoder->Shutdown();
3019 return MediaResult(NS_ERROR_FAILURE, "Failed to attach MediaSource");
3020 }
3021 ChangeDelayLoadStatus(false);
3022 nsresult rv = decoder->Load(mMediaSource->GetPrincipal());
3023 if (NS_FAILED(rv)) {
3024 decoder->Shutdown();
3025 LOG(LogLevel::Debug,
3026 ("%p Failed to load for decoder %p", this, decoder.get()));
3027 return MediaResult(rv, "Fail to load decoder");
3028 }
3029 rv = FinishDecoderSetup(decoder);
3030 return MediaResult(rv, "Failed to set up decoder");
3031 }
3032
3033 AssertReadyStateIsNothing();
3034
3035 RefPtr<ChannelLoader> loader = new ChannelLoader;
3036 nsresult rv = loader->Load(this);
3037 if (NS_SUCCEEDED(rv)) {
3038 mChannelLoader = std::move(loader);
3039 }
3040 return MediaResult(rv, "Failed to load channel");
3041 }
3042
LoadWithChannel(nsIChannel * aChannel,nsIStreamListener ** aListener)3043 nsresult HTMLMediaElement::LoadWithChannel(nsIChannel* aChannel,
3044 nsIStreamListener** aListener) {
3045 NS_ENSURE_ARG_POINTER(aChannel);
3046 NS_ENSURE_ARG_POINTER(aListener);
3047
3048 *aListener = nullptr;
3049
3050 // Make sure we don't reenter during synchronous abort events.
3051 if (mIsRunningLoadMethod) return NS_OK;
3052 mIsRunningLoadMethod = true;
3053 AbortExistingLoads();
3054 mIsRunningLoadMethod = false;
3055
3056 mLoadingSrcTriggeringPrincipal = nullptr;
3057 nsresult rv = aChannel->GetOriginalURI(getter_AddRefs(mLoadingSrc));
3058 NS_ENSURE_SUCCESS(rv, rv);
3059
3060 ChangeDelayLoadStatus(true);
3061 rv = InitializeDecoderForChannel(aChannel, aListener);
3062 if (NS_FAILED(rv)) {
3063 ChangeDelayLoadStatus(false);
3064 return rv;
3065 }
3066
3067 SetPlaybackRate(mDefaultPlaybackRate, IgnoreErrors());
3068 DispatchAsyncEvent(u"loadstart"_ns);
3069
3070 return NS_OK;
3071 }
3072
Seeking() const3073 bool HTMLMediaElement::Seeking() const {
3074 return mDecoder && mDecoder->IsSeeking();
3075 }
3076
CurrentTime() const3077 double HTMLMediaElement::CurrentTime() const {
3078 if (mMediaStreamRenderer) {
3079 return mMediaStreamRenderer->CurrentTime();
3080 }
3081
3082 if (mDefaultPlaybackStartPosition == 0.0 && mDecoder) {
3083 return mDecoder->GetCurrentTime();
3084 }
3085
3086 return mDefaultPlaybackStartPosition;
3087 }
3088
FastSeek(double aTime,ErrorResult & aRv)3089 void HTMLMediaElement::FastSeek(double aTime, ErrorResult& aRv) {
3090 LOG(LogLevel::Debug, ("%p FastSeek(%f) called by JS", this, aTime));
3091 Seek(aTime, SeekTarget::PrevSyncPoint, IgnoreErrors());
3092 }
3093
SeekToNextFrame(ErrorResult & aRv)3094 already_AddRefed<Promise> HTMLMediaElement::SeekToNextFrame(ErrorResult& aRv) {
3095 /* This will cause JIT code to be kept around longer, to help performance
3096 * when using SeekToNextFrame to iterate through every frame of a video.
3097 */
3098 nsPIDOMWindowInner* win = OwnerDoc()->GetInnerWindow();
3099
3100 if (win) {
3101 if (JSObject* obj = win->AsGlobal()->GetGlobalJSObject()) {
3102 js::NotifyAnimationActivity(obj);
3103 }
3104 }
3105
3106 Seek(CurrentTime(), SeekTarget::NextFrame, aRv);
3107 if (aRv.Failed()) {
3108 return nullptr;
3109 }
3110
3111 mSeekDOMPromise = CreateDOMPromise(aRv);
3112 if (NS_WARN_IF(aRv.Failed())) {
3113 return nullptr;
3114 }
3115
3116 return do_AddRef(mSeekDOMPromise);
3117 }
3118
SetCurrentTime(double aCurrentTime,ErrorResult & aRv)3119 void HTMLMediaElement::SetCurrentTime(double aCurrentTime, ErrorResult& aRv) {
3120 LOG(LogLevel::Debug,
3121 ("%p SetCurrentTime(%f) called by JS", this, aCurrentTime));
3122 Seek(aCurrentTime, SeekTarget::Accurate, IgnoreErrors());
3123 }
3124
3125 /**
3126 * Check if aValue is inside a range of aRanges, and if so returns true
3127 * and puts the range index in aIntervalIndex. If aValue is not
3128 * inside a range, returns false, and aIntervalIndex
3129 * is set to the index of the range which starts immediately after aValue
3130 * (and can be aRanges.Length() if aValue is after the last range).
3131 */
IsInRanges(TimeRanges & aRanges,double aValue,uint32_t & aIntervalIndex)3132 static bool IsInRanges(TimeRanges& aRanges, double aValue,
3133 uint32_t& aIntervalIndex) {
3134 uint32_t length = aRanges.Length();
3135
3136 for (uint32_t i = 0; i < length; i++) {
3137 double start = aRanges.Start(i);
3138 if (start > aValue) {
3139 aIntervalIndex = i;
3140 return false;
3141 }
3142 double end = aRanges.End(i);
3143 if (aValue <= end) {
3144 aIntervalIndex = i;
3145 return true;
3146 }
3147 }
3148 aIntervalIndex = length;
3149 return false;
3150 }
3151
Seek(double aTime,SeekTarget::Type aSeekType,ErrorResult & aRv)3152 void HTMLMediaElement::Seek(double aTime, SeekTarget::Type aSeekType,
3153 ErrorResult& aRv) {
3154 // Note: Seek is called both by synchronous code that expects errors thrown in
3155 // aRv, as well as asynchronous code that expects a promise. Make sure all
3156 // synchronous errors are returned using aRv, not promise rejections.
3157
3158 // aTime should be non-NaN.
3159 MOZ_ASSERT(!mozilla::IsNaN(aTime));
3160
3161 // Seeking step1, Set the media element's show poster flag to false.
3162 // https://html.spec.whatwg.org/multipage/media.html#dom-media-seek
3163 mShowPoster = false;
3164
3165 // Detect if user has interacted with element by seeking so that
3166 // play will not be blocked when initiated by a script.
3167 if (UserActivation::IsHandlingUserInput()) {
3168 mIsBlessed = true;
3169 }
3170
3171 StopSuspendingAfterFirstFrame();
3172
3173 if (mSrcAttrStream) {
3174 // do nothing since media streams have an empty Seekable range.
3175 aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
3176 return;
3177 }
3178
3179 if (mPlayed && mCurrentPlayRangeStart != -1.0) {
3180 double rangeEndTime = CurrentTime();
3181 LOG(LogLevel::Debug, ("%p Adding \'played\' a range : [%f, %f]", this,
3182 mCurrentPlayRangeStart, rangeEndTime));
3183 // Multiple seek without playing, or seek while playing.
3184 if (mCurrentPlayRangeStart != rangeEndTime) {
3185 mPlayed->Add(mCurrentPlayRangeStart, rangeEndTime);
3186 }
3187 // Reset the current played range start time. We'll re-set it once
3188 // the seek completes.
3189 mCurrentPlayRangeStart = -1.0;
3190 }
3191
3192 if (mReadyState == HAVE_NOTHING) {
3193 mDefaultPlaybackStartPosition = aTime;
3194 aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
3195 return;
3196 }
3197
3198 if (!mDecoder) {
3199 // mDecoder must always be set in order to reach this point.
3200 NS_ASSERTION(mDecoder, "SetCurrentTime failed: no decoder");
3201 aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
3202 return;
3203 }
3204
3205 // Clamp the seek target to inside the seekable ranges.
3206 media::TimeIntervals seekableIntervals = mDecoder->GetSeekable();
3207 if (seekableIntervals.IsInvalid()) {
3208 aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
3209 return;
3210 }
3211 RefPtr<TimeRanges> seekable =
3212 new TimeRanges(ToSupports(OwnerDoc()), seekableIntervals);
3213 uint32_t length = seekable->Length();
3214 if (length == 0) {
3215 aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
3216 return;
3217 }
3218
3219 // If the position we want to seek to is not in a seekable range, we seek
3220 // to the closest position in the seekable ranges instead. If two positions
3221 // are equally close, we seek to the closest position from the currentTime.
3222 // See seeking spec, point 7 :
3223 // http://www.whatwg.org/specs/web-apps/current-work/multipage/the-video-element.html#seeking
3224 uint32_t range = 0;
3225 bool isInRange = IsInRanges(*seekable, aTime, range);
3226 if (!isInRange) {
3227 if (range == 0) {
3228 // aTime is before the first range in |seekable|, the closest point we can
3229 // seek to is the start of the first range.
3230 aTime = seekable->Start(0);
3231 } else if (range == length) {
3232 // Seek target is after the end last range in seekable data.
3233 // Clamp the seek target to the end of the last seekable range.
3234 aTime = seekable->End(length - 1);
3235 } else {
3236 double leftBound = seekable->End(range - 1);
3237 double rightBound = seekable->Start(range);
3238 double distanceLeft = Abs(leftBound - aTime);
3239 double distanceRight = Abs(rightBound - aTime);
3240 if (distanceLeft == distanceRight) {
3241 double currentTime = CurrentTime();
3242 distanceLeft = Abs(leftBound - currentTime);
3243 distanceRight = Abs(rightBound - currentTime);
3244 }
3245 aTime = (distanceLeft < distanceRight) ? leftBound : rightBound;
3246 }
3247 }
3248
3249 // TODO: The spec requires us to update the current time to reflect the
3250 // actual seek target before beginning the synchronous section, but
3251 // that requires changing all MediaDecoderReaders to support telling
3252 // us the fastSeek target, and it's currently not possible to get
3253 // this information as we don't yet control the demuxer for all
3254 // MediaDecoderReaders.
3255
3256 mPlayingBeforeSeek = IsPotentiallyPlaying();
3257
3258 // The media backend is responsible for dispatching the timeupdate
3259 // event if it changes the playback position as a result of the seek.
3260 LOG(LogLevel::Debug, ("%p SetCurrentTime(%f) starting seek", this, aTime));
3261 mDecoder->Seek(aTime, aSeekType);
3262
3263 // We changed whether we're seeking so we need to AddRemoveSelfReference.
3264 AddRemoveSelfReference();
3265 }
3266
Duration() const3267 double HTMLMediaElement::Duration() const {
3268 if (mSrcStream) {
3269 if (mSrcStreamPlaybackEnded) {
3270 return CurrentTime();
3271 }
3272 return std::numeric_limits<double>::infinity();
3273 }
3274
3275 if (mDecoder) {
3276 return mDecoder->GetDuration();
3277 }
3278
3279 return std::numeric_limits<double>::quiet_NaN();
3280 }
3281
Seekable() const3282 already_AddRefed<TimeRanges> HTMLMediaElement::Seekable() const {
3283 media::TimeIntervals seekable =
3284 mDecoder ? mDecoder->GetSeekable() : media::TimeIntervals();
3285 RefPtr<TimeRanges> ranges = new TimeRanges(ToSupports(OwnerDoc()), seekable);
3286 return ranges.forget();
3287 }
3288
Played()3289 already_AddRefed<TimeRanges> HTMLMediaElement::Played() {
3290 RefPtr<TimeRanges> ranges = new TimeRanges(ToSupports(OwnerDoc()));
3291
3292 uint32_t timeRangeCount = 0;
3293 if (mPlayed) {
3294 timeRangeCount = mPlayed->Length();
3295 }
3296 for (uint32_t i = 0; i < timeRangeCount; i++) {
3297 double begin = mPlayed->Start(i);
3298 double end = mPlayed->End(i);
3299 ranges->Add(begin, end);
3300 }
3301
3302 if (mCurrentPlayRangeStart != -1.0) {
3303 double now = CurrentTime();
3304 if (mCurrentPlayRangeStart != now) {
3305 ranges->Add(mCurrentPlayRangeStart, now);
3306 }
3307 }
3308
3309 ranges->Normalize();
3310 return ranges.forget();
3311 }
3312
Pause(ErrorResult & aRv)3313 void HTMLMediaElement::Pause(ErrorResult& aRv) {
3314 LOG(LogLevel::Debug, ("%p Pause() called by JS", this));
3315 if (mNetworkState == NETWORK_EMPTY) {
3316 LOG(LogLevel::Debug, ("Loading due to Pause()"));
3317 DoLoad();
3318 }
3319 PauseInternal();
3320 }
3321
PauseInternal()3322 void HTMLMediaElement::PauseInternal() {
3323 if (mDecoder && mNetworkState != NETWORK_EMPTY) {
3324 mDecoder->Pause();
3325 }
3326 bool oldPaused = mPaused;
3327 mPaused = true;
3328 mAutoplaying = false;
3329 // We changed mPaused and mAutoplaying which can affect AddRemoveSelfReference
3330 AddRemoveSelfReference();
3331 UpdateSrcMediaStreamPlaying();
3332 if (mAudioChannelWrapper) {
3333 mAudioChannelWrapper->NotifyPlayStateChanged();
3334 }
3335
3336 // We don't need to resume media which is paused explicitly by user.
3337 ClearResumeDelayedMediaPlaybackAgentIfNeeded();
3338
3339 if (!oldPaused) {
3340 FireTimeUpdate(TimeupdateType::eMandatory);
3341 DispatchAsyncEvent(u"pause"_ns);
3342 AsyncRejectPendingPlayPromises(NS_ERROR_DOM_MEDIA_ABORT_ERR);
3343 }
3344 }
3345
SetVolume(double aVolume,ErrorResult & aRv)3346 void HTMLMediaElement::SetVolume(double aVolume, ErrorResult& aRv) {
3347 LOG(LogLevel::Debug, ("%p SetVolume(%f) called by JS", this, aVolume));
3348
3349 if (aVolume < 0.0 || aVolume > 1.0) {
3350 aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR);
3351 return;
3352 }
3353
3354 if (aVolume == mVolume) return;
3355
3356 mVolume = aVolume;
3357
3358 // Here we want just to update the volume.
3359 SetVolumeInternal();
3360
3361 DispatchAsyncEvent(u"volumechange"_ns);
3362
3363 // We allow inaudible autoplay. But changing our volume may make this
3364 // media audible. So pause if we are no longer supposed to be autoplaying.
3365 PauseIfShouldNotBePlaying();
3366 }
3367
MozGetMetadata(JSContext * cx,JS::MutableHandle<JSObject * > aRetval,ErrorResult & aRv)3368 void HTMLMediaElement::MozGetMetadata(JSContext* cx,
3369 JS::MutableHandle<JSObject*> aRetval,
3370 ErrorResult& aRv) {
3371 if (mReadyState < HAVE_METADATA) {
3372 aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
3373 return;
3374 }
3375
3376 JS::Rooted<JSObject*> tags(cx, JS_NewPlainObject(cx));
3377 if (!tags) {
3378 aRv.Throw(NS_ERROR_FAILURE);
3379 return;
3380 }
3381 if (mTags) {
3382 for (const auto& entry : *mTags) {
3383 nsString wideValue;
3384 CopyUTF8toUTF16(entry.GetData(), wideValue);
3385 JS::Rooted<JSString*> string(cx,
3386 JS_NewUCStringCopyZ(cx, wideValue.Data()));
3387 if (!string || !JS_DefineProperty(cx, tags, entry.GetKey().Data(), string,
3388 JSPROP_ENUMERATE)) {
3389 NS_WARNING("couldn't create metadata object!");
3390 aRv.Throw(NS_ERROR_FAILURE);
3391 return;
3392 }
3393 }
3394 }
3395
3396 aRetval.set(tags);
3397 }
3398
SetMutedInternal(uint32_t aMuted)3399 void HTMLMediaElement::SetMutedInternal(uint32_t aMuted) {
3400 uint32_t oldMuted = mMuted;
3401 mMuted = aMuted;
3402
3403 if (!!aMuted == !!oldMuted) {
3404 return;
3405 }
3406
3407 SetVolumeInternal();
3408 }
3409
PauseIfShouldNotBePlaying()3410 void HTMLMediaElement::PauseIfShouldNotBePlaying() {
3411 if (GetPaused()) {
3412 return;
3413 }
3414 if (!AutoplayPolicy::IsAllowedToPlay(*this)) {
3415 AUTOPLAY_LOG("pause because not allowed to play, element=%p", this);
3416 ErrorResult rv;
3417 Pause(rv);
3418 }
3419 }
3420
SetVolumeInternal()3421 void HTMLMediaElement::SetVolumeInternal() {
3422 float effectiveVolume = ComputedVolume();
3423
3424 if (mDecoder) {
3425 mDecoder->SetVolume(effectiveVolume);
3426 } else if (mMediaStreamRenderer) {
3427 mMediaStreamRenderer->SetAudioOutputVolume(effectiveVolume);
3428 }
3429
3430 NotifyAudioPlaybackChanged(
3431 AudioChannelService::AudibleChangedReasons::eVolumeChanged);
3432 }
3433
SetMuted(bool aMuted)3434 void HTMLMediaElement::SetMuted(bool aMuted) {
3435 LOG(LogLevel::Debug, ("%p SetMuted(%d) called by JS", this, aMuted));
3436 if (aMuted == Muted()) {
3437 return;
3438 }
3439
3440 if (aMuted) {
3441 SetMutedInternal(mMuted | MUTED_BY_CONTENT);
3442 } else {
3443 SetMutedInternal(mMuted & ~MUTED_BY_CONTENT);
3444 }
3445
3446 DispatchAsyncEvent(u"volumechange"_ns);
3447
3448 // We allow inaudible autoplay. But changing our mute status may make this
3449 // media audible. So pause if we are no longer supposed to be autoplaying.
3450 PauseIfShouldNotBePlaying();
3451 }
3452
GetAllEnabledMediaTracks(nsTArray<RefPtr<MediaTrack>> & aTracks)3453 void HTMLMediaElement::GetAllEnabledMediaTracks(
3454 nsTArray<RefPtr<MediaTrack>>& aTracks) {
3455 if (AudioTrackList* tracks = AudioTracks()) {
3456 for (size_t i = 0; i < tracks->Length(); ++i) {
3457 AudioTrack* track = (*tracks)[i];
3458 if (track->Enabled()) {
3459 aTracks.AppendElement(track);
3460 }
3461 }
3462 }
3463 if (IsVideo()) {
3464 if (VideoTrackList* tracks = VideoTracks()) {
3465 for (size_t i = 0; i < tracks->Length(); ++i) {
3466 VideoTrack* track = (*tracks)[i];
3467 if (track->Selected()) {
3468 aTracks.AppendElement(track);
3469 }
3470 }
3471 }
3472 }
3473 }
3474
SetCapturedOutputStreamsEnabled(bool aEnabled)3475 void HTMLMediaElement::SetCapturedOutputStreamsEnabled(bool aEnabled) {
3476 for (const auto& entry : mOutputTrackSources.Values()) {
3477 entry->SetEnabled(aEnabled);
3478 }
3479 }
3480
OutputTracksMuted()3481 HTMLMediaElement::OutputMuteState HTMLMediaElement::OutputTracksMuted() {
3482 return mPaused || mReadyState <= HAVE_CURRENT_DATA ? OutputMuteState::Muted
3483 : OutputMuteState::Unmuted;
3484 }
3485
UpdateOutputTracksMuting()3486 void HTMLMediaElement::UpdateOutputTracksMuting() {
3487 for (const auto& entry : mOutputTrackSources.Values()) {
3488 entry->SetMutedByElement(OutputTracksMuted());
3489 }
3490 }
3491
AddOutputTrackSourceToOutputStream(MediaElementTrackSource * aSource,OutputMediaStream & aOutputStream,AddTrackMode aMode)3492 void HTMLMediaElement::AddOutputTrackSourceToOutputStream(
3493 MediaElementTrackSource* aSource, OutputMediaStream& aOutputStream,
3494 AddTrackMode aMode) {
3495 if (aOutputStream.mStream == mSrcStream) {
3496 // Cycle detected. This can happen since tracks are added async.
3497 // We avoid forwarding it to the output here or we'd get into an infloop.
3498 LOG(LogLevel::Warning,
3499 ("NOT adding output track source %p to output stream "
3500 "%p -- cycle detected",
3501 aSource, aOutputStream.mStream.get()));
3502 return;
3503 }
3504
3505 LOG(LogLevel::Debug, ("Adding output track source %p to output stream %p",
3506 aSource, aOutputStream.mStream.get()));
3507
3508 RefPtr<MediaStreamTrack> domTrack;
3509 if (aSource->Track()->mType == MediaSegment::AUDIO) {
3510 domTrack = new AudioStreamTrack(
3511 aOutputStream.mStream->GetOwner(), aSource->Track(), aSource,
3512 MediaStreamTrackState::Live, aSource->Muted());
3513 } else {
3514 domTrack = new VideoStreamTrack(
3515 aOutputStream.mStream->GetOwner(), aSource->Track(), aSource,
3516 MediaStreamTrackState::Live, aSource->Muted());
3517 }
3518
3519 aOutputStream.mLiveTracks.AppendElement(domTrack);
3520
3521 switch (aMode) {
3522 case AddTrackMode::ASYNC:
3523 mMainThreadEventTarget->Dispatch(
3524 NewRunnableMethod<StoreRefPtrPassByPtr<MediaStreamTrack>>(
3525 "DOMMediaStream::AddTrackInternal", aOutputStream.mStream,
3526 &DOMMediaStream::AddTrackInternal, domTrack));
3527 break;
3528 case AddTrackMode::SYNC:
3529 aOutputStream.mStream->AddTrackInternal(domTrack);
3530 break;
3531 default:
3532 MOZ_CRASH("Unexpected mode");
3533 }
3534
3535 LOG(LogLevel::Debug,
3536 ("Created capture %s track %p",
3537 domTrack->AsAudioStreamTrack() ? "audio" : "video", domTrack.get()));
3538 }
3539
UpdateOutputTrackSources()3540 void HTMLMediaElement::UpdateOutputTrackSources() {
3541 // This updates the track sources in mOutputTrackSources so they're in sync
3542 // with the tracks being currently played, and state saying whether we should
3543 // be capturing tracks. This method is long so here is a breakdown:
3544 // - Figure out the tracks that should be captured
3545 // - Diff those against currently captured tracks (mOutputTrackSources), into
3546 // tracks-to-add, and tracks-to-remove
3547 // - Remove the tracks in tracks-to-remove and dispatch "removetrack" and
3548 // "ended" events for them
3549 // - If playback has ended, or there is no longer a media provider object,
3550 // remove any OutputMediaStreams that have the finish-when-ended flag set
3551 // - Create track sources for, and add to OutputMediaStreams, the tracks in
3552 // tracks-to-add
3553
3554 const bool shouldHaveTrackSources = mTracksCaptured.Ref() &&
3555 !IsPlaybackEnded() &&
3556 mReadyState >= HAVE_METADATA;
3557
3558 // Add track sources for all enabled/selected MediaTracks.
3559 nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
3560 if (!window) {
3561 return;
3562 }
3563
3564 if (mDecoder) {
3565 if (!mTracksCaptured.Ref()) {
3566 mDecoder->SetOutputCaptureState(MediaDecoder::OutputCaptureState::None);
3567 } else if (!AudioTracks() || !VideoTracks() || !shouldHaveTrackSources) {
3568 // We've been unlinked, or tracks are not yet known.
3569 mDecoder->SetOutputCaptureState(MediaDecoder::OutputCaptureState::Halt);
3570 } else {
3571 mDecoder->SetOutputCaptureState(MediaDecoder::OutputCaptureState::Capture,
3572 mTracksCaptured.Ref().get());
3573 }
3574 }
3575
3576 // Start with all MediaTracks
3577 AutoTArray<RefPtr<MediaTrack>, 4> mediaTracksToAdd;
3578 if (shouldHaveTrackSources) {
3579 GetAllEnabledMediaTracks(mediaTracksToAdd);
3580 }
3581
3582 // ...and all MediaElementTrackSources.
3583 auto trackSourcesToRemove =
3584 ToTArray<AutoTArray<nsString, 4>>(mOutputTrackSources.Keys());
3585
3586 // Then work out the differences.
3587 mediaTracksToAdd.RemoveLastElements(
3588 mediaTracksToAdd.end() -
3589 std::remove_if(mediaTracksToAdd.begin(), mediaTracksToAdd.end(),
3590 [this, &trackSourcesToRemove](const auto& track) {
3591 const bool remove =
3592 mOutputTrackSources.GetWeak(track->GetId());
3593 if (remove) {
3594 trackSourcesToRemove.RemoveElement(track->GetId());
3595 }
3596 return remove;
3597 }));
3598
3599 // First remove stale track sources.
3600 for (const auto& id : trackSourcesToRemove) {
3601 RefPtr<MediaElementTrackSource> source = mOutputTrackSources.GetWeak(id);
3602
3603 LOG(LogLevel::Debug, ("Removing output track source %p for track %s",
3604 source.get(), NS_ConvertUTF16toUTF8(id).get()));
3605
3606 if (mDecoder) {
3607 mDecoder->RemoveOutputTrack(source->Track());
3608 }
3609
3610 // The source of this track just ended. Force-notify that it ended.
3611 // If we bounce it to the MediaTrackGraph it might not be picked up,
3612 // for instance if the MediaInputPort was destroyed in the same
3613 // iteration as it was added.
3614 mMainThreadEventTarget->Dispatch(
3615 NewRunnableMethod("MediaElementTrackSource::OverrideEnded", source,
3616 &MediaElementTrackSource::OverrideEnded));
3617
3618 // Remove the track from the MediaStream after it ended.
3619 for (OutputMediaStream& ms : mOutputStreams) {
3620 if (source->Track()->mType == MediaSegment::VIDEO &&
3621 ms.mCapturingAudioOnly) {
3622 continue;
3623 }
3624 DebugOnly<size_t> length = ms.mLiveTracks.Length();
3625 ms.mLiveTracks.RemoveElementsBy(
3626 [&](const RefPtr<MediaStreamTrack>& aTrack) {
3627 if (&aTrack->GetSource() != source) {
3628 return false;
3629 }
3630 mMainThreadEventTarget->Dispatch(
3631 NewRunnableMethod<RefPtr<MediaStreamTrack>>(
3632 "DOMMediaStream::RemoveTrackInternal", ms.mStream,
3633 &DOMMediaStream::RemoveTrackInternal, aTrack));
3634 return true;
3635 });
3636 MOZ_ASSERT(ms.mLiveTracks.Length() == length - 1);
3637 }
3638
3639 mOutputTrackSources.Remove(id);
3640 }
3641
3642 // Then update finish-when-ended output streams as needed.
3643 for (int32_t i = mOutputStreams.Length() - 1; i >= 0; --i) {
3644 if (!mOutputStreams[i].mFinishWhenEnded) {
3645 continue;
3646 }
3647
3648 if (!mOutputStreams[i].mFinishWhenEndedLoadingSrc &&
3649 !mOutputStreams[i].mFinishWhenEndedAttrStream &&
3650 !mOutputStreams[i].mFinishWhenEndedMediaSource) {
3651 // This finish-when-ended stream has not seen any source loaded yet.
3652 // Update the loading src if it's time.
3653 if (!IsPlaybackEnded()) {
3654 if (mLoadingSrc) {
3655 mOutputStreams[i].mFinishWhenEndedLoadingSrc = mLoadingSrc;
3656 } else if (mSrcAttrStream) {
3657 mOutputStreams[i].mFinishWhenEndedAttrStream = mSrcAttrStream;
3658 } else if (mSrcMediaSource) {
3659 mOutputStreams[i].mFinishWhenEndedMediaSource = mSrcMediaSource;
3660 }
3661 }
3662 continue;
3663 }
3664
3665 // Discard finish-when-ended output streams with a loading src set as
3666 // needed.
3667 if (!IsPlaybackEnded() &&
3668 mLoadingSrc == mOutputStreams[i].mFinishWhenEndedLoadingSrc) {
3669 continue;
3670 }
3671 if (!IsPlaybackEnded() &&
3672 mSrcAttrStream == mOutputStreams[i].mFinishWhenEndedAttrStream) {
3673 continue;
3674 }
3675 if (!IsPlaybackEnded() &&
3676 mSrcMediaSource == mOutputStreams[i].mFinishWhenEndedMediaSource) {
3677 continue;
3678 }
3679 LOG(LogLevel::Debug,
3680 ("Playback ended or source changed. Discarding stream %p",
3681 mOutputStreams[i].mStream.get()));
3682 mOutputStreams.RemoveElementAt(i);
3683 if (mOutputStreams.IsEmpty()) {
3684 mTracksCaptured = nullptr;
3685 // mTracksCaptured is one of the Watchables triggering this method.
3686 // Unsetting it here means we'll run through this method again very soon.
3687 return;
3688 }
3689 }
3690
3691 // Finally add new MediaTracks.
3692 for (const auto& mediaTrack : mediaTracksToAdd) {
3693 nsAutoString id;
3694 mediaTrack->GetId(id);
3695
3696 MediaSegment::Type type;
3697 if (mediaTrack->AsAudioTrack()) {
3698 type = MediaSegment::AUDIO;
3699 } else if (mediaTrack->AsVideoTrack()) {
3700 type = MediaSegment::VIDEO;
3701 } else {
3702 MOZ_CRASH("Unknown track type");
3703 }
3704
3705 RefPtr<ProcessedMediaTrack> track;
3706 RefPtr<MediaElementTrackSource> source;
3707 if (mDecoder) {
3708 track = mTracksCaptured.Ref()->mTrack->Graph()->CreateForwardedInputTrack(
3709 type);
3710 RefPtr<nsIPrincipal> principal = GetCurrentPrincipal();
3711 if (!principal || IsCORSSameOrigin()) {
3712 principal = NodePrincipal();
3713 }
3714 source = MakeAndAddRef<MediaElementTrackSource>(
3715 mMainThreadEventTarget, track, principal, OutputTracksMuted(),
3716 type == MediaSegment::VIDEO
3717 ? HTMLVideoElement::FromNode(this)->HasAlpha()
3718 : false);
3719 mDecoder->AddOutputTrack(track);
3720 } else if (mSrcStream) {
3721 MediaStreamTrack* inputTrack;
3722 if (AudioTrack* t = mediaTrack->AsAudioTrack()) {
3723 inputTrack = t->GetAudioStreamTrack();
3724 } else if (VideoTrack* t = mediaTrack->AsVideoTrack()) {
3725 inputTrack = t->GetVideoStreamTrack();
3726 } else {
3727 MOZ_CRASH("Unknown track type");
3728 }
3729 MOZ_ASSERT(inputTrack);
3730 if (!inputTrack) {
3731 NS_ERROR("Input track not found in source stream");
3732 return;
3733 }
3734 MOZ_DIAGNOSTIC_ASSERT(!inputTrack->Ended());
3735
3736 track = inputTrack->Graph()->CreateForwardedInputTrack(type);
3737 RefPtr<MediaInputPort> port = inputTrack->ForwardTrackContentsTo(track);
3738 source = MakeAndAddRef<MediaElementTrackSource>(
3739 mMainThreadEventTarget, inputTrack, &inputTrack->GetSource(), track,
3740 port, OutputTracksMuted());
3741
3742 // Track is muted initially, so we don't leak data if it's added while
3743 // paused and an MTG iteration passes before the mute comes into effect.
3744 source->SetEnabled(mSrcStreamIsPlaying);
3745 } else {
3746 MOZ_CRASH("Unknown source");
3747 }
3748
3749 LOG(LogLevel::Debug, ("Adding output track source %p for track %s",
3750 source.get(), NS_ConvertUTF16toUTF8(id).get()));
3751
3752 track->QueueSetAutoend(false);
3753 MOZ_DIAGNOSTIC_ASSERT(!mOutputTrackSources.Contains(id));
3754 mOutputTrackSources.InsertOrUpdate(id, RefPtr{source});
3755
3756 // Add the new track source to any existing output streams
3757 for (OutputMediaStream& ms : mOutputStreams) {
3758 if (source->Track()->mType == MediaSegment::VIDEO &&
3759 ms.mCapturingAudioOnly) {
3760 // If the output stream is for audio only we ignore video sources.
3761 continue;
3762 }
3763 AddOutputTrackSourceToOutputStream(source, ms);
3764 }
3765 }
3766 }
3767
CanBeCaptured(StreamCaptureType aCaptureType)3768 bool HTMLMediaElement::CanBeCaptured(StreamCaptureType aCaptureType) {
3769 // Don't bother capturing when the document has gone away
3770 nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
3771 if (!window) {
3772 return false;
3773 }
3774
3775 // Prevent capturing restricted video
3776 if (aCaptureType == StreamCaptureType::CAPTURE_ALL_TRACKS &&
3777 ContainsRestrictedContent()) {
3778 return false;
3779 }
3780 return true;
3781 }
3782
CaptureStreamInternal(StreamCaptureBehavior aFinishBehavior,StreamCaptureType aStreamCaptureType,MediaTrackGraph * aGraph)3783 already_AddRefed<DOMMediaStream> HTMLMediaElement::CaptureStreamInternal(
3784 StreamCaptureBehavior aFinishBehavior, StreamCaptureType aStreamCaptureType,
3785 MediaTrackGraph* aGraph) {
3786 MOZ_ASSERT(CanBeCaptured(aStreamCaptureType));
3787
3788 MarkAsContentSource(CallerAPI::CAPTURE_STREAM);
3789 MarkAsTainted();
3790
3791 if (mTracksCaptured.Ref() &&
3792 aGraph != mTracksCaptured.Ref()->mTrack->Graph()) {
3793 return nullptr;
3794 }
3795
3796 if (!mTracksCaptured.Ref()) {
3797 // This is the first output stream, or there are no tracks. If the former,
3798 // start capturing all tracks. If the latter, they will be added later.
3799 mTracksCaptured = MakeRefPtr<SharedDummyTrack>(
3800 aGraph->CreateSourceTrack(MediaSegment::AUDIO));
3801 UpdateOutputTrackSources();
3802 }
3803
3804 nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
3805 OutputMediaStream* out = mOutputStreams.EmplaceBack(
3806 MakeRefPtr<DOMMediaStream>(window),
3807 aStreamCaptureType == StreamCaptureType::CAPTURE_AUDIO,
3808 aFinishBehavior == StreamCaptureBehavior::FINISH_WHEN_ENDED);
3809
3810 if (aFinishBehavior == StreamCaptureBehavior::FINISH_WHEN_ENDED &&
3811 !mOutputTrackSources.IsEmpty()) {
3812 // This output stream won't receive any more tracks when playback of the
3813 // current src of this media element ends, or when the src of this media
3814 // element changes. If we're currently playing something (i.e., if there are
3815 // tracks currently captured), set the current src on the output stream so
3816 // this can be tracked. If we're not playing anything,
3817 // UpdateOutputTrackSources will set the current src when it becomes
3818 // available later.
3819 if (mLoadingSrc) {
3820 out->mFinishWhenEndedLoadingSrc = mLoadingSrc;
3821 }
3822 if (mSrcAttrStream) {
3823 out->mFinishWhenEndedAttrStream = mSrcAttrStream;
3824 }
3825 if (mSrcMediaSource) {
3826 out->mFinishWhenEndedMediaSource = mSrcMediaSource;
3827 }
3828 MOZ_ASSERT(out->mFinishWhenEndedLoadingSrc ||
3829 out->mFinishWhenEndedAttrStream ||
3830 out->mFinishWhenEndedMediaSource);
3831 }
3832
3833 if (aStreamCaptureType == StreamCaptureType::CAPTURE_AUDIO) {
3834 if (mSrcStream) {
3835 // We don't support applying volume and mute to the captured stream, when
3836 // capturing a MediaStream.
3837 ReportToConsole(nsIScriptError::errorFlag,
3838 "MediaElementAudioCaptureOfMediaStreamError");
3839 }
3840
3841 // mAudioCaptured tells the user that the audio played by this media element
3842 // is being routed to the captureStreams *instead* of being played to
3843 // speakers.
3844 mAudioCaptured = true;
3845 }
3846
3847 for (const RefPtr<MediaElementTrackSource>& source :
3848 mOutputTrackSources.Values()) {
3849 if (source->Track()->mType == MediaSegment::VIDEO) {
3850 // Only add video tracks if we're a video element and the output stream
3851 // wants video.
3852 if (!IsVideo()) {
3853 continue;
3854 }
3855 if (out->mCapturingAudioOnly) {
3856 continue;
3857 }
3858 }
3859 AddOutputTrackSourceToOutputStream(source, *out, AddTrackMode::SYNC);
3860 }
3861
3862 return do_AddRef(out->mStream);
3863 }
3864
CaptureAudio(ErrorResult & aRv,MediaTrackGraph * aGraph)3865 already_AddRefed<DOMMediaStream> HTMLMediaElement::CaptureAudio(
3866 ErrorResult& aRv, MediaTrackGraph* aGraph) {
3867 MOZ_RELEASE_ASSERT(aGraph);
3868
3869 if (!CanBeCaptured(StreamCaptureType::CAPTURE_AUDIO)) {
3870 aRv.Throw(NS_ERROR_FAILURE);
3871 return nullptr;
3872 }
3873
3874 RefPtr<DOMMediaStream> stream =
3875 CaptureStreamInternal(StreamCaptureBehavior::CONTINUE_WHEN_ENDED,
3876 StreamCaptureType::CAPTURE_AUDIO, aGraph);
3877 if (!stream) {
3878 aRv.Throw(NS_ERROR_FAILURE);
3879 return nullptr;
3880 }
3881
3882 return stream.forget();
3883 }
3884
GetAllowedToPlayPromise()3885 RefPtr<GenericNonExclusivePromise> HTMLMediaElement::GetAllowedToPlayPromise() {
3886 MOZ_ASSERT(NS_IsMainThread());
3887 MOZ_ASSERT(!mOutputStreams.IsEmpty(),
3888 "This method should only be called during stream capturing!");
3889 if (AutoplayPolicy::IsAllowedToPlay(*this)) {
3890 AUTOPLAY_LOG("MediaElement %p has allowed to play, resolve promise", this);
3891 return GenericNonExclusivePromise::CreateAndResolve(true, __func__);
3892 }
3893 AUTOPLAY_LOG("create allow-to-play promise for MediaElement %p", this);
3894 return mAllowedToPlayPromise.Ensure(__func__);
3895 }
3896
MozCaptureStream(ErrorResult & aRv)3897 already_AddRefed<DOMMediaStream> HTMLMediaElement::MozCaptureStream(
3898 ErrorResult& aRv) {
3899 MediaTrackGraph::GraphDriverType graphDriverType =
3900 HasAudio() ? MediaTrackGraph::AUDIO_THREAD_DRIVER
3901 : MediaTrackGraph::SYSTEM_THREAD_DRIVER;
3902
3903 nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
3904 if (!window) {
3905 aRv.Throw(NS_ERROR_FAILURE);
3906 return nullptr;
3907 }
3908
3909 if (!CanBeCaptured(StreamCaptureType::CAPTURE_ALL_TRACKS)) {
3910 aRv.Throw(NS_ERROR_FAILURE);
3911 return nullptr;
3912 }
3913
3914 MediaTrackGraph* graph = MediaTrackGraph::GetInstance(
3915 graphDriverType, window, MediaTrackGraph::REQUEST_DEFAULT_SAMPLE_RATE,
3916 MediaTrackGraph::DEFAULT_OUTPUT_DEVICE);
3917
3918 RefPtr<DOMMediaStream> stream =
3919 CaptureStreamInternal(StreamCaptureBehavior::CONTINUE_WHEN_ENDED,
3920 StreamCaptureType::CAPTURE_ALL_TRACKS, graph);
3921 if (!stream) {
3922 aRv.Throw(NS_ERROR_FAILURE);
3923 return nullptr;
3924 }
3925
3926 return stream.forget();
3927 }
3928
MozCaptureStreamUntilEnded(ErrorResult & aRv)3929 already_AddRefed<DOMMediaStream> HTMLMediaElement::MozCaptureStreamUntilEnded(
3930 ErrorResult& aRv) {
3931 MediaTrackGraph::GraphDriverType graphDriverType =
3932 HasAudio() ? MediaTrackGraph::AUDIO_THREAD_DRIVER
3933 : MediaTrackGraph::SYSTEM_THREAD_DRIVER;
3934
3935 nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
3936 if (!window) {
3937 aRv.Throw(NS_ERROR_FAILURE);
3938 return nullptr;
3939 }
3940
3941 if (!CanBeCaptured(StreamCaptureType::CAPTURE_ALL_TRACKS)) {
3942 aRv.Throw(NS_ERROR_FAILURE);
3943 return nullptr;
3944 }
3945
3946 MediaTrackGraph* graph = MediaTrackGraph::GetInstance(
3947 graphDriverType, window, MediaTrackGraph::REQUEST_DEFAULT_SAMPLE_RATE,
3948 MediaTrackGraph::DEFAULT_OUTPUT_DEVICE);
3949
3950 RefPtr<DOMMediaStream> stream =
3951 CaptureStreamInternal(StreamCaptureBehavior::FINISH_WHEN_ENDED,
3952 StreamCaptureType::CAPTURE_ALL_TRACKS, graph);
3953 if (!stream) {
3954 aRv.Throw(NS_ERROR_FAILURE);
3955 return nullptr;
3956 }
3957
3958 return stream.forget();
3959 }
3960
3961 class MediaElementSetForURI : public nsURIHashKey {
3962 public:
MediaElementSetForURI(const nsIURI * aKey)3963 explicit MediaElementSetForURI(const nsIURI* aKey) : nsURIHashKey(aKey) {}
MediaElementSetForURI(MediaElementSetForURI && aOther)3964 MediaElementSetForURI(MediaElementSetForURI&& aOther)
3965 : nsURIHashKey(std::move(aOther)),
3966 mElements(std::move(aOther.mElements)) {}
3967 nsTArray<HTMLMediaElement*> mElements;
3968 };
3969
3970 typedef nsTHashtable<MediaElementSetForURI> MediaElementURITable;
3971 // Elements in this table must have non-null mDecoder and mLoadingSrc, and those
3972 // can't change while the element is in the table. The table is keyed by
3973 // the element's mLoadingSrc. Each entry has a list of all elements with the
3974 // same mLoadingSrc.
3975 static MediaElementURITable* gElementTable;
3976
3977 #ifdef DEBUG
URISafeEquals(nsIURI * a1,nsIURI * a2)3978 static bool URISafeEquals(nsIURI* a1, nsIURI* a2) {
3979 if (!a1 || !a2) {
3980 // Consider two empty URIs *not* equal!
3981 return false;
3982 }
3983 bool equal = false;
3984 nsresult rv = a1->Equals(a2, &equal);
3985 return NS_SUCCEEDED(rv) && equal;
3986 }
3987 // Returns the number of times aElement appears in the media element table
3988 // for aURI. If this returns other than 0 or 1, there's a bug somewhere!
MediaElementTableCount(HTMLMediaElement * aElement,nsIURI * aURI)3989 static unsigned MediaElementTableCount(HTMLMediaElement* aElement,
3990 nsIURI* aURI) {
3991 if (!gElementTable || !aElement) {
3992 return 0;
3993 }
3994 uint32_t uriCount = 0;
3995 uint32_t otherCount = 0;
3996 for (const auto& entry : *gElementTable) {
3997 uint32_t count = 0;
3998 for (const auto& elem : entry.mElements) {
3999 if (elem == aElement) {
4000 count++;
4001 }
4002 }
4003 if (URISafeEquals(aURI, entry.GetKey())) {
4004 uriCount = count;
4005 } else {
4006 otherCount += count;
4007 }
4008 }
4009 NS_ASSERTION(otherCount == 0, "Should not have entries for unknown URIs");
4010 return uriCount;
4011 }
4012 #endif
4013
AddMediaElementToURITable()4014 void HTMLMediaElement::AddMediaElementToURITable() {
4015 NS_ASSERTION(mDecoder, "Call this only with decoder Load called");
4016 NS_ASSERTION(
4017 MediaElementTableCount(this, mLoadingSrc) == 0,
4018 "Should not have entry for element in element table before addition");
4019 if (!gElementTable) {
4020 gElementTable = new MediaElementURITable();
4021 }
4022 MediaElementSetForURI* entry = gElementTable->PutEntry(mLoadingSrc);
4023 entry->mElements.AppendElement(this);
4024 NS_ASSERTION(
4025 MediaElementTableCount(this, mLoadingSrc) == 1,
4026 "Should have a single entry for element in element table after addition");
4027 }
4028
RemoveMediaElementFromURITable()4029 void HTMLMediaElement::RemoveMediaElementFromURITable() {
4030 if (!mDecoder || !mLoadingSrc || !gElementTable) {
4031 return;
4032 }
4033 MediaElementSetForURI* entry = gElementTable->GetEntry(mLoadingSrc);
4034 if (!entry) {
4035 return;
4036 }
4037 entry->mElements.RemoveElement(this);
4038 if (entry->mElements.IsEmpty()) {
4039 gElementTable->RemoveEntry(entry);
4040 if (gElementTable->Count() == 0) {
4041 delete gElementTable;
4042 gElementTable = nullptr;
4043 }
4044 }
4045 NS_ASSERTION(MediaElementTableCount(this, mLoadingSrc) == 0,
4046 "After remove, should no longer have an entry in element table");
4047 }
4048
LookupMediaElementURITable(nsIURI * aURI)4049 HTMLMediaElement* HTMLMediaElement::LookupMediaElementURITable(nsIURI* aURI) {
4050 if (!gElementTable) {
4051 return nullptr;
4052 }
4053 MediaElementSetForURI* entry = gElementTable->GetEntry(aURI);
4054 if (!entry) {
4055 return nullptr;
4056 }
4057 for (uint32_t i = 0; i < entry->mElements.Length(); ++i) {
4058 HTMLMediaElement* elem = entry->mElements[i];
4059 bool equal;
4060 // Look for elements that have the same principal and CORS mode.
4061 // Ditto for anything else that could cause us to send different headers.
4062 if (NS_SUCCEEDED(elem->NodePrincipal()->Equals(NodePrincipal(), &equal)) &&
4063 equal && elem->mCORSMode == mCORSMode) {
4064 // See SetupDecoder() below. We only add a element to the table when
4065 // mDecoder is a ChannelMediaDecoder.
4066 auto decoder = static_cast<ChannelMediaDecoder*>(elem->mDecoder.get());
4067 NS_ASSERTION(decoder, "Decoder gone");
4068 if (decoder->CanClone()) {
4069 return elem;
4070 }
4071 }
4072 }
4073 return nullptr;
4074 }
4075
4076 class HTMLMediaElement::ShutdownObserver : public nsIObserver {
4077 enum class Phase : int8_t { Init, Subscribed, Unsubscribed };
4078
4079 public:
4080 NS_DECL_ISUPPORTS
4081
Observe(nsISupports *,const char * aTopic,const char16_t *)4082 NS_IMETHOD Observe(nsISupports*, const char* aTopic,
4083 const char16_t*) override {
4084 if (mPhase != Phase::Subscribed) {
4085 // Bail out if we are not subscribed for this might be called even after
4086 // |nsContentUtils::UnregisterShutdownObserver(this)|.
4087 return NS_OK;
4088 }
4089 MOZ_DIAGNOSTIC_ASSERT(mWeak);
4090 if (strcmp(aTopic, NS_XPCOM_SHUTDOWN_OBSERVER_ID) == 0) {
4091 mWeak->NotifyShutdownEvent();
4092 }
4093 return NS_OK;
4094 }
Subscribe(HTMLMediaElement * aPtr)4095 void Subscribe(HTMLMediaElement* aPtr) {
4096 MOZ_DIAGNOSTIC_ASSERT(mPhase == Phase::Init);
4097 MOZ_DIAGNOSTIC_ASSERT(!mWeak);
4098 mWeak = aPtr;
4099 nsContentUtils::RegisterShutdownObserver(this);
4100 mPhase = Phase::Subscribed;
4101 }
Unsubscribe()4102 void Unsubscribe() {
4103 MOZ_DIAGNOSTIC_ASSERT(mPhase == Phase::Subscribed);
4104 MOZ_DIAGNOSTIC_ASSERT(mWeak);
4105 MOZ_DIAGNOSTIC_ASSERT(!mAddRefed,
4106 "ReleaseMediaElement should have been called first");
4107 mWeak = nullptr;
4108 nsContentUtils::UnregisterShutdownObserver(this);
4109 mPhase = Phase::Unsubscribed;
4110 }
AddRefMediaElement()4111 void AddRefMediaElement() {
4112 MOZ_DIAGNOSTIC_ASSERT(mWeak);
4113 MOZ_DIAGNOSTIC_ASSERT(!mAddRefed, "Should only ever AddRef once");
4114 mWeak->AddRef();
4115 mAddRefed = true;
4116 }
ReleaseMediaElement()4117 void ReleaseMediaElement() {
4118 MOZ_DIAGNOSTIC_ASSERT(mWeak);
4119 MOZ_DIAGNOSTIC_ASSERT(mAddRefed, "Should only release after AddRef");
4120 mWeak->Release();
4121 mAddRefed = false;
4122 }
4123
4124 private:
~ShutdownObserver()4125 virtual ~ShutdownObserver() {
4126 MOZ_DIAGNOSTIC_ASSERT(mPhase == Phase::Unsubscribed);
4127 MOZ_DIAGNOSTIC_ASSERT(!mWeak);
4128 MOZ_DIAGNOSTIC_ASSERT(!mAddRefed,
4129 "ReleaseMediaElement should have been called first");
4130 }
4131 // Guaranteed to be valid by HTMLMediaElement.
4132 HTMLMediaElement* mWeak = nullptr;
4133 Phase mPhase = Phase::Init;
4134 bool mAddRefed = false;
4135 };
4136
4137 NS_IMPL_ISUPPORTS(HTMLMediaElement::ShutdownObserver, nsIObserver)
4138
4139 class HTMLMediaElement::TitleChangeObserver final : public nsIObserver {
4140 public:
4141 NS_DECL_ISUPPORTS
4142
TitleChangeObserver(HTMLMediaElement * aElement)4143 explicit TitleChangeObserver(HTMLMediaElement* aElement)
4144 : mElement(aElement) {
4145 MOZ_ASSERT(NS_IsMainThread());
4146 MOZ_ASSERT(aElement);
4147 }
4148
Observe(nsISupports *,const char * aTopic,const char16_t *)4149 NS_IMETHOD Observe(nsISupports*, const char* aTopic,
4150 const char16_t*) override {
4151 if (mElement) {
4152 mElement->UpdateStreamName();
4153 }
4154
4155 return NS_OK;
4156 }
4157
Subscribe()4158 void Subscribe() {
4159 nsCOMPtr<nsIObserverService> observerService =
4160 mozilla::services::GetObserverService();
4161 if (observerService) {
4162 observerService->AddObserver(this, "document-title-changed", false);
4163 }
4164 }
4165
Unsubscribe()4166 void Unsubscribe() {
4167 nsCOMPtr<nsIObserverService> observerService =
4168 mozilla::services::GetObserverService();
4169 if (observerService) {
4170 observerService->RemoveObserver(this, "document-title-changed");
4171 }
4172 }
4173
4174 private:
4175 ~TitleChangeObserver() = default;
4176
4177 WeakPtr<HTMLMediaElement> mElement;
4178 };
4179
NS_IMPL_ISUPPORTS(HTMLMediaElement::TitleChangeObserver,nsIObserver)4180 NS_IMPL_ISUPPORTS(HTMLMediaElement::TitleChangeObserver, nsIObserver)
4181
4182 HTMLMediaElement::HTMLMediaElement(
4183 already_AddRefed<mozilla::dom::NodeInfo>&& aNodeInfo)
4184 : nsGenericHTMLElement(std::move(aNodeInfo)),
4185 mWatchManager(this,
4186 OwnerDoc()->AbstractMainThreadFor(TaskCategory::Other)),
4187 mMainThreadEventTarget(OwnerDoc()->EventTargetFor(TaskCategory::Other)),
4188 mAbstractMainThread(
4189 OwnerDoc()->AbstractMainThreadFor(TaskCategory::Other)),
4190 mShutdownObserver(new ShutdownObserver),
4191 mTitleChangeObserver(new TitleChangeObserver(this)),
4192 mEventBlocker(new EventBlocker(this)),
4193 mPlayed(new TimeRanges(ToSupports(OwnerDoc()))),
4194 mTracksCaptured(nullptr, "HTMLMediaElement::mTracksCaptured"),
4195 mErrorSink(new ErrorSink(this)),
4196 mAudioChannelWrapper(new AudioChannelAgentCallback(this)),
4197 mSink(std::pair(nsString(), RefPtr<AudioDeviceInfo>())),
4198 mShowPoster(IsVideo()),
4199 mMediaControlKeyListener(new MediaControlKeyListener(this)) {
4200 MOZ_ASSERT(mMainThreadEventTarget);
4201 MOZ_ASSERT(mAbstractMainThread);
4202 // Please don't add anything to this constructor or the initialization
4203 // list that can cause AddRef to be called. This prevents subclasses
4204 // from overriding AddRef in a way that works with our refcount
4205 // logging mechanisms. Put these things inside of the ::Init method
4206 // instead.
4207 }
4208
Init()4209 void HTMLMediaElement::Init() {
4210 MOZ_ASSERT(mRefCnt == 0 && !mRefCnt.IsPurple(),
4211 "HTMLMediaElement::Init called when AddRef has been called "
4212 "at least once already, probably in the constructor. Please "
4213 "see the documentation in the HTMLMediaElement constructor.");
4214 MOZ_ASSERT(!mRefCnt.IsPurple());
4215
4216 mAudioTrackList = new AudioTrackList(OwnerDoc()->GetParentObject(), this);
4217 mVideoTrackList = new VideoTrackList(OwnerDoc()->GetParentObject(), this);
4218
4219 DecoderDoctorLogger::LogConstruction(this);
4220
4221 mWatchManager.Watch(mPaused, &HTMLMediaElement::UpdateWakeLock);
4222 mWatchManager.Watch(mPaused, &HTMLMediaElement::UpdateOutputTracksMuting);
4223 mWatchManager.Watch(
4224 mPaused, &HTMLMediaElement::NotifyMediaControlPlaybackStateChanged);
4225 mWatchManager.Watch(mReadyState, &HTMLMediaElement::UpdateOutputTracksMuting);
4226
4227 mWatchManager.Watch(mTracksCaptured,
4228 &HTMLMediaElement::UpdateOutputTrackSources);
4229 mWatchManager.Watch(mReadyState, &HTMLMediaElement::UpdateOutputTrackSources);
4230
4231 mWatchManager.Watch(mDownloadSuspendedByCache,
4232 &HTMLMediaElement::UpdateReadyStateInternal);
4233 mWatchManager.Watch(mFirstFrameLoaded,
4234 &HTMLMediaElement::UpdateReadyStateInternal);
4235 mWatchManager.Watch(mSrcStreamPlaybackEnded,
4236 &HTMLMediaElement::UpdateReadyStateInternal);
4237
4238 ErrorResult rv;
4239
4240 double defaultVolume = Preferences::GetFloat("media.default_volume", 1.0);
4241 SetVolume(defaultVolume, rv);
4242
4243 RegisterActivityObserver();
4244 NotifyOwnerDocumentActivityChanged();
4245
4246 // We initialize the MediaShutdownManager as the HTMLMediaElement is always
4247 // constructed on the main thread, and not during stable state.
4248 // (MediaShutdownManager make use of nsIAsyncShutdownClient which is written
4249 // in JS)
4250 MediaShutdownManager::InitStatics();
4251
4252 #if defined(MOZ_WIDGET_ANDROID)
4253 GVAutoplayPermissionRequestor::AskForPermissionIfNeeded(
4254 OwnerDoc()->GetInnerWindow());
4255 #endif
4256
4257 OwnerDoc()->SetDocTreeHadMedia();
4258 mShutdownObserver->Subscribe(this);
4259 mInitialized = true;
4260 }
4261
~HTMLMediaElement()4262 HTMLMediaElement::~HTMLMediaElement() {
4263 MOZ_ASSERT(mInitialized,
4264 "HTMLMediaElement must be initialized before it is destroyed.");
4265 NS_ASSERTION(
4266 !mHasSelfReference,
4267 "How can we be destroyed if we're still holding a self reference?");
4268
4269 mWatchManager.Shutdown();
4270
4271 mShutdownObserver->Unsubscribe();
4272
4273 mTitleChangeObserver->Unsubscribe();
4274
4275 if (mVideoFrameContainer) {
4276 mVideoFrameContainer->ForgetElement();
4277 }
4278 UnregisterActivityObserver();
4279
4280 mSetCDMRequest.DisconnectIfExists();
4281 mAllowedToPlayPromise.RejectIfExists(NS_ERROR_FAILURE, __func__);
4282
4283 if (mDecoder) {
4284 ShutdownDecoder();
4285 }
4286 if (mProgressTimer) {
4287 StopProgress();
4288 }
4289 if (mSrcStream) {
4290 EndSrcMediaStreamPlayback();
4291 }
4292
4293 NS_ASSERTION(MediaElementTableCount(this, mLoadingSrc) == 0,
4294 "Destroyed media element should no longer be in element table");
4295
4296 if (mChannelLoader) {
4297 mChannelLoader->Cancel();
4298 }
4299
4300 if (mAudioChannelWrapper) {
4301 mAudioChannelWrapper->Shutdown();
4302 mAudioChannelWrapper = nullptr;
4303 }
4304
4305 if (mResumeDelayedPlaybackAgent) {
4306 mResumePlaybackRequest.DisconnectIfExists();
4307 mResumeDelayedPlaybackAgent = nullptr;
4308 }
4309
4310 mMediaControlKeyListener->StopIfNeeded();
4311 mMediaControlKeyListener = nullptr;
4312
4313 WakeLockRelease();
4314
4315 DecoderDoctorLogger::LogDestruction(this);
4316 }
4317
StopSuspendingAfterFirstFrame()4318 void HTMLMediaElement::StopSuspendingAfterFirstFrame() {
4319 mAllowSuspendAfterFirstFrame = false;
4320 if (!mSuspendedAfterFirstFrame) return;
4321 mSuspendedAfterFirstFrame = false;
4322 if (mDecoder) {
4323 mDecoder->Resume();
4324 }
4325 }
4326
SetPlayedOrSeeked(bool aValue)4327 void HTMLMediaElement::SetPlayedOrSeeked(bool aValue) {
4328 if (aValue == mHasPlayedOrSeeked) {
4329 return;
4330 }
4331
4332 mHasPlayedOrSeeked = aValue;
4333
4334 // Force a reflow so that the poster frame hides or shows immediately.
4335 nsIFrame* frame = GetPrimaryFrame();
4336 if (!frame) {
4337 return;
4338 }
4339 frame->PresShell()->FrameNeedsReflow(frame, IntrinsicDirty::TreeChange,
4340 NS_FRAME_IS_DIRTY);
4341 }
4342
NotifyXPCOMShutdown()4343 void HTMLMediaElement::NotifyXPCOMShutdown() { ShutdownDecoder(); }
4344
Play(ErrorResult & aRv)4345 already_AddRefed<Promise> HTMLMediaElement::Play(ErrorResult& aRv) {
4346 LOG(LogLevel::Debug,
4347 ("%p Play() called by JS readyState=%d", this, mReadyState.Ref()));
4348
4349 // 4.8.12.8
4350 // When the play() method on a media element is invoked, the user agent must
4351 // run the following steps.
4352
4353 RefPtr<PlayPromise> promise = CreatePlayPromise(aRv);
4354 if (NS_WARN_IF(aRv.Failed())) {
4355 return nullptr;
4356 }
4357
4358 // 4.8.12.8 - Step 1:
4359 // If the media element is not allowed to play, return a promise rejected
4360 // with a "NotAllowedError" DOMException and abort these steps.
4361 // NOTE: we may require requesting permission from the user, so we do the
4362 // "not allowed" check below.
4363
4364 // 4.8.12.8 - Step 2:
4365 // If the media element's error attribute is not null and its code
4366 // attribute has the value MEDIA_ERR_SRC_NOT_SUPPORTED, return a promise
4367 // rejected with a "NotSupportedError" DOMException and abort these steps.
4368 if (GetError() && GetError()->Code() == MEDIA_ERR_SRC_NOT_SUPPORTED) {
4369 LOG(LogLevel::Debug,
4370 ("%p Play() promise rejected because source not supported.", this));
4371 promise->MaybeReject(NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR);
4372 return promise.forget();
4373 }
4374
4375 // 4.8.12.8 - Step 3:
4376 // Let promise be a new promise and append promise to the list of pending
4377 // play promises.
4378 // Note: Promise appended to list of pending promises as needed below.
4379
4380 if (ShouldBeSuspendedByInactiveDocShell()) {
4381 LOG(LogLevel::Debug, ("%p no allow to play by the docShell for now", this));
4382 mPendingPlayPromises.AppendElement(promise);
4383 return promise.forget();
4384 }
4385
4386 // We may delay starting playback of a media resource for an unvisited tab
4387 // until it's going to foreground or being resumed by the play tab icon.
4388 if (MediaPlaybackDelayPolicy::ShouldDelayPlayback(this)) {
4389 CreateResumeDelayedMediaPlaybackAgentIfNeeded();
4390 LOG(LogLevel::Debug, ("%p delay Play() call", this));
4391 MaybeDoLoad();
4392 // When play is delayed, save a reference to the promise, and return it.
4393 // The promise will be resolved when we resume play by either the tab is
4394 // brought to the foreground, or the audio tab indicator is clicked.
4395 mPendingPlayPromises.AppendElement(promise);
4396 return promise.forget();
4397 }
4398
4399 const bool handlingUserInput = UserActivation::IsHandlingUserInput();
4400 mPendingPlayPromises.AppendElement(promise);
4401
4402 if (AutoplayPolicy::IsAllowedToPlay(*this)) {
4403 AUTOPLAY_LOG("allow MediaElement %p to play", this);
4404 mAllowedToPlayPromise.ResolveIfExists(true, __func__);
4405 PlayInternal(handlingUserInput);
4406 UpdateCustomPolicyAfterPlayed();
4407 } else {
4408 AUTOPLAY_LOG("reject MediaElement %p to play", this);
4409 AsyncRejectPendingPlayPromises(NS_ERROR_DOM_MEDIA_NOT_ALLOWED_ERR);
4410 }
4411 return promise.forget();
4412 }
4413
DispatchEventsWhenPlayWasNotAllowed()4414 void HTMLMediaElement::DispatchEventsWhenPlayWasNotAllowed() {
4415 if (StaticPrefs::media_autoplay_block_event_enabled()) {
4416 DispatchAsyncEvent(u"blocked"_ns);
4417 }
4418 DispatchBlockEventForVideoControl();
4419 MaybeNotifyAutoplayBlocked();
4420 ReportToConsole(nsIScriptError::warningFlag, "BlockAutoplayError");
4421 mHasEverBeenBlockedForAutoplay = true;
4422 }
4423
MaybeNotifyAutoplayBlocked()4424 void HTMLMediaElement::MaybeNotifyAutoplayBlocked() {
4425 // This event is used to notify front-end side that we've blocked autoplay,
4426 // so front-end side should show blocking icon as well.
4427 RefPtr<AsyncEventDispatcher> asyncDispatcher =
4428 new AsyncEventDispatcher(OwnerDoc(), u"GloballyAutoplayBlocked"_ns,
4429 CanBubble::eYes, ChromeOnlyDispatch::eYes);
4430 asyncDispatcher->PostDOMEvent();
4431 }
4432
DispatchBlockEventForVideoControl()4433 void HTMLMediaElement::DispatchBlockEventForVideoControl() {
4434 #if defined(MOZ_WIDGET_ANDROID)
4435 nsVideoFrame* videoFrame = do_QueryFrame(GetPrimaryFrame());
4436 if (!videoFrame || !videoFrame->GetVideoControls()) {
4437 return;
4438 }
4439
4440 RefPtr<AsyncEventDispatcher> asyncDispatcher = new AsyncEventDispatcher(
4441 videoFrame->GetVideoControls(), u"MozNoControlsBlockedVideo"_ns,
4442 CanBubble::eYes);
4443 asyncDispatcher->PostDOMEvent();
4444 #endif
4445 }
4446
PlayInternal(bool aHandlingUserInput)4447 void HTMLMediaElement::PlayInternal(bool aHandlingUserInput) {
4448 if (mPreloadAction == HTMLMediaElement::PRELOAD_NONE) {
4449 // The media load algorithm will be initiated by a user interaction.
4450 // We want to boost the channel priority for better responsiveness.
4451 // Note this must be done before UpdatePreloadAction() which will
4452 // update |mPreloadAction|.
4453 mUseUrgentStartForChannel = true;
4454 }
4455
4456 StopSuspendingAfterFirstFrame();
4457 SetPlayedOrSeeked(true);
4458
4459 // 4.8.12.8 - Step 4:
4460 // If the media element's networkState attribute has the value NETWORK_EMPTY,
4461 // invoke the media element's resource selection algorithm.
4462 MaybeDoLoad();
4463 if (mSuspendedForPreloadNone) {
4464 ResumeLoad(PRELOAD_ENOUGH);
4465 }
4466
4467 // 4.8.12.8 - Step 5:
4468 // If the playback has ended and the direction of playback is forwards,
4469 // seek to the earliest possible position of the media resource.
4470
4471 // Even if we just did Load() or ResumeLoad(), we could already have a decoder
4472 // here if we managed to clone an existing decoder.
4473 if (mDecoder) {
4474 if (mDecoder->IsEnded()) {
4475 SetCurrentTime(0);
4476 }
4477 if (!mSuspendedByInactiveDocOrDocshell) {
4478 mDecoder->Play();
4479 }
4480 }
4481
4482 if (mCurrentPlayRangeStart == -1.0) {
4483 mCurrentPlayRangeStart = CurrentTime();
4484 }
4485
4486 const bool oldPaused = mPaused;
4487 mPaused = false;
4488 mAutoplaying = false;
4489
4490 // We changed mPaused and mAutoplaying which can affect AddRemoveSelfReference
4491 // and our preload status.
4492 AddRemoveSelfReference();
4493 UpdatePreloadAction();
4494 UpdateSrcMediaStreamPlaying();
4495 StartMediaControlKeyListenerIfNeeded();
4496
4497 // Once play() has been called in a user generated event handler,
4498 // it is allowed to autoplay. Note: we can reach here when not in
4499 // a user generated event handler if our readyState has not yet
4500 // reached HAVE_METADATA.
4501 mIsBlessed |= aHandlingUserInput;
4502
4503 // TODO: If the playback has ended, then the user agent must set
4504 // seek to the effective start.
4505
4506 // 4.8.12.8 - Step 6:
4507 // If the media element's paused attribute is true, run the following steps:
4508 if (oldPaused) {
4509 // 6.1. Change the value of paused to false. (Already done.)
4510 // This step is uplifted because the "block-media-playback" feature needs
4511 // the mPaused to be false before UpdateAudioChannelPlayingState() being
4512 // called.
4513
4514 // 6.2. If the show poster flag is true, set the element's show poster flag
4515 // to false and run the time marches on steps.
4516 if (mShowPoster) {
4517 mShowPoster = false;
4518 if (mTextTrackManager) {
4519 mTextTrackManager->TimeMarchesOn();
4520 }
4521 }
4522
4523 // 6.3. Queue a task to fire a simple event named play at the element.
4524 DispatchAsyncEvent(u"play"_ns);
4525
4526 // 6.4. If the media element's readyState attribute has the value
4527 // HAVE_NOTHING, HAVE_METADATA, or HAVE_CURRENT_DATA, queue a task to
4528 // fire a simple event named waiting at the element.
4529 // Otherwise, the media element's readyState attribute has the value
4530 // HAVE_FUTURE_DATA or HAVE_ENOUGH_DATA: notify about playing for the
4531 // element.
4532 switch (mReadyState) {
4533 case HAVE_NOTHING:
4534 DispatchAsyncEvent(u"waiting"_ns);
4535 break;
4536 case HAVE_METADATA:
4537 case HAVE_CURRENT_DATA:
4538 DispatchAsyncEvent(u"waiting"_ns);
4539 break;
4540 case HAVE_FUTURE_DATA:
4541 case HAVE_ENOUGH_DATA:
4542 NotifyAboutPlaying();
4543 break;
4544 }
4545 } else if (mReadyState >= HAVE_FUTURE_DATA) {
4546 // 7. Otherwise, if the media element's readyState attribute has the value
4547 // HAVE_FUTURE_DATA or HAVE_ENOUGH_DATA, take pending play promises and
4548 // queue a task to resolve pending play promises with the result.
4549 AsyncResolvePendingPlayPromises();
4550 }
4551
4552 // 8. Set the media element's autoplaying flag to false. (Already done.)
4553
4554 // 9. Return promise.
4555 // (Done in caller.)
4556 }
4557
MaybeDoLoad()4558 void HTMLMediaElement::MaybeDoLoad() {
4559 if (mNetworkState == NETWORK_EMPTY) {
4560 DoLoad();
4561 }
4562 }
4563
UpdateWakeLock()4564 void HTMLMediaElement::UpdateWakeLock() {
4565 MOZ_ASSERT(NS_IsMainThread());
4566 // Ensure we have a wake lock if we're playing audibly. This ensures the
4567 // device doesn't sleep while playing.
4568 bool playing = !mPaused;
4569 bool isAudible = Volume() > 0.0 && !mMuted && mIsAudioTrackAudible;
4570 // WakeLock when playing audible media.
4571 if (playing && isAudible) {
4572 CreateAudioWakeLockIfNeeded();
4573 } else {
4574 ReleaseAudioWakeLockIfExists();
4575 }
4576 }
4577
CreateAudioWakeLockIfNeeded()4578 void HTMLMediaElement::CreateAudioWakeLockIfNeeded() {
4579 if (!mWakeLock) {
4580 RefPtr<power::PowerManagerService> pmService =
4581 power::PowerManagerService::GetInstance();
4582 NS_ENSURE_TRUE_VOID(pmService);
4583
4584 ErrorResult rv;
4585 mWakeLock = pmService->NewWakeLock(u"audio-playing"_ns,
4586 OwnerDoc()->GetInnerWindow(), rv);
4587 }
4588 }
4589
ReleaseAudioWakeLockIfExists()4590 void HTMLMediaElement::ReleaseAudioWakeLockIfExists() {
4591 if (mWakeLock) {
4592 ErrorResult rv;
4593 mWakeLock->Unlock(rv);
4594 rv.SuppressException();
4595 mWakeLock = nullptr;
4596 }
4597 }
4598
WakeLockRelease()4599 void HTMLMediaElement::WakeLockRelease() { ReleaseAudioWakeLockIfExists(); }
4600
GetEventTargetParent(EventChainPreVisitor & aVisitor)4601 void HTMLMediaElement::GetEventTargetParent(EventChainPreVisitor& aVisitor) {
4602 if (!this->Controls() || !aVisitor.mEvent->mFlags.mIsTrusted) {
4603 nsGenericHTMLElement::GetEventTargetParent(aVisitor);
4604 return;
4605 }
4606
4607 HTMLInputElement* el = nullptr;
4608 nsCOMPtr<nsINode> node;
4609
4610 // We will need to trap pointer, touch, and mouse events within the media
4611 // element, allowing media control exclusive consumption on these events,
4612 // and preventing the content from handling them.
4613 switch (aVisitor.mEvent->mMessage) {
4614 case ePointerDown:
4615 case ePointerUp:
4616 case eTouchEnd:
4617 // Always prevent touchmove captured in video element from being handled by
4618 // content, since we always do that for touchstart.
4619 case eTouchMove:
4620 case eTouchStart:
4621 case eMouseClick:
4622 case eMouseDoubleClick:
4623 case eMouseDown:
4624 case eMouseUp:
4625 aVisitor.mCanHandle = false;
4626 return;
4627
4628 // The *move events however are only comsumed when the range input is being
4629 // dragged.
4630 case ePointerMove:
4631 case eMouseMove:
4632 node = do_QueryInterface(aVisitor.mEvent->mOriginalTarget);
4633 if (node->IsInNativeAnonymousSubtree() || node->IsInUAWidget()) {
4634 if (node->IsHTMLElement(nsGkAtoms::input)) {
4635 // The node is a <input type="range">
4636 el = static_cast<HTMLInputElement*>(node.get());
4637 } else if (node->GetParentNode() &&
4638 node->GetParentNode()->IsHTMLElement(nsGkAtoms::input)) {
4639 // The node is a child of <input type="range">
4640 el = static_cast<HTMLInputElement*>(node->GetParentNode());
4641 }
4642 }
4643 if (el && el->IsDraggingRange()) {
4644 aVisitor.mCanHandle = false;
4645 return;
4646 }
4647 nsGenericHTMLElement::GetEventTargetParent(aVisitor);
4648 return;
4649
4650 default:
4651 nsGenericHTMLElement::GetEventTargetParent(aVisitor);
4652 return;
4653 }
4654 }
4655
ParseAttribute(int32_t aNamespaceID,nsAtom * aAttribute,const nsAString & aValue,nsIPrincipal * aMaybeScriptedPrincipal,nsAttrValue & aResult)4656 bool HTMLMediaElement::ParseAttribute(int32_t aNamespaceID, nsAtom* aAttribute,
4657 const nsAString& aValue,
4658 nsIPrincipal* aMaybeScriptedPrincipal,
4659 nsAttrValue& aResult) {
4660 // Mappings from 'preload' attribute strings to an enumeration.
4661 static const nsAttrValue::EnumTable kPreloadTable[] = {
4662 {"", HTMLMediaElement::PRELOAD_ATTR_EMPTY},
4663 {"none", HTMLMediaElement::PRELOAD_ATTR_NONE},
4664 {"metadata", HTMLMediaElement::PRELOAD_ATTR_METADATA},
4665 {"auto", HTMLMediaElement::PRELOAD_ATTR_AUTO},
4666 {nullptr, 0}};
4667
4668 if (aNamespaceID == kNameSpaceID_None) {
4669 if (aAttribute == nsGkAtoms::crossorigin) {
4670 ParseCORSValue(aValue, aResult);
4671 return true;
4672 }
4673 if (aAttribute == nsGkAtoms::preload) {
4674 return aResult.ParseEnumValue(aValue, kPreloadTable, false);
4675 }
4676 }
4677
4678 return nsGenericHTMLElement::ParseAttribute(aNamespaceID, aAttribute, aValue,
4679 aMaybeScriptedPrincipal, aResult);
4680 }
4681
DoneCreatingElement()4682 void HTMLMediaElement::DoneCreatingElement() {
4683 if (HasAttr(kNameSpaceID_None, nsGkAtoms::muted)) {
4684 mMuted |= MUTED_BY_CONTENT;
4685 }
4686 }
4687
IsHTMLFocusable(bool aWithMouse,bool * aIsFocusable,int32_t * aTabIndex)4688 bool HTMLMediaElement::IsHTMLFocusable(bool aWithMouse, bool* aIsFocusable,
4689 int32_t* aTabIndex) {
4690 if (nsGenericHTMLElement::IsHTMLFocusable(aWithMouse, aIsFocusable,
4691 aTabIndex)) {
4692 return true;
4693 }
4694
4695 *aIsFocusable = true;
4696 return false;
4697 }
4698
TabIndexDefault()4699 int32_t HTMLMediaElement::TabIndexDefault() { return 0; }
4700
AfterSetAttr(int32_t aNameSpaceID,nsAtom * aName,const nsAttrValue * aValue,const nsAttrValue * aOldValue,nsIPrincipal * aMaybeScriptedPrincipal,bool aNotify)4701 nsresult HTMLMediaElement::AfterSetAttr(int32_t aNameSpaceID, nsAtom* aName,
4702 const nsAttrValue* aValue,
4703 const nsAttrValue* aOldValue,
4704 nsIPrincipal* aMaybeScriptedPrincipal,
4705 bool aNotify) {
4706 if (aNameSpaceID == kNameSpaceID_None) {
4707 if (aName == nsGkAtoms::src) {
4708 mSrcMediaSource = nullptr;
4709 mSrcAttrTriggeringPrincipal = nsContentUtils::GetAttrTriggeringPrincipal(
4710 this, aValue ? aValue->GetStringValue() : EmptyString(),
4711 aMaybeScriptedPrincipal);
4712 if (aValue) {
4713 nsString srcStr = aValue->GetStringValue();
4714 nsCOMPtr<nsIURI> uri;
4715 NewURIFromString(srcStr, getter_AddRefs(uri));
4716 if (uri && IsMediaSourceURI(uri)) {
4717 nsresult rv = NS_GetSourceForMediaSourceURI(
4718 uri, getter_AddRefs(mSrcMediaSource));
4719 if (NS_FAILED(rv)) {
4720 nsAutoString spec;
4721 GetCurrentSrc(spec);
4722 AutoTArray<nsString, 1> params = {spec};
4723 ReportLoadError("MediaLoadInvalidURI", params);
4724 }
4725 }
4726 }
4727 } else if (aName == nsGkAtoms::autoplay) {
4728 if (aNotify) {
4729 if (aValue) {
4730 StopSuspendingAfterFirstFrame();
4731 CheckAutoplayDataReady();
4732 }
4733 // This attribute can affect AddRemoveSelfReference
4734 AddRemoveSelfReference();
4735 UpdatePreloadAction();
4736 }
4737 } else if (aName == nsGkAtoms::preload) {
4738 UpdatePreloadAction();
4739 } else if (aName == nsGkAtoms::loop) {
4740 if (mDecoder) {
4741 mDecoder->SetLooping(!!aValue);
4742 }
4743 } else if (aName == nsGkAtoms::controls && IsInComposedDoc()) {
4744 NotifyUAWidgetSetupOrChange();
4745 }
4746 }
4747
4748 // Since AfterMaybeChangeAttr may call DoLoad, make sure that it is called
4749 // *after* any possible changes to mSrcMediaSource.
4750 if (aValue) {
4751 AfterMaybeChangeAttr(aNameSpaceID, aName, aNotify);
4752 }
4753
4754 return nsGenericHTMLElement::AfterSetAttr(
4755 aNameSpaceID, aName, aValue, aOldValue, aMaybeScriptedPrincipal, aNotify);
4756 }
4757
OnAttrSetButNotChanged(int32_t aNamespaceID,nsAtom * aName,const nsAttrValueOrString & aValue,bool aNotify)4758 nsresult HTMLMediaElement::OnAttrSetButNotChanged(
4759 int32_t aNamespaceID, nsAtom* aName, const nsAttrValueOrString& aValue,
4760 bool aNotify) {
4761 AfterMaybeChangeAttr(aNamespaceID, aName, aNotify);
4762
4763 return nsGenericHTMLElement::OnAttrSetButNotChanged(aNamespaceID, aName,
4764 aValue, aNotify);
4765 }
4766
AfterMaybeChangeAttr(int32_t aNamespaceID,nsAtom * aName,bool aNotify)4767 void HTMLMediaElement::AfterMaybeChangeAttr(int32_t aNamespaceID, nsAtom* aName,
4768 bool aNotify) {
4769 if (aNamespaceID == kNameSpaceID_None) {
4770 if (aName == nsGkAtoms::src) {
4771 DoLoad();
4772 }
4773 }
4774 }
4775
BindToTree(BindContext & aContext,nsINode & aParent)4776 nsresult HTMLMediaElement::BindToTree(BindContext& aContext, nsINode& aParent) {
4777 nsresult rv = nsGenericHTMLElement::BindToTree(aContext, aParent);
4778
4779 if (IsInComposedDoc()) {
4780 // Construct Shadow Root so web content can be hidden in the DOM.
4781 AttachAndSetUAShadowRoot();
4782
4783 // The preload action depends on the value of the autoplay attribute.
4784 // It's value may have changed, so update it.
4785 UpdatePreloadAction();
4786 }
4787
4788 NotifyDecoderActivityChanges();
4789 mMediaControlKeyListener->UpdateOwnerBrowsingContextIfNeeded();
4790 return rv;
4791 }
4792
UnbindFromTree(bool aNullParent)4793 void HTMLMediaElement::UnbindFromTree(bool aNullParent) {
4794 mVisibilityState = Visibility::Untracked;
4795
4796 if (IsInComposedDoc()) {
4797 NotifyUAWidgetTeardown();
4798 }
4799
4800 nsGenericHTMLElement::UnbindFromTree(aNullParent);
4801
4802 MOZ_ASSERT(IsActuallyInvisible());
4803 NotifyDecoderActivityChanges();
4804
4805 // https://html.spec.whatwg.org/#playing-the-media-resource:remove-an-element-from-a-document
4806 //
4807 // Dispatch a task to run once we're in a stable state which ensures we're
4808 // paused if we're no longer in a document. Note that we need to dispatch this
4809 // even if there are other tasks in flight for this because these can be
4810 // cancelled if there's a new load.
4811 //
4812 // FIXME(emilio): Per that spec section, we should only do this if we used to
4813 // be connected, though other browsers match our current behavior...
4814 //
4815 // Also, https://github.com/whatwg/html/issues/4928
4816 nsCOMPtr<nsIRunnable> task =
4817 NS_NewRunnableFunction("dom::HTMLMediaElement::UnbindFromTree",
4818 [self = RefPtr<HTMLMediaElement>(this)]() {
4819 if (!self->IsInComposedDoc()) {
4820 self->PauseInternal();
4821 self->mMediaControlKeyListener->StopIfNeeded();
4822 }
4823 });
4824 RunInStableState(task);
4825 }
4826
4827 /* static */
GetCanPlay(const nsAString & aType,DecoderDoctorDiagnostics * aDiagnostics)4828 CanPlayStatus HTMLMediaElement::GetCanPlay(
4829 const nsAString& aType, DecoderDoctorDiagnostics* aDiagnostics) {
4830 Maybe<MediaContainerType> containerType = MakeMediaContainerType(aType);
4831 if (!containerType) {
4832 return CANPLAY_NO;
4833 }
4834 CanPlayStatus status =
4835 DecoderTraits::CanHandleContainerType(*containerType, aDiagnostics);
4836 if (status == CANPLAY_YES &&
4837 (*containerType).ExtendedType().Codecs().IsEmpty()) {
4838 // Per spec: 'Generally, a user agent should never return "probably" for a
4839 // type that allows the `codecs` parameter if that parameter is not
4840 // present.' As all our currently-supported types allow for `codecs`, we can
4841 // do this check here.
4842 // TODO: Instead, missing `codecs` should be checked in each decoder's
4843 // `IsSupportedType` call from `CanHandleCodecsType()`.
4844 // See bug 1399023.
4845 return CANPLAY_MAYBE;
4846 }
4847 return status;
4848 }
4849
CanPlayType(const nsAString & aType,nsAString & aResult)4850 void HTMLMediaElement::CanPlayType(const nsAString& aType, nsAString& aResult) {
4851 DecoderDoctorDiagnostics diagnostics;
4852 CanPlayStatus canPlay = GetCanPlay(aType, &diagnostics);
4853 diagnostics.StoreFormatDiagnostics(OwnerDoc(), aType, canPlay != CANPLAY_NO,
4854 __func__);
4855 switch (canPlay) {
4856 case CANPLAY_NO:
4857 aResult.Truncate();
4858 break;
4859 case CANPLAY_YES:
4860 aResult.AssignLiteral("probably");
4861 break;
4862 case CANPLAY_MAYBE:
4863 aResult.AssignLiteral("maybe");
4864 break;
4865 default:
4866 MOZ_ASSERT_UNREACHABLE("Unexpected case.");
4867 break;
4868 }
4869
4870 LOG(LogLevel::Debug,
4871 ("%p CanPlayType(%s) = \"%s\"", this, NS_ConvertUTF16toUTF8(aType).get(),
4872 NS_ConvertUTF16toUTF8(aResult).get()));
4873 }
4874
AssertReadyStateIsNothing()4875 void HTMLMediaElement::AssertReadyStateIsNothing() {
4876 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
4877 if (mReadyState != HAVE_NOTHING) {
4878 char buf[1024];
4879 SprintfLiteral(buf,
4880 "readyState=%d networkState=%d mLoadWaitStatus=%d "
4881 "mSourceLoadCandidate=%d "
4882 "mIsLoadingFromSourceChildren=%d mPreloadAction=%d "
4883 "mSuspendedForPreloadNone=%d error=%d",
4884 int(mReadyState), int(mNetworkState), int(mLoadWaitStatus),
4885 !!mSourceLoadCandidate, mIsLoadingFromSourceChildren,
4886 int(mPreloadAction), mSuspendedForPreloadNone,
4887 GetError() ? GetError()->Code() : 0);
4888 MOZ_CRASH_UNSAFE_PRINTF("ReadyState should be HAVE_NOTHING! %s", buf);
4889 }
4890 #endif
4891 }
4892
InitializeDecoderAsClone(ChannelMediaDecoder * aOriginal)4893 nsresult HTMLMediaElement::InitializeDecoderAsClone(
4894 ChannelMediaDecoder* aOriginal) {
4895 NS_ASSERTION(mLoadingSrc, "mLoadingSrc must already be set");
4896 NS_ASSERTION(mDecoder == nullptr, "Shouldn't have a decoder");
4897 AssertReadyStateIsNothing();
4898
4899 MediaDecoderInit decoderInit(
4900 this, this, mMuted ? 0.0 : mVolume, mPreservesPitch,
4901 ClampPlaybackRate(mPlaybackRate),
4902 mPreloadAction == HTMLMediaElement::PRELOAD_METADATA, mHasSuspendTaint,
4903 HasAttr(kNameSpaceID_None, nsGkAtoms::loop), aOriginal->ContainerType());
4904
4905 RefPtr<ChannelMediaDecoder> decoder = aOriginal->Clone(decoderInit);
4906 if (!decoder) return NS_ERROR_FAILURE;
4907
4908 LOG(LogLevel::Debug,
4909 ("%p Cloned decoder %p from %p", this, decoder.get(), aOriginal));
4910
4911 return FinishDecoderSetup(decoder);
4912 }
4913
4914 template <typename DecoderType, typename... LoadArgs>
SetupDecoder(DecoderType * aDecoder,LoadArgs &&...aArgs)4915 nsresult HTMLMediaElement::SetupDecoder(DecoderType* aDecoder,
4916 LoadArgs&&... aArgs) {
4917 LOG(LogLevel::Debug, ("%p Created decoder %p for type %s", this, aDecoder,
4918 aDecoder->ContainerType().OriginalString().Data()));
4919
4920 nsresult rv = aDecoder->Load(std::forward<LoadArgs>(aArgs)...);
4921 if (NS_FAILED(rv)) {
4922 aDecoder->Shutdown();
4923 LOG(LogLevel::Debug, ("%p Failed to load for decoder %p", this, aDecoder));
4924 return rv;
4925 }
4926
4927 rv = FinishDecoderSetup(aDecoder);
4928 // Only ChannelMediaDecoder supports resource cloning.
4929 if (std::is_same_v<DecoderType, ChannelMediaDecoder> && NS_SUCCEEDED(rv)) {
4930 AddMediaElementToURITable();
4931 NS_ASSERTION(
4932 MediaElementTableCount(this, mLoadingSrc) == 1,
4933 "Media element should have single table entry if decode initialized");
4934 }
4935
4936 return rv;
4937 }
4938
InitializeDecoderForChannel(nsIChannel * aChannel,nsIStreamListener ** aListener)4939 nsresult HTMLMediaElement::InitializeDecoderForChannel(
4940 nsIChannel* aChannel, nsIStreamListener** aListener) {
4941 NS_ASSERTION(mLoadingSrc, "mLoadingSrc must already be set");
4942 AssertReadyStateIsNothing();
4943
4944 DecoderDoctorDiagnostics diagnostics;
4945
4946 nsAutoCString mimeType;
4947 aChannel->GetContentType(mimeType);
4948 NS_ASSERTION(!mimeType.IsEmpty(), "We should have the Content-Type.");
4949 NS_ConvertUTF8toUTF16 mimeUTF16(mimeType);
4950
4951 RefPtr<HTMLMediaElement> self = this;
4952 auto reportCanPlay = [&, self](bool aCanPlay) {
4953 diagnostics.StoreFormatDiagnostics(self->OwnerDoc(), mimeUTF16, aCanPlay,
4954 __func__);
4955 if (!aCanPlay) {
4956 nsAutoString src;
4957 self->GetCurrentSrc(src);
4958 AutoTArray<nsString, 2> params = {mimeUTF16, src};
4959 self->ReportLoadError("MediaLoadUnsupportedMimeType", params);
4960 }
4961 };
4962
4963 auto onExit = MakeScopeExit([self] {
4964 if (self->mChannelLoader) {
4965 self->mChannelLoader->Done();
4966 self->mChannelLoader = nullptr;
4967 }
4968 });
4969
4970 Maybe<MediaContainerType> containerType = MakeMediaContainerType(mimeType);
4971 if (!containerType) {
4972 reportCanPlay(false);
4973 return NS_ERROR_FAILURE;
4974 }
4975
4976 MediaDecoderInit decoderInit(
4977 this, this, mMuted ? 0.0 : mVolume, mPreservesPitch,
4978 ClampPlaybackRate(mPlaybackRate),
4979 mPreloadAction == HTMLMediaElement::PRELOAD_METADATA, mHasSuspendTaint,
4980 HasAttr(kNameSpaceID_None, nsGkAtoms::loop), *containerType);
4981
4982 #ifdef MOZ_ANDROID_HLS_SUPPORT
4983 if (HLSDecoder::IsSupportedType(*containerType)) {
4984 RefPtr<HLSDecoder> decoder = HLSDecoder::Create(decoderInit);
4985 if (!decoder) {
4986 reportCanPlay(false);
4987 return NS_ERROR_OUT_OF_MEMORY;
4988 }
4989 reportCanPlay(true);
4990 return SetupDecoder(decoder.get(), aChannel);
4991 }
4992 #endif
4993
4994 RefPtr<ChannelMediaDecoder> decoder =
4995 ChannelMediaDecoder::Create(decoderInit, &diagnostics);
4996 if (!decoder) {
4997 reportCanPlay(false);
4998 return NS_ERROR_FAILURE;
4999 }
5000
5001 reportCanPlay(true);
5002 bool isPrivateBrowsing = NodePrincipal()->GetPrivateBrowsingId() > 0;
5003 return SetupDecoder(decoder.get(), aChannel, isPrivateBrowsing, aListener);
5004 }
5005
FinishDecoderSetup(MediaDecoder * aDecoder)5006 nsresult HTMLMediaElement::FinishDecoderSetup(MediaDecoder* aDecoder) {
5007 ChangeNetworkState(NETWORK_LOADING);
5008
5009 // Set mDecoder now so if methods like GetCurrentSrc get called between
5010 // here and Load(), they work.
5011 SetDecoder(aDecoder);
5012
5013 // Notify the decoder of the initial activity status.
5014 NotifyDecoderActivityChanges();
5015
5016 // Update decoder principal before we start decoding, since it
5017 // can affect how we feed data to MediaStreams
5018 NotifyDecoderPrincipalChanged();
5019
5020 // Set sink device if we have one. Otherwise the default is used.
5021 if (mSink.second) {
5022 mDecoder
5023 ->SetSink(mSink.second)
5024 #ifdef DEBUG
5025 ->Then(mAbstractMainThread, __func__,
5026 [](const GenericPromise::ResolveOrRejectValue& aValue) {
5027 MOZ_ASSERT(aValue.IsResolve() && !aValue.ResolveValue());
5028 });
5029 #else
5030 ;
5031 #endif
5032 }
5033
5034 if (mMediaKeys) {
5035 if (mMediaKeys->GetCDMProxy()) {
5036 mDecoder->SetCDMProxy(mMediaKeys->GetCDMProxy());
5037 } else {
5038 // CDM must have crashed.
5039 ShutdownDecoder();
5040 return NS_ERROR_FAILURE;
5041 }
5042 }
5043
5044 if (mChannelLoader) {
5045 mChannelLoader->Done();
5046 mChannelLoader = nullptr;
5047 }
5048
5049 // We may want to suspend the new stream now.
5050 // This will also do an AddRemoveSelfReference.
5051 NotifyOwnerDocumentActivityChanged();
5052
5053 if (!mDecoder) {
5054 // NotifyOwnerDocumentActivityChanged may shutdown the decoder if the
5055 // owning document is inactive and we're in the EME case. We could try and
5056 // handle this, but at the time of writing it's a pretty niche case, so just
5057 // bail.
5058 return NS_ERROR_FAILURE;
5059 }
5060
5061 if (mSuspendedByInactiveDocOrDocshell) {
5062 mDecoder->Suspend();
5063 }
5064
5065 if (!mPaused) {
5066 SetPlayedOrSeeked(true);
5067 if (!mSuspendedByInactiveDocOrDocshell) {
5068 mDecoder->Play();
5069 }
5070 }
5071
5072 MaybeBeginCloningVisually();
5073
5074 return NS_OK;
5075 }
5076
UpdateSrcMediaStreamPlaying(uint32_t aFlags)5077 void HTMLMediaElement::UpdateSrcMediaStreamPlaying(uint32_t aFlags) {
5078 if (!mSrcStream) {
5079 return;
5080 }
5081
5082 bool shouldPlay = !(aFlags & REMOVING_SRC_STREAM) && !mPaused &&
5083 !mSuspendedByInactiveDocOrDocshell;
5084 if (shouldPlay == mSrcStreamIsPlaying) {
5085 return;
5086 }
5087 mSrcStreamIsPlaying = shouldPlay;
5088
5089 LOG(LogLevel::Debug,
5090 ("MediaElement %p %s playback of DOMMediaStream %p", this,
5091 shouldPlay ? "Setting up" : "Removing", mSrcStream.get()));
5092
5093 if (shouldPlay) {
5094 mSrcStreamPlaybackEnded = false;
5095 mSrcStreamReportPlaybackEnded = false;
5096
5097 if (mMediaStreamRenderer) {
5098 mMediaStreamRenderer->Start();
5099 }
5100 if (mSecondaryMediaStreamRenderer) {
5101 mSecondaryMediaStreamRenderer->Start();
5102 }
5103
5104 SetCapturedOutputStreamsEnabled(true); // Unmute
5105 // If the input is a media stream, we don't check its data and always regard
5106 // it as audible when it's playing.
5107 SetAudibleState(true);
5108 } else {
5109 if (mMediaStreamRenderer) {
5110 mMediaStreamRenderer->Stop();
5111 }
5112 if (mSecondaryMediaStreamRenderer) {
5113 mSecondaryMediaStreamRenderer->Stop();
5114 }
5115 SetCapturedOutputStreamsEnabled(false); // Mute
5116 }
5117 }
5118
UpdateSrcStreamPotentiallyPlaying()5119 void HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying() {
5120 if (!mMediaStreamRenderer) {
5121 // Notifications are async, the renderer could have been cleared.
5122 return;
5123 }
5124
5125 mMediaStreamRenderer->SetProgressingCurrentTime(IsPotentiallyPlaying());
5126 }
5127
UpdateSrcStreamTime()5128 void HTMLMediaElement::UpdateSrcStreamTime() {
5129 MOZ_ASSERT(NS_IsMainThread());
5130
5131 if (mSrcStreamPlaybackEnded) {
5132 // We do a separate FireTimeUpdate() when this is set.
5133 return;
5134 }
5135
5136 FireTimeUpdate(TimeupdateType::ePeriodic);
5137 }
5138
SetupSrcMediaStreamPlayback(DOMMediaStream * aStream)5139 void HTMLMediaElement::SetupSrcMediaStreamPlayback(DOMMediaStream* aStream) {
5140 NS_ASSERTION(!mSrcStream, "Should have been ended already");
5141
5142 mLoadingSrc = nullptr;
5143 mSrcStream = aStream;
5144
5145 VideoFrameContainer* container = GetVideoFrameContainer();
5146 RefPtr<FirstFrameVideoOutput> firstFrameOutput =
5147 container
5148 ? MakeAndAddRef<FirstFrameVideoOutput>(container, mAbstractMainThread)
5149 : nullptr;
5150 mMediaStreamRenderer = MakeAndAddRef<MediaStreamRenderer>(
5151 mAbstractMainThread, container, firstFrameOutput, this);
5152 mWatchManager.Watch(mPaused,
5153 &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
5154 mWatchManager.Watch(mReadyState,
5155 &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
5156 mWatchManager.Watch(mSrcStreamPlaybackEnded,
5157 &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
5158 mWatchManager.Watch(mSrcStreamPlaybackEnded,
5159 &HTMLMediaElement::UpdateSrcStreamReportPlaybackEnded);
5160 mWatchManager.Watch(mMediaStreamRenderer->CurrentGraphTime(),
5161 &HTMLMediaElement::UpdateSrcStreamTime);
5162 SetVolumeInternal();
5163 if (mSink.second) {
5164 mMediaStreamRenderer->SetAudioOutputDevice(mSink.second);
5165 }
5166
5167 UpdateSrcMediaStreamPlaying();
5168 UpdateSrcStreamPotentiallyPlaying();
5169 mSrcStreamVideoPrincipal = NodePrincipal();
5170
5171 // If we pause this media element, track changes in the underlying stream
5172 // will continue to fire events at this element and alter its track list.
5173 // That's simpler than delaying the events, but probably confusing...
5174 nsTArray<RefPtr<MediaStreamTrack>> tracks;
5175 mSrcStream->GetTracks(tracks);
5176 for (const RefPtr<MediaStreamTrack>& track : tracks) {
5177 NotifyMediaStreamTrackAdded(track);
5178 }
5179
5180 mMediaStreamTrackListener = MakeUnique<MediaStreamTrackListener>(this);
5181 mSrcStream->RegisterTrackListener(mMediaStreamTrackListener.get());
5182
5183 ChangeNetworkState(NETWORK_IDLE);
5184 ChangeDelayLoadStatus(false);
5185
5186 // FirstFrameLoaded() will be called when the stream has tracks.
5187 }
5188
EndSrcMediaStreamPlayback()5189 void HTMLMediaElement::EndSrcMediaStreamPlayback() {
5190 MOZ_ASSERT(mSrcStream);
5191
5192 UpdateSrcMediaStreamPlaying(REMOVING_SRC_STREAM);
5193
5194 if (mSelectedVideoStreamTrack) {
5195 mSelectedVideoStreamTrack->RemovePrincipalChangeObserver(this);
5196 }
5197 mSelectedVideoStreamTrack = nullptr;
5198
5199 MOZ_ASSERT_IF(mSecondaryMediaStreamRenderer,
5200 !mMediaStreamRenderer == !mSecondaryMediaStreamRenderer);
5201 if (mMediaStreamRenderer) {
5202 mWatchManager.Unwatch(mPaused,
5203 &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
5204 mWatchManager.Unwatch(mReadyState,
5205 &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
5206 mWatchManager.Unwatch(mSrcStreamPlaybackEnded,
5207 &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
5208 mWatchManager.Unwatch(
5209 mSrcStreamPlaybackEnded,
5210 &HTMLMediaElement::UpdateSrcStreamReportPlaybackEnded);
5211 mWatchManager.Unwatch(mMediaStreamRenderer->CurrentGraphTime(),
5212 &HTMLMediaElement::UpdateSrcStreamTime);
5213 mMediaStreamRenderer->Shutdown();
5214 mMediaStreamRenderer = nullptr;
5215 }
5216 if (mSecondaryMediaStreamRenderer) {
5217 mSecondaryMediaStreamRenderer->Shutdown();
5218 mSecondaryMediaStreamRenderer = nullptr;
5219 }
5220
5221 mSrcStream->UnregisterTrackListener(mMediaStreamTrackListener.get());
5222 mMediaStreamTrackListener = nullptr;
5223 mSrcStreamPlaybackEnded = false;
5224 mSrcStreamReportPlaybackEnded = false;
5225 mSrcStreamVideoPrincipal = nullptr;
5226
5227 mSrcStream = nullptr;
5228 }
5229
CreateAudioTrack(AudioStreamTrack * aStreamTrack,nsIGlobalObject * aOwnerGlobal)5230 static already_AddRefed<AudioTrack> CreateAudioTrack(
5231 AudioStreamTrack* aStreamTrack, nsIGlobalObject* aOwnerGlobal) {
5232 nsAutoString id;
5233 nsAutoString label;
5234 aStreamTrack->GetId(id);
5235 aStreamTrack->GetLabel(label, CallerType::System);
5236
5237 return MediaTrackList::CreateAudioTrack(aOwnerGlobal, id, u"main"_ns, label,
5238 u""_ns, true, aStreamTrack);
5239 }
5240
CreateVideoTrack(VideoStreamTrack * aStreamTrack,nsIGlobalObject * aOwnerGlobal)5241 static already_AddRefed<VideoTrack> CreateVideoTrack(
5242 VideoStreamTrack* aStreamTrack, nsIGlobalObject* aOwnerGlobal) {
5243 nsAutoString id;
5244 nsAutoString label;
5245 aStreamTrack->GetId(id);
5246 aStreamTrack->GetLabel(label, CallerType::System);
5247
5248 return MediaTrackList::CreateVideoTrack(aOwnerGlobal, id, u"main"_ns, label,
5249 u""_ns, aStreamTrack);
5250 }
5251
NotifyMediaStreamTrackAdded(const RefPtr<MediaStreamTrack> & aTrack)5252 void HTMLMediaElement::NotifyMediaStreamTrackAdded(
5253 const RefPtr<MediaStreamTrack>& aTrack) {
5254 MOZ_ASSERT(aTrack);
5255
5256 if (aTrack->Ended()) {
5257 return;
5258 }
5259
5260 #ifdef DEBUG
5261 nsAutoString id;
5262 aTrack->GetId(id);
5263
5264 LOG(LogLevel::Debug, ("%p, Adding %sTrack with id %s", this,
5265 aTrack->AsAudioStreamTrack() ? "Audio" : "Video",
5266 NS_ConvertUTF16toUTF8(id).get()));
5267 #endif
5268
5269 if (AudioStreamTrack* t = aTrack->AsAudioStreamTrack()) {
5270 MOZ_DIAGNOSTIC_ASSERT(AudioTracks(), "Element can't have been unlinked");
5271 RefPtr<AudioTrack> audioTrack =
5272 CreateAudioTrack(t, AudioTracks()->GetOwnerGlobal());
5273 AudioTracks()->AddTrack(audioTrack);
5274 } else if (VideoStreamTrack* t = aTrack->AsVideoStreamTrack()) {
5275 // TODO: Fix this per the spec on bug 1273443.
5276 if (!IsVideo()) {
5277 return;
5278 }
5279 MOZ_DIAGNOSTIC_ASSERT(VideoTracks(), "Element can't have been unlinked");
5280 RefPtr<VideoTrack> videoTrack =
5281 CreateVideoTrack(t, VideoTracks()->GetOwnerGlobal());
5282 VideoTracks()->AddTrack(videoTrack);
5283 // New MediaStreamTrack added, set the new added video track as selected
5284 // video track when there is no selected track.
5285 if (VideoTracks()->SelectedIndex() == -1) {
5286 MOZ_ASSERT(!mSelectedVideoStreamTrack);
5287 videoTrack->SetEnabledInternal(true, dom::MediaTrack::FIRE_NO_EVENTS);
5288 }
5289 }
5290
5291 // The set of enabled AudioTracks and selected video track might have changed.
5292 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
5293 AbstractThread::DispatchDirectTask(
5294 NewRunnableMethod("HTMLMediaElement::FirstFrameLoaded", this,
5295 &HTMLMediaElement::FirstFrameLoaded));
5296 }
5297
NotifyMediaStreamTrackRemoved(const RefPtr<MediaStreamTrack> & aTrack)5298 void HTMLMediaElement::NotifyMediaStreamTrackRemoved(
5299 const RefPtr<MediaStreamTrack>& aTrack) {
5300 MOZ_ASSERT(aTrack);
5301
5302 nsAutoString id;
5303 aTrack->GetId(id);
5304
5305 LOG(LogLevel::Debug, ("%p, Removing %sTrack with id %s", this,
5306 aTrack->AsAudioStreamTrack() ? "Audio" : "Video",
5307 NS_ConvertUTF16toUTF8(id).get()));
5308
5309 MOZ_DIAGNOSTIC_ASSERT(AudioTracks() && VideoTracks(),
5310 "Element can't have been unlinked");
5311 if (dom::MediaTrack* t = AudioTracks()->GetTrackById(id)) {
5312 AudioTracks()->RemoveTrack(t);
5313 } else if (dom::MediaTrack* t = VideoTracks()->GetTrackById(id)) {
5314 VideoTracks()->RemoveTrack(t);
5315 } else {
5316 NS_ASSERTION(aTrack->AsVideoStreamTrack() && !IsVideo(),
5317 "MediaStreamTrack ended but did not exist in track lists. "
5318 "This is only allowed if a video element ends and we are an "
5319 "audio element.");
5320 return;
5321 }
5322 }
5323
ProcessMediaFragmentURI()5324 void HTMLMediaElement::ProcessMediaFragmentURI() {
5325 if (!mLoadingSrc) {
5326 mFragmentStart = mFragmentEnd = -1.0;
5327 return;
5328 }
5329 nsMediaFragmentURIParser parser(mLoadingSrc);
5330
5331 if (mDecoder && parser.HasEndTime()) {
5332 mFragmentEnd = parser.GetEndTime();
5333 }
5334
5335 if (parser.HasStartTime()) {
5336 SetCurrentTime(parser.GetStartTime());
5337 mFragmentStart = parser.GetStartTime();
5338 }
5339 }
5340
MetadataLoaded(const MediaInfo * aInfo,UniquePtr<const MetadataTags> aTags)5341 void HTMLMediaElement::MetadataLoaded(const MediaInfo* aInfo,
5342 UniquePtr<const MetadataTags> aTags) {
5343 MOZ_ASSERT(NS_IsMainThread());
5344
5345 if (mDecoder) {
5346 ConstructMediaTracks(aInfo);
5347 }
5348
5349 SetMediaInfo(*aInfo);
5350
5351 mIsEncrypted =
5352 aInfo->IsEncrypted() || mPendingEncryptedInitData.IsEncrypted();
5353 mTags = std::move(aTags);
5354 mLoadedDataFired = false;
5355 ChangeReadyState(HAVE_METADATA);
5356
5357 // Add output tracks synchronously now to be sure they're available in
5358 // "loadedmetadata" event handlers.
5359 UpdateOutputTrackSources();
5360
5361 DispatchAsyncEvent(u"durationchange"_ns);
5362 if (IsVideo() && HasVideo()) {
5363 DispatchAsyncEvent(u"resize"_ns);
5364 }
5365 NS_ASSERTION(!HasVideo() || (mMediaInfo.mVideo.mDisplay.width > 0 &&
5366 mMediaInfo.mVideo.mDisplay.height > 0),
5367 "Video resolution must be known on 'loadedmetadata'");
5368 DispatchAsyncEvent(u"loadedmetadata"_ns);
5369
5370 if (mDecoder && mDecoder->IsTransportSeekable() &&
5371 mDecoder->IsMediaSeekable()) {
5372 ProcessMediaFragmentURI();
5373 mDecoder->SetFragmentEndTime(mFragmentEnd);
5374 }
5375 if (mIsEncrypted) {
5376 // We only support playback of encrypted content via MSE by default.
5377 if (!mMediaSource && Preferences::GetBool("media.eme.mse-only", true)) {
5378 DecodeError(
5379 MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
5380 "Encrypted content not supported outside of MSE"));
5381 return;
5382 }
5383
5384 // Dispatch a distinct 'encrypted' event for each initData we have.
5385 for (const auto& initData : mPendingEncryptedInitData.mInitDatas) {
5386 DispatchEncrypted(initData.mInitData, initData.mType);
5387 }
5388 mPendingEncryptedInitData.Reset();
5389 }
5390
5391 if (IsVideo() && aInfo->HasVideo()) {
5392 // We are a video element playing video so update the screen wakelock
5393 NotifyOwnerDocumentActivityChanged();
5394 }
5395
5396 if (mDefaultPlaybackStartPosition != 0.0) {
5397 SetCurrentTime(mDefaultPlaybackStartPosition);
5398 mDefaultPlaybackStartPosition = 0.0;
5399 }
5400
5401 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
5402 }
5403
FirstFrameLoaded()5404 void HTMLMediaElement::FirstFrameLoaded() {
5405 LOG(LogLevel::Debug,
5406 ("%p, FirstFrameLoaded() mFirstFrameLoaded=%d mWaitingForKey=%d", this,
5407 mFirstFrameLoaded.Ref(), mWaitingForKey));
5408
5409 NS_ASSERTION(!mSuspendedAfterFirstFrame, "Should not have already suspended");
5410
5411 if (!mFirstFrameLoaded) {
5412 mFirstFrameLoaded = true;
5413 }
5414
5415 ChangeDelayLoadStatus(false);
5416
5417 if (mDecoder && mAllowSuspendAfterFirstFrame && mPaused &&
5418 !HasAttr(kNameSpaceID_None, nsGkAtoms::autoplay) &&
5419 mPreloadAction == HTMLMediaElement::PRELOAD_METADATA) {
5420 mSuspendedAfterFirstFrame = true;
5421 mDecoder->Suspend();
5422 }
5423 }
5424
NetworkError(const MediaResult & aError)5425 void HTMLMediaElement::NetworkError(const MediaResult& aError) {
5426 if (mReadyState == HAVE_NOTHING) {
5427 NoSupportedMediaSourceError(aError.Description());
5428 } else {
5429 Error(MEDIA_ERR_NETWORK);
5430 }
5431 }
5432
DecodeError(const MediaResult & aError)5433 void HTMLMediaElement::DecodeError(const MediaResult& aError) {
5434 nsAutoString src;
5435 GetCurrentSrc(src);
5436 AutoTArray<nsString, 1> params = {src};
5437 ReportLoadError("MediaLoadDecodeError", params);
5438
5439 DecoderDoctorDiagnostics diagnostics;
5440 diagnostics.StoreDecodeError(OwnerDoc(), aError, src, __func__);
5441
5442 if (mIsLoadingFromSourceChildren) {
5443 mErrorSink->ResetError();
5444 if (mSourceLoadCandidate) {
5445 DispatchAsyncSourceError(mSourceLoadCandidate);
5446 QueueLoadFromSourceTask();
5447 } else {
5448 NS_WARNING("Should know the source we were loading from!");
5449 }
5450 } else if (mReadyState == HAVE_NOTHING) {
5451 NoSupportedMediaSourceError(aError.Description());
5452 } else if (IsCORSSameOrigin()) {
5453 Error(MEDIA_ERR_DECODE, aError.Description());
5454 } else {
5455 Error(MEDIA_ERR_DECODE, "Failed to decode media"_ns);
5456 }
5457 }
5458
DecodeWarning(const MediaResult & aError)5459 void HTMLMediaElement::DecodeWarning(const MediaResult& aError) {
5460 nsAutoString src;
5461 GetCurrentSrc(src);
5462 DecoderDoctorDiagnostics diagnostics;
5463 diagnostics.StoreDecodeWarning(OwnerDoc(), aError, src, __func__);
5464 }
5465
HasError() const5466 bool HTMLMediaElement::HasError() const { return GetError(); }
5467
LoadAborted()5468 void HTMLMediaElement::LoadAborted() { Error(MEDIA_ERR_ABORTED); }
5469
Error(uint16_t aErrorCode,const nsACString & aErrorDetails)5470 void HTMLMediaElement::Error(uint16_t aErrorCode,
5471 const nsACString& aErrorDetails) {
5472 mErrorSink->SetError(aErrorCode, aErrorDetails);
5473 ChangeDelayLoadStatus(false);
5474 UpdateAudioChannelPlayingState();
5475 }
5476
PlaybackEnded()5477 void HTMLMediaElement::PlaybackEnded() {
5478 // We changed state which can affect AddRemoveSelfReference
5479 AddRemoveSelfReference();
5480
5481 NS_ASSERTION(!mDecoder || mDecoder->IsEnded(),
5482 "Decoder fired ended, but not in ended state");
5483
5484 // IsPlaybackEnded() became true.
5485 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
5486
5487 if (mSrcStream) {
5488 LOG(LogLevel::Debug,
5489 ("%p, got duration by reaching the end of the resource", this));
5490 mSrcStreamPlaybackEnded = true;
5491 DispatchAsyncEvent(u"durationchange"_ns);
5492 } else {
5493 // mediacapture-main:
5494 // Setting the loop attribute has no effect since a MediaStream has no
5495 // defined end and therefore cannot be looped.
5496 if (HasAttr(kNameSpaceID_None, nsGkAtoms::loop)) {
5497 SetCurrentTime(0);
5498 return;
5499 }
5500 }
5501
5502 FireTimeUpdate(TimeupdateType::eMandatory);
5503
5504 if (!mPaused) {
5505 Pause();
5506 }
5507
5508 if (mSrcStream) {
5509 // A MediaStream that goes from inactive to active shall be eligible for
5510 // autoplay again according to the mediacapture-main spec.
5511 mAutoplaying = true;
5512 }
5513
5514 if (StaticPrefs::media_mediacontrol_stopcontrol_aftermediaends()) {
5515 mMediaControlKeyListener->StopIfNeeded();
5516 }
5517 DispatchAsyncEvent(u"ended"_ns);
5518 }
5519
UpdateSrcStreamReportPlaybackEnded()5520 void HTMLMediaElement::UpdateSrcStreamReportPlaybackEnded() {
5521 mSrcStreamReportPlaybackEnded = mSrcStreamPlaybackEnded;
5522 }
5523
SeekStarted()5524 void HTMLMediaElement::SeekStarted() { DispatchAsyncEvent(u"seeking"_ns); }
5525
SeekCompleted()5526 void HTMLMediaElement::SeekCompleted() {
5527 mPlayingBeforeSeek = false;
5528 SetPlayedOrSeeked(true);
5529 if (mTextTrackManager) {
5530 mTextTrackManager->DidSeek();
5531 }
5532 // https://html.spec.whatwg.org/multipage/media.html#seeking:dom-media-seek
5533 // (Step 16)
5534 // TODO (bug 1688131): run these steps in a stable state.
5535 FireTimeUpdate(TimeupdateType::eMandatory);
5536 DispatchAsyncEvent(u"seeked"_ns);
5537 // We changed whether we're seeking so we need to AddRemoveSelfReference
5538 AddRemoveSelfReference();
5539 if (mCurrentPlayRangeStart == -1.0) {
5540 mCurrentPlayRangeStart = CurrentTime();
5541 }
5542
5543 if (mSeekDOMPromise) {
5544 mAbstractMainThread->Dispatch(NS_NewRunnableFunction(
5545 __func__, [promise = std::move(mSeekDOMPromise)] {
5546 promise->MaybeResolveWithUndefined();
5547 }));
5548 }
5549 MOZ_ASSERT(!mSeekDOMPromise);
5550 }
5551
SeekAborted()5552 void HTMLMediaElement::SeekAborted() {
5553 if (mSeekDOMPromise) {
5554 mAbstractMainThread->Dispatch(NS_NewRunnableFunction(
5555 __func__, [promise = std::move(mSeekDOMPromise)] {
5556 promise->MaybeReject(NS_ERROR_DOM_ABORT_ERR);
5557 }));
5558 }
5559 MOZ_ASSERT(!mSeekDOMPromise);
5560 }
5561
NotifySuspendedByCache(bool aSuspendedByCache)5562 void HTMLMediaElement::NotifySuspendedByCache(bool aSuspendedByCache) {
5563 LOG(LogLevel::Debug,
5564 ("%p, mDownloadSuspendedByCache=%d", this, aSuspendedByCache));
5565 mDownloadSuspendedByCache = aSuspendedByCache;
5566 }
5567
DownloadSuspended()5568 void HTMLMediaElement::DownloadSuspended() {
5569 if (mNetworkState == NETWORK_LOADING) {
5570 DispatchAsyncEvent(u"progress"_ns);
5571 }
5572 ChangeNetworkState(NETWORK_IDLE);
5573 }
5574
DownloadResumed()5575 void HTMLMediaElement::DownloadResumed() {
5576 ChangeNetworkState(NETWORK_LOADING);
5577 }
5578
CheckProgress(bool aHaveNewProgress)5579 void HTMLMediaElement::CheckProgress(bool aHaveNewProgress) {
5580 MOZ_ASSERT(NS_IsMainThread());
5581 MOZ_ASSERT(mNetworkState == NETWORK_LOADING);
5582
5583 TimeStamp now = TimeStamp::NowLoRes();
5584
5585 if (aHaveNewProgress) {
5586 mDataTime = now;
5587 }
5588
5589 // If this is the first progress, or PROGRESS_MS has passed since the last
5590 // progress event fired and more data has arrived since then, fire a
5591 // progress event.
5592 NS_ASSERTION(
5593 (mProgressTime.IsNull() && !aHaveNewProgress) || !mDataTime.IsNull(),
5594 "null TimeStamp mDataTime should not be used in comparison");
5595 if (mProgressTime.IsNull()
5596 ? aHaveNewProgress
5597 : (now - mProgressTime >=
5598 TimeDuration::FromMilliseconds(PROGRESS_MS) &&
5599 mDataTime > mProgressTime)) {
5600 DispatchAsyncEvent(u"progress"_ns);
5601 // Resolution() ensures that future data will have now > mProgressTime,
5602 // and so will trigger another event. mDataTime is not reset because it
5603 // is still required to detect stalled; it is similarly offset by
5604 // resolution to indicate the new data has not yet arrived.
5605 mProgressTime = now - TimeDuration::Resolution();
5606 if (mDataTime > mProgressTime) {
5607 mDataTime = mProgressTime;
5608 }
5609 if (!mProgressTimer) {
5610 NS_ASSERTION(aHaveNewProgress,
5611 "timer dispatched when there was no timer");
5612 // Were stalled. Restart timer.
5613 StartProgressTimer();
5614 if (!mLoadedDataFired) {
5615 ChangeDelayLoadStatus(true);
5616 }
5617 }
5618 // Download statistics may have been updated, force a recheck of the
5619 // readyState.
5620 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
5621 }
5622
5623 if (now - mDataTime >= TimeDuration::FromMilliseconds(STALL_MS)) {
5624 if (!mMediaSource) {
5625 DispatchAsyncEvent(u"stalled"_ns);
5626 } else {
5627 ChangeDelayLoadStatus(false);
5628 }
5629
5630 NS_ASSERTION(mProgressTimer, "detected stalled without timer");
5631 // Stop timer events, which prevents repeated stalled events until there
5632 // is more progress.
5633 StopProgress();
5634 }
5635
5636 AddRemoveSelfReference();
5637 }
5638
5639 /* static */
ProgressTimerCallback(nsITimer * aTimer,void * aClosure)5640 void HTMLMediaElement::ProgressTimerCallback(nsITimer* aTimer, void* aClosure) {
5641 auto decoder = static_cast<HTMLMediaElement*>(aClosure);
5642 decoder->CheckProgress(false);
5643 }
5644
StartProgressTimer()5645 void HTMLMediaElement::StartProgressTimer() {
5646 MOZ_ASSERT(NS_IsMainThread());
5647 MOZ_ASSERT(mNetworkState == NETWORK_LOADING);
5648 NS_ASSERTION(!mProgressTimer, "Already started progress timer.");
5649
5650 NS_NewTimerWithFuncCallback(
5651 getter_AddRefs(mProgressTimer), ProgressTimerCallback, this, PROGRESS_MS,
5652 nsITimer::TYPE_REPEATING_SLACK, "HTMLMediaElement::ProgressTimerCallback",
5653 mMainThreadEventTarget);
5654 }
5655
StartProgress()5656 void HTMLMediaElement::StartProgress() {
5657 // Record the time now for detecting stalled.
5658 mDataTime = TimeStamp::NowLoRes();
5659 // Reset mProgressTime so that mDataTime is not indicating bytes received
5660 // after the last progress event.
5661 mProgressTime = TimeStamp();
5662 StartProgressTimer();
5663 }
5664
StopProgress()5665 void HTMLMediaElement::StopProgress() {
5666 MOZ_ASSERT(NS_IsMainThread());
5667 if (!mProgressTimer) {
5668 return;
5669 }
5670
5671 mProgressTimer->Cancel();
5672 mProgressTimer = nullptr;
5673 }
5674
DownloadProgressed()5675 void HTMLMediaElement::DownloadProgressed() {
5676 if (mNetworkState != NETWORK_LOADING) {
5677 return;
5678 }
5679 CheckProgress(true);
5680 }
5681
ShouldCheckAllowOrigin()5682 bool HTMLMediaElement::ShouldCheckAllowOrigin() {
5683 return mCORSMode != CORS_NONE;
5684 }
5685
IsCORSSameOrigin()5686 bool HTMLMediaElement::IsCORSSameOrigin() {
5687 bool subsumes;
5688 RefPtr<nsIPrincipal> principal = GetCurrentPrincipal();
5689 return (NS_SUCCEEDED(NodePrincipal()->Subsumes(principal, &subsumes)) &&
5690 subsumes) ||
5691 ShouldCheckAllowOrigin();
5692 }
5693
UpdateReadyStateInternal()5694 void HTMLMediaElement::UpdateReadyStateInternal() {
5695 if (!mDecoder && !mSrcStream) {
5696 // Not initialized - bail out.
5697 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5698 "Not initialized",
5699 this));
5700 return;
5701 }
5702
5703 if (mDecoder && mReadyState < HAVE_METADATA) {
5704 // aNextFrame might have a next frame because the decoder can advance
5705 // on its own thread before MetadataLoaded gets a chance to run.
5706 // The arrival of more data can't change us out of this readyState.
5707 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5708 "Decoder ready state < HAVE_METADATA",
5709 this));
5710 return;
5711 }
5712
5713 if (mDecoder) {
5714 // IsPlaybackEnded() might have become false.
5715 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources);
5716 }
5717
5718 if (mSrcStream && mReadyState < HAVE_METADATA) {
5719 bool hasAudioTracks = AudioTracks() && !AudioTracks()->IsEmpty();
5720 bool hasVideoTracks = VideoTracks() && !VideoTracks()->IsEmpty();
5721 if (!hasAudioTracks && !hasVideoTracks) {
5722 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5723 "Stream with no tracks",
5724 this));
5725 // Give it one last chance to remove the self reference if needed.
5726 AddRemoveSelfReference();
5727 return;
5728 }
5729
5730 if (IsVideo() && hasVideoTracks && !HasVideo()) {
5731 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5732 "Stream waiting for video",
5733 this));
5734 return;
5735 }
5736
5737 LOG(LogLevel::Debug,
5738 ("MediaElement %p UpdateReadyStateInternal() Stream has "
5739 "metadata; audioTracks=%d, videoTracks=%d, "
5740 "hasVideoFrame=%d",
5741 this, AudioTracks()->Length(), VideoTracks()->Length(), HasVideo()));
5742
5743 // We are playing a stream that has video and a video frame is now set.
5744 // This means we have all metadata needed to change ready state.
5745 MediaInfo mediaInfo = mMediaInfo;
5746 if (hasAudioTracks) {
5747 mediaInfo.EnableAudio();
5748 }
5749 if (hasVideoTracks) {
5750 mediaInfo.EnableVideo();
5751 if (mSelectedVideoStreamTrack) {
5752 mediaInfo.mVideo.SetAlpha(mSelectedVideoStreamTrack->HasAlpha());
5753 }
5754 }
5755 MetadataLoaded(&mediaInfo, nullptr);
5756 }
5757
5758 if (mMediaSource) {
5759 // readyState has changed, assuming it's following the pending mediasource
5760 // operations. Notify the Mediasource that the operations have completed.
5761 mMediaSource->CompletePendingTransactions();
5762 }
5763
5764 enum NextFrameStatus nextFrameStatus = NextFrameStatus();
5765 if (mWaitingForKey == NOT_WAITING_FOR_KEY) {
5766 if (nextFrameStatus == NEXT_FRAME_UNAVAILABLE && mDecoder &&
5767 !mDecoder->IsEnded()) {
5768 nextFrameStatus = mDecoder->NextFrameBufferedStatus();
5769 }
5770 } else if (mWaitingForKey == WAITING_FOR_KEY) {
5771 if (nextFrameStatus == NEXT_FRAME_UNAVAILABLE ||
5772 nextFrameStatus == NEXT_FRAME_UNAVAILABLE_BUFFERING) {
5773 // http://w3c.github.io/encrypted-media/#wait-for-key
5774 // Continuing 7.3.4 Queue a "waitingforkey" Event
5775 // 4. Queue a task to fire a simple event named waitingforkey
5776 // at the media element.
5777 // 5. Set the readyState of media element to HAVE_METADATA.
5778 // NOTE: We'll change to HAVE_CURRENT_DATA or HAVE_METADATA
5779 // depending on whether we've loaded the first frame or not
5780 // below.
5781 // 6. Suspend playback.
5782 // Note: Playback will already be stalled, as the next frame is
5783 // unavailable.
5784 mWaitingForKey = WAITING_FOR_KEY_DISPATCHED;
5785 DispatchAsyncEvent(u"waitingforkey"_ns);
5786 }
5787 } else {
5788 MOZ_ASSERT(mWaitingForKey == WAITING_FOR_KEY_DISPATCHED);
5789 if (nextFrameStatus == NEXT_FRAME_AVAILABLE) {
5790 // We have new frames after dispatching "waitingforkey".
5791 // This means we've got the key and can reset mWaitingForKey now.
5792 mWaitingForKey = NOT_WAITING_FOR_KEY;
5793 }
5794 }
5795
5796 if (nextFrameStatus == MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING) {
5797 LOG(LogLevel::Debug,
5798 ("MediaElement %p UpdateReadyStateInternal() "
5799 "NEXT_FRAME_UNAVAILABLE_SEEKING; Forcing HAVE_METADATA",
5800 this));
5801 ChangeReadyState(HAVE_METADATA);
5802 return;
5803 }
5804
5805 if (IsVideo() && VideoTracks() && !VideoTracks()->IsEmpty() &&
5806 !IsPlaybackEnded() && GetImageContainer() &&
5807 !GetImageContainer()->HasCurrentImage()) {
5808 // Don't advance if we are playing video, but don't have a video frame.
5809 // Also, if video became available after advancing to HAVE_CURRENT_DATA
5810 // while we are still playing, we need to revert to HAVE_METADATA until
5811 // a video frame is available.
5812 LOG(LogLevel::Debug,
5813 ("MediaElement %p UpdateReadyStateInternal() "
5814 "Playing video but no video frame; Forcing HAVE_METADATA",
5815 this));
5816 ChangeReadyState(HAVE_METADATA);
5817 return;
5818 }
5819
5820 if (!mFirstFrameLoaded) {
5821 // We haven't yet loaded the first frame, making us unable to determine
5822 // if we have enough valid data at the present stage.
5823 return;
5824 }
5825
5826 if (nextFrameStatus == NEXT_FRAME_UNAVAILABLE_BUFFERING) {
5827 // Force HAVE_CURRENT_DATA when buffering.
5828 ChangeReadyState(HAVE_CURRENT_DATA);
5829 return;
5830 }
5831
5832 // TextTracks must be loaded for the HAVE_ENOUGH_DATA and
5833 // HAVE_FUTURE_DATA.
5834 // So force HAVE_CURRENT_DATA if text tracks not loaded.
5835 if (mTextTrackManager && !mTextTrackManager->IsLoaded()) {
5836 ChangeReadyState(HAVE_CURRENT_DATA);
5837 return;
5838 }
5839
5840 if (mDownloadSuspendedByCache && mDecoder && !mDecoder->IsEnded()) {
5841 // The decoder has signaled that the download has been suspended by the
5842 // media cache. So move readyState into HAVE_ENOUGH_DATA, in case there's
5843 // script waiting for a "canplaythrough" event; without this forced
5844 // transition, we will never fire the "canplaythrough" event if the
5845 // media cache is too small, and scripts are bound to fail. Don't force
5846 // this transition if the decoder is in ended state; the readyState
5847 // should remain at HAVE_CURRENT_DATA in this case.
5848 // Note that this state transition includes the case where we finished
5849 // downloaded the whole data stream.
5850 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5851 "Decoder download suspended by cache",
5852 this));
5853 ChangeReadyState(HAVE_ENOUGH_DATA);
5854 return;
5855 }
5856
5857 if (nextFrameStatus != MediaDecoderOwner::NEXT_FRAME_AVAILABLE) {
5858 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5859 "Next frame not available",
5860 this));
5861 ChangeReadyState(HAVE_CURRENT_DATA);
5862 return;
5863 }
5864
5865 if (mSrcStream) {
5866 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5867 "Stream HAVE_ENOUGH_DATA",
5868 this));
5869 ChangeReadyState(HAVE_ENOUGH_DATA);
5870 return;
5871 }
5872
5873 // Now see if we should set HAVE_ENOUGH_DATA.
5874 // If it's something we don't know the size of, then we can't
5875 // make a real estimate, so we go straight to HAVE_ENOUGH_DATA once
5876 // we've downloaded enough data that our download rate is considered
5877 // reliable. We have to move to HAVE_ENOUGH_DATA at some point or
5878 // autoplay elements for live streams will never play. Otherwise we
5879 // move to HAVE_ENOUGH_DATA if we can play through the entire media
5880 // without stopping to buffer.
5881 if (mDecoder->CanPlayThrough()) {
5882 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5883 "Decoder can play through",
5884 this));
5885 ChangeReadyState(HAVE_ENOUGH_DATA);
5886 return;
5887 }
5888 LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() "
5889 "Default; Decoder has future data",
5890 this));
5891 ChangeReadyState(HAVE_FUTURE_DATA);
5892 }
5893
5894 static const char* const gReadyStateToString[] = {
5895 "HAVE_NOTHING", "HAVE_METADATA", "HAVE_CURRENT_DATA", "HAVE_FUTURE_DATA",
5896 "HAVE_ENOUGH_DATA"};
5897
ChangeReadyState(nsMediaReadyState aState)5898 void HTMLMediaElement::ChangeReadyState(nsMediaReadyState aState) {
5899 if (mReadyState == aState) {
5900 return;
5901 }
5902
5903 nsMediaReadyState oldState = mReadyState;
5904 mReadyState = aState;
5905 LOG(LogLevel::Debug,
5906 ("%p Ready state changed to %s", this, gReadyStateToString[aState]));
5907
5908 DDLOG(DDLogCategory::Property, "ready_state", gReadyStateToString[aState]);
5909
5910 // https://html.spec.whatwg.org/multipage/media.html#text-track-cue-active-flag
5911 // The user agent must synchronously unset cues' active flag whenever the
5912 // media element's readyState is changed back to HAVE_NOTHING.
5913 if (mReadyState == HAVE_NOTHING && mTextTrackManager) {
5914 mTextTrackManager->NotifyReset();
5915 }
5916
5917 if (mNetworkState == NETWORK_EMPTY) {
5918 return;
5919 }
5920
5921 UpdateAudioChannelPlayingState();
5922
5923 // Handle raising of "waiting" event during seek (see 4.8.10.9)
5924 // or
5925 // 4.8.12.7 Ready states:
5926 // "If the previous ready state was HAVE_FUTURE_DATA or more, and the new
5927 // ready state is HAVE_CURRENT_DATA or less
5928 // If the media element was potentially playing before its readyState
5929 // attribute changed to a value lower than HAVE_FUTURE_DATA, and the element
5930 // has not ended playback, and playback has not stopped due to errors,
5931 // paused for user interaction, or paused for in-band content, the user agent
5932 // must queue a task to fire a simple event named timeupdate at the element,
5933 // and queue a task to fire a simple event named waiting at the element."
5934 if (mPlayingBeforeSeek && mReadyState < HAVE_FUTURE_DATA) {
5935 DispatchAsyncEvent(u"waiting"_ns);
5936 } else if (oldState >= HAVE_FUTURE_DATA && mReadyState < HAVE_FUTURE_DATA &&
5937 !Paused() && !Ended() && !mErrorSink->mError) {
5938 FireTimeUpdate(TimeupdateType::eMandatory);
5939 DispatchAsyncEvent(u"waiting"_ns);
5940 }
5941
5942 if (oldState < HAVE_CURRENT_DATA && mReadyState >= HAVE_CURRENT_DATA &&
5943 !mLoadedDataFired) {
5944 DispatchAsyncEvent(u"loadeddata"_ns);
5945 mLoadedDataFired = true;
5946 }
5947
5948 if (oldState < HAVE_FUTURE_DATA && mReadyState >= HAVE_FUTURE_DATA) {
5949 DispatchAsyncEvent(u"canplay"_ns);
5950 if (!mPaused) {
5951 if (mDecoder && !mSuspendedByInactiveDocOrDocshell) {
5952 MOZ_ASSERT(AutoplayPolicy::IsAllowedToPlay(*this));
5953 mDecoder->Play();
5954 }
5955 NotifyAboutPlaying();
5956 }
5957 }
5958
5959 CheckAutoplayDataReady();
5960
5961 if (oldState < HAVE_ENOUGH_DATA && mReadyState >= HAVE_ENOUGH_DATA) {
5962 DispatchAsyncEvent(u"canplaythrough"_ns);
5963 }
5964 }
5965
5966 static const char* const gNetworkStateToString[] = {"EMPTY", "IDLE", "LOADING",
5967 "NO_SOURCE"};
5968
ChangeNetworkState(nsMediaNetworkState aState)5969 void HTMLMediaElement::ChangeNetworkState(nsMediaNetworkState aState) {
5970 if (mNetworkState == aState) {
5971 return;
5972 }
5973
5974 nsMediaNetworkState oldState = mNetworkState;
5975 mNetworkState = aState;
5976 LOG(LogLevel::Debug,
5977 ("%p Network state changed to %s", this, gNetworkStateToString[aState]));
5978 DDLOG(DDLogCategory::Property, "network_state",
5979 gNetworkStateToString[aState]);
5980
5981 if (oldState == NETWORK_LOADING) {
5982 // Stop progress notification when exiting NETWORK_LOADING.
5983 StopProgress();
5984 }
5985
5986 if (mNetworkState == NETWORK_LOADING) {
5987 // Start progress notification when entering NETWORK_LOADING.
5988 StartProgress();
5989 } else if (mNetworkState == NETWORK_IDLE && !mErrorSink->mError) {
5990 // Fire 'suspend' event when entering NETWORK_IDLE and no error presented.
5991 DispatchAsyncEvent(u"suspend"_ns);
5992 }
5993
5994 // According to the resource selection (step2, step9-18), dedicated media
5995 // source failure step (step4) and aborting existing load (step4), set show
5996 // poster flag to true. https://html.spec.whatwg.org/multipage/media.html
5997 if (mNetworkState == NETWORK_NO_SOURCE || mNetworkState == NETWORK_EMPTY) {
5998 mShowPoster = true;
5999 }
6000
6001 // Changing mNetworkState affects AddRemoveSelfReference().
6002 AddRemoveSelfReference();
6003 }
6004
CanActivateAutoplay()6005 bool HTMLMediaElement::CanActivateAutoplay() {
6006 // We also activate autoplay when playing a media source since the data
6007 // download is controlled by the script and there is no way to evaluate
6008 // MediaDecoder::CanPlayThrough().
6009
6010 if (!HasAttr(kNameSpaceID_None, nsGkAtoms::autoplay)) {
6011 return false;
6012 }
6013
6014 if (!mAutoplaying) {
6015 return false;
6016 }
6017
6018 if (IsEditable()) {
6019 return false;
6020 }
6021
6022 if (!mPaused) {
6023 return false;
6024 }
6025
6026 if (mSuspendedByInactiveDocOrDocshell) {
6027 return false;
6028 }
6029
6030 // Static document is used for print preview and printing, should not be
6031 // autoplay
6032 if (OwnerDoc()->IsStaticDocument()) {
6033 return false;
6034 }
6035
6036 if (ShouldBeSuspendedByInactiveDocShell()) {
6037 LOG(LogLevel::Debug, ("%p prohibiting autoplay by the docShell", this));
6038 return false;
6039 }
6040
6041 if (MediaPlaybackDelayPolicy::ShouldDelayPlayback(this)) {
6042 CreateResumeDelayedMediaPlaybackAgentIfNeeded();
6043 LOG(LogLevel::Debug, ("%p delay playing from autoplay", this));
6044 return false;
6045 }
6046
6047 return mReadyState >= HAVE_ENOUGH_DATA;
6048 }
6049
CheckAutoplayDataReady()6050 void HTMLMediaElement::CheckAutoplayDataReady() {
6051 if (!CanActivateAutoplay()) {
6052 return;
6053 }
6054
6055 if (!AutoplayPolicy::IsAllowedToPlay(*this)) {
6056 DispatchEventsWhenPlayWasNotAllowed();
6057 return;
6058 }
6059
6060 mAllowedToPlayPromise.ResolveIfExists(true, __func__);
6061 mPaused = false;
6062 // We changed mPaused which can affect AddRemoveSelfReference
6063 AddRemoveSelfReference();
6064 UpdateSrcMediaStreamPlaying();
6065 UpdateAudioChannelPlayingState();
6066 StartMediaControlKeyListenerIfNeeded();
6067
6068 if (mDecoder) {
6069 SetPlayedOrSeeked(true);
6070 if (mCurrentPlayRangeStart == -1.0) {
6071 mCurrentPlayRangeStart = CurrentTime();
6072 }
6073 MOZ_ASSERT(!mSuspendedByInactiveDocOrDocshell);
6074 mDecoder->Play();
6075 } else if (mSrcStream) {
6076 SetPlayedOrSeeked(true);
6077 }
6078
6079 // https://html.spec.whatwg.org/multipage/media.html#ready-states:show-poster-flag
6080 if (mShowPoster) {
6081 mShowPoster = false;
6082 if (mTextTrackManager) {
6083 mTextTrackManager->TimeMarchesOn();
6084 }
6085 }
6086
6087 // For blocked media, the event would be pending until it is resumed.
6088 DispatchAsyncEvent(u"play"_ns);
6089
6090 DispatchAsyncEvent(u"playing"_ns);
6091 }
6092
IsActuallyInvisible() const6093 bool HTMLMediaElement::IsActuallyInvisible() const {
6094 // That means an element is not connected. It probably hasn't connected to a
6095 // document tree, or connects to a disconnected DOM tree.
6096 if (!IsInComposedDoc()) {
6097 return true;
6098 }
6099
6100 // An element is not in user's view port, which means it's either existing in
6101 // somewhere in the page where user hasn't seen yet, or is being set
6102 // `display:none`.
6103 if (!IsInViewPort()) {
6104 return true;
6105 }
6106
6107 // Element being used in picture-in-picture mode would be always visible.
6108 if (IsBeingUsedInPictureInPictureMode()) {
6109 return false;
6110 }
6111
6112 // That check is the page is in the background.
6113 return OwnerDoc()->Hidden();
6114 }
6115
IsInViewPort() const6116 bool HTMLMediaElement::IsInViewPort() const {
6117 return mVisibilityState == Visibility::ApproximatelyVisible;
6118 }
6119
GetVideoFrameContainer()6120 VideoFrameContainer* HTMLMediaElement::GetVideoFrameContainer() {
6121 if (mShuttingDown) {
6122 return nullptr;
6123 }
6124
6125 if (mVideoFrameContainer) return mVideoFrameContainer;
6126
6127 // Only video frames need an image container.
6128 if (!IsVideo()) {
6129 return nullptr;
6130 }
6131
6132 mVideoFrameContainer = new VideoFrameContainer(
6133 this, LayerManager::CreateImageContainer(ImageContainer::ASYNCHRONOUS));
6134
6135 return mVideoFrameContainer;
6136 }
6137
PrincipalChanged(MediaStreamTrack * aTrack)6138 void HTMLMediaElement::PrincipalChanged(MediaStreamTrack* aTrack) {
6139 if (aTrack != mSelectedVideoStreamTrack) {
6140 return;
6141 }
6142
6143 nsContentUtils::CombineResourcePrincipals(&mSrcStreamVideoPrincipal,
6144 aTrack->GetPrincipal());
6145
6146 LOG(LogLevel::Debug,
6147 ("HTMLMediaElement %p video track principal changed to %p (combined "
6148 "into %p). Waiting for it to reach VideoFrameContainer before setting.",
6149 this, aTrack->GetPrincipal(), mSrcStreamVideoPrincipal.get()));
6150
6151 if (mVideoFrameContainer) {
6152 UpdateSrcStreamVideoPrincipal(
6153 mVideoFrameContainer->GetLastPrincipalHandle());
6154 }
6155 }
6156
UpdateSrcStreamVideoPrincipal(const PrincipalHandle & aPrincipalHandle)6157 void HTMLMediaElement::UpdateSrcStreamVideoPrincipal(
6158 const PrincipalHandle& aPrincipalHandle) {
6159 nsTArray<RefPtr<VideoStreamTrack>> videoTracks;
6160 mSrcStream->GetVideoTracks(videoTracks);
6161
6162 PrincipalHandle handle(aPrincipalHandle);
6163 for (const RefPtr<VideoStreamTrack>& track : videoTracks) {
6164 if (PrincipalHandleMatches(handle, track->GetPrincipal()) &&
6165 !track->Ended()) {
6166 // When the PrincipalHandle for the VideoFrameContainer changes to that of
6167 // a live track in mSrcStream we know that a removed track was displayed
6168 // but is no longer so.
6169 LOG(LogLevel::Debug, ("HTMLMediaElement %p VideoFrameContainer's "
6170 "PrincipalHandle matches track %p. That's all we "
6171 "need.",
6172 this, track.get()));
6173 mSrcStreamVideoPrincipal = track->GetPrincipal();
6174 break;
6175 }
6176 }
6177 }
6178
PrincipalHandleChangedForVideoFrameContainer(VideoFrameContainer * aContainer,const PrincipalHandle & aNewPrincipalHandle)6179 void HTMLMediaElement::PrincipalHandleChangedForVideoFrameContainer(
6180 VideoFrameContainer* aContainer,
6181 const PrincipalHandle& aNewPrincipalHandle) {
6182 MOZ_ASSERT(NS_IsMainThread());
6183
6184 if (!mSrcStream) {
6185 return;
6186 }
6187
6188 LOG(LogLevel::Debug, ("HTMLMediaElement %p PrincipalHandle changed in "
6189 "VideoFrameContainer.",
6190 this));
6191
6192 UpdateSrcStreamVideoPrincipal(aNewPrincipalHandle);
6193 }
6194
GetEventRunner(const nsAString & aName,EventFlag aFlag)6195 already_AddRefed<nsMediaEventRunner> HTMLMediaElement::GetEventRunner(
6196 const nsAString& aName, EventFlag aFlag) {
6197 RefPtr<nsMediaEventRunner> runner;
6198 if (aName.EqualsLiteral("playing")) {
6199 runner = new nsNotifyAboutPlayingRunner(this, TakePendingPlayPromises());
6200 } else if (aName.EqualsLiteral("timeupdate")) {
6201 runner = new nsTimeupdateRunner(this, aFlag == EventFlag::eMandatory);
6202 } else {
6203 runner = new nsAsyncEventRunner(aName, this);
6204 }
6205 return runner.forget();
6206 }
6207
DispatchEvent(const nsAString & aName)6208 nsresult HTMLMediaElement::DispatchEvent(const nsAString& aName) {
6209 LOG_EVENT(LogLevel::Debug, ("%p Dispatching event %s", this,
6210 NS_ConvertUTF16toUTF8(aName).get()));
6211
6212 if (mEventBlocker->ShouldBlockEventDelivery()) {
6213 RefPtr<nsMediaEventRunner> runner = GetEventRunner(aName);
6214 mEventBlocker->PostponeEvent(runner);
6215 return NS_OK;
6216 }
6217
6218 return nsContentUtils::DispatchTrustedEvent(
6219 OwnerDoc(), static_cast<nsIContent*>(this), aName, CanBubble::eNo,
6220 Cancelable::eNo);
6221 }
6222
DispatchAsyncEvent(const nsAString & aName)6223 void HTMLMediaElement::DispatchAsyncEvent(const nsAString& aName) {
6224 RefPtr<nsMediaEventRunner> runner = GetEventRunner(aName);
6225 DispatchAsyncEvent(std::move(runner));
6226 }
6227
DispatchAsyncEvent(RefPtr<nsMediaEventRunner> aRunner)6228 void HTMLMediaElement::DispatchAsyncEvent(RefPtr<nsMediaEventRunner> aRunner) {
6229 NS_ConvertUTF16toUTF8 eventName(aRunner->EventName());
6230 LOG_EVENT(LogLevel::Debug, ("%p Queuing event %s", this, eventName.get()));
6231 DDLOG(DDLogCategory::Event, "HTMLMediaElement", nsCString(eventName.get()));
6232 if (mEventBlocker->ShouldBlockEventDelivery()) {
6233 mEventBlocker->PostponeEvent(aRunner);
6234 return;
6235 }
6236 mMainThreadEventTarget->Dispatch(aRunner.forget());
6237 }
6238
IsPotentiallyPlaying() const6239 bool HTMLMediaElement::IsPotentiallyPlaying() const {
6240 // TODO:
6241 // playback has not stopped due to errors,
6242 // and the element has not paused for user interaction
6243 return !mPaused &&
6244 (mReadyState == HAVE_ENOUGH_DATA || mReadyState == HAVE_FUTURE_DATA) &&
6245 !IsPlaybackEnded();
6246 }
6247
IsPlaybackEnded() const6248 bool HTMLMediaElement::IsPlaybackEnded() const {
6249 // TODO:
6250 // the current playback position is equal to the effective end of the media
6251 // resource. See bug 449157.
6252 if (mDecoder) {
6253 return mReadyState >= HAVE_METADATA && mDecoder->IsEnded();
6254 } else if (mSrcStream) {
6255 return mReadyState >= HAVE_METADATA && mSrcStreamPlaybackEnded;
6256 } else {
6257 return false;
6258 }
6259 }
6260
GetCurrentPrincipal()6261 already_AddRefed<nsIPrincipal> HTMLMediaElement::GetCurrentPrincipal() {
6262 if (mDecoder) {
6263 return mDecoder->GetCurrentPrincipal();
6264 }
6265 if (mSrcStream) {
6266 nsTArray<RefPtr<MediaStreamTrack>> tracks;
6267 mSrcStream->GetTracks(tracks);
6268 nsCOMPtr<nsIPrincipal> principal = mSrcStream->GetPrincipal();
6269 return principal.forget();
6270 }
6271 return nullptr;
6272 }
6273
HadCrossOriginRedirects()6274 bool HTMLMediaElement::HadCrossOriginRedirects() {
6275 if (mDecoder) {
6276 return mDecoder->HadCrossOriginRedirects();
6277 }
6278 return false;
6279 }
6280
GetCurrentVideoPrincipal()6281 already_AddRefed<nsIPrincipal> HTMLMediaElement::GetCurrentVideoPrincipal() {
6282 if (mDecoder) {
6283 return mDecoder->GetCurrentPrincipal();
6284 }
6285 if (mSrcStream) {
6286 nsCOMPtr<nsIPrincipal> principal = mSrcStreamVideoPrincipal;
6287 return principal.forget();
6288 }
6289 return nullptr;
6290 }
6291
NotifyDecoderPrincipalChanged()6292 void HTMLMediaElement::NotifyDecoderPrincipalChanged() {
6293 RefPtr<nsIPrincipal> principal = GetCurrentPrincipal();
6294 bool isSameOrigin = !principal || IsCORSSameOrigin();
6295 mDecoder->UpdateSameOriginStatus(isSameOrigin);
6296
6297 if (isSameOrigin) {
6298 principal = NodePrincipal();
6299 }
6300 for (const auto& entry : mOutputTrackSources.Values()) {
6301 entry->SetPrincipal(principal);
6302 }
6303 mDecoder->SetOutputTracksPrincipal(principal);
6304 }
6305
Invalidate(bool aImageSizeChanged,Maybe<nsIntSize> & aNewIntrinsicSize,bool aForceInvalidate)6306 void HTMLMediaElement::Invalidate(bool aImageSizeChanged,
6307 Maybe<nsIntSize>& aNewIntrinsicSize,
6308 bool aForceInvalidate) {
6309 nsIFrame* frame = GetPrimaryFrame();
6310 if (aNewIntrinsicSize) {
6311 UpdateMediaSize(aNewIntrinsicSize.value());
6312 if (frame) {
6313 nsPresContext* presContext = frame->PresContext();
6314 PresShell* presShell = presContext->PresShell();
6315 presShell->FrameNeedsReflow(frame, IntrinsicDirty::StyleChange,
6316 NS_FRAME_IS_DIRTY);
6317 }
6318 }
6319
6320 RefPtr<ImageContainer> imageContainer = GetImageContainer();
6321 bool asyncInvalidate =
6322 imageContainer && imageContainer->IsAsync() && !aForceInvalidate;
6323 if (frame) {
6324 if (aImageSizeChanged) {
6325 frame->InvalidateFrame();
6326 } else {
6327 frame->InvalidateLayer(DisplayItemType::TYPE_VIDEO, nullptr, nullptr,
6328 asyncInvalidate ? nsIFrame::UPDATE_IS_ASYNC : 0);
6329 }
6330 }
6331
6332 SVGObserverUtils::InvalidateDirectRenderingObservers(this);
6333 }
6334
UpdateMediaSize(const nsIntSize & aSize)6335 void HTMLMediaElement::UpdateMediaSize(const nsIntSize& aSize) {
6336 MOZ_ASSERT(NS_IsMainThread());
6337
6338 if (IsVideo() && mReadyState != HAVE_NOTHING &&
6339 mMediaInfo.mVideo.mDisplay != aSize) {
6340 DispatchAsyncEvent(u"resize"_ns);
6341 }
6342
6343 mMediaInfo.mVideo.mDisplay = aSize;
6344 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
6345 }
6346
SuspendOrResumeElement(bool aSuspendElement)6347 void HTMLMediaElement::SuspendOrResumeElement(bool aSuspendElement) {
6348 LOG(LogLevel::Debug, ("%p SuspendOrResumeElement(suspend=%d) docHidden=%d",
6349 this, aSuspendElement, OwnerDoc()->Hidden()));
6350
6351 if (aSuspendElement == mSuspendedByInactiveDocOrDocshell) {
6352 return;
6353 }
6354
6355 mSuspendedByInactiveDocOrDocshell = aSuspendElement;
6356 UpdateSrcMediaStreamPlaying();
6357 UpdateAudioChannelPlayingState();
6358
6359 if (aSuspendElement) {
6360 if (mDecoder) {
6361 mDecoder->Pause();
6362 mDecoder->Suspend();
6363 mDecoder->SetDelaySeekMode(true);
6364 }
6365 mEventBlocker->SetBlockEventDelivery(true);
6366 // We won't want to resume media element from the bfcache.
6367 ClearResumeDelayedMediaPlaybackAgentIfNeeded();
6368 mMediaControlKeyListener->StopIfNeeded();
6369 } else {
6370 if (mDecoder) {
6371 mDecoder->Resume();
6372 if (!mPaused && !mDecoder->IsEnded()) {
6373 mDecoder->Play();
6374 }
6375 mDecoder->SetDelaySeekMode(false);
6376 }
6377 mEventBlocker->SetBlockEventDelivery(false);
6378 // If the media element has been blocked and isn't still allowed to play
6379 // when it comes back from the bfcache, we would notify front end to show
6380 // the blocking icon in order to inform user that the site is still being
6381 // blocked.
6382 if (mHasEverBeenBlockedForAutoplay &&
6383 !AutoplayPolicy::IsAllowedToPlay(*this)) {
6384 MaybeNotifyAutoplayBlocked();
6385 }
6386 StartMediaControlKeyListenerIfNeeded();
6387 }
6388 if (StaticPrefs::media_testing_only_events()) {
6389 auto dispatcher = MakeRefPtr<AsyncEventDispatcher>(
6390 this, u"MozMediaSuspendChanged"_ns, CanBubble::eYes,
6391 ChromeOnlyDispatch::eYes);
6392 dispatcher->PostDOMEvent();
6393 }
6394 }
6395
IsBeingDestroyed()6396 bool HTMLMediaElement::IsBeingDestroyed() {
6397 nsIDocShell* docShell = OwnerDoc()->GetDocShell();
6398 bool isBeingDestroyed = false;
6399 if (docShell) {
6400 docShell->IsBeingDestroyed(&isBeingDestroyed);
6401 }
6402 return isBeingDestroyed;
6403 }
6404
ShouldBeSuspendedByInactiveDocShell() const6405 bool HTMLMediaElement::ShouldBeSuspendedByInactiveDocShell() const {
6406 BrowsingContext* bc = OwnerDoc()->GetBrowsingContext();
6407 return bc && !bc->IsActive() && bc->Top()->GetSuspendMediaWhenInactive();
6408 }
6409
NotifyOwnerDocumentActivityChanged()6410 void HTMLMediaElement::NotifyOwnerDocumentActivityChanged() {
6411 if (mDecoder && !IsBeingDestroyed()) {
6412 NotifyDecoderActivityChanges();
6413 }
6414
6415 // We would suspend media when the document is inactive, or its docshell has
6416 // been set to hidden and explicitly wants to suspend media. In those cases,
6417 // the media would be not visible and we don't want them to continue playing.
6418 bool shouldSuspend =
6419 !OwnerDoc()->IsActive() || ShouldBeSuspendedByInactiveDocShell();
6420 SuspendOrResumeElement(shouldSuspend);
6421
6422 // If the owning document has become inactive we should shutdown the CDM.
6423 if (!OwnerDoc()->IsCurrentActiveDocument() && mMediaKeys) {
6424 // We don't shutdown MediaKeys here because it also listens for document
6425 // activity and will take care of shutting down itself.
6426 DDUNLINKCHILD(mMediaKeys.get());
6427 mMediaKeys = nullptr;
6428 if (mDecoder) {
6429 ShutdownDecoder();
6430 }
6431 }
6432
6433 AddRemoveSelfReference();
6434 }
6435
NotifyFullScreenChanged()6436 void HTMLMediaElement::NotifyFullScreenChanged() {
6437 const bool isInFullScreen = IsInFullScreen();
6438 if (isInFullScreen) {
6439 StartMediaControlKeyListenerIfNeeded();
6440 if (!mMediaControlKeyListener->IsStarted()) {
6441 MEDIACONTROL_LOG("Failed to start the listener when entering fullscreen");
6442 }
6443 }
6444 // Updating controller fullscreen state no matter the listener starts or not.
6445 BrowsingContext* bc = OwnerDoc()->GetBrowsingContext();
6446 if (RefPtr<IMediaInfoUpdater> updater = ContentMediaAgent::Get(bc)) {
6447 updater->NotifyMediaFullScreenState(bc->Id(), isInFullScreen);
6448 }
6449 }
6450
AddRemoveSelfReference()6451 void HTMLMediaElement::AddRemoveSelfReference() {
6452 // XXX we could release earlier here in many situations if we examined
6453 // which event listeners are attached. Right now we assume there is a
6454 // potential listener for every event. We would also have to keep the
6455 // element alive if it was playing and producing audio output --- right now
6456 // that's covered by the !mPaused check.
6457 Document* ownerDoc = OwnerDoc();
6458
6459 // See the comment at the top of this file for the explanation of this
6460 // boolean expression.
6461 bool needSelfReference =
6462 !mShuttingDown && ownerDoc->IsActive() &&
6463 (mDelayingLoadEvent || (!mPaused && !Ended()) ||
6464 (mDecoder && mDecoder->IsSeeking()) || CanActivateAutoplay() ||
6465 (mMediaSource ? mProgressTimer : mNetworkState == NETWORK_LOADING));
6466
6467 if (needSelfReference != mHasSelfReference) {
6468 mHasSelfReference = needSelfReference;
6469 RefPtr<HTMLMediaElement> self = this;
6470 if (needSelfReference) {
6471 // The shutdown observer will hold a strong reference to us. This
6472 // will do to keep us alive. We need to know about shutdown so that
6473 // we can release our self-reference.
6474 mMainThreadEventTarget->Dispatch(NS_NewRunnableFunction(
6475 "dom::HTMLMediaElement::AddSelfReference",
6476 [self]() { self->mShutdownObserver->AddRefMediaElement(); }));
6477 } else {
6478 // Dispatch Release asynchronously so that we don't destroy this object
6479 // inside a call stack of method calls on this object
6480 mMainThreadEventTarget->Dispatch(NS_NewRunnableFunction(
6481 "dom::HTMLMediaElement::AddSelfReference",
6482 [self]() { self->mShutdownObserver->ReleaseMediaElement(); }));
6483 }
6484 }
6485 }
6486
NotifyShutdownEvent()6487 void HTMLMediaElement::NotifyShutdownEvent() {
6488 mShuttingDown = true;
6489 ResetState();
6490 AddRemoveSelfReference();
6491 }
6492
DispatchAsyncSourceError(nsIContent * aSourceElement)6493 void HTMLMediaElement::DispatchAsyncSourceError(nsIContent* aSourceElement) {
6494 LOG_EVENT(LogLevel::Debug, ("%p Queuing simple source error event", this));
6495
6496 nsCOMPtr<nsIRunnable> event =
6497 new nsSourceErrorEventRunner(this, aSourceElement);
6498 mMainThreadEventTarget->Dispatch(event.forget());
6499 }
6500
NotifyAddedSource()6501 void HTMLMediaElement::NotifyAddedSource() {
6502 // If a source element is inserted as a child of a media element
6503 // that has no src attribute and whose networkState has the value
6504 // NETWORK_EMPTY, the user agent must invoke the media element's
6505 // resource selection algorithm.
6506 if (!HasAttr(kNameSpaceID_None, nsGkAtoms::src) &&
6507 mNetworkState == NETWORK_EMPTY) {
6508 AssertReadyStateIsNothing();
6509 QueueSelectResourceTask();
6510 }
6511
6512 // A load was paused in the resource selection algorithm, waiting for
6513 // a new source child to be added, resume the resource selection algorithm.
6514 if (mLoadWaitStatus == WAITING_FOR_SOURCE) {
6515 // Rest the flag so we don't queue multiple LoadFromSourceTask() when
6516 // multiple <source> are attached in an event loop.
6517 mLoadWaitStatus = NOT_WAITING;
6518 QueueLoadFromSourceTask();
6519 }
6520 }
6521
GetNextSource()6522 Element* HTMLMediaElement::GetNextSource() {
6523 mSourceLoadCandidate = nullptr;
6524
6525 while (true) {
6526 if (mSourcePointer == nsINode::GetLastChild()) {
6527 return nullptr; // no more children
6528 }
6529
6530 if (!mSourcePointer) {
6531 mSourcePointer = nsINode::GetFirstChild();
6532 } else {
6533 mSourcePointer = mSourcePointer->GetNextSibling();
6534 }
6535 nsIContent* child = mSourcePointer;
6536
6537 // If child is a <source> element, it is the next candidate.
6538 if (child && child->IsHTMLElement(nsGkAtoms::source)) {
6539 mSourceLoadCandidate = child;
6540 return child->AsElement();
6541 }
6542 }
6543 MOZ_ASSERT_UNREACHABLE("Execution should not reach here!");
6544 return nullptr;
6545 }
6546
ChangeDelayLoadStatus(bool aDelay)6547 void HTMLMediaElement::ChangeDelayLoadStatus(bool aDelay) {
6548 if (mDelayingLoadEvent == aDelay) return;
6549
6550 mDelayingLoadEvent = aDelay;
6551
6552 LOG(LogLevel::Debug, ("%p ChangeDelayLoadStatus(%d) doc=0x%p", this, aDelay,
6553 mLoadBlockedDoc.get()));
6554 if (mDecoder) {
6555 mDecoder->SetLoadInBackground(!aDelay);
6556 }
6557 if (aDelay) {
6558 mLoadBlockedDoc = OwnerDoc();
6559 mLoadBlockedDoc->BlockOnload();
6560 } else {
6561 // mLoadBlockedDoc might be null due to GC unlinking
6562 if (mLoadBlockedDoc) {
6563 mLoadBlockedDoc->UnblockOnload(false);
6564 mLoadBlockedDoc = nullptr;
6565 }
6566 }
6567
6568 // We changed mDelayingLoadEvent which can affect AddRemoveSelfReference
6569 AddRemoveSelfReference();
6570 }
6571
GetDocumentLoadGroup()6572 already_AddRefed<nsILoadGroup> HTMLMediaElement::GetDocumentLoadGroup() {
6573 if (!OwnerDoc()->IsActive()) {
6574 NS_WARNING("Load group requested for media element in inactive document.");
6575 }
6576 return OwnerDoc()->GetDocumentLoadGroup();
6577 }
6578
CopyInnerTo(Element * aDest)6579 nsresult HTMLMediaElement::CopyInnerTo(Element* aDest) {
6580 nsresult rv = nsGenericHTMLElement::CopyInnerTo(aDest);
6581 NS_ENSURE_SUCCESS(rv, rv);
6582 if (aDest->OwnerDoc()->IsStaticDocument()) {
6583 HTMLMediaElement* dest = static_cast<HTMLMediaElement*>(aDest);
6584 dest->SetMediaInfo(mMediaInfo);
6585 }
6586 return rv;
6587 }
6588
Buffered() const6589 already_AddRefed<TimeRanges> HTMLMediaElement::Buffered() const {
6590 media::TimeIntervals buffered =
6591 mDecoder ? mDecoder->GetBuffered() : media::TimeIntervals();
6592 RefPtr<TimeRanges> ranges = new TimeRanges(ToSupports(OwnerDoc()), buffered);
6593 return ranges.forget();
6594 }
6595
SetRequestHeaders(nsIHttpChannel * aChannel)6596 void HTMLMediaElement::SetRequestHeaders(nsIHttpChannel* aChannel) {
6597 // Send Accept header for video and audio types only (Bug 489071)
6598 SetAcceptHeader(aChannel);
6599
6600 // Apache doesn't send Content-Length when gzip transfer encoding is used,
6601 // which prevents us from estimating the video length (if explicit
6602 // Content-Duration and a length spec in the container are not present either)
6603 // and from seeking. So, disable the standard "Accept-Encoding: gzip,deflate"
6604 // that we usually send. See bug 614760.
6605 DebugOnly<nsresult> rv =
6606 aChannel->SetRequestHeader("Accept-Encoding"_ns, ""_ns, false);
6607 MOZ_ASSERT(NS_SUCCEEDED(rv));
6608
6609 // Set the Referrer header
6610 //
6611 // FIXME: Shouldn't this use the Element constructor? Though I guess it
6612 // doesn't matter as no HTMLMediaElement supports the referrerinfo attribute.
6613 auto referrerInfo = MakeRefPtr<ReferrerInfo>(*OwnerDoc());
6614 rv = aChannel->SetReferrerInfoWithoutClone(referrerInfo);
6615 MOZ_ASSERT(NS_SUCCEEDED(rv));
6616 }
6617
LastTimeupdateDispatchTime() const6618 const TimeStamp& HTMLMediaElement::LastTimeupdateDispatchTime() const {
6619 MOZ_ASSERT(NS_IsMainThread());
6620 return mLastTimeUpdateDispatchTime;
6621 }
6622
UpdateLastTimeupdateDispatchTime()6623 void HTMLMediaElement::UpdateLastTimeupdateDispatchTime() {
6624 MOZ_ASSERT(NS_IsMainThread());
6625 mLastTimeUpdateDispatchTime = TimeStamp::Now();
6626 }
6627
ShouldQueueTimeupdateAsyncTask(TimeupdateType aType) const6628 bool HTMLMediaElement::ShouldQueueTimeupdateAsyncTask(
6629 TimeupdateType aType) const {
6630 NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
6631 // That means dispatching `timeupdate` is mandatorily required in the spec.
6632 if (aType == TimeupdateType::eMandatory) {
6633 return true;
6634 }
6635
6636 // The timeupdate only occurs when the current playback position changes.
6637 // https://html.spec.whatwg.org/multipage/media.html#event-media-timeupdate
6638 if (mLastCurrentTime == CurrentTime()) {
6639 return false;
6640 }
6641
6642 // Number of milliseconds between timeupdate events as defined by spec.
6643 if (!mQueueTimeUpdateRunnerTime.IsNull() &&
6644 TimeStamp::Now() - mQueueTimeUpdateRunnerTime <
6645 TimeDuration::FromMilliseconds(TIMEUPDATE_MS)) {
6646 return false;
6647 }
6648 return true;
6649 }
6650
FireTimeUpdate(TimeupdateType aType)6651 void HTMLMediaElement::FireTimeUpdate(TimeupdateType aType) {
6652 NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
6653
6654 if (ShouldQueueTimeupdateAsyncTask(aType)) {
6655 RefPtr<nsMediaEventRunner> runner =
6656 GetEventRunner(u"timeupdate"_ns, aType == TimeupdateType::eMandatory
6657 ? EventFlag::eMandatory
6658 : EventFlag::eNone);
6659 DispatchAsyncEvent(std::move(runner));
6660 mQueueTimeUpdateRunnerTime = TimeStamp::Now();
6661 mLastCurrentTime = CurrentTime();
6662 }
6663 if (mFragmentEnd >= 0.0 && CurrentTime() >= mFragmentEnd) {
6664 Pause();
6665 mFragmentEnd = -1.0;
6666 mFragmentStart = -1.0;
6667 mDecoder->SetFragmentEndTime(mFragmentEnd);
6668 }
6669
6670 // Update the cues displaying on the video.
6671 // Here mTextTrackManager can be null if the cycle collector has unlinked
6672 // us before our parent. In that case UnbindFromTree will call us
6673 // when our parent is unlinked.
6674 if (mTextTrackManager) {
6675 mTextTrackManager->TimeMarchesOn();
6676 }
6677 }
6678
GetError() const6679 MediaError* HTMLMediaElement::GetError() const { return mErrorSink->mError; }
6680
GetCurrentSpec(nsCString & aString)6681 void HTMLMediaElement::GetCurrentSpec(nsCString& aString) {
6682 // If playing a regular URL, an ObjectURL of a Blob/File, return that.
6683 if (mLoadingSrc) {
6684 mLoadingSrc->GetSpec(aString);
6685 } else if (mSrcMediaSource) {
6686 // If playing an ObjectURL, and it's a MediaSource, return the value of the
6687 // `src` attribute.
6688 nsAutoString src;
6689 GetSrc(src);
6690 CopyUTF16toUTF8(src, aString);
6691 } else {
6692 // Playing e.g. a MediaStream via an object URL - return an empty string
6693 aString.Truncate();
6694 }
6695 }
6696
MozFragmentEnd()6697 double HTMLMediaElement::MozFragmentEnd() {
6698 double duration = Duration();
6699
6700 // If there is no end fragment, or the fragment end is greater than the
6701 // duration, return the duration.
6702 return (mFragmentEnd < 0.0 || mFragmentEnd > duration) ? duration
6703 : mFragmentEnd;
6704 }
6705
SetDefaultPlaybackRate(double aDefaultPlaybackRate,ErrorResult & aRv)6706 void HTMLMediaElement::SetDefaultPlaybackRate(double aDefaultPlaybackRate,
6707 ErrorResult& aRv) {
6708 if (mSrcAttrStream) {
6709 return;
6710 }
6711
6712 if (aDefaultPlaybackRate < 0) {
6713 aRv.Throw(NS_ERROR_NOT_IMPLEMENTED);
6714 return;
6715 }
6716
6717 double defaultPlaybackRate = ClampPlaybackRate(aDefaultPlaybackRate);
6718
6719 if (mDefaultPlaybackRate == defaultPlaybackRate) {
6720 return;
6721 }
6722
6723 mDefaultPlaybackRate = defaultPlaybackRate;
6724 DispatchAsyncEvent(u"ratechange"_ns);
6725 }
6726
SetPlaybackRate(double aPlaybackRate,ErrorResult & aRv)6727 void HTMLMediaElement::SetPlaybackRate(double aPlaybackRate, ErrorResult& aRv) {
6728 if (mSrcAttrStream) {
6729 return;
6730 }
6731
6732 // Changing the playback rate of a media that has more than two channels is
6733 // not supported.
6734 if (aPlaybackRate < 0) {
6735 aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
6736 return;
6737 }
6738
6739 if (mPlaybackRate == aPlaybackRate) {
6740 return;
6741 }
6742
6743 mPlaybackRate = aPlaybackRate;
6744
6745 if (mPlaybackRate != 0.0 &&
6746 (mPlaybackRate > THRESHOLD_HIGH_PLAYBACKRATE_AUDIO ||
6747 mPlaybackRate < THRESHOLD_LOW_PLAYBACKRATE_AUDIO)) {
6748 SetMutedInternal(mMuted | MUTED_BY_INVALID_PLAYBACK_RATE);
6749 } else {
6750 SetMutedInternal(mMuted & ~MUTED_BY_INVALID_PLAYBACK_RATE);
6751 }
6752
6753 if (mDecoder) {
6754 mDecoder->SetPlaybackRate(ClampPlaybackRate(mPlaybackRate));
6755 }
6756 DispatchAsyncEvent(u"ratechange"_ns);
6757 }
6758
SetMozPreservesPitch(bool aPreservesPitch)6759 void HTMLMediaElement::SetMozPreservesPitch(bool aPreservesPitch) {
6760 mPreservesPitch = aPreservesPitch;
6761 if (mDecoder) {
6762 mDecoder->SetPreservesPitch(mPreservesPitch);
6763 }
6764 }
6765
GetImageContainer()6766 ImageContainer* HTMLMediaElement::GetImageContainer() {
6767 VideoFrameContainer* container = GetVideoFrameContainer();
6768 return container ? container->GetImageContainer() : nullptr;
6769 }
6770
UpdateAudioChannelPlayingState()6771 void HTMLMediaElement::UpdateAudioChannelPlayingState() {
6772 if (mAudioChannelWrapper) {
6773 mAudioChannelWrapper->UpdateAudioChannelPlayingState();
6774 }
6775 }
6776
VisibilityString(Visibility aVisibility)6777 static const char* VisibilityString(Visibility aVisibility) {
6778 switch (aVisibility) {
6779 case Visibility::Untracked: {
6780 return "Untracked";
6781 }
6782 case Visibility::ApproximatelyNonVisible: {
6783 return "ApproximatelyNonVisible";
6784 }
6785 case Visibility::ApproximatelyVisible: {
6786 return "ApproximatelyVisible";
6787 }
6788 }
6789
6790 return "NAN";
6791 }
6792
OnVisibilityChange(Visibility aNewVisibility)6793 void HTMLMediaElement::OnVisibilityChange(Visibility aNewVisibility) {
6794 LOG(LogLevel::Debug,
6795 ("OnVisibilityChange(): %s\n", VisibilityString(aNewVisibility)));
6796
6797 mVisibilityState = aNewVisibility;
6798 if (StaticPrefs::media_test_video_suspend()) {
6799 DispatchAsyncEvent(u"visibilitychanged"_ns);
6800 }
6801
6802 if (!mDecoder) {
6803 return;
6804 }
6805 NotifyDecoderActivityChanges();
6806 }
6807
GetMediaKeys() const6808 MediaKeys* HTMLMediaElement::GetMediaKeys() const { return mMediaKeys; }
6809
ContainsRestrictedContent()6810 bool HTMLMediaElement::ContainsRestrictedContent() {
6811 return GetMediaKeys() != nullptr;
6812 }
6813
SetCDMProxyFailure(const MediaResult & aResult)6814 void HTMLMediaElement::SetCDMProxyFailure(const MediaResult& aResult) {
6815 LOG(LogLevel::Debug, ("%s", __func__));
6816 MOZ_ASSERT(mSetMediaKeysDOMPromise);
6817
6818 ResetSetMediaKeysTempVariables();
6819
6820 mSetMediaKeysDOMPromise->MaybeReject(aResult.Code(), aResult.Message());
6821 }
6822
RemoveMediaKeys()6823 void HTMLMediaElement::RemoveMediaKeys() {
6824 LOG(LogLevel::Debug, ("%s", __func__));
6825 // 5.2.3 Stop using the CDM instance represented by the mediaKeys attribute
6826 // to decrypt media data and remove the association with the media element.
6827 if (mMediaKeys) {
6828 mMediaKeys->Unbind();
6829 }
6830 mMediaKeys = nullptr;
6831 }
6832
TryRemoveMediaKeysAssociation()6833 bool HTMLMediaElement::TryRemoveMediaKeysAssociation() {
6834 MOZ_ASSERT(mMediaKeys);
6835 LOG(LogLevel::Debug, ("%s", __func__));
6836 // 5.2.1 If the user agent or CDM do not support removing the association,
6837 // let this object's attaching media keys value be false and reject promise
6838 // with a new DOMException whose name is NotSupportedError.
6839 // 5.2.2 If the association cannot currently be removed, let this object's
6840 // attaching media keys value be false and reject promise with a new
6841 // DOMException whose name is InvalidStateError.
6842 if (mDecoder) {
6843 RefPtr<HTMLMediaElement> self = this;
6844 mDecoder->SetCDMProxy(nullptr)
6845 ->Then(
6846 mAbstractMainThread, __func__,
6847 [self]() {
6848 self->mSetCDMRequest.Complete();
6849
6850 self->RemoveMediaKeys();
6851 if (self->AttachNewMediaKeys()) {
6852 // No incoming MediaKeys object or MediaDecoder is not
6853 // created yet.
6854 self->MakeAssociationWithCDMResolved();
6855 }
6856 },
6857 [self](const MediaResult& aResult) {
6858 self->mSetCDMRequest.Complete();
6859 // 5.2.4 If the preceding step failed, let this object's
6860 // attaching media keys value be false and reject promise with
6861 // a new DOMException whose name is the appropriate error name.
6862 self->SetCDMProxyFailure(aResult);
6863 })
6864 ->Track(mSetCDMRequest);
6865 return false;
6866 }
6867
6868 RemoveMediaKeys();
6869 return true;
6870 }
6871
DetachExistingMediaKeys()6872 bool HTMLMediaElement::DetachExistingMediaKeys() {
6873 LOG(LogLevel::Debug, ("%s", __func__));
6874 MOZ_ASSERT(mSetMediaKeysDOMPromise);
6875 // 5.1 If mediaKeys is not null, CDM instance represented by mediaKeys is
6876 // already in use by another media element, and the user agent is unable
6877 // to use it with this element, let this object's attaching media keys
6878 // value be false and reject promise with a new DOMException whose name
6879 // is QuotaExceededError.
6880 if (mIncomingMediaKeys && mIncomingMediaKeys->IsBoundToMediaElement()) {
6881 SetCDMProxyFailure(MediaResult(
6882 NS_ERROR_DOM_QUOTA_EXCEEDED_ERR,
6883 "MediaKeys object is already bound to another HTMLMediaElement"));
6884 return false;
6885 }
6886
6887 // 5.2 If the mediaKeys attribute is not null, run the following steps:
6888 if (mMediaKeys) {
6889 return TryRemoveMediaKeysAssociation();
6890 }
6891 return true;
6892 }
6893
MakeAssociationWithCDMResolved()6894 void HTMLMediaElement::MakeAssociationWithCDMResolved() {
6895 LOG(LogLevel::Debug, ("%s", __func__));
6896 MOZ_ASSERT(mSetMediaKeysDOMPromise);
6897
6898 // 5.4 Set the mediaKeys attribute to mediaKeys.
6899 mMediaKeys = mIncomingMediaKeys;
6900 // 5.5 Let this object's attaching media keys value be false.
6901 ResetSetMediaKeysTempVariables();
6902 // 5.6 Resolve promise.
6903 mSetMediaKeysDOMPromise->MaybeResolveWithUndefined();
6904 mSetMediaKeysDOMPromise = nullptr;
6905 }
6906
TryMakeAssociationWithCDM(CDMProxy * aProxy)6907 bool HTMLMediaElement::TryMakeAssociationWithCDM(CDMProxy* aProxy) {
6908 LOG(LogLevel::Debug, ("%s", __func__));
6909 MOZ_ASSERT(aProxy);
6910
6911 // 5.3.3 Queue a task to run the "Attempt to Resume Playback If Necessary"
6912 // algorithm on the media element.
6913 // Note: Setting the CDMProxy on the MediaDecoder will unblock playback.
6914 if (mDecoder) {
6915 // CDMProxy is set asynchronously in MediaFormatReader, once it's done,
6916 // HTMLMediaElement should resolve or reject the DOM promise.
6917 RefPtr<HTMLMediaElement> self = this;
6918 mDecoder->SetCDMProxy(aProxy)
6919 ->Then(
6920 mAbstractMainThread, __func__,
6921 [self]() {
6922 self->mSetCDMRequest.Complete();
6923 self->MakeAssociationWithCDMResolved();
6924 },
6925 [self](const MediaResult& aResult) {
6926 self->mSetCDMRequest.Complete();
6927 self->SetCDMProxyFailure(aResult);
6928 })
6929 ->Track(mSetCDMRequest);
6930 return false;
6931 }
6932 return true;
6933 }
6934
AttachNewMediaKeys()6935 bool HTMLMediaElement::AttachNewMediaKeys() {
6936 LOG(LogLevel::Debug,
6937 ("%s incoming MediaKeys(%p)", __func__, mIncomingMediaKeys.get()));
6938 MOZ_ASSERT(mSetMediaKeysDOMPromise);
6939
6940 // 5.3. If mediaKeys is not null, run the following steps:
6941 if (mIncomingMediaKeys) {
6942 auto cdmProxy = mIncomingMediaKeys->GetCDMProxy();
6943 if (!cdmProxy) {
6944 SetCDMProxyFailure(MediaResult(
6945 NS_ERROR_DOM_INVALID_STATE_ERR,
6946 "CDM crashed before binding MediaKeys object to HTMLMediaElement"));
6947 return false;
6948 }
6949
6950 // 5.3.1 Associate the CDM instance represented by mediaKeys with the
6951 // media element for decrypting media data.
6952 if (NS_FAILED(mIncomingMediaKeys->Bind(this))) {
6953 // 5.3.2 If the preceding step failed, run the following steps:
6954
6955 // 5.3.2.1 Set the mediaKeys attribute to null.
6956 mMediaKeys = nullptr;
6957 // 5.3.2.2 Let this object's attaching media keys value be false.
6958 // 5.3.2.3 Reject promise with a new DOMException whose name is
6959 // the appropriate error name.
6960 SetCDMProxyFailure(
6961 MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,
6962 "Failed to bind MediaKeys object to HTMLMediaElement"));
6963 return false;
6964 }
6965 return TryMakeAssociationWithCDM(cdmProxy);
6966 }
6967 return true;
6968 }
6969
ResetSetMediaKeysTempVariables()6970 void HTMLMediaElement::ResetSetMediaKeysTempVariables() {
6971 mAttachingMediaKey = false;
6972 mIncomingMediaKeys = nullptr;
6973 }
6974
SetMediaKeys(mozilla::dom::MediaKeys * aMediaKeys,ErrorResult & aRv)6975 already_AddRefed<Promise> HTMLMediaElement::SetMediaKeys(
6976 mozilla::dom::MediaKeys* aMediaKeys, ErrorResult& aRv) {
6977 LOG(LogLevel::Debug, ("%p SetMediaKeys(%p) mMediaKeys=%p mDecoder=%p", this,
6978 aMediaKeys, mMediaKeys.get(), mDecoder.get()));
6979
6980 if (MozAudioCaptured()) {
6981 aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
6982 return nullptr;
6983 }
6984
6985 nsPIDOMWindowInner* win = OwnerDoc()->GetInnerWindow();
6986 if (!win) {
6987 aRv.Throw(NS_ERROR_UNEXPECTED);
6988 return nullptr;
6989 }
6990 RefPtr<DetailedPromise> promise = DetailedPromise::Create(
6991 win->AsGlobal(), aRv, "HTMLMediaElement.setMediaKeys"_ns);
6992 if (aRv.Failed()) {
6993 return nullptr;
6994 }
6995
6996 // 1. If mediaKeys and the mediaKeys attribute are the same object,
6997 // return a resolved promise.
6998 if (mMediaKeys == aMediaKeys) {
6999 promise->MaybeResolveWithUndefined();
7000 return promise.forget();
7001 }
7002
7003 // 2. If this object's attaching media keys value is true, return a
7004 // promise rejected with a new DOMException whose name is InvalidStateError.
7005 if (mAttachingMediaKey) {
7006 promise->MaybeRejectWithInvalidStateError(
7007 "A MediaKeys object is in attaching operation.");
7008 return promise.forget();
7009 }
7010
7011 // 3. Let this object's attaching media keys value be true.
7012 mAttachingMediaKey = true;
7013 mIncomingMediaKeys = aMediaKeys;
7014
7015 // 4. Let promise be a new promise.
7016 mSetMediaKeysDOMPromise = promise;
7017
7018 // 5. Run the following steps in parallel:
7019
7020 // 5.1 & 5.2 & 5.3
7021 if (!DetachExistingMediaKeys() || !AttachNewMediaKeys()) {
7022 return promise.forget();
7023 }
7024
7025 // 5.4, 5.5, 5.6
7026 MakeAssociationWithCDMResolved();
7027
7028 // 6. Return promise.
7029 return promise.forget();
7030 }
7031
GetOnencrypted()7032 EventHandlerNonNull* HTMLMediaElement::GetOnencrypted() {
7033 return EventTarget::GetEventHandler(nsGkAtoms::onencrypted);
7034 }
7035
SetOnencrypted(EventHandlerNonNull * aCallback)7036 void HTMLMediaElement::SetOnencrypted(EventHandlerNonNull* aCallback) {
7037 EventTarget::SetEventHandler(nsGkAtoms::onencrypted, aCallback);
7038 }
7039
GetOnwaitingforkey()7040 EventHandlerNonNull* HTMLMediaElement::GetOnwaitingforkey() {
7041 return EventTarget::GetEventHandler(nsGkAtoms::onwaitingforkey);
7042 }
7043
SetOnwaitingforkey(EventHandlerNonNull * aCallback)7044 void HTMLMediaElement::SetOnwaitingforkey(EventHandlerNonNull* aCallback) {
7045 EventTarget::SetEventHandler(nsGkAtoms::onwaitingforkey, aCallback);
7046 }
7047
DispatchEncrypted(const nsTArray<uint8_t> & aInitData,const nsAString & aInitDataType)7048 void HTMLMediaElement::DispatchEncrypted(const nsTArray<uint8_t>& aInitData,
7049 const nsAString& aInitDataType) {
7050 LOG(LogLevel::Debug, ("%p DispatchEncrypted initDataType='%s'", this,
7051 NS_ConvertUTF16toUTF8(aInitDataType).get()));
7052
7053 if (mReadyState == HAVE_NOTHING) {
7054 // Ready state not HAVE_METADATA (yet), don't dispatch encrypted now.
7055 // Queueing for later dispatch in MetadataLoaded.
7056 mPendingEncryptedInitData.AddInitData(aInitDataType, aInitData);
7057 return;
7058 }
7059
7060 RefPtr<MediaEncryptedEvent> event;
7061 if (IsCORSSameOrigin()) {
7062 event = MediaEncryptedEvent::Constructor(this, aInitDataType, aInitData);
7063 } else {
7064 event = MediaEncryptedEvent::Constructor(this);
7065 }
7066
7067 RefPtr<AsyncEventDispatcher> asyncDispatcher =
7068 new AsyncEventDispatcher(this, event);
7069 asyncDispatcher->PostDOMEvent();
7070 }
7071
IsEventAttributeNameInternal(nsAtom * aName)7072 bool HTMLMediaElement::IsEventAttributeNameInternal(nsAtom* aName) {
7073 return aName == nsGkAtoms::onencrypted ||
7074 nsGenericHTMLElement::IsEventAttributeNameInternal(aName);
7075 }
7076
NotifyWaitingForKey()7077 void HTMLMediaElement::NotifyWaitingForKey() {
7078 LOG(LogLevel::Debug, ("%p, NotifyWaitingForKey()", this));
7079
7080 // http://w3c.github.io/encrypted-media/#wait-for-key
7081 // 7.3.4 Queue a "waitingforkey" Event
7082 // 1. Let the media element be the specified HTMLMediaElement object.
7083 // 2. If the media element's waiting for key value is true, abort these steps.
7084 if (mWaitingForKey == NOT_WAITING_FOR_KEY) {
7085 // 3. Set the media element's waiting for key value to true.
7086 // Note: algorithm continues in UpdateReadyStateInternal() when all decoded
7087 // data enqueued in the MDSM is consumed.
7088 mWaitingForKey = WAITING_FOR_KEY;
7089 // mWaitingForKey changed outside of UpdateReadyStateInternal. This may
7090 // affect mReadyState.
7091 mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal);
7092 }
7093 }
7094
AudioTracks()7095 AudioTrackList* HTMLMediaElement::AudioTracks() { return mAudioTrackList; }
7096
VideoTracks()7097 VideoTrackList* HTMLMediaElement::VideoTracks() { return mVideoTrackList; }
7098
GetTextTracks()7099 TextTrackList* HTMLMediaElement::GetTextTracks() {
7100 return GetOrCreateTextTrackManager()->GetTextTracks();
7101 }
7102
AddTextTrack(TextTrackKind aKind,const nsAString & aLabel,const nsAString & aLanguage)7103 already_AddRefed<TextTrack> HTMLMediaElement::AddTextTrack(
7104 TextTrackKind aKind, const nsAString& aLabel, const nsAString& aLanguage) {
7105 return GetOrCreateTextTrackManager()->AddTextTrack(
7106 aKind, aLabel, aLanguage, TextTrackMode::Hidden,
7107 TextTrackReadyState::Loaded, TextTrackSource::AddTextTrack);
7108 }
7109
PopulatePendingTextTrackList()7110 void HTMLMediaElement::PopulatePendingTextTrackList() {
7111 if (mTextTrackManager) {
7112 mTextTrackManager->PopulatePendingList();
7113 }
7114 }
7115
GetOrCreateTextTrackManager()7116 TextTrackManager* HTMLMediaElement::GetOrCreateTextTrackManager() {
7117 if (!mTextTrackManager) {
7118 mTextTrackManager = new TextTrackManager(this);
7119 mTextTrackManager->AddListeners();
7120 }
7121 return mTextTrackManager;
7122 }
7123
NextFrameStatus()7124 MediaDecoderOwner::NextFrameStatus HTMLMediaElement::NextFrameStatus() {
7125 if (mDecoder) {
7126 return mDecoder->NextFrameStatus();
7127 }
7128 if (mSrcStream) {
7129 AutoTArray<RefPtr<MediaTrack>, 4> tracks;
7130 GetAllEnabledMediaTracks(tracks);
7131 if (!tracks.IsEmpty() && !mSrcStreamPlaybackEnded) {
7132 return NEXT_FRAME_AVAILABLE;
7133 }
7134 return NEXT_FRAME_UNAVAILABLE;
7135 }
7136 return NEXT_FRAME_UNINITIALIZED;
7137 }
7138
SetDecoder(MediaDecoder * aDecoder)7139 void HTMLMediaElement::SetDecoder(MediaDecoder* aDecoder) {
7140 MOZ_ASSERT(aDecoder); // Use ShutdownDecoder() to clear.
7141 if (mDecoder) {
7142 ShutdownDecoder();
7143 }
7144 mDecoder = aDecoder;
7145 DDLINKCHILD("decoder", mDecoder.get());
7146 if (mDecoder && mForcedHidden) {
7147 mDecoder->SetForcedHidden(mForcedHidden);
7148 }
7149 }
7150
ComputedVolume() const7151 float HTMLMediaElement::ComputedVolume() const {
7152 return mMuted ? 0.0f
7153 : mAudioChannelWrapper ? mAudioChannelWrapper->GetEffectiveVolume()
7154 : mVolume;
7155 }
7156
ComputedMuted() const7157 bool HTMLMediaElement::ComputedMuted() const {
7158 return (mMuted & MUTED_BY_AUDIO_CHANNEL);
7159 }
7160
IsSuspendedByInactiveDocOrDocShell() const7161 bool HTMLMediaElement::IsSuspendedByInactiveDocOrDocShell() const {
7162 return mSuspendedByInactiveDocOrDocshell;
7163 }
7164
IsCurrentlyPlaying() const7165 bool HTMLMediaElement::IsCurrentlyPlaying() const {
7166 // We have playable data, but we still need to check whether data is "real"
7167 // current data.
7168 return mReadyState >= HAVE_CURRENT_DATA && !IsPlaybackEnded();
7169 }
7170
SetAudibleState(bool aAudible)7171 void HTMLMediaElement::SetAudibleState(bool aAudible) {
7172 if (mIsAudioTrackAudible != aAudible) {
7173 mIsAudioTrackAudible = aAudible;
7174 NotifyAudioPlaybackChanged(
7175 AudioChannelService::AudibleChangedReasons::eDataAudibleChanged);
7176 }
7177 }
7178
NotifyAudioPlaybackChanged(AudibleChangedReasons aReason)7179 void HTMLMediaElement::NotifyAudioPlaybackChanged(
7180 AudibleChangedReasons aReason) {
7181 if (mAudioChannelWrapper) {
7182 mAudioChannelWrapper->NotifyAudioPlaybackChanged(aReason);
7183 }
7184 // We would start the listener after media becomes audible.
7185 const bool isAudible = IsAudible();
7186 if (isAudible && !mMediaControlKeyListener->IsStarted()) {
7187 StartMediaControlKeyListenerIfNeeded();
7188 }
7189 mMediaControlKeyListener->UpdateMediaAudibleState(isAudible);
7190 // only request wake lock for audible media.
7191 UpdateWakeLock();
7192 }
7193
SetMediaInfo(const MediaInfo & aInfo)7194 void HTMLMediaElement::SetMediaInfo(const MediaInfo& aInfo) {
7195 const bool oldHasAudio = mMediaInfo.HasAudio();
7196 mMediaInfo = aInfo;
7197 if ((aInfo.HasAudio() != oldHasAudio) && mResumeDelayedPlaybackAgent) {
7198 mResumeDelayedPlaybackAgent->UpdateAudibleState(this, IsAudible());
7199 }
7200 nsILoadContext* loadContext = OwnerDoc()->GetLoadContext();
7201 if (HasAudio() && loadContext && !loadContext->UsePrivateBrowsing()) {
7202 mTitleChangeObserver->Subscribe();
7203 UpdateStreamName();
7204 } else {
7205 mTitleChangeObserver->Unsubscribe();
7206 }
7207 if (mAudioChannelWrapper) {
7208 mAudioChannelWrapper->AudioCaptureTrackChangeIfNeeded();
7209 }
7210 UpdateWakeLock();
7211 }
7212
GetMediaInfo() const7213 MediaInfo HTMLMediaElement::GetMediaInfo() const { return mMediaInfo; }
7214
GetFrameStatistics() const7215 FrameStatistics* HTMLMediaElement::GetFrameStatistics() const {
7216 return mDecoder ? &(mDecoder->GetFrameStatistics()) : nullptr;
7217 }
7218
DispatchAsyncTestingEvent(const nsAString & aName)7219 void HTMLMediaElement::DispatchAsyncTestingEvent(const nsAString& aName) {
7220 if (!StaticPrefs::media_testing_only_events()) {
7221 return;
7222 }
7223 DispatchAsyncEvent(aName);
7224 }
7225
AudioCaptureTrackChange(bool aCapture)7226 void HTMLMediaElement::AudioCaptureTrackChange(bool aCapture) {
7227 // No need to capture a silent media element.
7228 if (!HasAudio()) {
7229 return;
7230 }
7231
7232 if (aCapture && !mStreamWindowCapturer) {
7233 nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow();
7234 if (!window) {
7235 return;
7236 }
7237
7238 MediaTrackGraph* mtg = MediaTrackGraph::GetInstance(
7239 MediaTrackGraph::AUDIO_THREAD_DRIVER, window,
7240 MediaTrackGraph::REQUEST_DEFAULT_SAMPLE_RATE,
7241 MediaTrackGraph::DEFAULT_OUTPUT_DEVICE);
7242 RefPtr<DOMMediaStream> stream =
7243 CaptureStreamInternal(StreamCaptureBehavior::CONTINUE_WHEN_ENDED,
7244 StreamCaptureType::CAPTURE_AUDIO, mtg);
7245 mStreamWindowCapturer =
7246 MakeUnique<MediaStreamWindowCapturer>(stream, window->WindowID());
7247 } else if (!aCapture && mStreamWindowCapturer) {
7248 for (size_t i = 0; i < mOutputStreams.Length(); i++) {
7249 if (mOutputStreams[i].mStream == mStreamWindowCapturer->mStream) {
7250 // We own this MediaStream, it is not exposed to JS.
7251 AutoTArray<RefPtr<MediaStreamTrack>, 2> tracks;
7252 mStreamWindowCapturer->mStream->GetTracks(tracks);
7253 for (auto& track : tracks) {
7254 track->Stop();
7255 }
7256 mOutputStreams.RemoveElementAt(i);
7257 break;
7258 }
7259 }
7260 mStreamWindowCapturer = nullptr;
7261 if (mOutputStreams.IsEmpty()) {
7262 mTracksCaptured = nullptr;
7263 }
7264 }
7265 }
7266
NotifyCueDisplayStatesChanged()7267 void HTMLMediaElement::NotifyCueDisplayStatesChanged() {
7268 if (!mTextTrackManager) {
7269 return;
7270 }
7271
7272 mTextTrackManager->DispatchUpdateCueDisplay();
7273 }
7274
MarkAsContentSource(CallerAPI aAPI)7275 void HTMLMediaElement::MarkAsContentSource(CallerAPI aAPI) {
7276 const bool isVisible = mVisibilityState == Visibility::ApproximatelyVisible;
7277
7278 LOG(LogLevel::Debug,
7279 ("%p Log VIDEO_AS_CONTENT_SOURCE: visibility = %u, API: '%d' and 'All'",
7280 this, isVisible, static_cast<int>(aAPI)));
7281
7282 if (!isVisible) {
7283 LOG(LogLevel::Debug,
7284 ("%p Log VIDEO_AS_CONTENT_SOURCE_IN_TREE_OR_NOT: inTree = %u, API: "
7285 "'%d' and 'All'",
7286 this, IsInComposedDoc(), static_cast<int>(aAPI)));
7287 }
7288 }
7289
UpdateCustomPolicyAfterPlayed()7290 void HTMLMediaElement::UpdateCustomPolicyAfterPlayed() {
7291 if (mAudioChannelWrapper) {
7292 mAudioChannelWrapper->NotifyPlayStateChanged();
7293 }
7294 }
7295
AbstractMainThread() const7296 AbstractThread* HTMLMediaElement::AbstractMainThread() const {
7297 MOZ_ASSERT(mAbstractMainThread);
7298
7299 return mAbstractMainThread;
7300 }
7301
TakePendingPlayPromises()7302 nsTArray<RefPtr<PlayPromise>> HTMLMediaElement::TakePendingPlayPromises() {
7303 return std::move(mPendingPlayPromises);
7304 }
7305
NotifyAboutPlaying()7306 void HTMLMediaElement::NotifyAboutPlaying() {
7307 // Stick to the DispatchAsyncEvent() call path for now because we want to
7308 // trigger some telemetry-related codes in the DispatchAsyncEvent() method.
7309 DispatchAsyncEvent(u"playing"_ns);
7310 }
7311
CreatePlayPromise(ErrorResult & aRv) const7312 already_AddRefed<PlayPromise> HTMLMediaElement::CreatePlayPromise(
7313 ErrorResult& aRv) const {
7314 nsPIDOMWindowInner* win = OwnerDoc()->GetInnerWindow();
7315
7316 if (!win) {
7317 aRv.Throw(NS_ERROR_UNEXPECTED);
7318 return nullptr;
7319 }
7320
7321 RefPtr<PlayPromise> promise = PlayPromise::Create(win->AsGlobal(), aRv);
7322 LOG(LogLevel::Debug, ("%p created PlayPromise %p", this, promise.get()));
7323
7324 return promise.forget();
7325 }
7326
CreateDOMPromise(ErrorResult & aRv) const7327 already_AddRefed<Promise> HTMLMediaElement::CreateDOMPromise(
7328 ErrorResult& aRv) const {
7329 nsPIDOMWindowInner* win = OwnerDoc()->GetInnerWindow();
7330
7331 if (!win) {
7332 aRv.Throw(NS_ERROR_UNEXPECTED);
7333 return nullptr;
7334 }
7335
7336 return Promise::Create(win->AsGlobal(), aRv);
7337 }
7338
AsyncResolvePendingPlayPromises()7339 void HTMLMediaElement::AsyncResolvePendingPlayPromises() {
7340 if (mShuttingDown) {
7341 return;
7342 }
7343
7344 nsCOMPtr<nsIRunnable> event = new nsResolveOrRejectPendingPlayPromisesRunner(
7345 this, TakePendingPlayPromises());
7346
7347 mMainThreadEventTarget->Dispatch(event.forget());
7348 }
7349
AsyncRejectPendingPlayPromises(nsresult aError)7350 void HTMLMediaElement::AsyncRejectPendingPlayPromises(nsresult aError) {
7351 if (!mPaused) {
7352 mPaused = true;
7353 DispatchAsyncEvent(u"pause"_ns);
7354 }
7355
7356 if (mShuttingDown) {
7357 return;
7358 }
7359
7360 if (aError == NS_ERROR_DOM_MEDIA_NOT_ALLOWED_ERR) {
7361 DispatchEventsWhenPlayWasNotAllowed();
7362 }
7363
7364 nsCOMPtr<nsIRunnable> event = new nsResolveOrRejectPendingPlayPromisesRunner(
7365 this, TakePendingPlayPromises(), aError);
7366
7367 mMainThreadEventTarget->Dispatch(event.forget());
7368 }
7369
GetEMEInfo(dom::EMEDebugInfo & aInfo)7370 void HTMLMediaElement::GetEMEInfo(dom::EMEDebugInfo& aInfo) {
7371 if (!mMediaKeys) {
7372 return;
7373 }
7374 mMediaKeys->GetKeySystem(aInfo.mKeySystem);
7375 mMediaKeys->GetSessionsInfo(aInfo.mSessionsInfo);
7376 }
7377
NotifyDecoderActivityChanges() const7378 void HTMLMediaElement::NotifyDecoderActivityChanges() const {
7379 if (mDecoder) {
7380 mDecoder->NotifyOwnerActivityChanged(IsActuallyInvisible(),
7381 IsInComposedDoc());
7382 }
7383 }
7384
GetDocument() const7385 Document* HTMLMediaElement::GetDocument() const { return OwnerDoc(); }
7386
IsAudible() const7387 bool HTMLMediaElement::IsAudible() const {
7388 // No audio track.
7389 if (!HasAudio()) {
7390 return false;
7391 }
7392
7393 // Muted or the volume should not be ~0
7394 if (mMuted || (std::fabs(Volume()) <= 1e-7)) {
7395 return false;
7396 }
7397
7398 return mIsAudioTrackAudible;
7399 }
7400
GetKeySystem() const7401 Maybe<nsAutoString> HTMLMediaElement::GetKeySystem() const {
7402 if (!mMediaKeys) {
7403 return Nothing();
7404 }
7405 nsAutoString keySystem;
7406 mMediaKeys->GetKeySystem(keySystem);
7407 return Some(keySystem);
7408 }
7409
ConstructMediaTracks(const MediaInfo * aInfo)7410 void HTMLMediaElement::ConstructMediaTracks(const MediaInfo* aInfo) {
7411 if (!aInfo) {
7412 return;
7413 }
7414
7415 AudioTrackList* audioList = AudioTracks();
7416 if (audioList && aInfo->HasAudio()) {
7417 const TrackInfo& info = aInfo->mAudio;
7418 RefPtr<AudioTrack> track = MediaTrackList::CreateAudioTrack(
7419 audioList->GetOwnerGlobal(), info.mId, info.mKind, info.mLabel,
7420 info.mLanguage, info.mEnabled);
7421
7422 audioList->AddTrack(track);
7423 }
7424
7425 VideoTrackList* videoList = VideoTracks();
7426 if (videoList && aInfo->HasVideo()) {
7427 const TrackInfo& info = aInfo->mVideo;
7428 RefPtr<VideoTrack> track = MediaTrackList::CreateVideoTrack(
7429 videoList->GetOwnerGlobal(), info.mId, info.mKind, info.mLabel,
7430 info.mLanguage);
7431
7432 videoList->AddTrack(track);
7433 track->SetEnabledInternal(info.mEnabled, MediaTrack::FIRE_NO_EVENTS);
7434 }
7435 }
7436
RemoveMediaTracks()7437 void HTMLMediaElement::RemoveMediaTracks() {
7438 if (mAudioTrackList) {
7439 mAudioTrackList->RemoveTracks();
7440 }
7441 if (mVideoTrackList) {
7442 mVideoTrackList->RemoveTracks();
7443 }
7444 }
7445
7446 class MediaElementGMPCrashHelper : public GMPCrashHelper {
7447 public:
MediaElementGMPCrashHelper(HTMLMediaElement * aElement)7448 explicit MediaElementGMPCrashHelper(HTMLMediaElement* aElement)
7449 : mElement(aElement) {
7450 MOZ_ASSERT(NS_IsMainThread()); // WeakPtr isn't thread safe.
7451 }
GetPluginCrashedEventTarget()7452 already_AddRefed<nsPIDOMWindowInner> GetPluginCrashedEventTarget() override {
7453 MOZ_ASSERT(NS_IsMainThread()); // WeakPtr isn't thread safe.
7454 if (!mElement) {
7455 return nullptr;
7456 }
7457 return do_AddRef(mElement->OwnerDoc()->GetInnerWindow());
7458 }
7459
7460 private:
7461 WeakPtr<HTMLMediaElement> mElement;
7462 };
7463
CreateGMPCrashHelper()7464 already_AddRefed<GMPCrashHelper> HTMLMediaElement::CreateGMPCrashHelper() {
7465 return MakeAndAddRef<MediaElementGMPCrashHelper>(this);
7466 }
7467
MarkAsTainted()7468 void HTMLMediaElement::MarkAsTainted() {
7469 mHasSuspendTaint = true;
7470
7471 if (mDecoder) {
7472 mDecoder->SetSuspendTaint(true);
7473 }
7474 }
7475
HasDebuggerOrTabsPrivilege(JSContext * aCx,JSObject * aObj)7476 bool HasDebuggerOrTabsPrivilege(JSContext* aCx, JSObject* aObj) {
7477 return nsContentUtils::CallerHasPermission(aCx, nsGkAtoms::debugger) ||
7478 nsContentUtils::CallerHasPermission(aCx, nsGkAtoms::tabs);
7479 }
7480
SetSinkId(const nsAString & aSinkId,ErrorResult & aRv)7481 already_AddRefed<Promise> HTMLMediaElement::SetSinkId(const nsAString& aSinkId,
7482 ErrorResult& aRv) {
7483 nsCOMPtr<nsPIDOMWindowInner> win = OwnerDoc()->GetInnerWindow();
7484 if (!win) {
7485 aRv.Throw(NS_ERROR_UNEXPECTED);
7486 return nullptr;
7487 }
7488
7489 RefPtr<Promise> promise = Promise::Create(win->AsGlobal(), aRv);
7490 if (aRv.Failed()) {
7491 return nullptr;
7492 }
7493
7494 if (mSink.first.Equals(aSinkId)) {
7495 promise->MaybeResolveWithUndefined();
7496 return promise.forget();
7497 }
7498
7499 RefPtr<MediaDevices> mediaDevices = win->Navigator()->GetMediaDevices(aRv);
7500 if (aRv.Failed()) {
7501 return nullptr;
7502 }
7503
7504 nsString sinkId(aSinkId);
7505 mediaDevices->GetSinkDevice(sinkId)
7506 ->Then(
7507 mAbstractMainThread, __func__,
7508 [self = RefPtr<HTMLMediaElement>(this),
7509 this](RefPtr<AudioDeviceInfo>&& aInfo) {
7510 // Sink found switch output device.
7511 MOZ_ASSERT(aInfo);
7512 if (mDecoder) {
7513 RefPtr<SinkInfoPromise> p = mDecoder->SetSink(aInfo)->Then(
7514 mAbstractMainThread, __func__,
7515 [aInfo](const GenericPromise::ResolveOrRejectValue& aValue) {
7516 if (aValue.IsResolve()) {
7517 return SinkInfoPromise::CreateAndResolve(aInfo, __func__);
7518 }
7519 return SinkInfoPromise::CreateAndReject(
7520 aValue.RejectValue(), __func__);
7521 });
7522 return p;
7523 }
7524 if (mSrcStream) {
7525 MOZ_ASSERT(mMediaStreamRenderer);
7526 RefPtr<SinkInfoPromise> p =
7527 mMediaStreamRenderer->SetAudioOutputDevice(aInfo)->Then(
7528 mAbstractMainThread, __func__,
7529 [aInfo](const GenericPromise::AllPromiseType::
7530 ResolveOrRejectValue& aValue) {
7531 if (aValue.IsResolve()) {
7532 return SinkInfoPromise::CreateAndResolve(aInfo,
7533 __func__);
7534 }
7535 return SinkInfoPromise::CreateAndReject(
7536 aValue.RejectValue(), __func__);
7537 });
7538 return p;
7539 }
7540 // No media attached to the element save it for later.
7541 return SinkInfoPromise::CreateAndResolve(aInfo, __func__);
7542 },
7543 [](nsresult res) {
7544 // Promise is rejected, sink not found.
7545 return SinkInfoPromise::CreateAndReject(res, __func__);
7546 })
7547 ->Then(mAbstractMainThread, __func__,
7548 [promise, self = RefPtr<HTMLMediaElement>(this), this,
7549 sinkId](const SinkInfoPromise::ResolveOrRejectValue& aValue) {
7550 if (aValue.IsResolve()) {
7551 mSink = std::pair(sinkId, aValue.ResolveValue());
7552 promise->MaybeResolveWithUndefined();
7553 } else {
7554 switch (aValue.RejectValue()) {
7555 case NS_ERROR_ABORT:
7556 promise->MaybeReject(NS_ERROR_DOM_ABORT_ERR);
7557 break;
7558 case NS_ERROR_NOT_AVAILABLE: {
7559 promise->MaybeRejectWithNotFoundError(
7560 "The object can not be found here.");
7561 break;
7562 }
7563 default:
7564 MOZ_ASSERT_UNREACHABLE("Invalid error.");
7565 }
7566 }
7567 });
7568
7569 aRv = NS_OK;
7570 return promise.forget();
7571 }
7572
NotifyTextTrackModeChanged()7573 void HTMLMediaElement::NotifyTextTrackModeChanged() {
7574 if (mPendingTextTrackChanged) {
7575 return;
7576 }
7577 mPendingTextTrackChanged = true;
7578 mAbstractMainThread->Dispatch(
7579 NS_NewRunnableFunction("HTMLMediaElement::NotifyTextTrackModeChanged",
7580 [this, self = RefPtr<HTMLMediaElement>(this)]() {
7581 mPendingTextTrackChanged = false;
7582 if (!mTextTrackManager) {
7583 return;
7584 }
7585 GetTextTracks()->CreateAndDispatchChangeEvent();
7586 // https://html.spec.whatwg.org/multipage/media.html#text-track-model:show-poster-flag
7587 if (!mShowPoster) {
7588 mTextTrackManager->TimeMarchesOn();
7589 }
7590 }));
7591 }
7592
CreateResumeDelayedMediaPlaybackAgentIfNeeded()7593 void HTMLMediaElement::CreateResumeDelayedMediaPlaybackAgentIfNeeded() {
7594 if (mResumeDelayedPlaybackAgent) {
7595 return;
7596 }
7597 mResumeDelayedPlaybackAgent =
7598 MediaPlaybackDelayPolicy::CreateResumeDelayedPlaybackAgent(this,
7599 IsAudible());
7600 if (!mResumeDelayedPlaybackAgent) {
7601 LOG(LogLevel::Debug,
7602 ("%p Failed to create a delayed playback agant", this));
7603 return;
7604 }
7605 mResumeDelayedPlaybackAgent->GetResumePromise()
7606 ->Then(
7607 mAbstractMainThread, __func__,
7608 [self = RefPtr<HTMLMediaElement>(this)]() {
7609 LOG(LogLevel::Debug, ("%p Resume delayed Play() call", self.get()));
7610 self->mResumePlaybackRequest.Complete();
7611 self->mResumeDelayedPlaybackAgent = nullptr;
7612 IgnoredErrorResult dummy;
7613 RefPtr<Promise> toBeIgnored = self->Play(dummy);
7614 },
7615 [self = RefPtr<HTMLMediaElement>(this)]() {
7616 LOG(LogLevel::Debug,
7617 ("%p Can not resume delayed Play() call", self.get()));
7618 self->mResumePlaybackRequest.Complete();
7619 self->mResumeDelayedPlaybackAgent = nullptr;
7620 })
7621 ->Track(mResumePlaybackRequest);
7622 }
7623
ClearResumeDelayedMediaPlaybackAgentIfNeeded()7624 void HTMLMediaElement::ClearResumeDelayedMediaPlaybackAgentIfNeeded() {
7625 if (mResumeDelayedPlaybackAgent) {
7626 mResumePlaybackRequest.DisconnectIfExists();
7627 mResumeDelayedPlaybackAgent = nullptr;
7628 }
7629 }
7630
NotifyMediaControlPlaybackStateChanged()7631 void HTMLMediaElement::NotifyMediaControlPlaybackStateChanged() {
7632 if (!mMediaControlKeyListener->IsStarted()) {
7633 return;
7634 }
7635 if (mPaused) {
7636 mMediaControlKeyListener->NotifyMediaStoppedPlaying();
7637 } else {
7638 mMediaControlKeyListener->NotifyMediaStartedPlaying();
7639 }
7640 }
7641
IsInFullScreen() const7642 bool HTMLMediaElement::IsInFullScreen() const {
7643 return State().HasState(NS_EVENT_STATE_FULLSCREEN);
7644 }
7645
IsPlayable() const7646 bool HTMLMediaElement::IsPlayable() const {
7647 return (mDecoder || mSrcStream) && !HasError();
7648 }
7649
ShouldStartMediaControlKeyListener() const7650 bool HTMLMediaElement::ShouldStartMediaControlKeyListener() const {
7651 if (!IsPlayable()) {
7652 MEDIACONTROL_LOG("Not start listener because media is not playable");
7653 return false;
7654 }
7655
7656 if (mSrcStream) {
7657 MEDIACONTROL_LOG("Not listening because media is real-time");
7658 return false;
7659 }
7660
7661 if (IsBeingUsedInPictureInPictureMode()) {
7662 MEDIACONTROL_LOG("Start listener because of being used in PiP mode");
7663 return true;
7664 }
7665
7666 if (IsInFullScreen()) {
7667 MEDIACONTROL_LOG("Start listener because of being used in fullscreen");
7668 return true;
7669 }
7670
7671 // In order to filter out notification-ish sound, we use this pref to set the
7672 // eligible media duration to prevent showing media control for those short
7673 // sound.
7674 if (Duration() <
7675 StaticPrefs::media_mediacontrol_eligible_media_duration_s()) {
7676 MEDIACONTROL_LOG("Not listening because media's duration %f is too short.",
7677 Duration());
7678 return false;
7679 }
7680
7681 // This includes cases such like `video is muted`, `video has zero volume`,
7682 // `video's audio track is still inaudible` and `tab is muted by audio channel
7683 // (tab sound indicator)`, all these cases would make media inaudible.
7684 // `ComputedVolume()` would return the final volume applied the affection made
7685 // by audio channel, which is used to detect if the tab is muted by audio
7686 // channel.
7687 if (!IsAudible() || ComputedVolume() == 0.0f) {
7688 MEDIACONTROL_LOG("Not listening because media is inaudible");
7689 return false;
7690 }
7691 return true;
7692 }
7693
StartMediaControlKeyListenerIfNeeded()7694 void HTMLMediaElement::StartMediaControlKeyListenerIfNeeded() {
7695 if (!ShouldStartMediaControlKeyListener()) {
7696 return;
7697 }
7698 mMediaControlKeyListener->Start();
7699 }
7700
UpdateStreamName()7701 void HTMLMediaElement::UpdateStreamName() {
7702 MOZ_ASSERT(NS_IsMainThread());
7703
7704 nsAutoString aTitle;
7705 OwnerDoc()->GetTitle(aTitle);
7706
7707 if (mDecoder) {
7708 mDecoder->SetStreamName(aTitle);
7709 }
7710 }
7711
SetSecondaryMediaStreamRenderer(VideoFrameContainer * aContainer,FirstFrameVideoOutput * aFirstFrameOutput)7712 void HTMLMediaElement::SetSecondaryMediaStreamRenderer(
7713 VideoFrameContainer* aContainer,
7714 FirstFrameVideoOutput* aFirstFrameOutput /* = nullptr */) {
7715 MOZ_ASSERT(mSrcStream);
7716 MOZ_ASSERT(mMediaStreamRenderer);
7717 if (mSecondaryMediaStreamRenderer) {
7718 mSecondaryMediaStreamRenderer->Shutdown();
7719 mSecondaryMediaStreamRenderer = nullptr;
7720 }
7721 if (aContainer) {
7722 mSecondaryMediaStreamRenderer = MakeAndAddRef<MediaStreamRenderer>(
7723 mAbstractMainThread, aContainer, aFirstFrameOutput, this);
7724 if (mSrcStreamIsPlaying) {
7725 mSecondaryMediaStreamRenderer->Start();
7726 }
7727 if (mSelectedVideoStreamTrack) {
7728 mSecondaryMediaStreamRenderer->AddTrack(mSelectedVideoStreamTrack);
7729 }
7730 }
7731 }
7732
UpdateMediaControlAfterPictureInPictureModeChanged()7733 void HTMLMediaElement::UpdateMediaControlAfterPictureInPictureModeChanged() {
7734 if (IsBeingUsedInPictureInPictureMode()) {
7735 // When media enters PIP mode, we should ensure that the listener has been
7736 // started because we always want to control PIP video.
7737 StartMediaControlKeyListenerIfNeeded();
7738 if (!mMediaControlKeyListener->IsStarted()) {
7739 MEDIACONTROL_LOG("Failed to start listener when entering PIP mode");
7740 }
7741 // Updating controller PIP state no matter the listener starts or not.
7742 mMediaControlKeyListener->SetPictureInPictureModeEnabled(true);
7743 } else {
7744 mMediaControlKeyListener->SetPictureInPictureModeEnabled(false);
7745 }
7746 }
7747
IsBeingUsedInPictureInPictureMode() const7748 bool HTMLMediaElement::IsBeingUsedInPictureInPictureMode() const {
7749 if (!IsVideo()) {
7750 return false;
7751 }
7752 return static_cast<const HTMLVideoElement*>(this)->IsCloningElementVisually();
7753 }
7754
NodeInfoChanged(Document * aOldDoc)7755 void HTMLMediaElement::NodeInfoChanged(Document* aOldDoc) {
7756 if (mMediaSource) {
7757 OwnerDoc()->AddMediaElementWithMSE();
7758 aOldDoc->RemoveMediaElementWithMSE();
7759 }
7760
7761 nsGenericHTMLElement::NodeInfoChanged(aOldDoc);
7762 }
7763
7764 } // namespace mozilla::dom
7765
7766 #undef LOG
7767 #undef LOG_EVENT
7768