1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
5 * You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "mozilla/dom/TextTrackManager.h"
8 #include "mozilla/ClearOnShutdown.h"
9 #include "mozilla/CycleCollectedJSContext.h"
10 #include "mozilla/Maybe.h"
11 #include "mozilla/Telemetry.h"
12 #include "mozilla/dom/Document.h"
13 #include "mozilla/dom/Event.h"
14 #include "mozilla/dom/HTMLMediaElement.h"
15 #include "mozilla/dom/HTMLTrackElement.h"
16 #include "mozilla/dom/HTMLVideoElement.h"
17 #include "mozilla/dom/TextTrack.h"
18 #include "mozilla/dom/TextTrackCue.h"
19 #include "nsComponentManagerUtils.h"
20 #include "nsGlobalWindow.h"
21 #include "nsIFrame.h"
22 #include "nsIWebVTTParserWrapper.h"
23 #include "nsVariant.h"
24 #include "nsVideoFrame.h"
25
26 mozilla::LazyLogModule gTextTrackLog("WebVTT");
27
28 #define WEBVTT_LOG(msg, ...) \
29 MOZ_LOG(gTextTrackLog, LogLevel::Debug, \
30 ("TextTrackManager=%p, " msg, this, ##__VA_ARGS__))
31 #define WEBVTT_LOGV(msg, ...) \
32 MOZ_LOG(gTextTrackLog, LogLevel::Verbose, \
33 ("TextTrackManager=%p, " msg, this, ##__VA_ARGS__))
34
35 namespace mozilla::dom {
36
37 NS_IMPL_ISUPPORTS(TextTrackManager::ShutdownObserverProxy, nsIObserver);
38
Unregister()39 void TextTrackManager::ShutdownObserverProxy::Unregister() {
40 nsContentUtils::UnregisterShutdownObserver(this);
41 mManager = nullptr;
42 }
43
CompareTextTracks(HTMLMediaElement * aMediaElement)44 CompareTextTracks::CompareTextTracks(HTMLMediaElement* aMediaElement) {
45 mMediaElement = aMediaElement;
46 }
47
TrackChildPosition(TextTrack * aTextTrack) const48 Maybe<uint32_t> CompareTextTracks::TrackChildPosition(
49 TextTrack* aTextTrack) const {
50 MOZ_DIAGNOSTIC_ASSERT(aTextTrack);
51 HTMLTrackElement* trackElement = aTextTrack->GetTrackElement();
52 if (!trackElement) {
53 return Nothing();
54 }
55 return mMediaElement->ComputeIndexOf(trackElement);
56 }
57
Equals(TextTrack * aOne,TextTrack * aTwo) const58 bool CompareTextTracks::Equals(TextTrack* aOne, TextTrack* aTwo) const {
59 // Two tracks can never be equal. If they have corresponding TrackElements
60 // they would need to occupy the same tree position (impossible) and in the
61 // case of tracks coming from AddTextTrack source we put the newest at the
62 // last position, so they won't be equal as well.
63 return false;
64 }
65
LessThan(TextTrack * aOne,TextTrack * aTwo) const66 bool CompareTextTracks::LessThan(TextTrack* aOne, TextTrack* aTwo) const {
67 // Protect against nullptr TextTrack objects; treat them as
68 // sorting toward the end.
69 if (!aOne) {
70 return false;
71 }
72 if (!aTwo) {
73 return true;
74 }
75 TextTrackSource sourceOne = aOne->GetTextTrackSource();
76 TextTrackSource sourceTwo = aTwo->GetTextTrackSource();
77 if (sourceOne != sourceTwo) {
78 return sourceOne == TextTrackSource::Track ||
79 (sourceOne == TextTrackSource::AddTextTrack &&
80 sourceTwo == TextTrackSource::MediaResourceSpecific);
81 }
82 switch (sourceOne) {
83 case TextTrackSource::Track: {
84 Maybe<uint32_t> positionOne = TrackChildPosition(aOne);
85 Maybe<uint32_t> positionTwo = TrackChildPosition(aTwo);
86 // If either position one or positiontwo are Nothing then something has
87 // gone wrong. In this case we should just put them at the back of the
88 // list.
89 return positionOne.isSome() && positionTwo.isSome() &&
90 *positionOne < *positionTwo;
91 }
92 case TextTrackSource::AddTextTrack:
93 // For AddTextTrack sources the tracks will already be in the correct
94 // relative order in the source array. Assume we're called in iteration
95 // order and can therefore always report aOne < aTwo to maintain the
96 // original temporal ordering.
97 return true;
98 case TextTrackSource::MediaResourceSpecific:
99 // No rules for Media Resource Specific tracks yet.
100 break;
101 }
102 return true;
103 }
104
105 NS_IMPL_CYCLE_COLLECTION(TextTrackManager, mMediaElement, mTextTracks,
106 mPendingTextTracks, mNewCues)
107
108 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(TextTrackManager)
109 NS_INTERFACE_MAP_ENTRY(nsIDOMEventListener)
110 NS_INTERFACE_MAP_END
111
112 NS_IMPL_CYCLE_COLLECTING_ADDREF(TextTrackManager)
113 NS_IMPL_CYCLE_COLLECTING_RELEASE(TextTrackManager)
114
115 StaticRefPtr<nsIWebVTTParserWrapper> TextTrackManager::sParserWrapper;
116
TextTrackManager(HTMLMediaElement * aMediaElement)117 TextTrackManager::TextTrackManager(HTMLMediaElement* aMediaElement)
118 : mMediaElement(aMediaElement),
119 mHasSeeked(false),
120 mLastTimeMarchesOnCalled(media::TimeUnit::Zero()),
121 mTimeMarchesOnDispatched(false),
122 mUpdateCueDisplayDispatched(false),
123 performedTrackSelection(false),
124 mShutdown(false) {
125 nsISupports* parentObject = mMediaElement->OwnerDoc()->GetParentObject();
126
127 NS_ENSURE_TRUE_VOID(parentObject);
128 WEBVTT_LOG("Create TextTrackManager");
129 nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(parentObject);
130 mNewCues = new TextTrackCueList(window);
131 mTextTracks = new TextTrackList(window, this);
132 mPendingTextTracks = new TextTrackList(window, this);
133
134 if (!sParserWrapper) {
135 nsCOMPtr<nsIWebVTTParserWrapper> parserWrapper =
136 do_CreateInstance(NS_WEBVTTPARSERWRAPPER_CONTRACTID);
137 MOZ_ASSERT(parserWrapper, "Can't create nsIWebVTTParserWrapper");
138 sParserWrapper = parserWrapper;
139 ClearOnShutdown(&sParserWrapper);
140 }
141 mShutdownProxy = new ShutdownObserverProxy(this);
142 }
143
~TextTrackManager()144 TextTrackManager::~TextTrackManager() {
145 WEBVTT_LOG("~TextTrackManager");
146 mShutdownProxy->Unregister();
147 }
148
GetTextTracks() const149 TextTrackList* TextTrackManager::GetTextTracks() const { return mTextTracks; }
150
AddTextTrack(TextTrackKind aKind,const nsAString & aLabel,const nsAString & aLanguage,TextTrackMode aMode,TextTrackReadyState aReadyState,TextTrackSource aTextTrackSource)151 already_AddRefed<TextTrack> TextTrackManager::AddTextTrack(
152 TextTrackKind aKind, const nsAString& aLabel, const nsAString& aLanguage,
153 TextTrackMode aMode, TextTrackReadyState aReadyState,
154 TextTrackSource aTextTrackSource) {
155 if (!mMediaElement || !mTextTracks) {
156 return nullptr;
157 }
158 RefPtr<TextTrack> track = mTextTracks->AddTextTrack(
159 aKind, aLabel, aLanguage, aMode, aReadyState, aTextTrackSource,
160 CompareTextTracks(mMediaElement));
161 WEBVTT_LOG("AddTextTrack %p kind %" PRIu32 " Label %s Language %s",
162 track.get(), static_cast<uint32_t>(aKind),
163 NS_ConvertUTF16toUTF8(aLabel).get(),
164 NS_ConvertUTF16toUTF8(aLanguage).get());
165 AddCues(track);
166
167 if (aTextTrackSource == TextTrackSource::Track) {
168 RefPtr<nsIRunnable> task = NewRunnableMethod(
169 "dom::TextTrackManager::HonorUserPreferencesForTrackSelection", this,
170 &TextTrackManager::HonorUserPreferencesForTrackSelection);
171 NS_DispatchToMainThread(task.forget());
172 }
173
174 return track.forget();
175 }
176
AddTextTrack(TextTrack * aTextTrack)177 void TextTrackManager::AddTextTrack(TextTrack* aTextTrack) {
178 if (!mMediaElement || !mTextTracks) {
179 return;
180 }
181 WEBVTT_LOG("AddTextTrack TextTrack %p", aTextTrack);
182 mTextTracks->AddTextTrack(aTextTrack, CompareTextTracks(mMediaElement));
183 AddCues(aTextTrack);
184
185 if (aTextTrack->GetTextTrackSource() == TextTrackSource::Track) {
186 RefPtr<nsIRunnable> task = NewRunnableMethod(
187 "dom::TextTrackManager::HonorUserPreferencesForTrackSelection", this,
188 &TextTrackManager::HonorUserPreferencesForTrackSelection);
189 NS_DispatchToMainThread(task.forget());
190 }
191 }
192
AddCues(TextTrack * aTextTrack)193 void TextTrackManager::AddCues(TextTrack* aTextTrack) {
194 if (!mNewCues) {
195 WEBVTT_LOG("AddCues mNewCues is null");
196 return;
197 }
198
199 TextTrackCueList* cueList = aTextTrack->GetCues();
200 if (cueList) {
201 bool dummy;
202 WEBVTT_LOGV("AddCues, CuesNum=%d", cueList->Length());
203 for (uint32_t i = 0; i < cueList->Length(); ++i) {
204 mNewCues->AddCue(*cueList->IndexedGetter(i, dummy));
205 }
206 MaybeRunTimeMarchesOn();
207 }
208 }
209
RemoveTextTrack(TextTrack * aTextTrack,bool aPendingListOnly)210 void TextTrackManager::RemoveTextTrack(TextTrack* aTextTrack,
211 bool aPendingListOnly) {
212 if (!mPendingTextTracks || !mTextTracks) {
213 return;
214 }
215
216 WEBVTT_LOG("RemoveTextTrack TextTrack %p", aTextTrack);
217 mPendingTextTracks->RemoveTextTrack(aTextTrack);
218 if (aPendingListOnly) {
219 return;
220 }
221
222 mTextTracks->RemoveTextTrack(aTextTrack);
223 // Remove the cues in mNewCues belong to aTextTrack.
224 TextTrackCueList* removeCueList = aTextTrack->GetCues();
225 if (removeCueList) {
226 WEBVTT_LOGV("RemoveTextTrack removeCuesNum=%d", removeCueList->Length());
227 for (uint32_t i = 0; i < removeCueList->Length(); ++i) {
228 mNewCues->RemoveCue(*((*removeCueList)[i]));
229 }
230 MaybeRunTimeMarchesOn();
231 }
232 }
233
DidSeek()234 void TextTrackManager::DidSeek() {
235 WEBVTT_LOG("DidSeek");
236 mHasSeeked = true;
237 }
238
UpdateCueDisplay()239 void TextTrackManager::UpdateCueDisplay() {
240 WEBVTT_LOG("UpdateCueDisplay");
241 mUpdateCueDisplayDispatched = false;
242
243 if (!mMediaElement || !mTextTracks || IsShutdown()) {
244 WEBVTT_LOG("Abort UpdateCueDisplay.");
245 return;
246 }
247
248 nsIFrame* frame = mMediaElement->GetPrimaryFrame();
249 nsVideoFrame* videoFrame = do_QueryFrame(frame);
250 if (!videoFrame) {
251 WEBVTT_LOG("Abort UpdateCueDisplay, because of no video frame.");
252 return;
253 }
254
255 nsCOMPtr<nsIContent> overlay = videoFrame->GetCaptionOverlay();
256 if (!overlay) {
257 WEBVTT_LOG("Abort UpdateCueDisplay, because of no overlay.");
258 return;
259 }
260
261 nsPIDOMWindowInner* window = mMediaElement->OwnerDoc()->GetInnerWindow();
262 if (!window) {
263 WEBVTT_LOG("Abort UpdateCueDisplay, because of no window.");
264 }
265
266 nsTArray<RefPtr<TextTrackCue>> showingCues;
267 mTextTracks->GetShowingCues(showingCues);
268
269 WEBVTT_LOG("UpdateCueDisplay, processCues, showingCuesNum=%zu",
270 showingCues.Length());
271 RefPtr<nsVariantCC> jsCues = new nsVariantCC();
272 jsCues->SetAsArray(nsIDataType::VTYPE_INTERFACE, &NS_GET_IID(EventTarget),
273 showingCues.Length(),
274 static_cast<void*>(showingCues.Elements()));
275 nsCOMPtr<nsIContent> controls = videoFrame->GetVideoControls();
276 sParserWrapper->ProcessCues(window, jsCues, overlay, controls);
277 }
278
NotifyCueAdded(TextTrackCue & aCue)279 void TextTrackManager::NotifyCueAdded(TextTrackCue& aCue) {
280 WEBVTT_LOG("NotifyCueAdded, cue=%p", &aCue);
281 if (mNewCues) {
282 mNewCues->AddCue(aCue);
283 }
284 MaybeRunTimeMarchesOn();
285 }
286
NotifyCueRemoved(TextTrackCue & aCue)287 void TextTrackManager::NotifyCueRemoved(TextTrackCue& aCue) {
288 WEBVTT_LOG("NotifyCueRemoved, cue=%p", &aCue);
289 if (mNewCues) {
290 mNewCues->RemoveCue(aCue);
291 }
292 MaybeRunTimeMarchesOn();
293 DispatchUpdateCueDisplay();
294 }
295
PopulatePendingList()296 void TextTrackManager::PopulatePendingList() {
297 if (!mTextTracks || !mPendingTextTracks || !mMediaElement) {
298 return;
299 }
300 uint32_t len = mTextTracks->Length();
301 bool dummy;
302 for (uint32_t index = 0; index < len; ++index) {
303 TextTrack* ttrack = mTextTracks->IndexedGetter(index, dummy);
304 if (ttrack && ttrack->Mode() != TextTrackMode::Disabled &&
305 ttrack->ReadyState() == TextTrackReadyState::Loading) {
306 mPendingTextTracks->AddTextTrack(ttrack,
307 CompareTextTracks(mMediaElement));
308 }
309 }
310 }
311
AddListeners()312 void TextTrackManager::AddListeners() {
313 if (mMediaElement) {
314 mMediaElement->AddEventListener(u"resizecaption"_ns, this, false, false);
315 mMediaElement->AddEventListener(u"resizevideocontrols"_ns, this, false,
316 false);
317 mMediaElement->AddEventListener(u"seeked"_ns, this, false, false);
318 mMediaElement->AddEventListener(u"controlbarchange"_ns, this, false, true);
319 }
320 }
321
HonorUserPreferencesForTrackSelection()322 void TextTrackManager::HonorUserPreferencesForTrackSelection() {
323 if (performedTrackSelection || !mTextTracks) {
324 return;
325 }
326 WEBVTT_LOG("HonorUserPreferencesForTrackSelection");
327 TextTrackKind ttKinds[] = {TextTrackKind::Captions, TextTrackKind::Subtitles};
328
329 // Steps 1 - 3: Perform automatic track selection for different TextTrack
330 // Kinds.
331 PerformTrackSelection(ttKinds, ArrayLength(ttKinds));
332 PerformTrackSelection(TextTrackKind::Descriptions);
333 PerformTrackSelection(TextTrackKind::Chapters);
334
335 // Step 4: Set all TextTracks with a kind of metadata that are disabled
336 // to hidden.
337 for (uint32_t i = 0; i < mTextTracks->Length(); i++) {
338 TextTrack* track = (*mTextTracks)[i];
339 if (track->Kind() == TextTrackKind::Metadata && TrackIsDefault(track) &&
340 track->Mode() == TextTrackMode::Disabled) {
341 track->SetMode(TextTrackMode::Hidden);
342 }
343 }
344
345 performedTrackSelection = true;
346 }
347
TrackIsDefault(TextTrack * aTextTrack)348 bool TextTrackManager::TrackIsDefault(TextTrack* aTextTrack) {
349 HTMLTrackElement* trackElement = aTextTrack->GetTrackElement();
350 if (!trackElement) {
351 return false;
352 }
353 return trackElement->Default();
354 }
355
PerformTrackSelection(TextTrackKind aTextTrackKind)356 void TextTrackManager::PerformTrackSelection(TextTrackKind aTextTrackKind) {
357 TextTrackKind ttKinds[] = {aTextTrackKind};
358 PerformTrackSelection(ttKinds, ArrayLength(ttKinds));
359 }
360
PerformTrackSelection(TextTrackKind aTextTrackKinds[],uint32_t size)361 void TextTrackManager::PerformTrackSelection(TextTrackKind aTextTrackKinds[],
362 uint32_t size) {
363 nsTArray<TextTrack*> candidates;
364 GetTextTracksOfKinds(aTextTrackKinds, size, candidates);
365
366 // Step 3: If any TextTracks in candidates are showing then abort these steps.
367 for (uint32_t i = 0; i < candidates.Length(); i++) {
368 if (candidates[i]->Mode() == TextTrackMode::Showing) {
369 WEBVTT_LOGV("PerformTrackSelection Showing return kind %d",
370 static_cast<int>(candidates[i]->Kind()));
371 return;
372 }
373 }
374
375 // Step 4: Honor user preferences for track selection, otherwise, set the
376 // first TextTrack in candidates with a default attribute to showing.
377 // TODO: Bug 981691 - Honor user preferences for text track selection.
378 for (uint32_t i = 0; i < candidates.Length(); i++) {
379 if (TrackIsDefault(candidates[i]) &&
380 candidates[i]->Mode() == TextTrackMode::Disabled) {
381 candidates[i]->SetMode(TextTrackMode::Showing);
382 WEBVTT_LOGV("PerformTrackSelection set Showing kind %d",
383 static_cast<int>(candidates[i]->Kind()));
384 return;
385 }
386 }
387 }
388
GetTextTracksOfKinds(TextTrackKind aTextTrackKinds[],uint32_t size,nsTArray<TextTrack * > & aTextTracks)389 void TextTrackManager::GetTextTracksOfKinds(TextTrackKind aTextTrackKinds[],
390 uint32_t size,
391 nsTArray<TextTrack*>& aTextTracks) {
392 for (uint32_t i = 0; i < size; i++) {
393 GetTextTracksOfKind(aTextTrackKinds[i], aTextTracks);
394 }
395 }
396
GetTextTracksOfKind(TextTrackKind aTextTrackKind,nsTArray<TextTrack * > & aTextTracks)397 void TextTrackManager::GetTextTracksOfKind(TextTrackKind aTextTrackKind,
398 nsTArray<TextTrack*>& aTextTracks) {
399 if (!mTextTracks) {
400 return;
401 }
402 for (uint32_t i = 0; i < mTextTracks->Length(); i++) {
403 TextTrack* textTrack = (*mTextTracks)[i];
404 if (textTrack->Kind() == aTextTrackKind) {
405 aTextTracks.AppendElement(textTrack);
406 }
407 }
408 }
409
410 NS_IMETHODIMP
HandleEvent(Event * aEvent)411 TextTrackManager::HandleEvent(Event* aEvent) {
412 if (!mTextTracks) {
413 return NS_OK;
414 }
415
416 nsAutoString type;
417 aEvent->GetType(type);
418 WEBVTT_LOG("Handle event %s", NS_ConvertUTF16toUTF8(type).get());
419
420 const bool setDirty = type.EqualsLiteral("seeked") ||
421 type.EqualsLiteral("resizecaption") ||
422 type.EqualsLiteral("resizevideocontrols");
423 const bool updateDisplay = type.EqualsLiteral("controlbarchange") ||
424 type.EqualsLiteral("resizecaption");
425
426 if (setDirty) {
427 for (uint32_t i = 0; i < mTextTracks->Length(); i++) {
428 ((*mTextTracks)[i])->SetCuesDirty();
429 }
430 }
431 if (updateDisplay) {
432 UpdateCueDisplay();
433 }
434
435 return NS_OK;
436 }
437
438 class SimpleTextTrackEvent : public Runnable {
439 public:
440 friend class CompareSimpleTextTrackEvents;
SimpleTextTrackEvent(const nsAString & aEventName,double aTime,TextTrack * aTrack,TextTrackCue * aCue)441 SimpleTextTrackEvent(const nsAString& aEventName, double aTime,
442 TextTrack* aTrack, TextTrackCue* aCue)
443 : Runnable("dom::SimpleTextTrackEvent"),
444 mName(aEventName),
445 mTime(aTime),
446 mTrack(aTrack),
447 mCue(aCue) {}
448
Run()449 NS_IMETHOD Run() override {
450 WEBVTT_LOGV("SimpleTextTrackEvent cue %p mName %s mTime %lf", mCue.get(),
451 NS_ConvertUTF16toUTF8(mName).get(), mTime);
452 mCue->DispatchTrustedEvent(mName);
453 return NS_OK;
454 }
455
Dispatch()456 void Dispatch() {
457 if (nsCOMPtr<nsIGlobalObject> global = mCue->GetOwnerGlobal()) {
458 global->Dispatch(TaskCategory::Other, do_AddRef(this));
459 } else {
460 NS_DispatchToMainThread(do_AddRef(this));
461 }
462 }
463
464 private:
465 nsString mName;
466 double mTime;
467 TextTrack* mTrack;
468 RefPtr<TextTrackCue> mCue;
469 };
470
471 class CompareSimpleTextTrackEvents {
472 private:
TrackChildPosition(SimpleTextTrackEvent * aEvent) const473 Maybe<uint32_t> TrackChildPosition(SimpleTextTrackEvent* aEvent) const {
474 if (aEvent->mTrack) {
475 HTMLTrackElement* trackElement = aEvent->mTrack->GetTrackElement();
476 if (trackElement) {
477 return mMediaElement->ComputeIndexOf(trackElement);
478 }
479 }
480 return Nothing();
481 }
482 HTMLMediaElement* mMediaElement;
483
484 public:
CompareSimpleTextTrackEvents(HTMLMediaElement * aMediaElement)485 explicit CompareSimpleTextTrackEvents(HTMLMediaElement* aMediaElement) {
486 mMediaElement = aMediaElement;
487 }
488
Equals(SimpleTextTrackEvent * aOne,SimpleTextTrackEvent * aTwo) const489 bool Equals(SimpleTextTrackEvent* aOne, SimpleTextTrackEvent* aTwo) const {
490 return false;
491 }
492
LessThan(SimpleTextTrackEvent * aOne,SimpleTextTrackEvent * aTwo) const493 bool LessThan(SimpleTextTrackEvent* aOne, SimpleTextTrackEvent* aTwo) const {
494 // TimeMarchesOn step 13.1.
495 if (aOne->mTime < aTwo->mTime) {
496 return true;
497 }
498 if (aOne->mTime > aTwo->mTime) {
499 return false;
500 }
501
502 // TimeMarchesOn step 13.2 text track cue order.
503 // TextTrack position in TextTrackList
504 TextTrack* t1 = aOne->mTrack;
505 TextTrack* t2 = aTwo->mTrack;
506 MOZ_ASSERT(t1, "CompareSimpleTextTrackEvents t1 is null");
507 MOZ_ASSERT(t2, "CompareSimpleTextTrackEvents t2 is null");
508 if (t1 != t2) {
509 TextTrackList* tList = t1->GetTextTrackList();
510 MOZ_ASSERT(tList, "CompareSimpleTextTrackEvents tList is null");
511 nsTArray<RefPtr<TextTrack>>& textTracks = tList->GetTextTrackArray();
512 auto index1 = textTracks.IndexOf(t1);
513 auto index2 = textTracks.IndexOf(t2);
514 if (index1 < index2) {
515 return true;
516 }
517 if (index1 > index2) {
518 return false;
519 }
520 }
521
522 MOZ_ASSERT(t1 == t2, "CompareSimpleTextTrackEvents t1 != t2");
523 // c1 and c2 are both belongs to t1.
524 TextTrackCue* c1 = aOne->mCue;
525 TextTrackCue* c2 = aTwo->mCue;
526 if (c1 != c2) {
527 if (c1->StartTime() < c2->StartTime()) {
528 return true;
529 }
530 if (c1->StartTime() > c2->StartTime()) {
531 return false;
532 }
533 if (c1->EndTime() < c2->EndTime()) {
534 return true;
535 }
536 if (c1->EndTime() > c2->EndTime()) {
537 return false;
538 }
539
540 TextTrackCueList* cueList = t1->GetCues();
541 MOZ_ASSERT(cueList);
542 nsTArray<RefPtr<TextTrackCue>>& cues = cueList->GetCuesArray();
543 auto index1 = cues.IndexOf(c1);
544 auto index2 = cues.IndexOf(c2);
545 if (index1 < index2) {
546 return true;
547 }
548 if (index1 > index2) {
549 return false;
550 }
551 }
552
553 // TimeMarchesOn step 13.3.
554 if (aOne->mName.EqualsLiteral("enter") ||
555 aTwo->mName.EqualsLiteral("exit")) {
556 return true;
557 }
558 return false;
559 }
560 };
561
562 class TextTrackListInternal {
563 public:
AddTextTrack(TextTrack * aTextTrack,const CompareTextTracks & aCompareTT)564 void AddTextTrack(TextTrack* aTextTrack,
565 const CompareTextTracks& aCompareTT) {
566 if (!mTextTracks.Contains(aTextTrack)) {
567 mTextTracks.InsertElementSorted(aTextTrack, aCompareTT);
568 }
569 }
Length() const570 uint32_t Length() const { return mTextTracks.Length(); }
operator [](uint32_t aIndex)571 TextTrack* operator[](uint32_t aIndex) {
572 return mTextTracks.SafeElementAt(aIndex, nullptr);
573 }
574
575 private:
576 nsTArray<RefPtr<TextTrack>> mTextTracks;
577 };
578
DispatchUpdateCueDisplay()579 void TextTrackManager::DispatchUpdateCueDisplay() {
580 if (!mUpdateCueDisplayDispatched && !IsShutdown()) {
581 WEBVTT_LOG("DispatchUpdateCueDisplay");
582 nsPIDOMWindowInner* win = mMediaElement->OwnerDoc()->GetInnerWindow();
583 if (win) {
584 nsGlobalWindowInner::Cast(win)->Dispatch(
585 TaskCategory::Other,
586 NewRunnableMethod("dom::TextTrackManager::UpdateCueDisplay", this,
587 &TextTrackManager::UpdateCueDisplay));
588 mUpdateCueDisplayDispatched = true;
589 }
590 }
591 }
592
DispatchTimeMarchesOn()593 void TextTrackManager::DispatchTimeMarchesOn() {
594 // Run the algorithm if no previous instance is still running, otherwise
595 // enqueue the current playback position and whether only that changed
596 // through its usual monotonic increase during normal playback; current
597 // executing call upon completion will check queue for further 'work'.
598 if (!mTimeMarchesOnDispatched && !IsShutdown()) {
599 WEBVTT_LOG("DispatchTimeMarchesOn");
600 nsPIDOMWindowInner* win = mMediaElement->OwnerDoc()->GetInnerWindow();
601 if (win) {
602 nsGlobalWindowInner::Cast(win)->Dispatch(
603 TaskCategory::Other,
604 NewRunnableMethod("dom::TextTrackManager::TimeMarchesOn", this,
605 &TextTrackManager::TimeMarchesOn));
606 mTimeMarchesOnDispatched = true;
607 }
608 }
609 }
610
611 // https://html.spec.whatwg.org/multipage/embedded-content.html#time-marches-on
TimeMarchesOn()612 void TextTrackManager::TimeMarchesOn() {
613 NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
614 mTimeMarchesOnDispatched = false;
615
616 CycleCollectedJSContext* context = CycleCollectedJSContext::Get();
617 if (context && context->IsInStableOrMetaStableState()) {
618 // FireTimeUpdate can be called while at stable state following a
619 // current position change which triggered a state watcher in MediaDecoder
620 // (see bug 1443429).
621 // TimeMarchesOn() will modify JS attributes which is forbidden while in
622 // stable state. So we dispatch a task to perform such operation later
623 // instead.
624 DispatchTimeMarchesOn();
625 return;
626 }
627 WEBVTT_LOG("TimeMarchesOn");
628
629 // Early return if we don't have any TextTracks or shutting down.
630 if (!mTextTracks || mTextTracks->Length() == 0 || IsShutdown() ||
631 !mMediaElement) {
632 return;
633 }
634
635 if (mMediaElement->ReadyState() == HTMLMediaElement_Binding::HAVE_NOTHING) {
636 WEBVTT_LOG(
637 "TimeMarchesOn return because media doesn't contain any data yet");
638 return;
639 }
640
641 if (mMediaElement->Seeking()) {
642 WEBVTT_LOG("TimeMarchesOn return during seeking");
643 return;
644 }
645
646 // Step 1, 2.
647 nsISupports* parentObject = mMediaElement->OwnerDoc()->GetParentObject();
648 if (NS_WARN_IF(!parentObject)) {
649 return;
650 }
651 nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(parentObject);
652 RefPtr<TextTrackCueList> currentCues = new TextTrackCueList(window);
653 RefPtr<TextTrackCueList> otherCues = new TextTrackCueList(window);
654
655 // Step 3.
656 auto currentPlaybackTime =
657 media::TimeUnit::FromSeconds(mMediaElement->CurrentTime());
658 bool hasNormalPlayback = !mHasSeeked;
659 mHasSeeked = false;
660 WEBVTT_LOG(
661 "TimeMarchesOn mLastTimeMarchesOnCalled %lf currentPlaybackTime %lf "
662 "hasNormalPlayback %d",
663 mLastTimeMarchesOnCalled.ToSeconds(), currentPlaybackTime.ToSeconds(),
664 hasNormalPlayback);
665
666 // The reason we collect other cues is (1) to change active cues to inactive,
667 // (2) find missing cues, so we actually no need to process all cues. We just
668 // need to handle cues which are in the time interval [lastTime:currentTime]
669 // or [currentTime:lastTime] (seeking forward). That can help us to reduce the
670 // size of other cues, which can improve execution time.
671 auto start = std::min(mLastTimeMarchesOnCalled, currentPlaybackTime);
672 auto end = std::max(mLastTimeMarchesOnCalled, currentPlaybackTime);
673 media::TimeInterval interval(start, end);
674 WEBVTT_LOGV("TimeMarchesOn Time interval [%f:%f]", start.ToSeconds(),
675 end.ToSeconds());
676 for (uint32_t idx = 0; idx < mTextTracks->Length(); ++idx) {
677 TextTrack* track = (*mTextTracks)[idx];
678 if (track) {
679 track->GetCurrentCuesAndOtherCues(currentCues, otherCues, interval);
680 }
681 }
682
683 // Step 4.
684 RefPtr<TextTrackCueList> missedCues = new TextTrackCueList(window);
685 if (hasNormalPlayback) {
686 for (uint32_t i = 0; i < otherCues->Length(); ++i) {
687 TextTrackCue* cue = (*otherCues)[i];
688 if (cue->StartTime() >= mLastTimeMarchesOnCalled.ToSeconds() &&
689 cue->EndTime() <= currentPlaybackTime.ToSeconds()) {
690 missedCues->AddCue(*cue);
691 }
692 }
693 }
694
695 WEBVTT_LOGV("TimeMarchesOn currentCues %d", currentCues->Length());
696 WEBVTT_LOGV("TimeMarchesOn otherCues %d", otherCues->Length());
697 WEBVTT_LOGV("TimeMarchesOn missedCues %d", missedCues->Length());
698 // Step 5. Empty now.
699 // TODO: Step 6: fire timeupdate?
700
701 // Step 7. Abort steps if condition 1, 2, 3 are satisfied.
702 // 1. All of the cues in current cues have their active flag set.
703 // 2. None of the cues in other cues have their active flag set.
704 // 3. Missed cues is empty.
705 bool c1 = true;
706 for (uint32_t i = 0; i < currentCues->Length(); ++i) {
707 if (!(*currentCues)[i]->GetActive()) {
708 c1 = false;
709 break;
710 }
711 }
712 bool c2 = true;
713 for (uint32_t i = 0; i < otherCues->Length(); ++i) {
714 if ((*otherCues)[i]->GetActive()) {
715 c2 = false;
716 break;
717 }
718 }
719 bool c3 = (missedCues->Length() == 0);
720 if (c1 && c2 && c3) {
721 mLastTimeMarchesOnCalled = currentPlaybackTime;
722 WEBVTT_LOG("TimeMarchesOn step 7 return, mLastTimeMarchesOnCalled %lf",
723 mLastTimeMarchesOnCalled.ToSeconds());
724 return;
725 }
726
727 // Step 8. Respect PauseOnExit flag if not seek.
728 if (hasNormalPlayback) {
729 for (uint32_t i = 0; i < otherCues->Length(); ++i) {
730 TextTrackCue* cue = (*otherCues)[i];
731 if (cue && cue->PauseOnExit() && cue->GetActive()) {
732 WEBVTT_LOG("TimeMarchesOn pause the MediaElement");
733 mMediaElement->Pause();
734 break;
735 }
736 }
737 for (uint32_t i = 0; i < missedCues->Length(); ++i) {
738 TextTrackCue* cue = (*missedCues)[i];
739 if (cue && cue->PauseOnExit()) {
740 WEBVTT_LOG("TimeMarchesOn pause the MediaElement");
741 mMediaElement->Pause();
742 break;
743 }
744 }
745 }
746
747 // Step 15.
748 // Sort text tracks in the same order as the text tracks appear
749 // in the media element's list of text tracks, and remove
750 // duplicates.
751 TextTrackListInternal affectedTracks;
752 // Step 13, 14.
753 nsTArray<RefPtr<SimpleTextTrackEvent>> eventList;
754 // Step 9, 10.
755 // For each text track cue in missed cues, prepare an event named
756 // enter for the TextTrackCue object with the cue start time.
757 for (uint32_t i = 0; i < missedCues->Length(); ++i) {
758 TextTrackCue* cue = (*missedCues)[i];
759 if (cue) {
760 WEBVTT_LOG("Prepare 'enter' event for cue %p [%f, %f] in missing cues",
761 cue, cue->StartTime(), cue->EndTime());
762 SimpleTextTrackEvent* event = new SimpleTextTrackEvent(
763 u"enter"_ns, cue->StartTime(), cue->GetTrack(), cue);
764 eventList.InsertElementSorted(
765 event, CompareSimpleTextTrackEvents(mMediaElement));
766 affectedTracks.AddTextTrack(cue->GetTrack(),
767 CompareTextTracks(mMediaElement));
768 }
769 }
770
771 // Step 11, 17.
772 for (uint32_t i = 0; i < otherCues->Length(); ++i) {
773 TextTrackCue* cue = (*otherCues)[i];
774 if (cue->GetActive() || missedCues->IsCueExist(cue)) {
775 double time =
776 cue->StartTime() > cue->EndTime() ? cue->StartTime() : cue->EndTime();
777 WEBVTT_LOG("Prepare 'exit' event for cue %p [%f, %f] in other cues", cue,
778 cue->StartTime(), cue->EndTime());
779 SimpleTextTrackEvent* event =
780 new SimpleTextTrackEvent(u"exit"_ns, time, cue->GetTrack(), cue);
781 eventList.InsertElementSorted(
782 event, CompareSimpleTextTrackEvents(mMediaElement));
783 affectedTracks.AddTextTrack(cue->GetTrack(),
784 CompareTextTracks(mMediaElement));
785 }
786 cue->SetActive(false);
787 }
788
789 // Step 12, 17.
790 for (uint32_t i = 0; i < currentCues->Length(); ++i) {
791 TextTrackCue* cue = (*currentCues)[i];
792 if (!cue->GetActive()) {
793 WEBVTT_LOG("Prepare 'enter' event for cue %p [%f, %f] in current cues",
794 cue, cue->StartTime(), cue->EndTime());
795 SimpleTextTrackEvent* event = new SimpleTextTrackEvent(
796 u"enter"_ns, cue->StartTime(), cue->GetTrack(), cue);
797 eventList.InsertElementSorted(
798 event, CompareSimpleTextTrackEvents(mMediaElement));
799 affectedTracks.AddTextTrack(cue->GetTrack(),
800 CompareTextTracks(mMediaElement));
801 }
802 cue->SetActive(true);
803 }
804
805 // Fire the eventList
806 for (uint32_t i = 0; i < eventList.Length(); ++i) {
807 eventList[i]->Dispatch();
808 }
809
810 // Step 16.
811 for (uint32_t i = 0; i < affectedTracks.Length(); ++i) {
812 TextTrack* ttrack = affectedTracks[i];
813 if (ttrack) {
814 ttrack->DispatchAsyncTrustedEvent(u"cuechange"_ns);
815 HTMLTrackElement* trackElement = ttrack->GetTrackElement();
816 if (trackElement) {
817 trackElement->DispatchTrackRunnable(u"cuechange"_ns);
818 }
819 }
820 }
821
822 mLastTimeMarchesOnCalled = currentPlaybackTime;
823
824 // Step 18.
825 UpdateCueDisplay();
826 }
827
NotifyCueUpdated(TextTrackCue * aCue)828 void TextTrackManager::NotifyCueUpdated(TextTrackCue* aCue) {
829 // TODO: Add/Reorder the cue to mNewCues if we have some optimization?
830 WEBVTT_LOG("NotifyCueUpdated, cue=%p", aCue);
831 MaybeRunTimeMarchesOn();
832 // For the case "Texttrack.mode = hidden/showing", if the mode
833 // changing between showing and hidden, TimeMarchesOn
834 // doesn't render the cue. Call DispatchUpdateCueDisplay() explicitly.
835 DispatchUpdateCueDisplay();
836 }
837
NotifyReset()838 void TextTrackManager::NotifyReset() {
839 // https://html.spec.whatwg.org/multipage/media.html#text-track-cue-active-flag
840 // This will unset all cues' active flag and update the cue display.
841 WEBVTT_LOG("NotifyReset");
842 mLastTimeMarchesOnCalled = media::TimeUnit::Zero();
843 for (uint32_t idx = 0; idx < mTextTracks->Length(); ++idx) {
844 (*mTextTracks)[idx]->SetCuesInactive();
845 }
846 UpdateCueDisplay();
847 }
848
IsLoaded()849 bool TextTrackManager::IsLoaded() {
850 return mTextTracks ? mTextTracks->AreTextTracksLoaded() : true;
851 }
852
IsShutdown() const853 bool TextTrackManager::IsShutdown() const {
854 return (mShutdown || !sParserWrapper);
855 }
856
MaybeRunTimeMarchesOn()857 void TextTrackManager::MaybeRunTimeMarchesOn() {
858 MOZ_ASSERT(mMediaElement);
859 // According to spec, we should check media element's show poster flag before
860 // running `TimeMarchesOn` in following situations, (1) add cue (2) remove cue
861 // (3) cue's start time changes (4) cues's end time changes
862 // https://html.spec.whatwg.org/multipage/media.html#playing-the-media-resource:time-marches-on
863 // https://html.spec.whatwg.org/multipage/media.html#text-track-api:time-marches-on
864 if (mMediaElement->GetShowPosterFlag()) {
865 return;
866 }
867 TimeMarchesOn();
868 }
869
870 } // namespace mozilla::dom
871