1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
5 * You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "mozilla/dom/TextTrackManager.h"
8 #include "mozilla/ClearOnShutdown.h"
9 #include "mozilla/CycleCollectedJSContext.h"
10 #include "mozilla/Telemetry.h"
11 #include "mozilla/dom/Document.h"
12 #include "mozilla/dom/Event.h"
13 #include "mozilla/dom/HTMLMediaElement.h"
14 #include "mozilla/dom/HTMLTrackElement.h"
15 #include "mozilla/dom/HTMLVideoElement.h"
16 #include "mozilla/dom/TextTrack.h"
17 #include "mozilla/dom/TextTrackCue.h"
18 #include "nsComponentManagerUtils.h"
19 #include "nsGlobalWindow.h"
20 #include "nsIFrame.h"
21 #include "nsIWebVTTParserWrapper.h"
22 #include "nsVariant.h"
23 #include "nsVideoFrame.h"
24
25 mozilla::LazyLogModule gTextTrackLog("WebVTT");
26
27 #define WEBVTT_LOG(msg, ...) \
28 MOZ_LOG(gTextTrackLog, LogLevel::Debug, \
29 ("TextTrackManager=%p, " msg, this, ##__VA_ARGS__))
30 #define WEBVTT_LOGV(msg, ...) \
31 MOZ_LOG(gTextTrackLog, LogLevel::Verbose, \
32 ("TextTrackManager=%p, " msg, this, ##__VA_ARGS__))
33
34 namespace mozilla::dom {
35
36 NS_IMPL_ISUPPORTS(TextTrackManager::ShutdownObserverProxy, nsIObserver);
37
Unregister()38 void TextTrackManager::ShutdownObserverProxy::Unregister() {
39 nsContentUtils::UnregisterShutdownObserver(this);
40 mManager = nullptr;
41 }
42
CompareTextTracks(HTMLMediaElement * aMediaElement)43 CompareTextTracks::CompareTextTracks(HTMLMediaElement* aMediaElement) {
44 mMediaElement = aMediaElement;
45 }
46
TrackChildPosition(TextTrack * aTextTrack) const47 int32_t CompareTextTracks::TrackChildPosition(TextTrack* aTextTrack) const {
48 MOZ_DIAGNOSTIC_ASSERT(aTextTrack);
49 HTMLTrackElement* trackElement = aTextTrack->GetTrackElement();
50 if (!trackElement) {
51 return -1;
52 }
53 return mMediaElement->ComputeIndexOf(trackElement);
54 }
55
Equals(TextTrack * aOne,TextTrack * aTwo) const56 bool CompareTextTracks::Equals(TextTrack* aOne, TextTrack* aTwo) const {
57 // Two tracks can never be equal. If they have corresponding TrackElements
58 // they would need to occupy the same tree position (impossible) and in the
59 // case of tracks coming from AddTextTrack source we put the newest at the
60 // last position, so they won't be equal as well.
61 return false;
62 }
63
LessThan(TextTrack * aOne,TextTrack * aTwo) const64 bool CompareTextTracks::LessThan(TextTrack* aOne, TextTrack* aTwo) const {
65 // Protect against nullptr TextTrack objects; treat them as
66 // sorting toward the end.
67 if (!aOne) {
68 return false;
69 }
70 if (!aTwo) {
71 return true;
72 }
73 TextTrackSource sourceOne = aOne->GetTextTrackSource();
74 TextTrackSource sourceTwo = aTwo->GetTextTrackSource();
75 if (sourceOne != sourceTwo) {
76 return sourceOne == TextTrackSource::Track ||
77 (sourceOne == TextTrackSource::AddTextTrack &&
78 sourceTwo == TextTrackSource::MediaResourceSpecific);
79 }
80 switch (sourceOne) {
81 case TextTrackSource::Track: {
82 int32_t positionOne = TrackChildPosition(aOne);
83 int32_t positionTwo = TrackChildPosition(aTwo);
84 // If either position one or positiontwo are -1 then something has gone
85 // wrong. In this case we should just put them at the back of the list.
86 return positionOne != -1 && positionTwo != -1 &&
87 positionOne < positionTwo;
88 }
89 case TextTrackSource::AddTextTrack:
90 // For AddTextTrack sources the tracks will already be in the correct
91 // relative order in the source array. Assume we're called in iteration
92 // order and can therefore always report aOne < aTwo to maintain the
93 // original temporal ordering.
94 return true;
95 case TextTrackSource::MediaResourceSpecific:
96 // No rules for Media Resource Specific tracks yet.
97 break;
98 }
99 return true;
100 }
101
102 NS_IMPL_CYCLE_COLLECTION(TextTrackManager, mMediaElement, mTextTracks,
103 mPendingTextTracks, mNewCues)
104
105 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(TextTrackManager)
106 NS_INTERFACE_MAP_ENTRY(nsIDOMEventListener)
107 NS_INTERFACE_MAP_END
108
109 NS_IMPL_CYCLE_COLLECTING_ADDREF(TextTrackManager)
110 NS_IMPL_CYCLE_COLLECTING_RELEASE(TextTrackManager)
111
112 StaticRefPtr<nsIWebVTTParserWrapper> TextTrackManager::sParserWrapper;
113
TextTrackManager(HTMLMediaElement * aMediaElement)114 TextTrackManager::TextTrackManager(HTMLMediaElement* aMediaElement)
115 : mMediaElement(aMediaElement),
116 mHasSeeked(false),
117 mLastTimeMarchesOnCalled(media::TimeUnit::Zero()),
118 mTimeMarchesOnDispatched(false),
119 mUpdateCueDisplayDispatched(false),
120 performedTrackSelection(false),
121 mShutdown(false) {
122 nsISupports* parentObject = mMediaElement->OwnerDoc()->GetParentObject();
123
124 NS_ENSURE_TRUE_VOID(parentObject);
125 WEBVTT_LOG("Create TextTrackManager");
126 nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(parentObject);
127 mNewCues = new TextTrackCueList(window);
128 mTextTracks = new TextTrackList(window, this);
129 mPendingTextTracks = new TextTrackList(window, this);
130
131 if (!sParserWrapper) {
132 nsCOMPtr<nsIWebVTTParserWrapper> parserWrapper =
133 do_CreateInstance(NS_WEBVTTPARSERWRAPPER_CONTRACTID);
134 MOZ_ASSERT(parserWrapper, "Can't create nsIWebVTTParserWrapper");
135 sParserWrapper = parserWrapper;
136 ClearOnShutdown(&sParserWrapper);
137 }
138 mShutdownProxy = new ShutdownObserverProxy(this);
139 }
140
~TextTrackManager()141 TextTrackManager::~TextTrackManager() {
142 WEBVTT_LOG("~TextTrackManager");
143 mShutdownProxy->Unregister();
144 }
145
GetTextTracks() const146 TextTrackList* TextTrackManager::GetTextTracks() const { return mTextTracks; }
147
AddTextTrack(TextTrackKind aKind,const nsAString & aLabel,const nsAString & aLanguage,TextTrackMode aMode,TextTrackReadyState aReadyState,TextTrackSource aTextTrackSource)148 already_AddRefed<TextTrack> TextTrackManager::AddTextTrack(
149 TextTrackKind aKind, const nsAString& aLabel, const nsAString& aLanguage,
150 TextTrackMode aMode, TextTrackReadyState aReadyState,
151 TextTrackSource aTextTrackSource) {
152 if (!mMediaElement || !mTextTracks) {
153 return nullptr;
154 }
155 RefPtr<TextTrack> track = mTextTracks->AddTextTrack(
156 aKind, aLabel, aLanguage, aMode, aReadyState, aTextTrackSource,
157 CompareTextTracks(mMediaElement));
158 WEBVTT_LOG("AddTextTrack %p kind %" PRIu32 " Label %s Language %s",
159 track.get(), static_cast<uint32_t>(aKind),
160 NS_ConvertUTF16toUTF8(aLabel).get(),
161 NS_ConvertUTF16toUTF8(aLanguage).get());
162 AddCues(track);
163 ReportTelemetryForTrack(track);
164
165 if (aTextTrackSource == TextTrackSource::Track) {
166 RefPtr<nsIRunnable> task = NewRunnableMethod(
167 "dom::TextTrackManager::HonorUserPreferencesForTrackSelection", this,
168 &TextTrackManager::HonorUserPreferencesForTrackSelection);
169 NS_DispatchToMainThread(task.forget());
170 }
171
172 return track.forget();
173 }
174
AddTextTrack(TextTrack * aTextTrack)175 void TextTrackManager::AddTextTrack(TextTrack* aTextTrack) {
176 if (!mMediaElement || !mTextTracks) {
177 return;
178 }
179 WEBVTT_LOG("AddTextTrack TextTrack %p", aTextTrack);
180 mTextTracks->AddTextTrack(aTextTrack, CompareTextTracks(mMediaElement));
181 AddCues(aTextTrack);
182 ReportTelemetryForTrack(aTextTrack);
183
184 if (aTextTrack->GetTextTrackSource() == TextTrackSource::Track) {
185 RefPtr<nsIRunnable> task = NewRunnableMethod(
186 "dom::TextTrackManager::HonorUserPreferencesForTrackSelection", this,
187 &TextTrackManager::HonorUserPreferencesForTrackSelection);
188 NS_DispatchToMainThread(task.forget());
189 }
190 }
191
AddCues(TextTrack * aTextTrack)192 void TextTrackManager::AddCues(TextTrack* aTextTrack) {
193 if (!mNewCues) {
194 WEBVTT_LOG("AddCues mNewCues is null");
195 return;
196 }
197
198 TextTrackCueList* cueList = aTextTrack->GetCues();
199 if (cueList) {
200 bool dummy;
201 WEBVTT_LOGV("AddCues, CuesNum=%d", cueList->Length());
202 for (uint32_t i = 0; i < cueList->Length(); ++i) {
203 mNewCues->AddCue(*cueList->IndexedGetter(i, dummy));
204 }
205 MaybeRunTimeMarchesOn();
206 }
207 }
208
RemoveTextTrack(TextTrack * aTextTrack,bool aPendingListOnly)209 void TextTrackManager::RemoveTextTrack(TextTrack* aTextTrack,
210 bool aPendingListOnly) {
211 if (!mPendingTextTracks || !mTextTracks) {
212 return;
213 }
214
215 WEBVTT_LOG("RemoveTextTrack TextTrack %p", aTextTrack);
216 mPendingTextTracks->RemoveTextTrack(aTextTrack);
217 if (aPendingListOnly) {
218 return;
219 }
220
221 mTextTracks->RemoveTextTrack(aTextTrack);
222 // Remove the cues in mNewCues belong to aTextTrack.
223 TextTrackCueList* removeCueList = aTextTrack->GetCues();
224 if (removeCueList) {
225 WEBVTT_LOGV("RemoveTextTrack removeCuesNum=%d", removeCueList->Length());
226 for (uint32_t i = 0; i < removeCueList->Length(); ++i) {
227 mNewCues->RemoveCue(*((*removeCueList)[i]));
228 }
229 MaybeRunTimeMarchesOn();
230 }
231 }
232
DidSeek()233 void TextTrackManager::DidSeek() {
234 WEBVTT_LOG("DidSeek");
235 mHasSeeked = true;
236 }
237
UpdateCueDisplay()238 void TextTrackManager::UpdateCueDisplay() {
239 WEBVTT_LOG("UpdateCueDisplay");
240 mUpdateCueDisplayDispatched = false;
241
242 if (!mMediaElement || !mTextTracks || IsShutdown()) {
243 WEBVTT_LOG("Abort UpdateCueDisplay.");
244 return;
245 }
246
247 nsIFrame* frame = mMediaElement->GetPrimaryFrame();
248 nsVideoFrame* videoFrame = do_QueryFrame(frame);
249 if (!videoFrame) {
250 WEBVTT_LOG("Abort UpdateCueDisplay, because of no video frame.");
251 return;
252 }
253
254 nsCOMPtr<nsIContent> overlay = videoFrame->GetCaptionOverlay();
255 if (!overlay) {
256 WEBVTT_LOG("Abort UpdateCueDisplay, because of no overlay.");
257 return;
258 }
259
260 nsPIDOMWindowInner* window = mMediaElement->OwnerDoc()->GetInnerWindow();
261 if (!window) {
262 WEBVTT_LOG("Abort UpdateCueDisplay, because of no window.");
263 }
264
265 nsTArray<RefPtr<TextTrackCue>> showingCues;
266 mTextTracks->GetShowingCues(showingCues);
267
268 WEBVTT_LOG("UpdateCueDisplay, processCues, showingCuesNum=%zu",
269 showingCues.Length());
270 RefPtr<nsVariantCC> jsCues = new nsVariantCC();
271 jsCues->SetAsArray(nsIDataType::VTYPE_INTERFACE, &NS_GET_IID(EventTarget),
272 showingCues.Length(),
273 static_cast<void*>(showingCues.Elements()));
274 nsCOMPtr<nsIContent> controls = videoFrame->GetVideoControls();
275 sParserWrapper->ProcessCues(window, jsCues, overlay, controls);
276 }
277
NotifyCueAdded(TextTrackCue & aCue)278 void TextTrackManager::NotifyCueAdded(TextTrackCue& aCue) {
279 WEBVTT_LOG("NotifyCueAdded, cue=%p", &aCue);
280 if (mNewCues) {
281 mNewCues->AddCue(aCue);
282 }
283 MaybeRunTimeMarchesOn();
284 }
285
NotifyCueRemoved(TextTrackCue & aCue)286 void TextTrackManager::NotifyCueRemoved(TextTrackCue& aCue) {
287 WEBVTT_LOG("NotifyCueRemoved, cue=%p", &aCue);
288 if (mNewCues) {
289 mNewCues->RemoveCue(aCue);
290 }
291 MaybeRunTimeMarchesOn();
292 DispatchUpdateCueDisplay();
293 }
294
PopulatePendingList()295 void TextTrackManager::PopulatePendingList() {
296 if (!mTextTracks || !mPendingTextTracks || !mMediaElement) {
297 return;
298 }
299 uint32_t len = mTextTracks->Length();
300 bool dummy;
301 for (uint32_t index = 0; index < len; ++index) {
302 TextTrack* ttrack = mTextTracks->IndexedGetter(index, dummy);
303 if (ttrack && ttrack->Mode() != TextTrackMode::Disabled &&
304 ttrack->ReadyState() == TextTrackReadyState::Loading) {
305 mPendingTextTracks->AddTextTrack(ttrack,
306 CompareTextTracks(mMediaElement));
307 }
308 }
309 }
310
AddListeners()311 void TextTrackManager::AddListeners() {
312 if (mMediaElement) {
313 mMediaElement->AddEventListener(u"resizecaption"_ns, this, false, false);
314 mMediaElement->AddEventListener(u"resizevideocontrols"_ns, this, false,
315 false);
316 mMediaElement->AddEventListener(u"seeked"_ns, this, false, false);
317 mMediaElement->AddEventListener(u"controlbarchange"_ns, this, false, true);
318 }
319 }
320
HonorUserPreferencesForTrackSelection()321 void TextTrackManager::HonorUserPreferencesForTrackSelection() {
322 if (performedTrackSelection || !mTextTracks) {
323 return;
324 }
325 WEBVTT_LOG("HonorUserPreferencesForTrackSelection");
326 TextTrackKind ttKinds[] = {TextTrackKind::Captions, TextTrackKind::Subtitles};
327
328 // Steps 1 - 3: Perform automatic track selection for different TextTrack
329 // Kinds.
330 PerformTrackSelection(ttKinds, ArrayLength(ttKinds));
331 PerformTrackSelection(TextTrackKind::Descriptions);
332 PerformTrackSelection(TextTrackKind::Chapters);
333
334 // Step 4: Set all TextTracks with a kind of metadata that are disabled
335 // to hidden.
336 for (uint32_t i = 0; i < mTextTracks->Length(); i++) {
337 TextTrack* track = (*mTextTracks)[i];
338 if (track->Kind() == TextTrackKind::Metadata && TrackIsDefault(track) &&
339 track->Mode() == TextTrackMode::Disabled) {
340 track->SetMode(TextTrackMode::Hidden);
341 }
342 }
343
344 performedTrackSelection = true;
345 }
346
TrackIsDefault(TextTrack * aTextTrack)347 bool TextTrackManager::TrackIsDefault(TextTrack* aTextTrack) {
348 HTMLTrackElement* trackElement = aTextTrack->GetTrackElement();
349 if (!trackElement) {
350 return false;
351 }
352 return trackElement->Default();
353 }
354
PerformTrackSelection(TextTrackKind aTextTrackKind)355 void TextTrackManager::PerformTrackSelection(TextTrackKind aTextTrackKind) {
356 TextTrackKind ttKinds[] = {aTextTrackKind};
357 PerformTrackSelection(ttKinds, ArrayLength(ttKinds));
358 }
359
PerformTrackSelection(TextTrackKind aTextTrackKinds[],uint32_t size)360 void TextTrackManager::PerformTrackSelection(TextTrackKind aTextTrackKinds[],
361 uint32_t size) {
362 nsTArray<TextTrack*> candidates;
363 GetTextTracksOfKinds(aTextTrackKinds, size, candidates);
364
365 // Step 3: If any TextTracks in candidates are showing then abort these steps.
366 for (uint32_t i = 0; i < candidates.Length(); i++) {
367 if (candidates[i]->Mode() == TextTrackMode::Showing) {
368 WEBVTT_LOGV("PerformTrackSelection Showing return kind %d",
369 static_cast<int>(candidates[i]->Kind()));
370 return;
371 }
372 }
373
374 // Step 4: Honor user preferences for track selection, otherwise, set the
375 // first TextTrack in candidates with a default attribute to showing.
376 // TODO: Bug 981691 - Honor user preferences for text track selection.
377 for (uint32_t i = 0; i < candidates.Length(); i++) {
378 if (TrackIsDefault(candidates[i]) &&
379 candidates[i]->Mode() == TextTrackMode::Disabled) {
380 candidates[i]->SetMode(TextTrackMode::Showing);
381 WEBVTT_LOGV("PerformTrackSelection set Showing kind %d",
382 static_cast<int>(candidates[i]->Kind()));
383 return;
384 }
385 }
386 }
387
GetTextTracksOfKinds(TextTrackKind aTextTrackKinds[],uint32_t size,nsTArray<TextTrack * > & aTextTracks)388 void TextTrackManager::GetTextTracksOfKinds(TextTrackKind aTextTrackKinds[],
389 uint32_t size,
390 nsTArray<TextTrack*>& aTextTracks) {
391 for (uint32_t i = 0; i < size; i++) {
392 GetTextTracksOfKind(aTextTrackKinds[i], aTextTracks);
393 }
394 }
395
GetTextTracksOfKind(TextTrackKind aTextTrackKind,nsTArray<TextTrack * > & aTextTracks)396 void TextTrackManager::GetTextTracksOfKind(TextTrackKind aTextTrackKind,
397 nsTArray<TextTrack*>& aTextTracks) {
398 if (!mTextTracks) {
399 return;
400 }
401 for (uint32_t i = 0; i < mTextTracks->Length(); i++) {
402 TextTrack* textTrack = (*mTextTracks)[i];
403 if (textTrack->Kind() == aTextTrackKind) {
404 aTextTracks.AppendElement(textTrack);
405 }
406 }
407 }
408
409 NS_IMETHODIMP
HandleEvent(Event * aEvent)410 TextTrackManager::HandleEvent(Event* aEvent) {
411 if (!mTextTracks) {
412 return NS_OK;
413 }
414
415 nsAutoString type;
416 aEvent->GetType(type);
417 WEBVTT_LOG("Handle event %s", NS_ConvertUTF16toUTF8(type).get());
418
419 const bool setDirty = type.EqualsLiteral("seeked") ||
420 type.EqualsLiteral("resizecaption") ||
421 type.EqualsLiteral("resizevideocontrols");
422 const bool updateDisplay = type.EqualsLiteral("controlbarchange") ||
423 type.EqualsLiteral("resizecaption");
424
425 if (setDirty) {
426 for (uint32_t i = 0; i < mTextTracks->Length(); i++) {
427 ((*mTextTracks)[i])->SetCuesDirty();
428 }
429 }
430 if (updateDisplay) {
431 UpdateCueDisplay();
432 }
433
434 return NS_OK;
435 }
436
437 class SimpleTextTrackEvent : public Runnable {
438 public:
439 friend class CompareSimpleTextTrackEvents;
SimpleTextTrackEvent(const nsAString & aEventName,double aTime,TextTrack * aTrack,TextTrackCue * aCue)440 SimpleTextTrackEvent(const nsAString& aEventName, double aTime,
441 TextTrack* aTrack, TextTrackCue* aCue)
442 : Runnable("dom::SimpleTextTrackEvent"),
443 mName(aEventName),
444 mTime(aTime),
445 mTrack(aTrack),
446 mCue(aCue) {}
447
Run()448 NS_IMETHOD Run() override {
449 WEBVTT_LOGV("SimpleTextTrackEvent cue %p mName %s mTime %lf", mCue.get(),
450 NS_ConvertUTF16toUTF8(mName).get(), mTime);
451 mCue->DispatchTrustedEvent(mName);
452 return NS_OK;
453 }
454
Dispatch()455 void Dispatch() {
456 if (nsCOMPtr<nsIGlobalObject> global = mCue->GetOwnerGlobal()) {
457 global->Dispatch(TaskCategory::Other, do_AddRef(this));
458 } else {
459 NS_DispatchToMainThread(do_AddRef(this));
460 }
461 }
462
463 private:
464 nsString mName;
465 double mTime;
466 TextTrack* mTrack;
467 RefPtr<TextTrackCue> mCue;
468 };
469
470 class CompareSimpleTextTrackEvents {
471 private:
TrackChildPosition(SimpleTextTrackEvent * aEvent) const472 int32_t TrackChildPosition(SimpleTextTrackEvent* aEvent) const {
473 if (aEvent->mTrack) {
474 HTMLTrackElement* trackElement = aEvent->mTrack->GetTrackElement();
475 if (trackElement) {
476 return mMediaElement->ComputeIndexOf(trackElement);
477 }
478 }
479 return -1;
480 }
481 HTMLMediaElement* mMediaElement;
482
483 public:
CompareSimpleTextTrackEvents(HTMLMediaElement * aMediaElement)484 explicit CompareSimpleTextTrackEvents(HTMLMediaElement* aMediaElement) {
485 mMediaElement = aMediaElement;
486 }
487
Equals(SimpleTextTrackEvent * aOne,SimpleTextTrackEvent * aTwo) const488 bool Equals(SimpleTextTrackEvent* aOne, SimpleTextTrackEvent* aTwo) const {
489 return false;
490 }
491
LessThan(SimpleTextTrackEvent * aOne,SimpleTextTrackEvent * aTwo) const492 bool LessThan(SimpleTextTrackEvent* aOne, SimpleTextTrackEvent* aTwo) const {
493 // TimeMarchesOn step 13.1.
494 if (aOne->mTime < aTwo->mTime) {
495 return true;
496 }
497 if (aOne->mTime > aTwo->mTime) {
498 return false;
499 }
500
501 // TimeMarchesOn step 13.2 text track cue order.
502 // TextTrack position in TextTrackList
503 TextTrack* t1 = aOne->mTrack;
504 TextTrack* t2 = aTwo->mTrack;
505 MOZ_ASSERT(t1, "CompareSimpleTextTrackEvents t1 is null");
506 MOZ_ASSERT(t2, "CompareSimpleTextTrackEvents t2 is null");
507 if (t1 != t2) {
508 TextTrackList* tList = t1->GetTextTrackList();
509 MOZ_ASSERT(tList, "CompareSimpleTextTrackEvents tList is null");
510 nsTArray<RefPtr<TextTrack>>& textTracks = tList->GetTextTrackArray();
511 auto index1 = textTracks.IndexOf(t1);
512 auto index2 = textTracks.IndexOf(t2);
513 if (index1 < index2) {
514 return true;
515 }
516 if (index1 > index2) {
517 return false;
518 }
519 }
520
521 MOZ_ASSERT(t1 == t2, "CompareSimpleTextTrackEvents t1 != t2");
522 // c1 and c2 are both belongs to t1.
523 TextTrackCue* c1 = aOne->mCue;
524 TextTrackCue* c2 = aTwo->mCue;
525 if (c1 != c2) {
526 if (c1->StartTime() < c2->StartTime()) {
527 return true;
528 }
529 if (c1->StartTime() > c2->StartTime()) {
530 return false;
531 }
532 if (c1->EndTime() < c2->EndTime()) {
533 return true;
534 }
535 if (c1->EndTime() > c2->EndTime()) {
536 return false;
537 }
538
539 TextTrackCueList* cueList = t1->GetCues();
540 MOZ_ASSERT(cueList);
541 nsTArray<RefPtr<TextTrackCue>>& cues = cueList->GetCuesArray();
542 auto index1 = cues.IndexOf(c1);
543 auto index2 = cues.IndexOf(c2);
544 if (index1 < index2) {
545 return true;
546 }
547 if (index1 > index2) {
548 return false;
549 }
550 }
551
552 // TimeMarchesOn step 13.3.
553 if (aOne->mName.EqualsLiteral("enter") ||
554 aTwo->mName.EqualsLiteral("exit")) {
555 return true;
556 }
557 return false;
558 }
559 };
560
561 class TextTrackListInternal {
562 public:
AddTextTrack(TextTrack * aTextTrack,const CompareTextTracks & aCompareTT)563 void AddTextTrack(TextTrack* aTextTrack,
564 const CompareTextTracks& aCompareTT) {
565 if (!mTextTracks.Contains(aTextTrack)) {
566 mTextTracks.InsertElementSorted(aTextTrack, aCompareTT);
567 }
568 }
Length() const569 uint32_t Length() const { return mTextTracks.Length(); }
operator [](uint32_t aIndex)570 TextTrack* operator[](uint32_t aIndex) {
571 return mTextTracks.SafeElementAt(aIndex, nullptr);
572 }
573
574 private:
575 nsTArray<RefPtr<TextTrack>> mTextTracks;
576 };
577
DispatchUpdateCueDisplay()578 void TextTrackManager::DispatchUpdateCueDisplay() {
579 if (!mUpdateCueDisplayDispatched && !IsShutdown()) {
580 WEBVTT_LOG("DispatchUpdateCueDisplay");
581 nsPIDOMWindowInner* win = mMediaElement->OwnerDoc()->GetInnerWindow();
582 if (win) {
583 nsGlobalWindowInner::Cast(win)->Dispatch(
584 TaskCategory::Other,
585 NewRunnableMethod("dom::TextTrackManager::UpdateCueDisplay", this,
586 &TextTrackManager::UpdateCueDisplay));
587 mUpdateCueDisplayDispatched = true;
588 }
589 }
590 }
591
DispatchTimeMarchesOn()592 void TextTrackManager::DispatchTimeMarchesOn() {
593 // Run the algorithm if no previous instance is still running, otherwise
594 // enqueue the current playback position and whether only that changed
595 // through its usual monotonic increase during normal playback; current
596 // executing call upon completion will check queue for further 'work'.
597 if (!mTimeMarchesOnDispatched && !IsShutdown()) {
598 WEBVTT_LOG("DispatchTimeMarchesOn");
599 nsPIDOMWindowInner* win = mMediaElement->OwnerDoc()->GetInnerWindow();
600 if (win) {
601 nsGlobalWindowInner::Cast(win)->Dispatch(
602 TaskCategory::Other,
603 NewRunnableMethod("dom::TextTrackManager::TimeMarchesOn", this,
604 &TextTrackManager::TimeMarchesOn));
605 mTimeMarchesOnDispatched = true;
606 }
607 }
608 }
609
610 // https://html.spec.whatwg.org/multipage/embedded-content.html#time-marches-on
TimeMarchesOn()611 void TextTrackManager::TimeMarchesOn() {
612 NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
613 mTimeMarchesOnDispatched = false;
614
615 CycleCollectedJSContext* context = CycleCollectedJSContext::Get();
616 if (context && context->IsInStableOrMetaStableState()) {
617 // FireTimeUpdate can be called while at stable state following a
618 // current position change which triggered a state watcher in MediaDecoder
619 // (see bug 1443429).
620 // TimeMarchesOn() will modify JS attributes which is forbidden while in
621 // stable state. So we dispatch a task to perform such operation later
622 // instead.
623 DispatchTimeMarchesOn();
624 return;
625 }
626 WEBVTT_LOG("TimeMarchesOn");
627
628 // Early return if we don't have any TextTracks or shutting down.
629 if (!mTextTracks || mTextTracks->Length() == 0 || IsShutdown() ||
630 !mMediaElement) {
631 return;
632 }
633
634 if (mMediaElement->ReadyState() == HTMLMediaElement_Binding::HAVE_NOTHING) {
635 WEBVTT_LOG(
636 "TimeMarchesOn return because media doesn't contain any data yet");
637 return;
638 }
639
640 if (mMediaElement->Seeking()) {
641 WEBVTT_LOG("TimeMarchesOn return during seeking");
642 return;
643 }
644
645 // Step 1, 2.
646 nsISupports* parentObject = mMediaElement->OwnerDoc()->GetParentObject();
647 if (NS_WARN_IF(!parentObject)) {
648 return;
649 }
650 nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(parentObject);
651 RefPtr<TextTrackCueList> currentCues = new TextTrackCueList(window);
652 RefPtr<TextTrackCueList> otherCues = new TextTrackCueList(window);
653
654 // Step 3.
655 auto currentPlaybackTime =
656 media::TimeUnit::FromSeconds(mMediaElement->CurrentTime());
657 bool hasNormalPlayback = !mHasSeeked;
658 mHasSeeked = false;
659 WEBVTT_LOG(
660 "TimeMarchesOn mLastTimeMarchesOnCalled %lf currentPlaybackTime %lf "
661 "hasNormalPlayback %d",
662 mLastTimeMarchesOnCalled.ToSeconds(), currentPlaybackTime.ToSeconds(),
663 hasNormalPlayback);
664
665 // The reason we collect other cues is (1) to change active cues to inactive,
666 // (2) find missing cues, so we actually no need to process all cues. We just
667 // need to handle cues which are in the time interval [lastTime:currentTime]
668 // or [currentTime:lastTime] (seeking forward). That can help us to reduce the
669 // size of other cues, which can improve execution time.
670 auto start = std::min(mLastTimeMarchesOnCalled, currentPlaybackTime);
671 auto end = std::max(mLastTimeMarchesOnCalled, currentPlaybackTime);
672 media::TimeInterval interval(start, end);
673 WEBVTT_LOGV("TimeMarchesOn Time interval [%f:%f]", start.ToSeconds(),
674 end.ToSeconds());
675 for (uint32_t idx = 0; idx < mTextTracks->Length(); ++idx) {
676 TextTrack* track = (*mTextTracks)[idx];
677 if (track) {
678 track->GetCurrentCuesAndOtherCues(currentCues, otherCues, interval);
679 }
680 }
681
682 // Step 4.
683 RefPtr<TextTrackCueList> missedCues = new TextTrackCueList(window);
684 if (hasNormalPlayback) {
685 for (uint32_t i = 0; i < otherCues->Length(); ++i) {
686 TextTrackCue* cue = (*otherCues)[i];
687 if (cue->StartTime() >= mLastTimeMarchesOnCalled.ToSeconds() &&
688 cue->EndTime() <= currentPlaybackTime.ToSeconds()) {
689 missedCues->AddCue(*cue);
690 }
691 }
692 }
693
694 WEBVTT_LOGV("TimeMarchesOn currentCues %d", currentCues->Length());
695 WEBVTT_LOGV("TimeMarchesOn otherCues %d", otherCues->Length());
696 WEBVTT_LOGV("TimeMarchesOn missedCues %d", missedCues->Length());
697 // Step 5. Empty now.
698 // TODO: Step 6: fire timeupdate?
699
700 // Step 7. Abort steps if condition 1, 2, 3 are satisfied.
701 // 1. All of the cues in current cues have their active flag set.
702 // 2. None of the cues in other cues have their active flag set.
703 // 3. Missed cues is empty.
704 bool c1 = true;
705 for (uint32_t i = 0; i < currentCues->Length(); ++i) {
706 if (!(*currentCues)[i]->GetActive()) {
707 c1 = false;
708 break;
709 }
710 }
711 bool c2 = true;
712 for (uint32_t i = 0; i < otherCues->Length(); ++i) {
713 if ((*otherCues)[i]->GetActive()) {
714 c2 = false;
715 break;
716 }
717 }
718 bool c3 = (missedCues->Length() == 0);
719 if (c1 && c2 && c3) {
720 mLastTimeMarchesOnCalled = currentPlaybackTime;
721 WEBVTT_LOG("TimeMarchesOn step 7 return, mLastTimeMarchesOnCalled %lf",
722 mLastTimeMarchesOnCalled.ToSeconds());
723 return;
724 }
725
726 // Step 8. Respect PauseOnExit flag if not seek.
727 if (hasNormalPlayback) {
728 for (uint32_t i = 0; i < otherCues->Length(); ++i) {
729 TextTrackCue* cue = (*otherCues)[i];
730 if (cue && cue->PauseOnExit() && cue->GetActive()) {
731 WEBVTT_LOG("TimeMarchesOn pause the MediaElement");
732 mMediaElement->Pause();
733 break;
734 }
735 }
736 for (uint32_t i = 0; i < missedCues->Length(); ++i) {
737 TextTrackCue* cue = (*missedCues)[i];
738 if (cue && cue->PauseOnExit()) {
739 WEBVTT_LOG("TimeMarchesOn pause the MediaElement");
740 mMediaElement->Pause();
741 break;
742 }
743 }
744 }
745
746 // Step 15.
747 // Sort text tracks in the same order as the text tracks appear
748 // in the media element's list of text tracks, and remove
749 // duplicates.
750 TextTrackListInternal affectedTracks;
751 // Step 13, 14.
752 nsTArray<RefPtr<SimpleTextTrackEvent>> eventList;
753 // Step 9, 10.
754 // For each text track cue in missed cues, prepare an event named
755 // enter for the TextTrackCue object with the cue start time.
756 for (uint32_t i = 0; i < missedCues->Length(); ++i) {
757 TextTrackCue* cue = (*missedCues)[i];
758 if (cue) {
759 WEBVTT_LOG("Prepare 'enter' event for cue %p [%f, %f] in missing cues",
760 cue, cue->StartTime(), cue->EndTime());
761 SimpleTextTrackEvent* event = new SimpleTextTrackEvent(
762 u"enter"_ns, cue->StartTime(), cue->GetTrack(), cue);
763 eventList.InsertElementSorted(
764 event, CompareSimpleTextTrackEvents(mMediaElement));
765 affectedTracks.AddTextTrack(cue->GetTrack(),
766 CompareTextTracks(mMediaElement));
767 }
768 }
769
770 // Step 11, 17.
771 for (uint32_t i = 0; i < otherCues->Length(); ++i) {
772 TextTrackCue* cue = (*otherCues)[i];
773 if (cue->GetActive() || missedCues->IsCueExist(cue)) {
774 double time =
775 cue->StartTime() > cue->EndTime() ? cue->StartTime() : cue->EndTime();
776 WEBVTT_LOG("Prepare 'exit' event for cue %p [%f, %f] in other cues", cue,
777 cue->StartTime(), cue->EndTime());
778 SimpleTextTrackEvent* event =
779 new SimpleTextTrackEvent(u"exit"_ns, time, cue->GetTrack(), cue);
780 eventList.InsertElementSorted(
781 event, CompareSimpleTextTrackEvents(mMediaElement));
782 affectedTracks.AddTextTrack(cue->GetTrack(),
783 CompareTextTracks(mMediaElement));
784 }
785 cue->SetActive(false);
786 }
787
788 // Step 12, 17.
789 for (uint32_t i = 0; i < currentCues->Length(); ++i) {
790 TextTrackCue* cue = (*currentCues)[i];
791 if (!cue->GetActive()) {
792 WEBVTT_LOG("Prepare 'enter' event for cue %p [%f, %f] in current cues",
793 cue, cue->StartTime(), cue->EndTime());
794 SimpleTextTrackEvent* event = new SimpleTextTrackEvent(
795 u"enter"_ns, cue->StartTime(), cue->GetTrack(), cue);
796 eventList.InsertElementSorted(
797 event, CompareSimpleTextTrackEvents(mMediaElement));
798 affectedTracks.AddTextTrack(cue->GetTrack(),
799 CompareTextTracks(mMediaElement));
800 }
801 cue->SetActive(true);
802 }
803
804 // Fire the eventList
805 for (uint32_t i = 0; i < eventList.Length(); ++i) {
806 eventList[i]->Dispatch();
807 }
808
809 // Step 16.
810 for (uint32_t i = 0; i < affectedTracks.Length(); ++i) {
811 TextTrack* ttrack = affectedTracks[i];
812 if (ttrack) {
813 ttrack->DispatchAsyncTrustedEvent(u"cuechange"_ns);
814 HTMLTrackElement* trackElement = ttrack->GetTrackElement();
815 if (trackElement) {
816 trackElement->DispatchTrackRunnable(u"cuechange"_ns);
817 }
818 }
819 }
820
821 mLastTimeMarchesOnCalled = currentPlaybackTime;
822
823 // Step 18.
824 UpdateCueDisplay();
825 }
826
NotifyCueUpdated(TextTrackCue * aCue)827 void TextTrackManager::NotifyCueUpdated(TextTrackCue* aCue) {
828 // TODO: Add/Reorder the cue to mNewCues if we have some optimization?
829 WEBVTT_LOG("NotifyCueUpdated, cue=%p", aCue);
830 MaybeRunTimeMarchesOn();
831 // For the case "Texttrack.mode = hidden/showing", if the mode
832 // changing between showing and hidden, TimeMarchesOn
833 // doesn't render the cue. Call DispatchUpdateCueDisplay() explicitly.
834 DispatchUpdateCueDisplay();
835 }
836
NotifyReset()837 void TextTrackManager::NotifyReset() {
838 // https://html.spec.whatwg.org/multipage/media.html#text-track-cue-active-flag
839 // This will unset all cues' active flag and update the cue display.
840 WEBVTT_LOG("NotifyReset");
841 mLastTimeMarchesOnCalled = media::TimeUnit::Zero();
842 for (uint32_t idx = 0; idx < mTextTracks->Length(); ++idx) {
843 (*mTextTracks)[idx]->SetCuesInactive();
844 }
845 UpdateCueDisplay();
846 }
847
ReportTelemetryForTrack(TextTrack * aTextTrack) const848 void TextTrackManager::ReportTelemetryForTrack(TextTrack* aTextTrack) const {
849 MOZ_ASSERT(NS_IsMainThread());
850 MOZ_ASSERT(aTextTrack);
851 MOZ_ASSERT(mTextTracks->Length() > 0);
852
853 TextTrackKind kind = aTextTrack->Kind();
854 Telemetry::Accumulate(Telemetry::WEBVTT_TRACK_KINDS, uint32_t(kind));
855 }
856
IsLoaded()857 bool TextTrackManager::IsLoaded() {
858 return mTextTracks ? mTextTracks->AreTextTracksLoaded() : true;
859 }
860
IsShutdown() const861 bool TextTrackManager::IsShutdown() const {
862 return (mShutdown || !sParserWrapper);
863 }
864
MaybeRunTimeMarchesOn()865 void TextTrackManager::MaybeRunTimeMarchesOn() {
866 MOZ_ASSERT(mMediaElement);
867 // According to spec, we should check media element's show poster flag before
868 // running `TimeMarchesOn` in following situations, (1) add cue (2) remove cue
869 // (3) cue's start time changes (4) cues's end time changes
870 // https://html.spec.whatwg.org/multipage/media.html#playing-the-media-resource:time-marches-on
871 // https://html.spec.whatwg.org/multipage/media.html#text-track-api:time-marches-on
872 if (mMediaElement->GetShowPosterFlag()) {
873 return;
874 }
875 TimeMarchesOn();
876 }
877
878 } // namespace mozilla::dom
879