1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
5 * You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "mozilla/dom/TextTrackManager.h"
8 #include "mozilla/dom/HTMLMediaElement.h"
9 #include "mozilla/dom/HTMLTrackElement.h"
10 #include "mozilla/dom/HTMLVideoElement.h"
11 #include "mozilla/dom/TextTrack.h"
12 #include "mozilla/dom/TextTrackCue.h"
13 #include "mozilla/dom/Event.h"
14 #include "mozilla/ClearOnShutdown.h"
15 #include "mozilla/Telemetry.h"
16 #include "nsComponentManagerUtils.h"
17 #include "nsVariant.h"
18 #include "nsVideoFrame.h"
19 #include "nsIFrame.h"
20 #include "nsTArrayHelpers.h"
21 #include "nsIWebVTTParserWrapper.h"
22
23
24 static mozilla::LazyLogModule gTextTrackLog("TextTrackManager");
25 #define WEBVTT_LOG(...) MOZ_LOG(gTextTrackLog, LogLevel::Debug, (__VA_ARGS__))
26 #define WEBVTT_LOGV(...) MOZ_LOG(gTextTrackLog, LogLevel::Verbose, (__VA_ARGS__))
27
28 namespace mozilla {
29 namespace dom {
30
31 NS_IMPL_ISUPPORTS(TextTrackManager::ShutdownObserverProxy, nsIObserver);
32
CompareTextTracks(HTMLMediaElement * aMediaElement)33 CompareTextTracks::CompareTextTracks(HTMLMediaElement* aMediaElement)
34 {
35 mMediaElement = aMediaElement;
36 }
37
38 int32_t
TrackChildPosition(TextTrack * aTextTrack) const39 CompareTextTracks::TrackChildPosition(TextTrack* aTextTrack) const {
40 MOZ_DIAGNOSTIC_ASSERT(aTextTrack);
41 HTMLTrackElement* trackElement = aTextTrack->GetTrackElement();
42 if (!trackElement) {
43 return -1;
44 }
45 return mMediaElement->IndexOf(trackElement);
46 }
47
48 bool
Equals(TextTrack * aOne,TextTrack * aTwo) const49 CompareTextTracks::Equals(TextTrack* aOne, TextTrack* aTwo) const {
50 // Two tracks can never be equal. If they have corresponding TrackElements
51 // they would need to occupy the same tree position (impossible) and in the
52 // case of tracks coming from AddTextTrack source we put the newest at the
53 // last position, so they won't be equal as well.
54 return false;
55 }
56
57 bool
LessThan(TextTrack * aOne,TextTrack * aTwo) const58 CompareTextTracks::LessThan(TextTrack* aOne, TextTrack* aTwo) const
59 {
60 // Protect against nullptr TextTrack objects; treat them as
61 // sorting toward the end.
62 if (!aOne) {
63 return false;
64 }
65 if (!aTwo) {
66 return true;
67 }
68 TextTrackSource sourceOne = aOne->GetTextTrackSource();
69 TextTrackSource sourceTwo = aTwo->GetTextTrackSource();
70 if (sourceOne != sourceTwo) {
71 return sourceOne == TextTrackSource::Track ||
72 (sourceOne == AddTextTrack && sourceTwo == MediaResourceSpecific);
73 }
74 switch (sourceOne) {
75 case Track: {
76 int32_t positionOne = TrackChildPosition(aOne);
77 int32_t positionTwo = TrackChildPosition(aTwo);
78 // If either position one or positiontwo are -1 then something has gone
79 // wrong. In this case we should just put them at the back of the list.
80 return positionOne != -1 && positionTwo != -1 &&
81 positionOne < positionTwo;
82 }
83 case AddTextTrack:
84 // For AddTextTrack sources the tracks will already be in the correct relative
85 // order in the source array. Assume we're called in iteration order and can
86 // therefore always report aOne < aTwo to maintain the original temporal ordering.
87 return true;
88 case MediaResourceSpecific:
89 // No rules for Media Resource Specific tracks yet.
90 break;
91 }
92 return true;
93 }
94
95 NS_IMPL_CYCLE_COLLECTION(TextTrackManager, mMediaElement, mTextTracks,
96 mPendingTextTracks, mNewCues,
97 mLastActiveCues)
98
99 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(TextTrackManager)
100 NS_INTERFACE_MAP_ENTRY(nsIDOMEventListener)
101 NS_INTERFACE_MAP_END
102
103 NS_IMPL_CYCLE_COLLECTING_ADDREF(TextTrackManager)
104 NS_IMPL_CYCLE_COLLECTING_RELEASE(TextTrackManager)
105
106 StaticRefPtr<nsIWebVTTParserWrapper> TextTrackManager::sParserWrapper;
107
TextTrackManager(HTMLMediaElement * aMediaElement)108 TextTrackManager::TextTrackManager(HTMLMediaElement *aMediaElement)
109 : mMediaElement(aMediaElement)
110 , mHasSeeked(false)
111 , mLastTimeMarchesOnCalled(0.0)
112 , mTimeMarchesOnDispatched(false)
113 , mUpdateCueDisplayDispatched(false)
114 , performedTrackSelection(false)
115 , mCueTelemetryReported(false)
116 , mShutdown(false)
117 {
118 nsISupports* parentObject =
119 mMediaElement->OwnerDoc()->GetParentObject();
120
121 NS_ENSURE_TRUE_VOID(parentObject);
122 WEBVTT_LOG("%p Create TextTrackManager",this);
123 nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(parentObject);
124 mNewCues = new TextTrackCueList(window);
125 mLastActiveCues = new TextTrackCueList(window);
126 mTextTracks = new TextTrackList(window, this);
127 mPendingTextTracks = new TextTrackList(window, this);
128
129 if (!sParserWrapper) {
130 nsCOMPtr<nsIWebVTTParserWrapper> parserWrapper =
131 do_CreateInstance(NS_WEBVTTPARSERWRAPPER_CONTRACTID);
132 sParserWrapper = parserWrapper;
133 ClearOnShutdown(&sParserWrapper);
134 }
135 mShutdownProxy = new ShutdownObserverProxy(this);
136 }
137
~TextTrackManager()138 TextTrackManager::~TextTrackManager()
139 {
140 WEBVTT_LOG("%p ~TextTrackManager",this);
141 nsContentUtils::UnregisterShutdownObserver(mShutdownProxy);
142 }
143
144 TextTrackList*
GetTextTracks() const145 TextTrackManager::GetTextTracks() const
146 {
147 return mTextTracks;
148 }
149
150 already_AddRefed<TextTrack>
AddTextTrack(TextTrackKind aKind,const nsAString & aLabel,const nsAString & aLanguage,TextTrackMode aMode,TextTrackReadyState aReadyState,TextTrackSource aTextTrackSource)151 TextTrackManager::AddTextTrack(TextTrackKind aKind, const nsAString& aLabel,
152 const nsAString& aLanguage,
153 TextTrackMode aMode,
154 TextTrackReadyState aReadyState,
155 TextTrackSource aTextTrackSource)
156 {
157 if (!mMediaElement || !mTextTracks) {
158 return nullptr;
159 }
160 WEBVTT_LOG("%p AddTextTrack",this);
161 WEBVTT_LOGV("AddTextTrack kind %d Label %s Language %s",aKind,
162 NS_ConvertUTF16toUTF8(aLabel).get(), NS_ConvertUTF16toUTF8(aLanguage).get());
163 RefPtr<TextTrack> track =
164 mTextTracks->AddTextTrack(aKind, aLabel, aLanguage, aMode, aReadyState,
165 aTextTrackSource, CompareTextTracks(mMediaElement));
166 AddCues(track);
167 ReportTelemetryForTrack(track);
168
169 if (aTextTrackSource == TextTrackSource::Track) {
170 RefPtr<nsIRunnable> task =
171 NewRunnableMethod(this, &TextTrackManager::HonorUserPreferencesForTrackSelection);
172 nsContentUtils::RunInStableState(task.forget());
173 }
174
175 return track.forget();
176 }
177
178 void
AddTextTrack(TextTrack * aTextTrack)179 TextTrackManager::AddTextTrack(TextTrack* aTextTrack)
180 {
181 if (!mMediaElement || !mTextTracks) {
182 return;
183 }
184 WEBVTT_LOG("%p AddTextTrack TextTrack %p",this, aTextTrack);
185 mTextTracks->AddTextTrack(aTextTrack, CompareTextTracks(mMediaElement));
186 AddCues(aTextTrack);
187 ReportTelemetryForTrack(aTextTrack);
188
189 if (aTextTrack->GetTextTrackSource() == TextTrackSource::Track) {
190 RefPtr<nsIRunnable> task =
191 NewRunnableMethod(this, &TextTrackManager::HonorUserPreferencesForTrackSelection);
192 nsContentUtils::RunInStableState(task.forget());
193 }
194 }
195
196 void
AddCues(TextTrack * aTextTrack)197 TextTrackManager::AddCues(TextTrack* aTextTrack)
198 {
199 if (!mNewCues) {
200 WEBVTT_LOG("AddCues mNewCues is null");
201 return;
202 }
203
204 TextTrackCueList* cueList = aTextTrack->GetCues();
205 if (cueList) {
206 bool dummy;
207 WEBVTT_LOGV("AddCues cueList->Length() %d",cueList->Length());
208 for (uint32_t i = 0; i < cueList->Length(); ++i) {
209 mNewCues->AddCue(*cueList->IndexedGetter(i, dummy));
210 }
211 DispatchTimeMarchesOn();
212 }
213 }
214
215 void
RemoveTextTrack(TextTrack * aTextTrack,bool aPendingListOnly)216 TextTrackManager::RemoveTextTrack(TextTrack* aTextTrack, bool aPendingListOnly)
217 {
218 if (!mPendingTextTracks || !mTextTracks) {
219 return;
220 }
221
222 WEBVTT_LOG("%p RemoveTextTrack TextTrack %p", this, aTextTrack);
223 mPendingTextTracks->RemoveTextTrack(aTextTrack);
224 if (aPendingListOnly) {
225 return;
226 }
227
228 mTextTracks->RemoveTextTrack(aTextTrack);
229 // Remove the cues in mNewCues belong to aTextTrack.
230 TextTrackCueList* removeCueList = aTextTrack->GetCues();
231 if (removeCueList) {
232 WEBVTT_LOGV("RemoveTextTrack removeCueList->Length() %d", removeCueList->Length());
233 for (uint32_t i = 0; i < removeCueList->Length(); ++i) {
234 mNewCues->RemoveCue(*((*removeCueList)[i]));
235 }
236 DispatchTimeMarchesOn();
237 }
238 }
239
240 void
DidSeek()241 TextTrackManager::DidSeek()
242 {
243 WEBVTT_LOG("%p DidSeek",this);
244 if (mTextTracks) {
245 mTextTracks->DidSeek();
246 }
247 if (mMediaElement) {
248 mLastTimeMarchesOnCalled = mMediaElement->CurrentTime();
249 WEBVTT_LOGV("DidSeek set mLastTimeMarchesOnCalled %lf",mLastTimeMarchesOnCalled);
250 }
251 mHasSeeked = true;
252 }
253
254 void
UpdateCueDisplay()255 TextTrackManager::UpdateCueDisplay()
256 {
257 WEBVTT_LOG("UpdateCueDisplay");
258 mUpdateCueDisplayDispatched = false;
259
260 if (!mMediaElement || !mTextTracks) {
261 return;
262 }
263
264 nsIFrame* frame = mMediaElement->GetPrimaryFrame();
265 nsVideoFrame* videoFrame = do_QueryFrame(frame);
266 if (!videoFrame) {
267 return;
268 }
269
270 nsCOMPtr<nsIContent> overlay = videoFrame->GetCaptionOverlay();
271 nsCOMPtr<nsIContent> controls = videoFrame->GetVideoControls();
272 if (!overlay) {
273 return;
274 }
275
276 nsTArray<RefPtr<TextTrackCue> > activeCues;
277 mTextTracks->GetShowingCues(activeCues);
278
279 if (activeCues.Length() > 0) {
280 WEBVTT_LOG("UpdateCueDisplay ProcessCues");
281 WEBVTT_LOGV("UpdateCueDisplay activeCues.Length() %d",activeCues.Length());
282 RefPtr<nsVariantCC> jsCues = new nsVariantCC();
283
284 jsCues->SetAsArray(nsIDataType::VTYPE_INTERFACE,
285 &NS_GET_IID(nsIDOMEventTarget),
286 activeCues.Length(),
287 static_cast<void*>(activeCues.Elements()));
288 nsPIDOMWindowInner* window = mMediaElement->OwnerDoc()->GetInnerWindow();
289 if (window) {
290 sParserWrapper->ProcessCues(window, jsCues, overlay, controls);
291 }
292 } else if (overlay->Length() > 0) {
293 WEBVTT_LOG("UpdateCueDisplay EmptyString");
294 nsContentUtils::SetNodeTextContent(overlay, EmptyString(), true);
295 }
296 }
297
298 void
NotifyCueAdded(TextTrackCue & aCue)299 TextTrackManager::NotifyCueAdded(TextTrackCue& aCue)
300 {
301 WEBVTT_LOG("NotifyCueAdded");
302 if (mNewCues) {
303 mNewCues->AddCue(aCue);
304 }
305 DispatchTimeMarchesOn();
306 ReportTelemetryForCue();
307 }
308
309 void
NotifyCueRemoved(TextTrackCue & aCue)310 TextTrackManager::NotifyCueRemoved(TextTrackCue& aCue)
311 {
312 WEBVTT_LOG("NotifyCueRemoved");
313 if (mNewCues) {
314 mNewCues->RemoveCue(aCue);
315 }
316 DispatchTimeMarchesOn();
317 if (aCue.GetActive()) {
318 // We remove an active cue, need to update the display.
319 DispatchUpdateCueDisplay();
320 }
321 }
322
323 void
PopulatePendingList()324 TextTrackManager::PopulatePendingList()
325 {
326 if (!mTextTracks || !mPendingTextTracks || !mMediaElement) {
327 return;
328 }
329 uint32_t len = mTextTracks->Length();
330 bool dummy;
331 for (uint32_t index = 0; index < len; ++index) {
332 TextTrack* ttrack = mTextTracks->IndexedGetter(index, dummy);
333 if (ttrack && ttrack->Mode() != TextTrackMode::Disabled &&
334 ttrack->ReadyState() == TextTrackReadyState::Loading) {
335 mPendingTextTracks->AddTextTrack(ttrack,
336 CompareTextTracks(mMediaElement));
337 }
338 }
339 }
340
341 void
AddListeners()342 TextTrackManager::AddListeners()
343 {
344 if (mMediaElement) {
345 mMediaElement->AddEventListener(NS_LITERAL_STRING("resizevideocontrols"),
346 this, false, false);
347 mMediaElement->AddEventListener(NS_LITERAL_STRING("seeked"),
348 this, false, false);
349 mMediaElement->AddEventListener(NS_LITERAL_STRING("controlbarchange"),
350 this, false, true);
351 }
352 }
353
354 void
HonorUserPreferencesForTrackSelection()355 TextTrackManager::HonorUserPreferencesForTrackSelection()
356 {
357 if (performedTrackSelection || !mTextTracks) {
358 return;
359 }
360 WEBVTT_LOG("HonorUserPreferencesForTrackSelection");
361 TextTrackKind ttKinds[] = { TextTrackKind::Captions,
362 TextTrackKind::Subtitles };
363
364 // Steps 1 - 3: Perform automatic track selection for different TextTrack
365 // Kinds.
366 PerformTrackSelection(ttKinds, ArrayLength(ttKinds));
367 PerformTrackSelection(TextTrackKind::Descriptions);
368 PerformTrackSelection(TextTrackKind::Chapters);
369
370 // Step 4: Set all TextTracks with a kind of metadata that are disabled
371 // to hidden.
372 for (uint32_t i = 0; i < mTextTracks->Length(); i++) {
373 TextTrack* track = (*mTextTracks)[i];
374 if (track->Kind() == TextTrackKind::Metadata && TrackIsDefault(track) &&
375 track->Mode() == TextTrackMode::Disabled) {
376 track->SetMode(TextTrackMode::Hidden);
377 }
378 }
379
380 performedTrackSelection = true;
381 }
382
383 bool
TrackIsDefault(TextTrack * aTextTrack)384 TextTrackManager::TrackIsDefault(TextTrack* aTextTrack)
385 {
386 HTMLTrackElement* trackElement = aTextTrack->GetTrackElement();
387 if (!trackElement) {
388 return false;
389 }
390 return trackElement->Default();
391 }
392
393 void
PerformTrackSelection(TextTrackKind aTextTrackKind)394 TextTrackManager::PerformTrackSelection(TextTrackKind aTextTrackKind)
395 {
396 TextTrackKind ttKinds[] = { aTextTrackKind };
397 PerformTrackSelection(ttKinds, ArrayLength(ttKinds));
398 }
399
400 void
PerformTrackSelection(TextTrackKind aTextTrackKinds[],uint32_t size)401 TextTrackManager::PerformTrackSelection(TextTrackKind aTextTrackKinds[],
402 uint32_t size)
403 {
404 nsTArray<TextTrack*> candidates;
405 GetTextTracksOfKinds(aTextTrackKinds, size, candidates);
406
407 // Step 3: If any TextTracks in candidates are showing then abort these steps.
408 for (uint32_t i = 0; i < candidates.Length(); i++) {
409 if (candidates[i]->Mode() == TextTrackMode::Showing) {
410 WEBVTT_LOGV("PerformTrackSelection Showing return kind %d",candidates[i]->Kind());
411 return;
412 }
413 }
414
415 // Step 4: Honor user preferences for track selection, otherwise, set the
416 // first TextTrack in candidates with a default attribute to showing.
417 // TODO: Bug 981691 - Honor user preferences for text track selection.
418 for (uint32_t i = 0; i < candidates.Length(); i++) {
419 if (TrackIsDefault(candidates[i]) &&
420 candidates[i]->Mode() == TextTrackMode::Disabled) {
421 candidates[i]->SetMode(TextTrackMode::Showing);
422 WEBVTT_LOGV("PerformTrackSelection set Showing kind %d", candidates[i]->Kind());
423 return;
424 }
425 }
426 }
427
428 void
GetTextTracksOfKinds(TextTrackKind aTextTrackKinds[],uint32_t size,nsTArray<TextTrack * > & aTextTracks)429 TextTrackManager::GetTextTracksOfKinds(TextTrackKind aTextTrackKinds[],
430 uint32_t size,
431 nsTArray<TextTrack*>& aTextTracks)
432 {
433 for (uint32_t i = 0; i < size; i++) {
434 GetTextTracksOfKind(aTextTrackKinds[i], aTextTracks);
435 }
436 }
437
438 void
GetTextTracksOfKind(TextTrackKind aTextTrackKind,nsTArray<TextTrack * > & aTextTracks)439 TextTrackManager::GetTextTracksOfKind(TextTrackKind aTextTrackKind,
440 nsTArray<TextTrack*>& aTextTracks)
441 {
442 if (!mTextTracks) {
443 return;
444 }
445 for (uint32_t i = 0; i < mTextTracks->Length(); i++) {
446 TextTrack* textTrack = (*mTextTracks)[i];
447 if (textTrack->Kind() == aTextTrackKind) {
448 aTextTracks.AppendElement(textTrack);
449 }
450 }
451 }
452
453 NS_IMETHODIMP
HandleEvent(nsIDOMEvent * aEvent)454 TextTrackManager::HandleEvent(nsIDOMEvent* aEvent)
455 {
456 if (!mTextTracks) {
457 return NS_OK;
458 }
459
460 nsAutoString type;
461 aEvent->GetType(type);
462 if (type.EqualsLiteral("resizevideocontrols") ||
463 type.EqualsLiteral("seeked")) {
464 for (uint32_t i = 0; i< mTextTracks->Length(); i++) {
465 ((*mTextTracks)[i])->SetCuesDirty();
466 }
467 }
468
469 if (type.EqualsLiteral("controlbarchange")) {
470 UpdateCueDisplay();
471 }
472
473 return NS_OK;
474 }
475
476
477 class SimpleTextTrackEvent : public Runnable
478 {
479 public:
480 friend class CompareSimpleTextTrackEvents;
SimpleTextTrackEvent(const nsAString & aEventName,double aTime,TextTrack * aTrack,TextTrackCue * aCue)481 SimpleTextTrackEvent(const nsAString& aEventName, double aTime,
482 TextTrack* aTrack, TextTrackCue* aCue)
483 : mName(aEventName),
484 mTime(aTime),
485 mTrack(aTrack),
486 mCue(aCue)
487 {}
488
Run()489 NS_IMETHOD Run() {
490 WEBVTT_LOGV("SimpleTextTrackEvent cue %p mName %s mTime %lf",
491 mCue.get(), NS_ConvertUTF16toUTF8(mName).get(), mTime);
492 mCue->DispatchTrustedEvent(mName);
493 return NS_OK;
494 }
495
496 private:
497 nsString mName;
498 double mTime;
499 TextTrack* mTrack;
500 RefPtr<TextTrackCue> mCue;
501 };
502
503 class CompareSimpleTextTrackEvents {
504 private:
TrackChildPosition(SimpleTextTrackEvent * aEvent) const505 int32_t TrackChildPosition(SimpleTextTrackEvent* aEvent) const
506 {
507 if (aEvent->mTrack) {
508 HTMLTrackElement* trackElement = aEvent->mTrack->GetTrackElement();
509 if (trackElement) {
510 return mMediaElement->IndexOf(trackElement);
511 }
512 }
513 return -1;
514 }
515 HTMLMediaElement* mMediaElement;
516 public:
CompareSimpleTextTrackEvents(HTMLMediaElement * aMediaElement)517 explicit CompareSimpleTextTrackEvents(HTMLMediaElement* aMediaElement)
518 {
519 mMediaElement = aMediaElement;
520 }
521
Equals(SimpleTextTrackEvent * aOne,SimpleTextTrackEvent * aTwo) const522 bool Equals(SimpleTextTrackEvent* aOne, SimpleTextTrackEvent* aTwo) const
523 {
524 return false;
525 }
526
LessThan(SimpleTextTrackEvent * aOne,SimpleTextTrackEvent * aTwo) const527 bool LessThan(SimpleTextTrackEvent* aOne, SimpleTextTrackEvent* aTwo) const
528 {
529 if (aOne->mTime < aTwo->mTime) {
530 return true;
531 } else if (aOne->mTime > aTwo->mTime) {
532 return false;
533 }
534
535 int32_t positionOne = TrackChildPosition(aOne);
536 int32_t positionTwo = TrackChildPosition(aTwo);
537 if (positionOne < positionTwo) {
538 return true;
539 } else if (positionOne > positionTwo) {
540 return false;
541 }
542
543 if (aOne->mName.EqualsLiteral("enter") ||
544 aTwo->mName.EqualsLiteral("exit")) {
545 return true;
546 }
547 return false;
548 }
549 };
550
551 class TextTrackListInternal
552 {
553 public:
AddTextTrack(TextTrack * aTextTrack,const CompareTextTracks & aCompareTT)554 void AddTextTrack(TextTrack* aTextTrack,
555 const CompareTextTracks& aCompareTT)
556 {
557 if (!mTextTracks.Contains(aTextTrack)) {
558 mTextTracks.InsertElementSorted(aTextTrack, aCompareTT);
559 }
560 }
Length() const561 uint32_t Length() const
562 {
563 return mTextTracks.Length();
564 }
operator [](uint32_t aIndex)565 TextTrack* operator[](uint32_t aIndex)
566 {
567 return mTextTracks.SafeElementAt(aIndex, nullptr);
568 }
569 private:
570 nsTArray<RefPtr<TextTrack>> mTextTracks;
571 };
572
573 void
DispatchUpdateCueDisplay()574 TextTrackManager::DispatchUpdateCueDisplay()
575 {
576 if (!mUpdateCueDisplayDispatched && !mShutdown &&
577 (mMediaElement->GetHasUserInteraction() || mMediaElement->IsCurrentlyPlaying())) {
578 WEBVTT_LOG("DispatchUpdateCueDisplay");
579 NS_DispatchToMainThread(NewRunnableMethod(this, &TextTrackManager::UpdateCueDisplay));
580 mUpdateCueDisplayDispatched = true;
581 }
582 }
583
584 void
DispatchTimeMarchesOn()585 TextTrackManager::DispatchTimeMarchesOn()
586 {
587 // Run the algorithm if no previous instance is still running, otherwise
588 // enqueue the current playback position and whether only that changed
589 // through its usual monotonic increase during normal playback; current
590 // executing call upon completion will check queue for further 'work'.
591 if (!mTimeMarchesOnDispatched && !mShutdown &&
592 (mMediaElement->GetHasUserInteraction() || mMediaElement->IsCurrentlyPlaying())) {
593 WEBVTT_LOG("DispatchTimeMarchesOn");
594 NS_DispatchToMainThread(NewRunnableMethod(this, &TextTrackManager::TimeMarchesOn));
595 mTimeMarchesOnDispatched = true;
596 }
597 }
598
599 // https://html.spec.whatwg.org/multipage/embedded-content.html#time-marches-on
600 void
TimeMarchesOn()601 TextTrackManager::TimeMarchesOn()
602 {
603 NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
604 WEBVTT_LOG("TimeMarchesOn");
605 mTimeMarchesOnDispatched = false;
606
607 // Early return if we don't have any TextTracks or shutting down.
608 if (!mTextTracks || mTextTracks->Length() == 0 || mShutdown) {
609 return;
610 }
611
612 nsISupports* parentObject =
613 mMediaElement->OwnerDoc()->GetParentObject();
614 if (NS_WARN_IF(!parentObject)) {
615 return;
616 }
617 nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(parentObject);
618
619 if (mMediaElement &&
620 (!(mMediaElement->GetPlayedOrSeeked()) || mMediaElement->Seeking())) {
621 WEBVTT_LOG("TimeMarchesOn seeking or post return");
622 return;
623 }
624
625 // Step 3.
626 double currentPlaybackTime = mMediaElement->CurrentTime();
627 bool hasNormalPlayback = !mHasSeeked;
628 mHasSeeked = false;
629 WEBVTT_LOG("TimeMarchesOn mLastTimeMarchesOnCalled %lf currentPlaybackTime %lf hasNormalPlayback %d"
630 , mLastTimeMarchesOnCalled, currentPlaybackTime, hasNormalPlayback);
631
632 // Step 1, 2.
633 RefPtr<TextTrackCueList> currentCues =
634 new TextTrackCueList(window);
635 RefPtr<TextTrackCueList> otherCues =
636 new TextTrackCueList(window);
637 bool dummy;
638 for (uint32_t index = 0; index < mTextTracks->Length(); ++index) {
639 TextTrack* ttrack = mTextTracks->IndexedGetter(index, dummy);
640 if (ttrack && dummy) {
641 // TODO: call GetCueListByTimeInterval on mNewCues?
642 ttrack->UpdateActiveCueList();
643 TextTrackCueList* activeCueList = ttrack->GetActiveCues();
644 if (activeCueList) {
645 for (uint32_t i = 0; i < activeCueList->Length(); ++i) {
646 currentCues->AddCue(*((*activeCueList)[i]));
647 }
648 }
649 }
650 }
651 WEBVTT_LOGV("TimeMarchesOn currentCues %d", currentCues->Length());
652 // Populate otherCues with 'non-active" cues.
653 if (hasNormalPlayback) {
654 if (currentPlaybackTime < mLastTimeMarchesOnCalled) {
655 // TODO: Add log and find the root cause why the
656 // playback position goes backward.
657 mLastTimeMarchesOnCalled = currentPlaybackTime;
658 }
659 media::Interval<double> interval(mLastTimeMarchesOnCalled,
660 currentPlaybackTime);
661 otherCues = mNewCues->GetCueListByTimeInterval(interval);;
662 } else {
663 // Seek case. Put the mLastActiveCues into otherCues.
664 otherCues = mLastActiveCues;
665 }
666 for (uint32_t i = 0; i < currentCues->Length(); ++i) {
667 TextTrackCue* cue = (*currentCues)[i];
668 otherCues->RemoveCue(*cue);
669 }
670 WEBVTT_LOGV("TimeMarchesOn otherCues %d", otherCues->Length());
671 // Step 4.
672 RefPtr<TextTrackCueList> missedCues = new TextTrackCueList(window);
673 if (hasNormalPlayback) {
674 for (uint32_t i = 0; i < otherCues->Length(); ++i) {
675 TextTrackCue* cue = (*otherCues)[i];
676 if (cue->StartTime() >= mLastTimeMarchesOnCalled &&
677 cue->EndTime() <= currentPlaybackTime) {
678 missedCues->AddCue(*cue);
679 }
680 }
681 }
682 WEBVTT_LOGV("TimeMarchesOn missedCues %d", missedCues->Length());
683 // Step 5. Empty now.
684 // TODO: Step 6: fire timeupdate?
685
686 // Step 7. Abort steps if condition 1, 2, 3 are satisfied.
687 // 1. All of the cues in current cues have their active flag set.
688 // 2. None of the cues in other cues have their active flag set.
689 // 3. Missed cues is empty.
690 bool c1 = true;
691 for (uint32_t i = 0; i < currentCues->Length(); ++i) {
692 if (!(*currentCues)[i]->GetActive()) {
693 c1 = false;
694 break;
695 }
696 }
697 bool c2 = true;
698 for (uint32_t i = 0; i < otherCues->Length(); ++i) {
699 if ((*otherCues)[i]->GetActive()) {
700 c2 = false;
701 break;
702 }
703 }
704 bool c3 = (missedCues->Length() == 0);
705 if (c1 && c2 && c3) {
706 mLastTimeMarchesOnCalled = currentPlaybackTime;
707 WEBVTT_LOG("TimeMarchesOn step 7 return, mLastTimeMarchesOnCalled %lf", mLastTimeMarchesOnCalled);
708 return;
709 }
710
711 // Step 8. Respect PauseOnExit flag if not seek.
712 if (hasNormalPlayback) {
713 for (uint32_t i = 0; i < otherCues->Length(); ++i) {
714 TextTrackCue* cue = (*otherCues)[i];
715 if (cue && cue->PauseOnExit() && cue->GetActive()) {
716 WEBVTT_LOG("TimeMarchesOn pause the MediaElement");
717 mMediaElement->Pause();
718 break;
719 }
720 }
721 for (uint32_t i = 0; i < missedCues->Length(); ++i) {
722 TextTrackCue* cue = (*missedCues)[i];
723 if (cue && cue->PauseOnExit()) {
724 WEBVTT_LOG("TimeMarchesOn pause the MediaElement");
725 mMediaElement->Pause();
726 break;
727 }
728 }
729 }
730
731 // Step 15.
732 // Sort text tracks in the same order as the text tracks appear
733 // in the media element's list of text tracks, and remove
734 // duplicates.
735 TextTrackListInternal affectedTracks;
736 // Step 13, 14.
737 nsTArray<RefPtr<SimpleTextTrackEvent>> eventList;
738 // Step 9, 10.
739 // For each text track cue in missed cues, prepare an event named
740 // enter for the TextTrackCue object with the cue start time.
741 for (uint32_t i = 0; i < missedCues->Length(); ++i) {
742 TextTrackCue* cue = (*missedCues)[i];
743 if (cue) {
744 SimpleTextTrackEvent* event =
745 new SimpleTextTrackEvent(NS_LITERAL_STRING("enter"),
746 cue->StartTime(), cue->GetTrack(),
747 cue);
748 eventList.InsertElementSorted(event,
749 CompareSimpleTextTrackEvents(mMediaElement));
750 affectedTracks.AddTextTrack(cue->GetTrack(), CompareTextTracks(mMediaElement));
751 }
752 }
753
754 // Step 11, 17.
755 for (uint32_t i = 0; i < otherCues->Length(); ++i) {
756 TextTrackCue* cue = (*otherCues)[i];
757 if (cue->GetActive() || missedCues->IsCueExist(cue)) {
758 double time = cue->StartTime() > cue->EndTime() ? cue->StartTime()
759 : cue->EndTime();
760 SimpleTextTrackEvent* event =
761 new SimpleTextTrackEvent(NS_LITERAL_STRING("exit"), time,
762 cue->GetTrack(), cue);
763 eventList.InsertElementSorted(event,
764 CompareSimpleTextTrackEvents(mMediaElement));
765 affectedTracks.AddTextTrack(cue->GetTrack(), CompareTextTracks(mMediaElement));
766 }
767 cue->SetActive(false);
768 }
769
770 // Step 12, 17.
771 for (uint32_t i = 0; i < currentCues->Length(); ++i) {
772 TextTrackCue* cue = (*currentCues)[i];
773 if (!cue->GetActive()) {
774 SimpleTextTrackEvent* event =
775 new SimpleTextTrackEvent(NS_LITERAL_STRING("enter"),
776 cue->StartTime(), cue->GetTrack(),
777 cue);
778 eventList.InsertElementSorted(event,
779 CompareSimpleTextTrackEvents(mMediaElement));
780 affectedTracks.AddTextTrack(cue->GetTrack(), CompareTextTracks(mMediaElement));
781 }
782 cue->SetActive(true);
783 }
784
785 // Fire the eventList
786 for (uint32_t i = 0; i < eventList.Length(); ++i) {
787 NS_DispatchToMainThread(eventList[i].forget());
788 }
789
790 // Step 16.
791 for (uint32_t i = 0; i < affectedTracks.Length(); ++i) {
792 TextTrack* ttrack = affectedTracks[i];
793 if (ttrack) {
794 ttrack->DispatchAsyncTrustedEvent(NS_LITERAL_STRING("cuechange"));
795 HTMLTrackElement* trackElement = ttrack->GetTrackElement();
796 if (trackElement) {
797 trackElement->DispatchTrackRunnable(NS_LITERAL_STRING("cuechange"));
798 }
799 }
800 }
801
802 mLastTimeMarchesOnCalled = currentPlaybackTime;
803 mLastActiveCues = currentCues;
804
805 // Step 18.
806 UpdateCueDisplay();
807 }
808
809 void
NotifyCueUpdated(TextTrackCue * aCue)810 TextTrackManager::NotifyCueUpdated(TextTrackCue *aCue)
811 {
812 // TODO: Add/Reorder the cue to mNewCues if we have some optimization?
813 WEBVTT_LOG("NotifyCueUpdated");
814 DispatchTimeMarchesOn();
815 }
816
817 void
NotifyReset()818 TextTrackManager::NotifyReset()
819 {
820 WEBVTT_LOG("NotifyReset");
821 mLastTimeMarchesOnCalled = 0.0;
822 }
823
824 void
ReportTelemetryForTrack(TextTrack * aTextTrack) const825 TextTrackManager::ReportTelemetryForTrack(TextTrack* aTextTrack) const
826 {
827 MOZ_ASSERT(NS_IsMainThread());
828 MOZ_ASSERT(aTextTrack);
829 MOZ_ASSERT(mTextTracks->Length() > 0);
830
831 TextTrackKind kind = aTextTrack->Kind();
832 Telemetry::Accumulate(Telemetry::WEBVTT_TRACK_KINDS, uint32_t(kind));
833 }
834
835 void
ReportTelemetryForCue()836 TextTrackManager::ReportTelemetryForCue()
837 {
838 MOZ_ASSERT(NS_IsMainThread());
839 MOZ_ASSERT(!mNewCues->IsEmpty() || !mLastActiveCues->IsEmpty());
840
841 if (!mCueTelemetryReported) {
842 Telemetry::Accumulate(Telemetry::WEBVTT_USED_VTT_CUES, 1);
843 mCueTelemetryReported = true;
844 }
845 }
846
847 } // namespace dom
848 } // namespace mozilla
849