1 /*
2  * Copyright (C) 2013 Google Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions are
6  * met:
7  *
8  *     * Redistributions of source code must retain the above copyright
9  * notice, this list of conditions and the following disclaimer.
10  *     * Redistributions in binary form must reproduce the above
11  * copyright notice, this list of conditions and the following disclaimer
12  * in the documentation and/or other materials provided with the
13  * distribution.
14  *     * Neither the name of Google Inc. nor the names of its
15  * contributors may be used to endorse or promote products derived from
16  * this software without specific prior written permission.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29  */
30 
31 #include "third_party/blink/renderer/modules/mediasource/source_buffer.h"
32 
33 #include <limits>
34 #include <memory>
35 #include <sstream>
36 
37 #include "media/base/logging_override_if_enabled.h"
38 #include "third_party/blink/public/platform/task_type.h"
39 #include "third_party/blink/public/platform/web_source_buffer.h"
40 #include "third_party/blink/renderer/core/dom/document.h"
41 #include "third_party/blink/renderer/core/dom/events/event.h"
42 #include "third_party/blink/renderer/core/dom/events/event_queue.h"
43 #include "third_party/blink/renderer/core/execution_context/execution_context.h"
44 #include "third_party/blink/renderer/core/frame/deprecation.h"
45 #include "third_party/blink/renderer/core/html/media/html_media_element.h"
46 #include "third_party/blink/renderer/core/html/time_ranges.h"
47 #include "third_party/blink/renderer/core/html/track/audio_track.h"
48 #include "third_party/blink/renderer/core/html/track/audio_track_list.h"
49 #include "third_party/blink/renderer/core/html/track/video_track.h"
50 #include "third_party/blink/renderer/core/html/track/video_track_list.h"
51 #include "third_party/blink/renderer/core/typed_arrays/dom_array_buffer.h"
52 #include "third_party/blink/renderer/core/typed_arrays/dom_array_buffer_view.h"
53 #include "third_party/blink/renderer/modules/mediasource/media_source_impl.h"
54 #include "third_party/blink/renderer/modules/mediasource/source_buffer_track_base_supplement.h"
55 #include "third_party/blink/renderer/platform/bindings/exception_messages.h"
56 #include "third_party/blink/renderer/platform/bindings/exception_state.h"
57 #include "third_party/blink/renderer/platform/heap/heap.h"
58 #include "third_party/blink/renderer/platform/instrumentation/tracing/trace_event.h"
59 #include "third_party/blink/renderer/platform/instrumentation/use_counter.h"
60 #include "third_party/blink/renderer/platform/network/mime/content_type.h"
61 #include "third_party/blink/renderer/platform/runtime_enabled_features.h"
62 #include "third_party/blink/renderer/platform/wtf/functional.h"
63 #include "third_party/blink/renderer/platform/wtf/math_extras.h"
64 
65 using blink::WebSourceBuffer;
66 
67 namespace blink {
68 
69 namespace {
70 
ThrowExceptionIfRemovedOrUpdating(bool is_removed,bool is_updating,ExceptionState & exception_state)71 static bool ThrowExceptionIfRemovedOrUpdating(bool is_removed,
72                                               bool is_updating,
73                                               ExceptionState& exception_state) {
74   if (is_removed) {
75     MediaSourceImpl::LogAndThrowDOMException(
76         exception_state, DOMExceptionCode::kInvalidStateError,
77         "This SourceBuffer has been removed from the parent media source.");
78     return true;
79   }
80   if (is_updating) {
81     MediaSourceImpl::LogAndThrowDOMException(
82         exception_state, DOMExceptionCode::kInvalidStateError,
83         "This SourceBuffer is still processing an 'appendBuffer' or "
84         "'remove' operation.");
85     return true;
86   }
87 
88   return false;
89 }
90 
WebTimeRangesToString(const WebTimeRanges & ranges)91 WTF::String WebTimeRangesToString(const WebTimeRanges& ranges) {
92   StringBuilder string_builder;
93   string_builder.Append('{');
94   for (auto& r : ranges) {
95     string_builder.Append(" [");
96     string_builder.AppendNumber(r.start);
97     string_builder.Append(';');
98     string_builder.AppendNumber(r.end);
99     string_builder.Append(']');
100   }
101   string_builder.Append(" }");
102   return string_builder.ToString();
103 }
104 
105 }  // namespace
106 
SourceBuffer(std::unique_ptr<WebSourceBuffer> web_source_buffer,MediaSourceImpl * source,EventQueue * async_event_queue)107 SourceBuffer::SourceBuffer(std::unique_ptr<WebSourceBuffer> web_source_buffer,
108                            MediaSourceImpl* source,
109                            EventQueue* async_event_queue)
110     : ExecutionContextLifecycleObserver(source->GetExecutionContext()),
111       web_source_buffer_(std::move(web_source_buffer)),
112       source_(source),
113       track_defaults_(MakeGarbageCollected<TrackDefaultList>()),
114       async_event_queue_(async_event_queue),
115       mode_(SegmentsKeyword()),
116       updating_(false),
117       timestamp_offset_(0),
118       append_window_start_(0),
119       append_window_end_(std::numeric_limits<double>::infinity()),
120       first_initialization_segment_received_(false),
121       pending_append_data_offset_(0),
122       pending_remove_start_(-1),
123       pending_remove_end_(-1) {
124   DVLOG(1) << __func__ << " this=" << this;
125 
126   DCHECK(web_source_buffer_);
127   DCHECK(source_);
128   DCHECK(source_->MediaElement());
129   audio_tracks_ =
130       MakeGarbageCollected<AudioTrackList>(*source_->MediaElement());
131   video_tracks_ =
132       MakeGarbageCollected<VideoTrackList>(*source_->MediaElement());
133   web_source_buffer_->SetClient(this);
134 }
135 
~SourceBuffer()136 SourceBuffer::~SourceBuffer() {
137   DVLOG(1) << __func__ << " this=" << this;
138 }
139 
Dispose()140 void SourceBuffer::Dispose() {
141   // Promptly clears a raw reference from content/ to an on-heap object
142   // so that content/ doesn't access it in a lazy sweeping phase.
143   web_source_buffer_.reset();
144 }
145 
SegmentsKeyword()146 const AtomicString& SourceBuffer::SegmentsKeyword() {
147   DEFINE_STATIC_LOCAL(const AtomicString, segments, ("segments"));
148   return segments;
149 }
150 
SequenceKeyword()151 const AtomicString& SourceBuffer::SequenceKeyword() {
152   DEFINE_STATIC_LOCAL(const AtomicString, sequence, ("sequence"));
153   return sequence;
154 }
155 
setMode(const AtomicString & new_mode,ExceptionState & exception_state)156 void SourceBuffer::setMode(const AtomicString& new_mode,
157                            ExceptionState& exception_state) {
158   DVLOG(3) << __func__ << " this=" << this << " new_mode=" << new_mode;
159   // Section 3.1 On setting mode attribute steps.
160   // https://www.w3.org/TR/media-source/#dom-sourcebuffer-mode
161   // 1. If this object has been removed from the sourceBuffers attribute of the
162   //    parent media source, then throw an INVALID_STATE_ERR exception and abort
163   //    these steps.
164   // 2. If the updating attribute equals true, then throw an INVALID_STATE_ERR
165   //    exception and abort these steps.
166   // 3. Let new mode equal the new value being assigned to this attribute.
167   if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
168                                         exception_state)) {
169     return;
170   }
171 
172   // 4. If generate timestamps flag equals true and new mode equals "segments",
173   //    then throw a TypeError exception and abort these steps.
174   if (web_source_buffer_->GetGenerateTimestampsFlag() &&
175       new_mode == SegmentsKeyword()) {
176     MediaSourceImpl::LogAndThrowTypeError(
177         exception_state, "The mode value provided (" + SegmentsKeyword() +
178                              ") is invalid for a byte stream format that uses "
179                              "generated timestamps.");
180     return;
181   }
182 
183   // 5. If the readyState attribute of the parent media source is in the "ended"
184   //    state then run the following steps:
185   // 5.1 Set the readyState attribute of the parent media source to "open"
186   // 5.2 Queue a task to fire a simple event named sourceopen at the parent
187   //     media source.
188   source_->OpenIfInEndedState();
189 
190   // 6. If the append state equals PARSING_MEDIA_SEGMENT, then throw an
191   //    INVALID_STATE_ERR and abort these steps.
192   // 7. If the new mode equals "sequence", then set the group start timestamp to
193   //    the highest presentation end timestamp.
194   WebSourceBuffer::AppendMode append_mode =
195       WebSourceBuffer::kAppendModeSegments;
196   if (new_mode == SequenceKeyword())
197     append_mode = WebSourceBuffer::kAppendModeSequence;
198   if (!web_source_buffer_->SetMode(append_mode)) {
199     MediaSourceImpl::LogAndThrowDOMException(
200         exception_state, DOMExceptionCode::kInvalidStateError,
201         "The mode may not be set while the SourceBuffer's append state is "
202         "'PARSING_MEDIA_SEGMENT'.");
203     return;
204   }
205 
206   // 8. Update the attribute to new mode.
207   mode_ = new_mode;
208 }
209 
buffered(ExceptionState & exception_state) const210 TimeRanges* SourceBuffer::buffered(ExceptionState& exception_state) const {
211   // Section 3.1 buffered attribute steps.
212   // 1. If this object has been removed from the sourceBuffers attribute of the
213   //    parent media source then throw an InvalidStateError exception and abort
214   //    these steps.
215   if (IsRemoved()) {
216     MediaSourceImpl::LogAndThrowDOMException(
217         exception_state, DOMExceptionCode::kInvalidStateError,
218         "This SourceBuffer has been removed from the parent media source.");
219     return nullptr;
220   }
221 
222   // 2. Return a new static normalized TimeRanges object for the media segments
223   //    buffered.
224   return MakeGarbageCollected<TimeRanges>(web_source_buffer_->Buffered());
225 }
226 
buffered() const227 WebTimeRanges SourceBuffer::buffered() const {
228   return web_source_buffer_->Buffered();
229 }
230 
timestampOffset() const231 double SourceBuffer::timestampOffset() const {
232   return timestamp_offset_;
233 }
234 
setTimestampOffset(double offset,ExceptionState & exception_state)235 void SourceBuffer::setTimestampOffset(double offset,
236                                       ExceptionState& exception_state) {
237   DVLOG(3) << __func__ << " this=" << this << " offset=" << offset;
238   // Section 3.1 timestampOffset attribute setter steps.
239   // https://dvcs.w3.org/hg/html-media/raw-file/tip/media-source/media-source.html#widl-SourceBuffer-timestampOffset
240   // 1. Let new timestamp offset equal the new value being assigned to this
241   //    attribute.
242   // 2. If this object has been removed from the sourceBuffers attribute of the
243   //    parent media source, then throw an InvalidStateError exception and abort
244   //    these steps.
245   // 3. If the updating attribute equals true, then throw an InvalidStateError
246   //    exception and abort these steps.
247   if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
248                                         exception_state))
249     return;
250 
251   // 4. If the readyState attribute of the parent media source is in the "ended"
252   //    state then run the following steps:
253   // 4.1 Set the readyState attribute of the parent media source to "open"
254   // 4.2 Queue a task to fire a simple event named sourceopen at the parent
255   //     media source.
256   source_->OpenIfInEndedState();
257 
258   // 5. If the append state equals PARSING_MEDIA_SEGMENT, then throw an
259   //    INVALID_STATE_ERR and abort these steps.
260   // 6. If the mode attribute equals "sequence", then set the group start
261   //    timestamp to new timestamp offset.
262   if (!web_source_buffer_->SetTimestampOffset(offset)) {
263     MediaSourceImpl::LogAndThrowDOMException(
264         exception_state, DOMExceptionCode::kInvalidStateError,
265         "The timestamp offset may not be set while the SourceBuffer's append "
266         "state is 'PARSING_MEDIA_SEGMENT'.");
267     return;
268   }
269 
270   // 7. Update the attribute to new timestamp offset.
271   timestamp_offset_ = offset;
272 }
273 
audioTracks()274 AudioTrackList& SourceBuffer::audioTracks() {
275   DCHECK(HTMLMediaElement::MediaTracksEnabledInternally());
276   return *audio_tracks_;
277 }
278 
videoTracks()279 VideoTrackList& SourceBuffer::videoTracks() {
280   DCHECK(HTMLMediaElement::MediaTracksEnabledInternally());
281   return *video_tracks_;
282 }
283 
appendWindowStart() const284 double SourceBuffer::appendWindowStart() const {
285   return append_window_start_;
286 }
287 
setAppendWindowStart(double start,ExceptionState & exception_state)288 void SourceBuffer::setAppendWindowStart(double start,
289                                         ExceptionState& exception_state) {
290   DVLOG(3) << __func__ << " this=" << this << " start=" << start;
291   // Section 3.1 appendWindowStart attribute setter steps.
292   // https://www.w3.org/TR/media-source/#widl-SourceBuffer-appendWindowStart
293   // 1. If this object has been removed from the sourceBuffers attribute of the
294   //    parent media source then throw an InvalidStateError exception and abort
295   //    these steps.
296   // 2. If the updating attribute equals true, then throw an InvalidStateError
297   //    exception and abort these steps.
298   if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
299                                         exception_state))
300     return;
301 
302   // 3. If the new value is less than 0 or greater than or equal to
303   //    appendWindowEnd then throw a TypeError exception and abort these steps.
304   if (start < 0 || start >= append_window_end_) {
305     MediaSourceImpl::LogAndThrowTypeError(
306         exception_state,
307         ExceptionMessages::IndexOutsideRange(
308             "value", start, 0.0, ExceptionMessages::kExclusiveBound,
309             append_window_end_, ExceptionMessages::kInclusiveBound));
310     return;
311   }
312 
313   web_source_buffer_->SetAppendWindowStart(start);
314 
315   // 4. Update the attribute to the new value.
316   append_window_start_ = start;
317 }
318 
appendWindowEnd() const319 double SourceBuffer::appendWindowEnd() const {
320   return append_window_end_;
321 }
322 
setAppendWindowEnd(double end,ExceptionState & exception_state)323 void SourceBuffer::setAppendWindowEnd(double end,
324                                       ExceptionState& exception_state) {
325   DVLOG(3) << __func__ << " this=" << this << " end=" << end;
326   // Section 3.1 appendWindowEnd attribute setter steps.
327   // https://www.w3.org/TR/media-source/#widl-SourceBuffer-appendWindowEnd
328   // 1. If this object has been removed from the sourceBuffers attribute of the
329   //    parent media source then throw an InvalidStateError exception and abort
330   //    these steps.
331   // 2. If the updating attribute equals true, then throw an InvalidStateError
332   //    exception and abort these steps.
333   if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
334                                         exception_state))
335     return;
336 
337   // 3. If the new value equals NaN, then throw a TypeError and abort these
338   //    steps.
339   if (std::isnan(end)) {
340     MediaSourceImpl::LogAndThrowTypeError(
341         exception_state, ExceptionMessages::NotAFiniteNumber(end));
342     return;
343   }
344   // 4. If the new value is less than or equal to appendWindowStart then throw a
345   //    TypeError exception and abort these steps.
346   if (end <= append_window_start_) {
347     MediaSourceImpl::LogAndThrowTypeError(
348         exception_state, ExceptionMessages::IndexExceedsMinimumBound(
349                              "value", end, append_window_start_));
350     return;
351   }
352 
353   web_source_buffer_->SetAppendWindowEnd(end);
354 
355   // 5. Update the attribute to the new value.
356   append_window_end_ = end;
357 }
358 
appendBuffer(DOMArrayBuffer * data,ExceptionState & exception_state)359 void SourceBuffer::appendBuffer(DOMArrayBuffer* data,
360                                 ExceptionState& exception_state) {
361   double media_time = GetMediaTime();
362   DVLOG(2) << __func__ << " this=" << this << " media_time=" << media_time
363            << " size=" << data->ByteLengthAsSizeT();
364   // Section 3.2 appendBuffer()
365   // https://dvcs.w3.org/hg/html-media/raw-file/default/media-source/media-source.html#widl-SourceBuffer-appendBuffer-void-ArrayBufferView-data
366   AppendBufferInternal(media_time,
367                        static_cast<const unsigned char*>(data->Data()),
368                        data->ByteLengthAsSizeT(), exception_state);
369 }
370 
appendBuffer(NotShared<DOMArrayBufferView> data,ExceptionState & exception_state)371 void SourceBuffer::appendBuffer(NotShared<DOMArrayBufferView> data,
372                                 ExceptionState& exception_state) {
373   double media_time = GetMediaTime();
374   DVLOG(3) << __func__ << " this=" << this << " media_time=" << media_time
375            << " size=" << data.View()->byteLengthAsSizeT();
376   // Section 3.2 appendBuffer()
377   // https://dvcs.w3.org/hg/html-media/raw-file/default/media-source/media-source.html#widl-SourceBuffer-appendBuffer-void-ArrayBufferView-data
378   AppendBufferInternal(
379       media_time, static_cast<const unsigned char*>(data.View()->BaseAddress()),
380       data.View()->byteLengthAsSizeT(), exception_state);
381 }
382 
abort(ExceptionState & exception_state)383 void SourceBuffer::abort(ExceptionState& exception_state) {
384   DVLOG(2) << __func__ << " this=" << this;
385   // http://w3c.github.io/media-source/#widl-SourceBuffer-abort-void
386   // 1. If this object has been removed from the sourceBuffers attribute of the
387   //    parent media source then throw an InvalidStateError exception and abort
388   //    these steps.
389   // 2. If the readyState attribute of the parent media source is not in the
390   //    "open" state then throw an InvalidStateError exception and abort these
391   //    steps.
392   if (IsRemoved()) {
393     MediaSourceImpl::LogAndThrowDOMException(
394         exception_state, DOMExceptionCode::kInvalidStateError,
395         "This SourceBuffer has been removed from the parent media source.");
396     return;
397   }
398   if (!source_->IsOpen()) {
399     MediaSourceImpl::LogAndThrowDOMException(
400         exception_state, DOMExceptionCode::kInvalidStateError,
401         "The parent media source's readyState is not 'open'.");
402     return;
403   }
404 
405   // 3. If the range removal algorithm is running, then throw an
406   //    InvalidStateError exception and abort these steps.
407   if (pending_remove_start_ != -1) {
408     DCHECK(updating_);
409     // Throwing the exception and aborting these steps is new behavior that
410     // is implemented behind the MediaSourceNewAbortAndDuration
411     // RuntimeEnabledFeature.
412     if (RuntimeEnabledFeatures::MediaSourceNewAbortAndDurationEnabled()) {
413       MediaSourceImpl::LogAndThrowDOMException(
414           exception_state, DOMExceptionCode::kInvalidStateError,
415           "Aborting asynchronous remove() operation is disallowed.");
416       return;
417     }
418 
419     Deprecation::CountDeprecation(source_->MediaElement()->GetDocument(),
420                                   WebFeature::kMediaSourceAbortRemove);
421     CancelRemove();
422   }
423 
424   // 4. If the sourceBuffer.updating attribute equals true, then run the
425   //    following steps: ...
426   AbortIfUpdating();
427 
428   // 5. Run the reset parser state algorithm.
429   web_source_buffer_->ResetParserState();
430 
431   // 6. Set appendWindowStart to 0.
432   setAppendWindowStart(0, exception_state);
433 
434   // 7. Set appendWindowEnd to positive Infinity.
435   setAppendWindowEnd(std::numeric_limits<double>::infinity(), exception_state);
436 }
437 
remove(double start,double end,ExceptionState & exception_state)438 void SourceBuffer::remove(double start,
439                           double end,
440                           ExceptionState& exception_state) {
441   DVLOG(2) << __func__ << " this=" << this << " start=" << start
442            << " end=" << end;
443 
444   // Section 3.2 remove() method steps.
445   // https://www.w3.org/TR/media-source/#widl-SourceBuffer-remove-void-double-start-unrestricted-double-end
446   // 1. If this object has been removed from the sourceBuffers attribute of the
447   //    parent media source then throw an InvalidStateError exception and abort
448   //    these steps.
449   // 2. If the updating attribute equals true, then throw an InvalidStateError
450   //    exception and abort these steps.
451   if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
452                                         exception_state))
453     return;
454 
455   // 3. If duration equals NaN, then throw a TypeError exception and abort these
456   //    steps.
457   // 4. If start is negative or greater than duration, then throw a TypeError
458   //    exception and abort these steps.
459   if (start < 0 || std::isnan(source_->duration()) ||
460       start > source_->duration()) {
461     MediaSourceImpl::LogAndThrowTypeError(
462         exception_state,
463         ExceptionMessages::IndexOutsideRange(
464             "start", start, 0.0, ExceptionMessages::kExclusiveBound,
465             std::isnan(source_->duration()) ? 0 : source_->duration(),
466             ExceptionMessages::kExclusiveBound));
467     return;
468   }
469 
470   // 5. If end is less than or equal to start or end equals NaN, then throw a
471   //    TypeError exception and abort these steps.
472   if (end <= start || std::isnan(end)) {
473     MediaSourceImpl::LogAndThrowTypeError(
474         exception_state,
475         "The end value provided (" + String::Number(end) +
476             ") must be greater than the start value provided (" +
477             String::Number(start) + ").");
478     return;
479   }
480 
481   TRACE_EVENT_NESTABLE_ASYNC_BEGIN0("media", "SourceBuffer::remove",
482                                     TRACE_ID_LOCAL(this));
483 
484   // 6. If the readyState attribute of the parent media source is in the "ended"
485   //    state then run the following steps:
486   // 6.1. Set the readyState attribute of the parent media source to "open"
487   // 6.2. Queue a task to fire a simple event named sourceopen at the parent
488   //      media source .
489   source_->OpenIfInEndedState();
490 
491   // 7. Run the range removal algorithm with start and end as the start and end
492   //    of the removal range.
493   // 7.3. Set the updating attribute to true.
494   updating_ = true;
495 
496   // 7.4. Queue a task to fire a simple event named updatestart at this
497   //      SourceBuffer object.
498   ScheduleEvent(event_type_names::kUpdatestart);
499 
500   // 7.5. Return control to the caller and run the rest of the steps
501   //      asynchronously.
502   pending_remove_start_ = start;
503   pending_remove_end_ = end;
504   remove_async_task_handle_ = PostCancellableTask(
505       *GetExecutionContext()->GetTaskRunner(TaskType::kMediaElementEvent),
506       FROM_HERE,
507       WTF::Bind(&SourceBuffer::RemoveAsyncPart, WrapPersistent(this)));
508 }
509 
changeType(const String & type,ExceptionState & exception_state)510 void SourceBuffer::changeType(const String& type,
511                               ExceptionState& exception_state) {
512   DVLOG(2) << __func__ << " this=" << this << " type=" << type;
513 
514   // Per 30 May 2018 Codec Switching feature incubation spec:
515   // https://rawgit.com/WICG/media-source/3b3742ea788999bb7ae4a4553ac7d574b0547dbe/index.html#dom-sourcebuffer-changetype
516   // 1. If type is an empty string then throw a TypeError exception and abort
517   //    these steps.
518   if (type.IsEmpty()) {
519     MediaSourceImpl::LogAndThrowTypeError(exception_state,
520                                           "The type provided is empty");
521     return;
522   }
523 
524   // 2. If this object has been removed from the sourceBuffers attribute of the
525   //    parent media source, then throw an InvalidStateError exception and abort
526   //    these steps.
527   // 3. If the updating attribute equals true, then throw an InvalidStateError
528   //    exception and abort these steps.
529   if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
530                                         exception_state))
531     return;
532 
533   // 4. If type contains a MIME type that is not supported or contains a MIME
534   //    type that is not supported with the types specified (currently or
535   //    previously) of SourceBuffer objects in the sourceBuffers attribute of
536   //    the parent media source, then throw a NotSupportedError exception and
537   //    abort these steps.
538   ContentType content_type(type);
539   String codecs = content_type.Parameter("codecs");
540   // TODO(wolenetz): Refactor and use a less-strict version of isTypeSupported
541   // here. As part of that, CanChangeType in Chromium should inherit relaxation
542   // of impl's StreamParserFactory (since it returns true iff a stream parser
543   // can be constructed with |type|). See https://crbug.com/535738.
544   if (!MediaSourceImpl::isTypeSupported(type) ||
545       !web_source_buffer_->CanChangeType(content_type.GetType(), codecs)) {
546     MediaSourceImpl::LogAndThrowDOMException(
547         exception_state, DOMExceptionCode::kNotSupportedError,
548         "Changing to the type provided ('" + type + "') is not supported.");
549     return;
550   }
551 
552   // 5. If the readyState attribute of the parent media source is in the "ended"
553   //    state then run the following steps:
554   //    1. Set the readyState attribute of the parent media source to "open"
555   //    2. Queue a task to fire a simple event named sourceopen at the parent
556   //       media source.
557   source_->OpenIfInEndedState();
558 
559   // 6. Run the reset parser state algorithm.
560   web_source_buffer_->ResetParserState();
561 
562   // 7. Update the generate timestamps flag on this SourceBuffer object to the
563   //    value in the "Generate Timestamps Flag" column of the byte stream format
564   //    registry entry that is associated with type.
565   // This call also updates the pipeline to switch bytestream parser and codecs.
566   web_source_buffer_->ChangeType(content_type.GetType(), codecs);
567 
568   // 8. If the generate timestamps flag equals true: Set the mode attribute on
569   //    this SourceBuffer object to "sequence", including running the associated
570   //    steps for that attribute being set. Otherwise: keep the previous value
571   //    of the mode attribute on this SourceBuffer object, without running any
572   //    associated steps for that attribute being set.
573   if (web_source_buffer_->GetGenerateTimestampsFlag())
574     setMode(SequenceKeyword(), exception_state);
575 
576   // 9. Set pending initialization segment for changeType flag to true.
577   // The logic for this flag is handled by the pipeline (the new bytestream
578   // parser will expect an initialization segment first).
579 }
580 
setTrackDefaults(TrackDefaultList * track_defaults,ExceptionState & exception_state)581 void SourceBuffer::setTrackDefaults(TrackDefaultList* track_defaults,
582                                     ExceptionState& exception_state) {
583   // Per 02 Dec 2014 Editor's Draft
584   // http://w3c.github.io/media-source/#widl-SourceBuffer-trackDefaults
585   // 1. If this object has been removed from the sourceBuffers attribute of
586   //    the parent media source, then throw an InvalidStateError exception
587   //    and abort these steps.
588   // 2. If the updating attribute equals true, then throw an InvalidStateError
589   //    exception and abort these steps.
590   if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
591                                         exception_state))
592     return;
593 
594   // 3. Update the attribute to the new value.
595   track_defaults_ = track_defaults;
596 }
597 
CancelRemove()598 void SourceBuffer::CancelRemove() {
599   DCHECK(updating_);
600   DCHECK_NE(pending_remove_start_, -1);
601   remove_async_task_handle_.Cancel();
602   pending_remove_start_ = -1;
603   pending_remove_end_ = -1;
604   updating_ = false;
605 
606   if (!RuntimeEnabledFeatures::MediaSourceNewAbortAndDurationEnabled()) {
607     ScheduleEvent(event_type_names::kAbort);
608     ScheduleEvent(event_type_names::kUpdateend);
609   }
610 
611   TRACE_EVENT_NESTABLE_ASYNC_END0("media", "SourceBuffer::remove",
612                                   TRACE_ID_LOCAL(this));
613 }
614 
AbortIfUpdating()615 void SourceBuffer::AbortIfUpdating() {
616   // Section 3.2 abort() method step 4 substeps.
617   // http://w3c.github.io/media-source/#widl-SourceBuffer-abort-void
618 
619   if (!updating_)
620     return;
621 
622   DCHECK_EQ(pending_remove_start_, -1);
623 
624   const char* trace_event_name = "SourceBuffer::appendBuffer";
625 
626   // 4.1. Abort the buffer append and stream append loop algorithms if they are
627   //      running.
628   append_buffer_async_task_handle_.Cancel();
629   pending_append_data_.clear();
630   pending_append_data_offset_ = 0;
631 
632   // 4.2. Set the updating attribute to false.
633   updating_ = false;
634 
635   // 4.3. Queue a task to fire a simple event named abort at this SourceBuffer
636   //      object.
637   ScheduleEvent(event_type_names::kAbort);
638 
639   // 4.4. Queue a task to fire a simple event named updateend at this
640   //      SourceBuffer object.
641   ScheduleEvent(event_type_names::kUpdateend);
642 
643   TRACE_EVENT_NESTABLE_ASYNC_END0("media", trace_event_name,
644                                   TRACE_ID_LOCAL(this));
645 }
646 
RemovedFromMediaSource()647 void SourceBuffer::RemovedFromMediaSource() {
648   if (IsRemoved())
649     return;
650 
651   DVLOG(3) << __func__ << " this=" << this;
652   if (pending_remove_start_ != -1) {
653     CancelRemove();
654   } else {
655     AbortIfUpdating();
656   }
657 
658   if (HTMLMediaElement::MediaTracksEnabledInternally()) {
659     DCHECK(source_);
660     if (source_->MediaElement()->audioTracks().length() > 0 ||
661         source_->MediaElement()->videoTracks().length() > 0) {
662       RemoveMediaTracks();
663     }
664   }
665 
666   web_source_buffer_->RemovedFromMediaSource();
667   web_source_buffer_.reset();
668   source_ = nullptr;
669   async_event_queue_ = nullptr;
670 }
671 
HighestPresentationTimestamp()672 double SourceBuffer::HighestPresentationTimestamp() {
673   DCHECK(!IsRemoved());
674 
675   double pts = web_source_buffer_->HighestPresentationTimestamp();
676   DVLOG(3) << __func__ << " this=" << this << ", pts=" << pts;
677   return pts;
678 }
679 
RemoveMediaTracks()680 void SourceBuffer::RemoveMediaTracks() {
681   DCHECK(HTMLMediaElement::MediaTracksEnabledInternally());
682   // Spec:
683   // http://w3c.github.io/media-source/#widl-MediaSource-removeSourceBuffer-void-SourceBuffer-sourceBuffer
684   DCHECK(source_);
685 
686   HTMLMediaElement* media_element = source_->MediaElement();
687   DCHECK(media_element);
688   // 3. Let SourceBuffer audioTracks list equal the AudioTrackList object
689   //    returned by sourceBuffer.audioTracks.
690   // 4. If the SourceBuffer audioTracks list is not empty, then run the
691   //    following steps:
692   // 4.1 Let HTMLMediaElement audioTracks list equal the AudioTrackList object
693   //     returned by the audioTracks attribute on the HTMLMediaElement.
694   // 4.2 Let the removed enabled audio track flag equal false.
695   bool removed_enabled_audio_track = false;
696   // 4.3 For each AudioTrack object in the SourceBuffer audioTracks list, run
697   //     the following steps:
698   while (audioTracks().length() > 0) {
699     AudioTrack* audio_track = audioTracks().AnonymousIndexedGetter(0);
700     // 4.3.1 Set the sourceBuffer attribute on the AudioTrack object to null.
701     SourceBufferTrackBaseSupplement::SetSourceBuffer(*audio_track, nullptr);
702     // 4.3.2 If the enabled attribute on the AudioTrack object is true, then set
703     //       the removed enabled audio track flag to true.
704     if (audio_track->enabled())
705       removed_enabled_audio_track = true;
706     // 4.3.3 Remove the AudioTrack object from the HTMLMediaElement audioTracks
707     //       list.
708     // 4.3.4 Queue a task to fire a trusted event named removetrack, that does
709     //       not bubble and is not cancelable, and that uses the TrackEvent
710     //       interface, at the HTMLMediaElement audioTracks list.
711     media_element->audioTracks().Remove(audio_track->id());
712     // 4.3.5 Remove the AudioTrack object from the SourceBuffer audioTracks
713     //       list.
714     // 4.3.6 Queue a task to fire a trusted event named removetrack, that does
715     //       not bubble and is not cancelable, and that uses the TrackEvent
716     //       interface, at the SourceBuffer audioTracks list.
717     audioTracks().Remove(audio_track->id());
718   }
719   // 4.4 If the removed enabled audio track flag equals true, then queue a task
720   //     to fire a simple event named change at the HTMLMediaElement audioTracks
721   //     list.
722   if (removed_enabled_audio_track) {
723     Event* event = Event::Create(event_type_names::kChange);
724     event->SetTarget(&media_element->audioTracks());
725     media_element->ScheduleEvent(event);
726   }
727 
728   // 5. Let SourceBuffer videoTracks list equal the VideoTrackList object
729   //    returned by sourceBuffer.videoTracks.
730   // 6. If the SourceBuffer videoTracks list is not empty, then run the
731   //    following steps:
732   // 6.1 Let HTMLMediaElement videoTracks list equal the VideoTrackList object
733   //     returned by the videoTracks attribute on the HTMLMediaElement.
734   // 6.2 Let the removed selected video track flag equal false.
735   bool removed_selected_video_track = false;
736   // 6.3 For each VideoTrack object in the SourceBuffer videoTracks list, run
737   //     the following steps:
738   while (videoTracks().length() > 0) {
739     VideoTrack* video_track = videoTracks().AnonymousIndexedGetter(0);
740     // 6.3.1 Set the sourceBuffer attribute on the VideoTrack object to null.
741     SourceBufferTrackBaseSupplement::SetSourceBuffer(*video_track, nullptr);
742     // 6.3.2 If the selected attribute on the VideoTrack object is true, then
743     //       set the removed selected video track flag to true.
744     if (video_track->selected())
745       removed_selected_video_track = true;
746     // 6.3.3 Remove the VideoTrack object from the HTMLMediaElement videoTracks
747     //       list.
748     // 6.3.4 Queue a task to fire a trusted event named removetrack, that does
749     //       not bubble and is not cancelable, and that uses the TrackEvent
750     //       interface, at the HTMLMediaElement videoTracks list.
751     media_element->videoTracks().Remove(video_track->id());
752     // 6.3.5 Remove the VideoTrack object from the SourceBuffer videoTracks
753     //       list.
754     // 6.3.6 Queue a task to fire a trusted event named removetrack, that does
755     //       not bubble and is not cancelable, and that uses the TrackEvent
756     //       interface, at the SourceBuffer videoTracks list.
757     videoTracks().Remove(video_track->id());
758   }
759   // 6.4 If the removed selected video track flag equals true, then queue a task
760   //     to fire a simple event named change at the HTMLMediaElement videoTracks
761   //     list.
762   if (removed_selected_video_track) {
763     Event* event = Event::Create(event_type_names::kChange);
764     event->SetTarget(&media_element->videoTracks());
765     media_element->ScheduleEvent(event);
766   }
767 
768   // 7-8. TODO(servolk): Remove text tracks once SourceBuffer has text tracks.
769 }
770 
GetMediaTime()771 double SourceBuffer::GetMediaTime() {
772   double media_time = std::numeric_limits<float>::quiet_NaN();
773   if (source_ && source_->MediaElement())
774     media_time = source_->MediaElement()->currentTime();
775   return media_time;
776 }
777 
778 template <class T>
FindExistingTrackById(const TrackListBase<T> & track_list,const String & id)779 T* FindExistingTrackById(const TrackListBase<T>& track_list, const String& id) {
780   // According to MSE specification
781   // (https://w3c.github.io/media-source/#sourcebuffer-init-segment-received)
782   // step 3.1:
783   // > If more than one track for a single type are present (ie 2 audio tracks),
784   // then the Track IDs match the ones in the first initialization segment.
785   // I.e. we only need to search by TrackID if there is more than one track,
786   // otherwise we can assume that the only track of the given type is the same
787   // one that we had in previous init segments.
788   if (track_list.length() == 1)
789     return track_list.AnonymousIndexedGetter(0);
790   return track_list.getTrackById(id);
791 }
792 
GetTrackDefault(const AtomicString & track_type,const AtomicString & byte_stream_track_id) const793 const TrackDefault* SourceBuffer::GetTrackDefault(
794     const AtomicString& track_type,
795     const AtomicString& byte_stream_track_id) const {
796   // This is a helper for implementation of default track label and default
797   // track language algorithms.
798   // defaultTrackLabel spec:
799   // https://w3c.github.io/media-source/#sourcebuffer-default-track-label
800   // defaultTrackLanguage spec:
801   // https://w3c.github.io/media-source/#sourcebuffer-default-track-language
802 
803   // 1. If trackDefaults contains a TrackDefault object with a type attribute
804   //    equal to type and a byteStreamTrackID attribute equal to
805   //    byteStreamTrackID, then return the value of the label/language attribute
806   //    on this matching object and abort these steps.
807   // 2. If trackDefaults contains a TrackDefault object with a type attribute
808   //    equal to type and a byteStreamTrackID attribute equal to an empty
809   //    string, then return the value of the label/language attribute on this
810   //    matching object and abort these steps.
811   // 3. Return an empty string to the caller
812   const TrackDefault* track_default_with_empty_bytestream_id = nullptr;
813   for (unsigned i = 0; i < track_defaults_->length(); ++i) {
814     const TrackDefault* track_default = track_defaults_->item(i);
815     if (track_default->type() != track_type)
816       continue;
817     if (track_default->byteStreamTrackID() == byte_stream_track_id)
818       return track_default;
819     if (!track_default_with_empty_bytestream_id &&
820         track_default->byteStreamTrackID() == "")
821       track_default_with_empty_bytestream_id = track_default;
822   }
823   return track_default_with_empty_bytestream_id;
824 }
825 
DefaultTrackLabel(const AtomicString & track_type,const AtomicString & byte_stream_track_id) const826 AtomicString SourceBuffer::DefaultTrackLabel(
827     const AtomicString& track_type,
828     const AtomicString& byte_stream_track_id) const {
829   // Spec: https://w3c.github.io/media-source/#sourcebuffer-default-track-label
830   const TrackDefault* track_default =
831       GetTrackDefault(track_type, byte_stream_track_id);
832   return track_default ? AtomicString(track_default->label()) : "";
833 }
834 
DefaultTrackLanguage(const AtomicString & track_type,const AtomicString & byte_stream_track_id) const835 AtomicString SourceBuffer::DefaultTrackLanguage(
836     const AtomicString& track_type,
837     const AtomicString& byte_stream_track_id) const {
838   // Spec:
839   // https://w3c.github.io/media-source/#sourcebuffer-default-track-language
840   const TrackDefault* track_default =
841       GetTrackDefault(track_type, byte_stream_track_id);
842   return track_default ? AtomicString(track_default->language()) : "";
843 }
844 
InitializationSegmentReceived(const WebVector<MediaTrackInfo> & new_tracks)845 bool SourceBuffer::InitializationSegmentReceived(
846     const WebVector<MediaTrackInfo>& new_tracks) {
847   DVLOG(3) << __func__ << " this=" << this << " tracks=" << new_tracks.size();
848   DCHECK(source_);
849   DCHECK(source_->MediaElement());
850   DCHECK(updating_);
851 
852   if (!HTMLMediaElement::MediaTracksEnabledInternally()) {
853     if (!first_initialization_segment_received_) {
854       source_->SetSourceBufferActive(this, true);
855       first_initialization_segment_received_ = true;
856     }
857     return true;
858   }
859 
860   // Implementation of Initialization Segment Received, see
861   // https://w3c.github.io/media-source/#sourcebuffer-init-segment-received
862 
863   // Sort newTracks into audio and video tracks to facilitate implementation
864   // of subsequent steps of this algorithm.
865   Vector<MediaTrackInfo> new_audio_tracks;
866   Vector<MediaTrackInfo> new_video_tracks;
867   for (const MediaTrackInfo& track_info : new_tracks) {
868     const TrackBase* track = nullptr;
869     if (track_info.track_type == WebMediaPlayer::kAudioTrack) {
870       new_audio_tracks.push_back(track_info);
871       if (first_initialization_segment_received_)
872         track = FindExistingTrackById(audioTracks(), track_info.id);
873     } else if (track_info.track_type == WebMediaPlayer::kVideoTrack) {
874       new_video_tracks.push_back(track_info);
875       if (first_initialization_segment_received_)
876         track = FindExistingTrackById(videoTracks(), track_info.id);
877     } else {
878       DVLOG(3) << __func__ << " this=" << this
879                << " failed: unsupported track type " << track_info.track_type;
880       // TODO(servolk): Add handling of text tracks.
881       NOTREACHED();
882     }
883     if (first_initialization_segment_received_ && !track) {
884       DVLOG(3) << __func__ << " this=" << this
885                << " failed: tracks mismatch the first init segment.";
886       return false;
887     }
888 #if DCHECK_IS_ON()
889     const char* log_track_type_str =
890         (track_info.track_type == WebMediaPlayer::kAudioTrack) ? "audio"
891                                                                : "video";
892     DVLOG(3) << __func__ << " this=" << this << " : " << log_track_type_str
893              << " track "
894              << " id=" << String(track_info.id)
895              << " byteStreamTrackID=" << String(track_info.byte_stream_track_id)
896              << " kind=" << String(track_info.kind)
897              << " label=" << String(track_info.label)
898              << " language=" << String(track_info.language);
899 #endif
900   }
901 
902   // 1. Update the duration attribute if it currently equals NaN:
903   // TODO(servolk): Pass also stream duration into initSegmentReceived.
904 
905   // 2. If the initialization segment has no audio, video, or text tracks, then
906   //    run the append error algorithm with the decode error parameter set to
907   //    true and abort these steps.
908   if (new_tracks.empty()) {
909     DVLOG(3) << __func__ << " this=" << this
910              << " failed: no tracks found in the init segment.";
911     // The append error algorithm will be called at the top level after we
912     // return false here to indicate failure.
913     return false;
914   }
915 
916   // 3. If the first initialization segment received flag is true, then run the
917   //    following steps:
918   if (first_initialization_segment_received_) {
919     // 3.1 Verify the following properties. If any of the checks fail then run
920     //     the append error algorithm with the decode error parameter set to
921     //     true and abort these steps.
922     bool tracks_match_first_init_segment = true;
923     // - The number of audio, video, and text tracks match what was in the first
924     //   initialization segment.
925     if (new_audio_tracks.size() != audioTracks().length() ||
926         new_video_tracks.size() != videoTracks().length()) {
927       tracks_match_first_init_segment = false;
928     }
929     // - The codecs for each track, match what was specified in the first
930     //   initialization segment.
931     // This is currently done in MediaSourceState::OnNewConfigs.
932     // - If more than one track for a single type are present (ie 2 audio
933     //   tracks), then the Track IDs match the ones in the first initialization
934     //   segment.
935     if (tracks_match_first_init_segment && new_audio_tracks.size() > 1) {
936       for (wtf_size_t i = 0; i < new_audio_tracks.size(); ++i) {
937         const String& new_track_id = new_video_tracks[i].id;
938         if (new_track_id !=
939             String(audioTracks().AnonymousIndexedGetter(i)->id())) {
940           tracks_match_first_init_segment = false;
941           break;
942         }
943       }
944     }
945 
946     if (tracks_match_first_init_segment && new_video_tracks.size() > 1) {
947       for (wtf_size_t i = 0; i < new_video_tracks.size(); ++i) {
948         const String& new_track_id = new_video_tracks[i].id;
949         if (new_track_id !=
950             String(videoTracks().AnonymousIndexedGetter(i)->id())) {
951           tracks_match_first_init_segment = false;
952           break;
953         }
954       }
955     }
956 
957     if (!tracks_match_first_init_segment) {
958       DVLOG(3) << __func__ << " this=" << this
959                << " failed: tracks mismatch the first init segment.";
960       // The append error algorithm will be called at the top level after we
961       // return false here to indicate failure.
962       return false;
963     }
964 
965     // 3.2 Add the appropriate track descriptions from this initialization
966     //     segment to each of the track buffers.  This is done in Chromium code
967     //     in stream parsers and demuxer implementations.
968 
969     // 3.3 Set the need random access point flag on all track buffers to true.
970     // This is done in Chromium code, see MediaSourceState::OnNewConfigs.
971   }
972 
973   // 4. Let active track flag equal false.
974   bool active_track = false;
975 
976   // 5. If the first initialization segment received flag is false, then run the
977   //    following steps:
978   if (!first_initialization_segment_received_) {
979     // 5.1 If the initialization segment contains tracks with codecs the user
980     //     agent does not support, then run the append error algorithm with the
981     //     decode error parameter set to true and abort these steps.
982     // This is done in Chromium code, see MediaSourceState::OnNewConfigs.
983 
984     // 5.2 For each audio track in the initialization segment, run following
985     //     steps:
986     for (const MediaTrackInfo& track_info : new_audio_tracks) {
987       // 5.2.1 Let audio byte stream track ID be the Track ID for the current
988       //       track being processed.
989       const auto& byte_stream_track_id = track_info.byte_stream_track_id;
990       // 5.2.2 Let audio language be a BCP 47 language tag for the language
991       //       specified in the initialization segment for this track or an
992       //       empty string if no language info is present.
993       WebString language = track_info.language;
994       // 5.2.3 If audio language equals an empty string or the 'und' BCP 47
995       //       value, then run the default track language algorithm with
996       //       byteStreamTrackID set to audio byte stream track ID and type set
997       //       to "audio" and assign the value returned by the algorithm to
998       //       audio language.
999       if (language.IsEmpty() || language == "und")
1000         language = DefaultTrackLanguage(TrackDefault::AudioKeyword(),
1001                                         byte_stream_track_id);
1002       // 5.2.4 Let audio label be a label specified in the initialization
1003       //       segment for this track or an empty string if no label info is
1004       //       present.
1005       WebString label = track_info.label;
1006       // 5.3.5 If audio label equals an empty string, then run the default track
1007       //       label algorithm with byteStreamTrackID set to audio byte stream
1008       //       track ID and type set to "audio" and assign the value returned by
1009       //       the algorithm to audio label.
1010       if (label.IsEmpty())
1011         label = DefaultTrackLabel(TrackDefault::AudioKeyword(),
1012                                   byte_stream_track_id);
1013       // 5.2.6 Let audio kinds be an array of kind strings specified in the
1014       //       initialization segment for this track or an empty array if no
1015       //       kind information is provided.
1016       const auto& kind = track_info.kind;
1017       // 5.2.7 TODO(servolk): Implement track kind processing.
1018       // 5.2.8.2 Let new audio track be a new AudioTrack object.
1019       auto* audio_track = MakeGarbageCollected<AudioTrack>(
1020           track_info.id, kind, label, language, false);
1021       SourceBufferTrackBaseSupplement::SetSourceBuffer(*audio_track, this);
1022       // 5.2.8.7 If audioTracks.length equals 0, then run the following steps:
1023       if (audioTracks().length() == 0) {
1024         // 5.2.8.7.1 Set the enabled property on new audio track to true.
1025         audio_track->setEnabled(true);
1026         // 5.2.8.7.2 Set active track flag to true.
1027         active_track = true;
1028       }
1029       // 5.2.8.8 Add new audio track to the audioTracks attribute on this
1030       //         SourceBuffer object.
1031       // 5.2.8.9 Queue a task to fire a trusted event named addtrack, that does
1032       //         not bubble and is not cancelable, and that uses the TrackEvent
1033       //         interface, at the AudioTrackList object referenced by the
1034       //         audioTracks attribute on this SourceBuffer object.
1035       audioTracks().Add(audio_track);
1036       // 5.2.8.10 Add new audio track to the audioTracks attribute on the
1037       //          HTMLMediaElement.
1038       // 5.2.8.11 Queue a task to fire a trusted event named addtrack, that does
1039       //          not bubble and is not cancelable, and that uses the TrackEvent
1040       //          interface, at the AudioTrackList object referenced by the
1041       //          audioTracks attribute on the HTMLMediaElement.
1042       source_->MediaElement()->audioTracks().Add(audio_track);
1043     }
1044 
1045     // 5.3. For each video track in the initialization segment, run following
1046     //      steps:
1047     for (const MediaTrackInfo& track_info : new_video_tracks) {
1048       // 5.3.1 Let video byte stream track ID be the Track ID for the current
1049       //       track being processed.
1050       const auto& byte_stream_track_id = track_info.byte_stream_track_id;
1051       // 5.3.2 Let video language be a BCP 47 language tag for the language
1052       //       specified in the initialization segment for this track or an
1053       //       empty string if no language info is present.
1054       WebString language = track_info.language;
1055       // 5.3.3 If video language equals an empty string or the 'und' BCP 47
1056       //       value, then run the default track language algorithm with
1057       //       byteStreamTrackID set to video byte stream track ID and type set
1058       //       to "video" and assign the value returned by the algorithm to
1059       //       video language.
1060       if (language.IsEmpty() || language == "und")
1061         language = DefaultTrackLanguage(TrackDefault::VideoKeyword(),
1062                                         byte_stream_track_id);
1063       // 5.3.4 Let video label be a label specified in the initialization
1064       //       segment for this track or an empty string if no label info is
1065       //       present.
1066       WebString label = track_info.label;
1067       // 5.3.5 If video label equals an empty string, then run the default track
1068       //       label algorithm with byteStreamTrackID set to video byte stream
1069       //       track ID and type set to "video" and assign the value returned by
1070       //       the algorithm to video label.
1071       if (label.IsEmpty())
1072         label = DefaultTrackLabel(TrackDefault::VideoKeyword(),
1073                                   byte_stream_track_id);
1074       // 5.3.6 Let video kinds be an array of kind strings specified in the
1075       //       initialization segment for this track or an empty array if no
1076       //       kind information is provided.
1077       const auto& kind = track_info.kind;
1078       // 5.3.7 TODO(servolk): Implement track kind processing.
1079       // 5.3.8.2 Let new video track be a new VideoTrack object.
1080       auto* video_track = MakeGarbageCollected<VideoTrack>(
1081           track_info.id, kind, label, language, false);
1082       SourceBufferTrackBaseSupplement::SetSourceBuffer(*video_track, this);
1083       // 5.3.8.7 If videoTracks.length equals 0, then run the following steps:
1084       if (videoTracks().length() == 0) {
1085         // 5.3.8.7.1 Set the selected property on new audio track to true.
1086         video_track->setSelected(true);
1087         // 5.3.8.7.2 Set active track flag to true.
1088         active_track = true;
1089       }
1090       // 5.3.8.8 Add new video track to the videoTracks attribute on this
1091       //         SourceBuffer object.
1092       // 5.3.8.9 Queue a task to fire a trusted event named addtrack, that does
1093       //         not bubble and is not cancelable, and that uses the TrackEvent
1094       //         interface, at the VideoTrackList object referenced by the
1095       //         videoTracks attribute on this SourceBuffer object.
1096       videoTracks().Add(video_track);
1097       // 5.3.8.10 Add new video track to the videoTracks attribute on the
1098       //          HTMLMediaElement.
1099       // 5.3.8.11 Queue a task to fire a trusted event named addtrack, that does
1100       //          not bubble and is not cancelable, and that uses the TrackEvent
1101       //          interface, at the VideoTrackList object referenced by the
1102       //          videoTracks attribute on the HTMLMediaElement.
1103       source_->MediaElement()->videoTracks().Add(video_track);
1104     }
1105 
1106     // 5.4 TODO(servolk): Add text track processing here.
1107 
1108     // 5.5 If active track flag equals true, then run the following steps:
1109     // activesourcebuffers.
1110     if (active_track) {
1111       // 5.5.1 Add this SourceBuffer to activeSourceBuffers.
1112       // 5.5.2 Queue a task to fire a simple event named addsourcebuffer at
1113       //       activeSourceBuffers
1114       source_->SetSourceBufferActive(this, true);
1115     }
1116 
1117     // 5.6. Set first initialization segment received flag to true.
1118     first_initialization_segment_received_ = true;
1119   }
1120 
1121   return true;
1122 }
1123 
NotifyParseWarning(const ParseWarning warning)1124 void SourceBuffer::NotifyParseWarning(const ParseWarning warning) {
1125   switch (warning) {
1126     case WebSourceBufferClient::kKeyframeTimeGreaterThanDependant:
1127       // Report this problematic GOP structure to help inform follow-up work.
1128       // Media engine also records RAPPOR for these, up to once per track, at
1129       // Media.OriginUrl.MSE.KeyframeTimeGreaterThanDependant.
1130       // TODO(wolenetz): Use the data to scope additional work. See
1131       // https://crbug.com/739931.
1132       UseCounter::Count(
1133           source_->MediaElement()->GetDocument(),
1134           WebFeature::kMediaSourceKeyframeTimeGreaterThanDependant);
1135       break;
1136     case WebSourceBufferClient::kMuxedSequenceMode:
1137       // Report this problematic API usage to help inform follow-up work.
1138       // Media engine also records RAPPOR for these, up to once per
1139       // SourceBuffer, at Media.OriginUrl.MSE.MuxedSequenceModeSourceBuffer.
1140       // TODO(wolenetz): Use the data to scope additional work. See
1141       // https://crbug.com/737757.
1142       UseCounter::Count(source_->MediaElement()->GetDocument(),
1143                         WebFeature::kMediaSourceMuxedSequenceMode);
1144       break;
1145     case WebSourceBufferClient::kGroupEndTimestampDecreaseWithinMediaSegment:
1146       // Report this problematic Media Segment structure usage to help inform
1147       // follow-up work.
1148       // TODO(wolenetz): Use the data to scope additional work. See
1149       // https://crbug.com/920853 and
1150       // https://github.com/w3c/media-source/issues/203.
1151       UseCounter::Count(
1152           source_->MediaElement()->GetDocument(),
1153           WebFeature::kMediaSourceGroupEndTimestampDecreaseWithinMediaSegment);
1154       break;
1155   }
1156 }
1157 
HasPendingActivity() const1158 bool SourceBuffer::HasPendingActivity() const {
1159   return updating_ || append_buffer_async_task_handle_.IsActive() ||
1160          remove_async_task_handle_.IsActive() ||
1161          (async_event_queue_ && async_event_queue_->HasPendingEvents());
1162 }
1163 
ContextDestroyed()1164 void SourceBuffer::ContextDestroyed() {
1165   append_buffer_async_task_handle_.Cancel();
1166   remove_async_task_handle_.Cancel();
1167   updating_ = false;
1168 }
1169 
GetExecutionContext() const1170 ExecutionContext* SourceBuffer::GetExecutionContext() const {
1171   return ExecutionContextLifecycleObserver::GetExecutionContext();
1172 }
1173 
InterfaceName() const1174 const AtomicString& SourceBuffer::InterfaceName() const {
1175   return event_target_names::kSourceBuffer;
1176 }
1177 
IsRemoved() const1178 bool SourceBuffer::IsRemoved() const {
1179   return !source_;
1180 }
1181 
ScheduleEvent(const AtomicString & event_name)1182 void SourceBuffer::ScheduleEvent(const AtomicString& event_name) {
1183   DCHECK(async_event_queue_);
1184 
1185   Event* event = Event::Create(event_name);
1186   event->SetTarget(this);
1187 
1188   async_event_queue_->EnqueueEvent(FROM_HERE, *event);
1189 }
1190 
PrepareAppend(double media_time,size_t new_data_size,ExceptionState & exception_state)1191 bool SourceBuffer::PrepareAppend(double media_time,
1192                                  size_t new_data_size,
1193                                  ExceptionState& exception_state) {
1194   TRACE_EVENT_NESTABLE_ASYNC_BEGIN0("media", "SourceBuffer::prepareAppend",
1195                                     TRACE_ID_LOCAL(this));
1196   // http://w3c.github.io/media-source/#sourcebuffer-prepare-append
1197   // 3.5.4 Prepare Append Algorithm
1198   // 1. If the SourceBuffer has been removed from the sourceBuffers attribute of
1199   //    the parent media source then throw an InvalidStateError exception and
1200   //    abort these steps.
1201   // 2. If the updating attribute equals true, then throw an InvalidStateError
1202   //    exception and abort these steps.
1203   if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
1204                                         exception_state)) {
1205     TRACE_EVENT_NESTABLE_ASYNC_END0("media", "SourceBuffer::prepareAppend",
1206                                     TRACE_ID_LOCAL(this));
1207     return false;
1208   }
1209 
1210   // 3. If the HTMLMediaElement.error attribute is not null, then throw an
1211   //    InvalidStateError exception and abort these steps.
1212   DCHECK(source_);
1213   DCHECK(source_->MediaElement());
1214   if (source_->MediaElement()->error()) {
1215     MediaSourceImpl::LogAndThrowDOMException(
1216         exception_state, DOMExceptionCode::kInvalidStateError,
1217         "The HTMLMediaElement.error attribute is not null.");
1218     TRACE_EVENT_NESTABLE_ASYNC_END0("media", "SourceBuffer::prepareAppend",
1219                                     TRACE_ID_LOCAL(this));
1220     return false;
1221   }
1222 
1223   // 4. If the readyState attribute of the parent media source is in the "ended"
1224   //    state then run the following steps:
1225   //    1. Set the readyState attribute of the parent media source to "open"
1226   //    2. Queue a task to fire a simple event named sourceopen at the parent
1227   //       media source.
1228   source_->OpenIfInEndedState();
1229 
1230   // 5. Run the coded frame eviction algorithm.
1231   if (!EvictCodedFrames(media_time, new_data_size) ||
1232       !base::CheckedNumeric<wtf_size_t>(new_data_size).IsValid()) {
1233     // 6. If the buffer full flag equals true, then throw a QUOTA_EXCEEDED_ERR
1234     //    exception and abort these steps.
1235     //    If the incoming data exceeds wtf_size_t::max, then our implementation
1236     //    cannot deal with it, so we also throw a QuotaExceededError.
1237     DVLOG(3) << __func__ << " this=" << this << " -> throw QuotaExceededError";
1238     MediaSourceImpl::LogAndThrowDOMException(
1239         exception_state, DOMExceptionCode::kQuotaExceededError,
1240         "The SourceBuffer is full, and cannot free space to append additional "
1241         "buffers.");
1242     TRACE_EVENT_NESTABLE_ASYNC_END0("media", "SourceBuffer::prepareAppend",
1243                                     TRACE_ID_LOCAL(this));
1244     return false;
1245   }
1246 
1247   TRACE_EVENT_NESTABLE_ASYNC_END0("media", "SourceBuffer::prepareAppend",
1248                                   TRACE_ID_LOCAL(this));
1249   return true;
1250 }
1251 
EvictCodedFrames(double media_time,size_t new_data_size)1252 bool SourceBuffer::EvictCodedFrames(double media_time, size_t new_data_size) {
1253   DCHECK(source_);
1254   DCHECK(source_->MediaElement());
1255 
1256   // Nothing to do if the mediaElement does not yet have frames to evict.
1257   if (source_->MediaElement()->getReadyState() <
1258       HTMLMediaElement::kHaveMetadata) {
1259     return true;
1260   }
1261 
1262   bool result = web_source_buffer_->EvictCodedFrames(media_time, new_data_size);
1263   if (!result) {
1264     DVLOG(3) << __func__ << " this=" << this
1265              << " failed. newDataSize=" << new_data_size
1266              << " media_time=" << media_time << " buffered="
1267              << WebTimeRangesToString(web_source_buffer_->Buffered());
1268   }
1269   return result;
1270 }
1271 
AppendBufferInternal(double media_time,const unsigned char * data,size_t size,ExceptionState & exception_state)1272 void SourceBuffer::AppendBufferInternal(double media_time,
1273                                         const unsigned char* data,
1274                                         size_t size,
1275                                         ExceptionState& exception_state) {
1276   TRACE_EVENT_NESTABLE_ASYNC_BEGIN1("media", "SourceBuffer::appendBuffer",
1277                                     TRACE_ID_LOCAL(this), "size", size);
1278   // Section 3.2 appendBuffer()
1279   // https://dvcs.w3.org/hg/html-media/raw-file/default/media-source/media-source.html#widl-SourceBuffer-appendBuffer-void-ArrayBufferView-data
1280 
1281   // 1. Run the prepare append algorithm.
1282   if (!PrepareAppend(media_time, size, exception_state)) {
1283     TRACE_EVENT_NESTABLE_ASYNC_END0("media", "SourceBuffer::appendBuffer",
1284                                     TRACE_ID_LOCAL(this));
1285     return;
1286   }
1287   TRACE_EVENT_NESTABLE_ASYNC_BEGIN0("media", "prepareAppend",
1288                                     TRACE_ID_LOCAL(this));
1289 
1290   // 2. Add data to the end of the input buffer.
1291   DCHECK(data || size == 0);
1292   if (data)
1293     pending_append_data_.Append(data, base::checked_cast<wtf_size_t>(size));
1294   pending_append_data_offset_ = 0;
1295 
1296   // 3. Set the updating attribute to true.
1297   updating_ = true;
1298 
1299   // 4. Queue a task to fire a simple event named updatestart at this
1300   //    SourceBuffer object.
1301   ScheduleEvent(event_type_names::kUpdatestart);
1302 
1303   // 5. Asynchronously run the buffer append algorithm.
1304   append_buffer_async_task_handle_ = PostCancellableTask(
1305       *GetExecutionContext()->GetTaskRunner(TaskType::kMediaElementEvent),
1306       FROM_HERE,
1307       WTF::Bind(&SourceBuffer::AppendBufferAsyncPart, WrapPersistent(this)));
1308 
1309   TRACE_EVENT_NESTABLE_ASYNC_END0("media", "prepareAppend",
1310                                   TRACE_ID_LOCAL(this));
1311   TRACE_EVENT_NESTABLE_ASYNC_BEGIN1("media", "delay", TRACE_ID_LOCAL(this),
1312                                     "type", "initialDelay");
1313 }
1314 
AppendBufferAsyncPart()1315 void SourceBuffer::AppendBufferAsyncPart() {
1316   DCHECK(updating_);
1317 
1318   // Section 3.5.4 Buffer Append Algorithm
1319   // https://dvcs.w3.org/hg/html-media/raw-file/default/media-source/media-source.html#sourcebuffer-buffer-append
1320 
1321   // 1. Run the segment parser loop algorithm.
1322   // Step 2 doesn't apply since we run Step 1 synchronously here.
1323   DCHECK_GE(pending_append_data_.size(), pending_append_data_offset_);
1324   wtf_size_t append_size =
1325       pending_append_data_.size() - pending_append_data_offset_;
1326 
1327   // Impose an arbitrary max size for a single append() call so that an append
1328   // doesn't block the renderer event loop very long. This value was selected
1329   // by looking at YouTube SourceBuffer usage across a variety of bitrates.
1330   // This value allows relatively large appends while keeping append() call
1331   // duration in the  ~5-15ms range.
1332   const wtf_size_t kMaxAppendSize = 128 * 1024;
1333   if (append_size > kMaxAppendSize)
1334     append_size = kMaxAppendSize;
1335 
1336   TRACE_EVENT_NESTABLE_ASYNC_END0("media", "delay", TRACE_ID_LOCAL(this));
1337   TRACE_EVENT_NESTABLE_ASYNC_BEGIN1("media", "appending", TRACE_ID_LOCAL(this),
1338                                     "appendSize", append_size);
1339 
1340   // |zero| is used for 0 byte appends so we always have a valid pointer.
1341   // We need to convey all appends, even 0 byte ones to |m_webSourceBuffer|
1342   // so that it can clear its end of stream state if necessary.
1343   unsigned char zero = 0;
1344   unsigned char* append_data = &zero;
1345   if (append_size)
1346     append_data = pending_append_data_.data() + pending_append_data_offset_;
1347 
1348   bool append_success =
1349       web_source_buffer_->Append(append_data, append_size, &timestamp_offset_);
1350 
1351   if (!append_success) {
1352     pending_append_data_.clear();
1353     pending_append_data_offset_ = 0;
1354     AppendError();
1355   } else {
1356     pending_append_data_offset_ += append_size;
1357 
1358     if (pending_append_data_offset_ < pending_append_data_.size()) {
1359       append_buffer_async_task_handle_ = PostCancellableTask(
1360           *GetExecutionContext()->GetTaskRunner(TaskType::kMediaElementEvent),
1361           FROM_HERE,
1362           WTF::Bind(&SourceBuffer::AppendBufferAsyncPart,
1363                     WrapPersistent(this)));
1364       TRACE_EVENT_NESTABLE_ASYNC_END0("media", "appending",
1365                                       TRACE_ID_LOCAL(this));
1366       TRACE_EVENT_NESTABLE_ASYNC_BEGIN1("media", "delay", TRACE_ID_LOCAL(this),
1367                                         "type", "nextPieceDelay");
1368       return;
1369     }
1370 
1371     // 3. Set the updating attribute to false.
1372     updating_ = false;
1373     pending_append_data_.clear();
1374     pending_append_data_offset_ = 0;
1375 
1376     // 4. Queue a task to fire a simple event named update at this SourceBuffer
1377     //    object.
1378     ScheduleEvent(event_type_names::kUpdate);
1379 
1380     // 5. Queue a task to fire a simple event named updateend at this
1381     //    SourceBuffer object.
1382     ScheduleEvent(event_type_names::kUpdateend);
1383   }
1384 
1385   TRACE_EVENT_NESTABLE_ASYNC_END0("media", "appending", TRACE_ID_LOCAL(this));
1386   TRACE_EVENT_NESTABLE_ASYNC_END0("media", "SourceBuffer::appendBuffer",
1387                                   TRACE_ID_LOCAL(this));
1388 
1389   double media_time = GetMediaTime();
1390   DVLOG(3) << __func__ << " done. this=" << this << " media_time=" << media_time
1391            << " buffered="
1392            << WebTimeRangesToString(web_source_buffer_->Buffered());
1393 }
1394 
RemoveAsyncPart()1395 void SourceBuffer::RemoveAsyncPart() {
1396   DCHECK(updating_);
1397   DCHECK_GE(pending_remove_start_, 0);
1398   DCHECK_LT(pending_remove_start_, pending_remove_end_);
1399 
1400   // Section 3.2 remove() method steps
1401   // https://dvcs.w3.org/hg/html-media/raw-file/default/media-source/media-source.html#widl-SourceBuffer-remove-void-double-start-double-end
1402 
1403   // 9. Run the coded frame removal algorithm with start and end as the start
1404   //    and end of the removal range.
1405   web_source_buffer_->Remove(pending_remove_start_, pending_remove_end_);
1406 
1407   // 10. Set the updating attribute to false.
1408   updating_ = false;
1409   pending_remove_start_ = -1;
1410   pending_remove_end_ = -1;
1411 
1412   // 11. Queue a task to fire a simple event named update at this SourceBuffer
1413   //     object.
1414   ScheduleEvent(event_type_names::kUpdate);
1415 
1416   // 12. Queue a task to fire a simple event named updateend at this
1417   //     SourceBuffer object.
1418   ScheduleEvent(event_type_names::kUpdateend);
1419 }
1420 
AppendError()1421 void SourceBuffer::AppendError() {
1422   DVLOG(3) << __func__ << " this=" << this;
1423   // Section 3.5.3 Append Error Algorithm
1424   // https://dvcs.w3.org/hg/html-media/raw-file/default/media-source/media-source.html#sourcebuffer-append-error
1425 
1426   // 1. Run the reset parser state algorithm.
1427   web_source_buffer_->ResetParserState();
1428 
1429   // 2. Set the updating attribute to false.
1430   updating_ = false;
1431 
1432   // 3. Queue a task to fire a simple event named error at this SourceBuffer
1433   //    object.
1434   ScheduleEvent(event_type_names::kError);
1435 
1436   // 4. Queue a task to fire a simple event named updateend at this SourceBuffer
1437   //    object.
1438   ScheduleEvent(event_type_names::kUpdateend);
1439 
1440   // 5. If decode error is true, then run the end of stream algorithm with the
1441   // error parameter set to "decode".
1442   source_->EndOfStreamAlgorithm(WebMediaSource::kEndOfStreamStatusDecodeError);
1443 }
1444 
Trace(Visitor * visitor)1445 void SourceBuffer::Trace(Visitor* visitor) {
1446   visitor->Trace(source_);
1447   visitor->Trace(track_defaults_);
1448   visitor->Trace(async_event_queue_);
1449   visitor->Trace(audio_tracks_);
1450   visitor->Trace(video_tracks_);
1451   EventTargetWithInlineData::Trace(visitor);
1452   ExecutionContextLifecycleObserver::Trace(visitor);
1453 }
1454 
1455 }  // namespace blink
1456