1 /*
2 * Copyright (C) 2013 Google Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions are
6 * met:
7 *
8 * * Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * * Redistributions in binary form must reproduce the above
11 * copyright notice, this list of conditions and the following disclaimer
12 * in the documentation and/or other materials provided with the
13 * distribution.
14 * * Neither the name of Google Inc. nor the names of its
15 * contributors may be used to endorse or promote products derived from
16 * this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31 #include "third_party/blink/renderer/modules/mediasource/source_buffer.h"
32
33 #include <limits>
34 #include <memory>
35 #include <sstream>
36
37 #include "media/base/logging_override_if_enabled.h"
38 #include "third_party/blink/public/platform/task_type.h"
39 #include "third_party/blink/public/platform/web_source_buffer.h"
40 #include "third_party/blink/renderer/core/dom/document.h"
41 #include "third_party/blink/renderer/core/dom/events/event.h"
42 #include "third_party/blink/renderer/core/dom/events/event_queue.h"
43 #include "third_party/blink/renderer/core/execution_context/execution_context.h"
44 #include "third_party/blink/renderer/core/frame/deprecation.h"
45 #include "third_party/blink/renderer/core/html/media/html_media_element.h"
46 #include "third_party/blink/renderer/core/html/time_ranges.h"
47 #include "third_party/blink/renderer/core/html/track/audio_track.h"
48 #include "third_party/blink/renderer/core/html/track/audio_track_list.h"
49 #include "third_party/blink/renderer/core/html/track/video_track.h"
50 #include "third_party/blink/renderer/core/html/track/video_track_list.h"
51 #include "third_party/blink/renderer/core/typed_arrays/dom_array_buffer.h"
52 #include "third_party/blink/renderer/core/typed_arrays/dom_array_buffer_view.h"
53 #include "third_party/blink/renderer/modules/mediasource/media_source_impl.h"
54 #include "third_party/blink/renderer/modules/mediasource/source_buffer_track_base_supplement.h"
55 #include "third_party/blink/renderer/platform/bindings/exception_messages.h"
56 #include "third_party/blink/renderer/platform/bindings/exception_state.h"
57 #include "third_party/blink/renderer/platform/heap/heap.h"
58 #include "third_party/blink/renderer/platform/instrumentation/tracing/trace_event.h"
59 #include "third_party/blink/renderer/platform/instrumentation/use_counter.h"
60 #include "third_party/blink/renderer/platform/network/mime/content_type.h"
61 #include "third_party/blink/renderer/platform/runtime_enabled_features.h"
62 #include "third_party/blink/renderer/platform/wtf/functional.h"
63 #include "third_party/blink/renderer/platform/wtf/math_extras.h"
64
65 using blink::WebSourceBuffer;
66
67 namespace blink {
68
69 namespace {
70
ThrowExceptionIfRemovedOrUpdating(bool is_removed,bool is_updating,ExceptionState & exception_state)71 static bool ThrowExceptionIfRemovedOrUpdating(bool is_removed,
72 bool is_updating,
73 ExceptionState& exception_state) {
74 if (is_removed) {
75 MediaSourceImpl::LogAndThrowDOMException(
76 exception_state, DOMExceptionCode::kInvalidStateError,
77 "This SourceBuffer has been removed from the parent media source.");
78 return true;
79 }
80 if (is_updating) {
81 MediaSourceImpl::LogAndThrowDOMException(
82 exception_state, DOMExceptionCode::kInvalidStateError,
83 "This SourceBuffer is still processing an 'appendBuffer' or "
84 "'remove' operation.");
85 return true;
86 }
87
88 return false;
89 }
90
WebTimeRangesToString(const WebTimeRanges & ranges)91 WTF::String WebTimeRangesToString(const WebTimeRanges& ranges) {
92 StringBuilder string_builder;
93 string_builder.Append('{');
94 for (auto& r : ranges) {
95 string_builder.Append(" [");
96 string_builder.AppendNumber(r.start);
97 string_builder.Append(';');
98 string_builder.AppendNumber(r.end);
99 string_builder.Append(']');
100 }
101 string_builder.Append(" }");
102 return string_builder.ToString();
103 }
104
105 } // namespace
106
SourceBuffer(std::unique_ptr<WebSourceBuffer> web_source_buffer,MediaSourceImpl * source,EventQueue * async_event_queue)107 SourceBuffer::SourceBuffer(std::unique_ptr<WebSourceBuffer> web_source_buffer,
108 MediaSourceImpl* source,
109 EventQueue* async_event_queue)
110 : ExecutionContextLifecycleObserver(source->GetExecutionContext()),
111 web_source_buffer_(std::move(web_source_buffer)),
112 source_(source),
113 track_defaults_(MakeGarbageCollected<TrackDefaultList>()),
114 async_event_queue_(async_event_queue),
115 mode_(SegmentsKeyword()),
116 updating_(false),
117 timestamp_offset_(0),
118 append_window_start_(0),
119 append_window_end_(std::numeric_limits<double>::infinity()),
120 first_initialization_segment_received_(false),
121 pending_append_data_offset_(0),
122 pending_remove_start_(-1),
123 pending_remove_end_(-1) {
124 DVLOG(1) << __func__ << " this=" << this;
125
126 DCHECK(web_source_buffer_);
127 DCHECK(source_);
128 DCHECK(source_->MediaElement());
129 audio_tracks_ =
130 MakeGarbageCollected<AudioTrackList>(*source_->MediaElement());
131 video_tracks_ =
132 MakeGarbageCollected<VideoTrackList>(*source_->MediaElement());
133 web_source_buffer_->SetClient(this);
134 }
135
~SourceBuffer()136 SourceBuffer::~SourceBuffer() {
137 DVLOG(1) << __func__ << " this=" << this;
138 }
139
Dispose()140 void SourceBuffer::Dispose() {
141 // Promptly clears a raw reference from content/ to an on-heap object
142 // so that content/ doesn't access it in a lazy sweeping phase.
143 web_source_buffer_.reset();
144 }
145
SegmentsKeyword()146 const AtomicString& SourceBuffer::SegmentsKeyword() {
147 DEFINE_STATIC_LOCAL(const AtomicString, segments, ("segments"));
148 return segments;
149 }
150
SequenceKeyword()151 const AtomicString& SourceBuffer::SequenceKeyword() {
152 DEFINE_STATIC_LOCAL(const AtomicString, sequence, ("sequence"));
153 return sequence;
154 }
155
setMode(const AtomicString & new_mode,ExceptionState & exception_state)156 void SourceBuffer::setMode(const AtomicString& new_mode,
157 ExceptionState& exception_state) {
158 DVLOG(3) << __func__ << " this=" << this << " new_mode=" << new_mode;
159 // Section 3.1 On setting mode attribute steps.
160 // https://www.w3.org/TR/media-source/#dom-sourcebuffer-mode
161 // 1. If this object has been removed from the sourceBuffers attribute of the
162 // parent media source, then throw an INVALID_STATE_ERR exception and abort
163 // these steps.
164 // 2. If the updating attribute equals true, then throw an INVALID_STATE_ERR
165 // exception and abort these steps.
166 // 3. Let new mode equal the new value being assigned to this attribute.
167 if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
168 exception_state)) {
169 return;
170 }
171
172 // 4. If generate timestamps flag equals true and new mode equals "segments",
173 // then throw a TypeError exception and abort these steps.
174 if (web_source_buffer_->GetGenerateTimestampsFlag() &&
175 new_mode == SegmentsKeyword()) {
176 MediaSourceImpl::LogAndThrowTypeError(
177 exception_state, "The mode value provided (" + SegmentsKeyword() +
178 ") is invalid for a byte stream format that uses "
179 "generated timestamps.");
180 return;
181 }
182
183 // 5. If the readyState attribute of the parent media source is in the "ended"
184 // state then run the following steps:
185 // 5.1 Set the readyState attribute of the parent media source to "open"
186 // 5.2 Queue a task to fire a simple event named sourceopen at the parent
187 // media source.
188 source_->OpenIfInEndedState();
189
190 // 6. If the append state equals PARSING_MEDIA_SEGMENT, then throw an
191 // INVALID_STATE_ERR and abort these steps.
192 // 7. If the new mode equals "sequence", then set the group start timestamp to
193 // the highest presentation end timestamp.
194 WebSourceBuffer::AppendMode append_mode =
195 WebSourceBuffer::kAppendModeSegments;
196 if (new_mode == SequenceKeyword())
197 append_mode = WebSourceBuffer::kAppendModeSequence;
198 if (!web_source_buffer_->SetMode(append_mode)) {
199 MediaSourceImpl::LogAndThrowDOMException(
200 exception_state, DOMExceptionCode::kInvalidStateError,
201 "The mode may not be set while the SourceBuffer's append state is "
202 "'PARSING_MEDIA_SEGMENT'.");
203 return;
204 }
205
206 // 8. Update the attribute to new mode.
207 mode_ = new_mode;
208 }
209
buffered(ExceptionState & exception_state) const210 TimeRanges* SourceBuffer::buffered(ExceptionState& exception_state) const {
211 // Section 3.1 buffered attribute steps.
212 // 1. If this object has been removed from the sourceBuffers attribute of the
213 // parent media source then throw an InvalidStateError exception and abort
214 // these steps.
215 if (IsRemoved()) {
216 MediaSourceImpl::LogAndThrowDOMException(
217 exception_state, DOMExceptionCode::kInvalidStateError,
218 "This SourceBuffer has been removed from the parent media source.");
219 return nullptr;
220 }
221
222 // 2. Return a new static normalized TimeRanges object for the media segments
223 // buffered.
224 return MakeGarbageCollected<TimeRanges>(web_source_buffer_->Buffered());
225 }
226
buffered() const227 WebTimeRanges SourceBuffer::buffered() const {
228 return web_source_buffer_->Buffered();
229 }
230
timestampOffset() const231 double SourceBuffer::timestampOffset() const {
232 return timestamp_offset_;
233 }
234
setTimestampOffset(double offset,ExceptionState & exception_state)235 void SourceBuffer::setTimestampOffset(double offset,
236 ExceptionState& exception_state) {
237 DVLOG(3) << __func__ << " this=" << this << " offset=" << offset;
238 // Section 3.1 timestampOffset attribute setter steps.
239 // https://dvcs.w3.org/hg/html-media/raw-file/tip/media-source/media-source.html#widl-SourceBuffer-timestampOffset
240 // 1. Let new timestamp offset equal the new value being assigned to this
241 // attribute.
242 // 2. If this object has been removed from the sourceBuffers attribute of the
243 // parent media source, then throw an InvalidStateError exception and abort
244 // these steps.
245 // 3. If the updating attribute equals true, then throw an InvalidStateError
246 // exception and abort these steps.
247 if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
248 exception_state))
249 return;
250
251 // 4. If the readyState attribute of the parent media source is in the "ended"
252 // state then run the following steps:
253 // 4.1 Set the readyState attribute of the parent media source to "open"
254 // 4.2 Queue a task to fire a simple event named sourceopen at the parent
255 // media source.
256 source_->OpenIfInEndedState();
257
258 // 5. If the append state equals PARSING_MEDIA_SEGMENT, then throw an
259 // INVALID_STATE_ERR and abort these steps.
260 // 6. If the mode attribute equals "sequence", then set the group start
261 // timestamp to new timestamp offset.
262 if (!web_source_buffer_->SetTimestampOffset(offset)) {
263 MediaSourceImpl::LogAndThrowDOMException(
264 exception_state, DOMExceptionCode::kInvalidStateError,
265 "The timestamp offset may not be set while the SourceBuffer's append "
266 "state is 'PARSING_MEDIA_SEGMENT'.");
267 return;
268 }
269
270 // 7. Update the attribute to new timestamp offset.
271 timestamp_offset_ = offset;
272 }
273
audioTracks()274 AudioTrackList& SourceBuffer::audioTracks() {
275 DCHECK(HTMLMediaElement::MediaTracksEnabledInternally());
276 return *audio_tracks_;
277 }
278
videoTracks()279 VideoTrackList& SourceBuffer::videoTracks() {
280 DCHECK(HTMLMediaElement::MediaTracksEnabledInternally());
281 return *video_tracks_;
282 }
283
appendWindowStart() const284 double SourceBuffer::appendWindowStart() const {
285 return append_window_start_;
286 }
287
setAppendWindowStart(double start,ExceptionState & exception_state)288 void SourceBuffer::setAppendWindowStart(double start,
289 ExceptionState& exception_state) {
290 DVLOG(3) << __func__ << " this=" << this << " start=" << start;
291 // Section 3.1 appendWindowStart attribute setter steps.
292 // https://www.w3.org/TR/media-source/#widl-SourceBuffer-appendWindowStart
293 // 1. If this object has been removed from the sourceBuffers attribute of the
294 // parent media source then throw an InvalidStateError exception and abort
295 // these steps.
296 // 2. If the updating attribute equals true, then throw an InvalidStateError
297 // exception and abort these steps.
298 if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
299 exception_state))
300 return;
301
302 // 3. If the new value is less than 0 or greater than or equal to
303 // appendWindowEnd then throw a TypeError exception and abort these steps.
304 if (start < 0 || start >= append_window_end_) {
305 MediaSourceImpl::LogAndThrowTypeError(
306 exception_state,
307 ExceptionMessages::IndexOutsideRange(
308 "value", start, 0.0, ExceptionMessages::kExclusiveBound,
309 append_window_end_, ExceptionMessages::kInclusiveBound));
310 return;
311 }
312
313 web_source_buffer_->SetAppendWindowStart(start);
314
315 // 4. Update the attribute to the new value.
316 append_window_start_ = start;
317 }
318
appendWindowEnd() const319 double SourceBuffer::appendWindowEnd() const {
320 return append_window_end_;
321 }
322
setAppendWindowEnd(double end,ExceptionState & exception_state)323 void SourceBuffer::setAppendWindowEnd(double end,
324 ExceptionState& exception_state) {
325 DVLOG(3) << __func__ << " this=" << this << " end=" << end;
326 // Section 3.1 appendWindowEnd attribute setter steps.
327 // https://www.w3.org/TR/media-source/#widl-SourceBuffer-appendWindowEnd
328 // 1. If this object has been removed from the sourceBuffers attribute of the
329 // parent media source then throw an InvalidStateError exception and abort
330 // these steps.
331 // 2. If the updating attribute equals true, then throw an InvalidStateError
332 // exception and abort these steps.
333 if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
334 exception_state))
335 return;
336
337 // 3. If the new value equals NaN, then throw a TypeError and abort these
338 // steps.
339 if (std::isnan(end)) {
340 MediaSourceImpl::LogAndThrowTypeError(
341 exception_state, ExceptionMessages::NotAFiniteNumber(end));
342 return;
343 }
344 // 4. If the new value is less than or equal to appendWindowStart then throw a
345 // TypeError exception and abort these steps.
346 if (end <= append_window_start_) {
347 MediaSourceImpl::LogAndThrowTypeError(
348 exception_state, ExceptionMessages::IndexExceedsMinimumBound(
349 "value", end, append_window_start_));
350 return;
351 }
352
353 web_source_buffer_->SetAppendWindowEnd(end);
354
355 // 5. Update the attribute to the new value.
356 append_window_end_ = end;
357 }
358
appendBuffer(DOMArrayBuffer * data,ExceptionState & exception_state)359 void SourceBuffer::appendBuffer(DOMArrayBuffer* data,
360 ExceptionState& exception_state) {
361 double media_time = GetMediaTime();
362 DVLOG(2) << __func__ << " this=" << this << " media_time=" << media_time
363 << " size=" << data->ByteLengthAsSizeT();
364 // Section 3.2 appendBuffer()
365 // https://dvcs.w3.org/hg/html-media/raw-file/default/media-source/media-source.html#widl-SourceBuffer-appendBuffer-void-ArrayBufferView-data
366 AppendBufferInternal(media_time,
367 static_cast<const unsigned char*>(data->Data()),
368 data->ByteLengthAsSizeT(), exception_state);
369 }
370
appendBuffer(NotShared<DOMArrayBufferView> data,ExceptionState & exception_state)371 void SourceBuffer::appendBuffer(NotShared<DOMArrayBufferView> data,
372 ExceptionState& exception_state) {
373 double media_time = GetMediaTime();
374 DVLOG(3) << __func__ << " this=" << this << " media_time=" << media_time
375 << " size=" << data.View()->byteLengthAsSizeT();
376 // Section 3.2 appendBuffer()
377 // https://dvcs.w3.org/hg/html-media/raw-file/default/media-source/media-source.html#widl-SourceBuffer-appendBuffer-void-ArrayBufferView-data
378 AppendBufferInternal(
379 media_time, static_cast<const unsigned char*>(data.View()->BaseAddress()),
380 data.View()->byteLengthAsSizeT(), exception_state);
381 }
382
abort(ExceptionState & exception_state)383 void SourceBuffer::abort(ExceptionState& exception_state) {
384 DVLOG(2) << __func__ << " this=" << this;
385 // http://w3c.github.io/media-source/#widl-SourceBuffer-abort-void
386 // 1. If this object has been removed from the sourceBuffers attribute of the
387 // parent media source then throw an InvalidStateError exception and abort
388 // these steps.
389 // 2. If the readyState attribute of the parent media source is not in the
390 // "open" state then throw an InvalidStateError exception and abort these
391 // steps.
392 if (IsRemoved()) {
393 MediaSourceImpl::LogAndThrowDOMException(
394 exception_state, DOMExceptionCode::kInvalidStateError,
395 "This SourceBuffer has been removed from the parent media source.");
396 return;
397 }
398 if (!source_->IsOpen()) {
399 MediaSourceImpl::LogAndThrowDOMException(
400 exception_state, DOMExceptionCode::kInvalidStateError,
401 "The parent media source's readyState is not 'open'.");
402 return;
403 }
404
405 // 3. If the range removal algorithm is running, then throw an
406 // InvalidStateError exception and abort these steps.
407 if (pending_remove_start_ != -1) {
408 DCHECK(updating_);
409 // Throwing the exception and aborting these steps is new behavior that
410 // is implemented behind the MediaSourceNewAbortAndDuration
411 // RuntimeEnabledFeature.
412 if (RuntimeEnabledFeatures::MediaSourceNewAbortAndDurationEnabled()) {
413 MediaSourceImpl::LogAndThrowDOMException(
414 exception_state, DOMExceptionCode::kInvalidStateError,
415 "Aborting asynchronous remove() operation is disallowed.");
416 return;
417 }
418
419 Deprecation::CountDeprecation(source_->MediaElement()->GetDocument(),
420 WebFeature::kMediaSourceAbortRemove);
421 CancelRemove();
422 }
423
424 // 4. If the sourceBuffer.updating attribute equals true, then run the
425 // following steps: ...
426 AbortIfUpdating();
427
428 // 5. Run the reset parser state algorithm.
429 web_source_buffer_->ResetParserState();
430
431 // 6. Set appendWindowStart to 0.
432 setAppendWindowStart(0, exception_state);
433
434 // 7. Set appendWindowEnd to positive Infinity.
435 setAppendWindowEnd(std::numeric_limits<double>::infinity(), exception_state);
436 }
437
remove(double start,double end,ExceptionState & exception_state)438 void SourceBuffer::remove(double start,
439 double end,
440 ExceptionState& exception_state) {
441 DVLOG(2) << __func__ << " this=" << this << " start=" << start
442 << " end=" << end;
443
444 // Section 3.2 remove() method steps.
445 // https://www.w3.org/TR/media-source/#widl-SourceBuffer-remove-void-double-start-unrestricted-double-end
446 // 1. If this object has been removed from the sourceBuffers attribute of the
447 // parent media source then throw an InvalidStateError exception and abort
448 // these steps.
449 // 2. If the updating attribute equals true, then throw an InvalidStateError
450 // exception and abort these steps.
451 if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
452 exception_state))
453 return;
454
455 // 3. If duration equals NaN, then throw a TypeError exception and abort these
456 // steps.
457 // 4. If start is negative or greater than duration, then throw a TypeError
458 // exception and abort these steps.
459 if (start < 0 || std::isnan(source_->duration()) ||
460 start > source_->duration()) {
461 MediaSourceImpl::LogAndThrowTypeError(
462 exception_state,
463 ExceptionMessages::IndexOutsideRange(
464 "start", start, 0.0, ExceptionMessages::kExclusiveBound,
465 std::isnan(source_->duration()) ? 0 : source_->duration(),
466 ExceptionMessages::kExclusiveBound));
467 return;
468 }
469
470 // 5. If end is less than or equal to start or end equals NaN, then throw a
471 // TypeError exception and abort these steps.
472 if (end <= start || std::isnan(end)) {
473 MediaSourceImpl::LogAndThrowTypeError(
474 exception_state,
475 "The end value provided (" + String::Number(end) +
476 ") must be greater than the start value provided (" +
477 String::Number(start) + ").");
478 return;
479 }
480
481 TRACE_EVENT_NESTABLE_ASYNC_BEGIN0("media", "SourceBuffer::remove",
482 TRACE_ID_LOCAL(this));
483
484 // 6. If the readyState attribute of the parent media source is in the "ended"
485 // state then run the following steps:
486 // 6.1. Set the readyState attribute of the parent media source to "open"
487 // 6.2. Queue a task to fire a simple event named sourceopen at the parent
488 // media source .
489 source_->OpenIfInEndedState();
490
491 // 7. Run the range removal algorithm with start and end as the start and end
492 // of the removal range.
493 // 7.3. Set the updating attribute to true.
494 updating_ = true;
495
496 // 7.4. Queue a task to fire a simple event named updatestart at this
497 // SourceBuffer object.
498 ScheduleEvent(event_type_names::kUpdatestart);
499
500 // 7.5. Return control to the caller and run the rest of the steps
501 // asynchronously.
502 pending_remove_start_ = start;
503 pending_remove_end_ = end;
504 remove_async_task_handle_ = PostCancellableTask(
505 *GetExecutionContext()->GetTaskRunner(TaskType::kMediaElementEvent),
506 FROM_HERE,
507 WTF::Bind(&SourceBuffer::RemoveAsyncPart, WrapPersistent(this)));
508 }
509
changeType(const String & type,ExceptionState & exception_state)510 void SourceBuffer::changeType(const String& type,
511 ExceptionState& exception_state) {
512 DVLOG(2) << __func__ << " this=" << this << " type=" << type;
513
514 // Per 30 May 2018 Codec Switching feature incubation spec:
515 // https://rawgit.com/WICG/media-source/3b3742ea788999bb7ae4a4553ac7d574b0547dbe/index.html#dom-sourcebuffer-changetype
516 // 1. If type is an empty string then throw a TypeError exception and abort
517 // these steps.
518 if (type.IsEmpty()) {
519 MediaSourceImpl::LogAndThrowTypeError(exception_state,
520 "The type provided is empty");
521 return;
522 }
523
524 // 2. If this object has been removed from the sourceBuffers attribute of the
525 // parent media source, then throw an InvalidStateError exception and abort
526 // these steps.
527 // 3. If the updating attribute equals true, then throw an InvalidStateError
528 // exception and abort these steps.
529 if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
530 exception_state))
531 return;
532
533 // 4. If type contains a MIME type that is not supported or contains a MIME
534 // type that is not supported with the types specified (currently or
535 // previously) of SourceBuffer objects in the sourceBuffers attribute of
536 // the parent media source, then throw a NotSupportedError exception and
537 // abort these steps.
538 ContentType content_type(type);
539 String codecs = content_type.Parameter("codecs");
540 // TODO(wolenetz): Refactor and use a less-strict version of isTypeSupported
541 // here. As part of that, CanChangeType in Chromium should inherit relaxation
542 // of impl's StreamParserFactory (since it returns true iff a stream parser
543 // can be constructed with |type|). See https://crbug.com/535738.
544 if (!MediaSourceImpl::isTypeSupported(type) ||
545 !web_source_buffer_->CanChangeType(content_type.GetType(), codecs)) {
546 MediaSourceImpl::LogAndThrowDOMException(
547 exception_state, DOMExceptionCode::kNotSupportedError,
548 "Changing to the type provided ('" + type + "') is not supported.");
549 return;
550 }
551
552 // 5. If the readyState attribute of the parent media source is in the "ended"
553 // state then run the following steps:
554 // 1. Set the readyState attribute of the parent media source to "open"
555 // 2. Queue a task to fire a simple event named sourceopen at the parent
556 // media source.
557 source_->OpenIfInEndedState();
558
559 // 6. Run the reset parser state algorithm.
560 web_source_buffer_->ResetParserState();
561
562 // 7. Update the generate timestamps flag on this SourceBuffer object to the
563 // value in the "Generate Timestamps Flag" column of the byte stream format
564 // registry entry that is associated with type.
565 // This call also updates the pipeline to switch bytestream parser and codecs.
566 web_source_buffer_->ChangeType(content_type.GetType(), codecs);
567
568 // 8. If the generate timestamps flag equals true: Set the mode attribute on
569 // this SourceBuffer object to "sequence", including running the associated
570 // steps for that attribute being set. Otherwise: keep the previous value
571 // of the mode attribute on this SourceBuffer object, without running any
572 // associated steps for that attribute being set.
573 if (web_source_buffer_->GetGenerateTimestampsFlag())
574 setMode(SequenceKeyword(), exception_state);
575
576 // 9. Set pending initialization segment for changeType flag to true.
577 // The logic for this flag is handled by the pipeline (the new bytestream
578 // parser will expect an initialization segment first).
579 }
580
setTrackDefaults(TrackDefaultList * track_defaults,ExceptionState & exception_state)581 void SourceBuffer::setTrackDefaults(TrackDefaultList* track_defaults,
582 ExceptionState& exception_state) {
583 // Per 02 Dec 2014 Editor's Draft
584 // http://w3c.github.io/media-source/#widl-SourceBuffer-trackDefaults
585 // 1. If this object has been removed from the sourceBuffers attribute of
586 // the parent media source, then throw an InvalidStateError exception
587 // and abort these steps.
588 // 2. If the updating attribute equals true, then throw an InvalidStateError
589 // exception and abort these steps.
590 if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
591 exception_state))
592 return;
593
594 // 3. Update the attribute to the new value.
595 track_defaults_ = track_defaults;
596 }
597
CancelRemove()598 void SourceBuffer::CancelRemove() {
599 DCHECK(updating_);
600 DCHECK_NE(pending_remove_start_, -1);
601 remove_async_task_handle_.Cancel();
602 pending_remove_start_ = -1;
603 pending_remove_end_ = -1;
604 updating_ = false;
605
606 if (!RuntimeEnabledFeatures::MediaSourceNewAbortAndDurationEnabled()) {
607 ScheduleEvent(event_type_names::kAbort);
608 ScheduleEvent(event_type_names::kUpdateend);
609 }
610
611 TRACE_EVENT_NESTABLE_ASYNC_END0("media", "SourceBuffer::remove",
612 TRACE_ID_LOCAL(this));
613 }
614
AbortIfUpdating()615 void SourceBuffer::AbortIfUpdating() {
616 // Section 3.2 abort() method step 4 substeps.
617 // http://w3c.github.io/media-source/#widl-SourceBuffer-abort-void
618
619 if (!updating_)
620 return;
621
622 DCHECK_EQ(pending_remove_start_, -1);
623
624 const char* trace_event_name = "SourceBuffer::appendBuffer";
625
626 // 4.1. Abort the buffer append and stream append loop algorithms if they are
627 // running.
628 append_buffer_async_task_handle_.Cancel();
629 pending_append_data_.clear();
630 pending_append_data_offset_ = 0;
631
632 // 4.2. Set the updating attribute to false.
633 updating_ = false;
634
635 // 4.3. Queue a task to fire a simple event named abort at this SourceBuffer
636 // object.
637 ScheduleEvent(event_type_names::kAbort);
638
639 // 4.4. Queue a task to fire a simple event named updateend at this
640 // SourceBuffer object.
641 ScheduleEvent(event_type_names::kUpdateend);
642
643 TRACE_EVENT_NESTABLE_ASYNC_END0("media", trace_event_name,
644 TRACE_ID_LOCAL(this));
645 }
646
RemovedFromMediaSource()647 void SourceBuffer::RemovedFromMediaSource() {
648 if (IsRemoved())
649 return;
650
651 DVLOG(3) << __func__ << " this=" << this;
652 if (pending_remove_start_ != -1) {
653 CancelRemove();
654 } else {
655 AbortIfUpdating();
656 }
657
658 if (HTMLMediaElement::MediaTracksEnabledInternally()) {
659 DCHECK(source_);
660 if (source_->MediaElement()->audioTracks().length() > 0 ||
661 source_->MediaElement()->videoTracks().length() > 0) {
662 RemoveMediaTracks();
663 }
664 }
665
666 web_source_buffer_->RemovedFromMediaSource();
667 web_source_buffer_.reset();
668 source_ = nullptr;
669 async_event_queue_ = nullptr;
670 }
671
HighestPresentationTimestamp()672 double SourceBuffer::HighestPresentationTimestamp() {
673 DCHECK(!IsRemoved());
674
675 double pts = web_source_buffer_->HighestPresentationTimestamp();
676 DVLOG(3) << __func__ << " this=" << this << ", pts=" << pts;
677 return pts;
678 }
679
RemoveMediaTracks()680 void SourceBuffer::RemoveMediaTracks() {
681 DCHECK(HTMLMediaElement::MediaTracksEnabledInternally());
682 // Spec:
683 // http://w3c.github.io/media-source/#widl-MediaSource-removeSourceBuffer-void-SourceBuffer-sourceBuffer
684 DCHECK(source_);
685
686 HTMLMediaElement* media_element = source_->MediaElement();
687 DCHECK(media_element);
688 // 3. Let SourceBuffer audioTracks list equal the AudioTrackList object
689 // returned by sourceBuffer.audioTracks.
690 // 4. If the SourceBuffer audioTracks list is not empty, then run the
691 // following steps:
692 // 4.1 Let HTMLMediaElement audioTracks list equal the AudioTrackList object
693 // returned by the audioTracks attribute on the HTMLMediaElement.
694 // 4.2 Let the removed enabled audio track flag equal false.
695 bool removed_enabled_audio_track = false;
696 // 4.3 For each AudioTrack object in the SourceBuffer audioTracks list, run
697 // the following steps:
698 while (audioTracks().length() > 0) {
699 AudioTrack* audio_track = audioTracks().AnonymousIndexedGetter(0);
700 // 4.3.1 Set the sourceBuffer attribute on the AudioTrack object to null.
701 SourceBufferTrackBaseSupplement::SetSourceBuffer(*audio_track, nullptr);
702 // 4.3.2 If the enabled attribute on the AudioTrack object is true, then set
703 // the removed enabled audio track flag to true.
704 if (audio_track->enabled())
705 removed_enabled_audio_track = true;
706 // 4.3.3 Remove the AudioTrack object from the HTMLMediaElement audioTracks
707 // list.
708 // 4.3.4 Queue a task to fire a trusted event named removetrack, that does
709 // not bubble and is not cancelable, and that uses the TrackEvent
710 // interface, at the HTMLMediaElement audioTracks list.
711 media_element->audioTracks().Remove(audio_track->id());
712 // 4.3.5 Remove the AudioTrack object from the SourceBuffer audioTracks
713 // list.
714 // 4.3.6 Queue a task to fire a trusted event named removetrack, that does
715 // not bubble and is not cancelable, and that uses the TrackEvent
716 // interface, at the SourceBuffer audioTracks list.
717 audioTracks().Remove(audio_track->id());
718 }
719 // 4.4 If the removed enabled audio track flag equals true, then queue a task
720 // to fire a simple event named change at the HTMLMediaElement audioTracks
721 // list.
722 if (removed_enabled_audio_track) {
723 Event* event = Event::Create(event_type_names::kChange);
724 event->SetTarget(&media_element->audioTracks());
725 media_element->ScheduleEvent(event);
726 }
727
728 // 5. Let SourceBuffer videoTracks list equal the VideoTrackList object
729 // returned by sourceBuffer.videoTracks.
730 // 6. If the SourceBuffer videoTracks list is not empty, then run the
731 // following steps:
732 // 6.1 Let HTMLMediaElement videoTracks list equal the VideoTrackList object
733 // returned by the videoTracks attribute on the HTMLMediaElement.
734 // 6.2 Let the removed selected video track flag equal false.
735 bool removed_selected_video_track = false;
736 // 6.3 For each VideoTrack object in the SourceBuffer videoTracks list, run
737 // the following steps:
738 while (videoTracks().length() > 0) {
739 VideoTrack* video_track = videoTracks().AnonymousIndexedGetter(0);
740 // 6.3.1 Set the sourceBuffer attribute on the VideoTrack object to null.
741 SourceBufferTrackBaseSupplement::SetSourceBuffer(*video_track, nullptr);
742 // 6.3.2 If the selected attribute on the VideoTrack object is true, then
743 // set the removed selected video track flag to true.
744 if (video_track->selected())
745 removed_selected_video_track = true;
746 // 6.3.3 Remove the VideoTrack object from the HTMLMediaElement videoTracks
747 // list.
748 // 6.3.4 Queue a task to fire a trusted event named removetrack, that does
749 // not bubble and is not cancelable, and that uses the TrackEvent
750 // interface, at the HTMLMediaElement videoTracks list.
751 media_element->videoTracks().Remove(video_track->id());
752 // 6.3.5 Remove the VideoTrack object from the SourceBuffer videoTracks
753 // list.
754 // 6.3.6 Queue a task to fire a trusted event named removetrack, that does
755 // not bubble and is not cancelable, and that uses the TrackEvent
756 // interface, at the SourceBuffer videoTracks list.
757 videoTracks().Remove(video_track->id());
758 }
759 // 6.4 If the removed selected video track flag equals true, then queue a task
760 // to fire a simple event named change at the HTMLMediaElement videoTracks
761 // list.
762 if (removed_selected_video_track) {
763 Event* event = Event::Create(event_type_names::kChange);
764 event->SetTarget(&media_element->videoTracks());
765 media_element->ScheduleEvent(event);
766 }
767
768 // 7-8. TODO(servolk): Remove text tracks once SourceBuffer has text tracks.
769 }
770
GetMediaTime()771 double SourceBuffer::GetMediaTime() {
772 double media_time = std::numeric_limits<float>::quiet_NaN();
773 if (source_ && source_->MediaElement())
774 media_time = source_->MediaElement()->currentTime();
775 return media_time;
776 }
777
778 template <class T>
FindExistingTrackById(const TrackListBase<T> & track_list,const String & id)779 T* FindExistingTrackById(const TrackListBase<T>& track_list, const String& id) {
780 // According to MSE specification
781 // (https://w3c.github.io/media-source/#sourcebuffer-init-segment-received)
782 // step 3.1:
783 // > If more than one track for a single type are present (ie 2 audio tracks),
784 // then the Track IDs match the ones in the first initialization segment.
785 // I.e. we only need to search by TrackID if there is more than one track,
786 // otherwise we can assume that the only track of the given type is the same
787 // one that we had in previous init segments.
788 if (track_list.length() == 1)
789 return track_list.AnonymousIndexedGetter(0);
790 return track_list.getTrackById(id);
791 }
792
GetTrackDefault(const AtomicString & track_type,const AtomicString & byte_stream_track_id) const793 const TrackDefault* SourceBuffer::GetTrackDefault(
794 const AtomicString& track_type,
795 const AtomicString& byte_stream_track_id) const {
796 // This is a helper for implementation of default track label and default
797 // track language algorithms.
798 // defaultTrackLabel spec:
799 // https://w3c.github.io/media-source/#sourcebuffer-default-track-label
800 // defaultTrackLanguage spec:
801 // https://w3c.github.io/media-source/#sourcebuffer-default-track-language
802
803 // 1. If trackDefaults contains a TrackDefault object with a type attribute
804 // equal to type and a byteStreamTrackID attribute equal to
805 // byteStreamTrackID, then return the value of the label/language attribute
806 // on this matching object and abort these steps.
807 // 2. If trackDefaults contains a TrackDefault object with a type attribute
808 // equal to type and a byteStreamTrackID attribute equal to an empty
809 // string, then return the value of the label/language attribute on this
810 // matching object and abort these steps.
811 // 3. Return an empty string to the caller
812 const TrackDefault* track_default_with_empty_bytestream_id = nullptr;
813 for (unsigned i = 0; i < track_defaults_->length(); ++i) {
814 const TrackDefault* track_default = track_defaults_->item(i);
815 if (track_default->type() != track_type)
816 continue;
817 if (track_default->byteStreamTrackID() == byte_stream_track_id)
818 return track_default;
819 if (!track_default_with_empty_bytestream_id &&
820 track_default->byteStreamTrackID() == "")
821 track_default_with_empty_bytestream_id = track_default;
822 }
823 return track_default_with_empty_bytestream_id;
824 }
825
DefaultTrackLabel(const AtomicString & track_type,const AtomicString & byte_stream_track_id) const826 AtomicString SourceBuffer::DefaultTrackLabel(
827 const AtomicString& track_type,
828 const AtomicString& byte_stream_track_id) const {
829 // Spec: https://w3c.github.io/media-source/#sourcebuffer-default-track-label
830 const TrackDefault* track_default =
831 GetTrackDefault(track_type, byte_stream_track_id);
832 return track_default ? AtomicString(track_default->label()) : "";
833 }
834
DefaultTrackLanguage(const AtomicString & track_type,const AtomicString & byte_stream_track_id) const835 AtomicString SourceBuffer::DefaultTrackLanguage(
836 const AtomicString& track_type,
837 const AtomicString& byte_stream_track_id) const {
838 // Spec:
839 // https://w3c.github.io/media-source/#sourcebuffer-default-track-language
840 const TrackDefault* track_default =
841 GetTrackDefault(track_type, byte_stream_track_id);
842 return track_default ? AtomicString(track_default->language()) : "";
843 }
844
InitializationSegmentReceived(const WebVector<MediaTrackInfo> & new_tracks)845 bool SourceBuffer::InitializationSegmentReceived(
846 const WebVector<MediaTrackInfo>& new_tracks) {
847 DVLOG(3) << __func__ << " this=" << this << " tracks=" << new_tracks.size();
848 DCHECK(source_);
849 DCHECK(source_->MediaElement());
850 DCHECK(updating_);
851
852 if (!HTMLMediaElement::MediaTracksEnabledInternally()) {
853 if (!first_initialization_segment_received_) {
854 source_->SetSourceBufferActive(this, true);
855 first_initialization_segment_received_ = true;
856 }
857 return true;
858 }
859
860 // Implementation of Initialization Segment Received, see
861 // https://w3c.github.io/media-source/#sourcebuffer-init-segment-received
862
863 // Sort newTracks into audio and video tracks to facilitate implementation
864 // of subsequent steps of this algorithm.
865 Vector<MediaTrackInfo> new_audio_tracks;
866 Vector<MediaTrackInfo> new_video_tracks;
867 for (const MediaTrackInfo& track_info : new_tracks) {
868 const TrackBase* track = nullptr;
869 if (track_info.track_type == WebMediaPlayer::kAudioTrack) {
870 new_audio_tracks.push_back(track_info);
871 if (first_initialization_segment_received_)
872 track = FindExistingTrackById(audioTracks(), track_info.id);
873 } else if (track_info.track_type == WebMediaPlayer::kVideoTrack) {
874 new_video_tracks.push_back(track_info);
875 if (first_initialization_segment_received_)
876 track = FindExistingTrackById(videoTracks(), track_info.id);
877 } else {
878 DVLOG(3) << __func__ << " this=" << this
879 << " failed: unsupported track type " << track_info.track_type;
880 // TODO(servolk): Add handling of text tracks.
881 NOTREACHED();
882 }
883 if (first_initialization_segment_received_ && !track) {
884 DVLOG(3) << __func__ << " this=" << this
885 << " failed: tracks mismatch the first init segment.";
886 return false;
887 }
888 #if DCHECK_IS_ON()
889 const char* log_track_type_str =
890 (track_info.track_type == WebMediaPlayer::kAudioTrack) ? "audio"
891 : "video";
892 DVLOG(3) << __func__ << " this=" << this << " : " << log_track_type_str
893 << " track "
894 << " id=" << String(track_info.id)
895 << " byteStreamTrackID=" << String(track_info.byte_stream_track_id)
896 << " kind=" << String(track_info.kind)
897 << " label=" << String(track_info.label)
898 << " language=" << String(track_info.language);
899 #endif
900 }
901
902 // 1. Update the duration attribute if it currently equals NaN:
903 // TODO(servolk): Pass also stream duration into initSegmentReceived.
904
905 // 2. If the initialization segment has no audio, video, or text tracks, then
906 // run the append error algorithm with the decode error parameter set to
907 // true and abort these steps.
908 if (new_tracks.empty()) {
909 DVLOG(3) << __func__ << " this=" << this
910 << " failed: no tracks found in the init segment.";
911 // The append error algorithm will be called at the top level after we
912 // return false here to indicate failure.
913 return false;
914 }
915
916 // 3. If the first initialization segment received flag is true, then run the
917 // following steps:
918 if (first_initialization_segment_received_) {
919 // 3.1 Verify the following properties. If any of the checks fail then run
920 // the append error algorithm with the decode error parameter set to
921 // true and abort these steps.
922 bool tracks_match_first_init_segment = true;
923 // - The number of audio, video, and text tracks match what was in the first
924 // initialization segment.
925 if (new_audio_tracks.size() != audioTracks().length() ||
926 new_video_tracks.size() != videoTracks().length()) {
927 tracks_match_first_init_segment = false;
928 }
929 // - The codecs for each track, match what was specified in the first
930 // initialization segment.
931 // This is currently done in MediaSourceState::OnNewConfigs.
932 // - If more than one track for a single type are present (ie 2 audio
933 // tracks), then the Track IDs match the ones in the first initialization
934 // segment.
935 if (tracks_match_first_init_segment && new_audio_tracks.size() > 1) {
936 for (wtf_size_t i = 0; i < new_audio_tracks.size(); ++i) {
937 const String& new_track_id = new_video_tracks[i].id;
938 if (new_track_id !=
939 String(audioTracks().AnonymousIndexedGetter(i)->id())) {
940 tracks_match_first_init_segment = false;
941 break;
942 }
943 }
944 }
945
946 if (tracks_match_first_init_segment && new_video_tracks.size() > 1) {
947 for (wtf_size_t i = 0; i < new_video_tracks.size(); ++i) {
948 const String& new_track_id = new_video_tracks[i].id;
949 if (new_track_id !=
950 String(videoTracks().AnonymousIndexedGetter(i)->id())) {
951 tracks_match_first_init_segment = false;
952 break;
953 }
954 }
955 }
956
957 if (!tracks_match_first_init_segment) {
958 DVLOG(3) << __func__ << " this=" << this
959 << " failed: tracks mismatch the first init segment.";
960 // The append error algorithm will be called at the top level after we
961 // return false here to indicate failure.
962 return false;
963 }
964
965 // 3.2 Add the appropriate track descriptions from this initialization
966 // segment to each of the track buffers. This is done in Chromium code
967 // in stream parsers and demuxer implementations.
968
969 // 3.3 Set the need random access point flag on all track buffers to true.
970 // This is done in Chromium code, see MediaSourceState::OnNewConfigs.
971 }
972
973 // 4. Let active track flag equal false.
974 bool active_track = false;
975
976 // 5. If the first initialization segment received flag is false, then run the
977 // following steps:
978 if (!first_initialization_segment_received_) {
979 // 5.1 If the initialization segment contains tracks with codecs the user
980 // agent does not support, then run the append error algorithm with the
981 // decode error parameter set to true and abort these steps.
982 // This is done in Chromium code, see MediaSourceState::OnNewConfigs.
983
984 // 5.2 For each audio track in the initialization segment, run following
985 // steps:
986 for (const MediaTrackInfo& track_info : new_audio_tracks) {
987 // 5.2.1 Let audio byte stream track ID be the Track ID for the current
988 // track being processed.
989 const auto& byte_stream_track_id = track_info.byte_stream_track_id;
990 // 5.2.2 Let audio language be a BCP 47 language tag for the language
991 // specified in the initialization segment for this track or an
992 // empty string if no language info is present.
993 WebString language = track_info.language;
994 // 5.2.3 If audio language equals an empty string or the 'und' BCP 47
995 // value, then run the default track language algorithm with
996 // byteStreamTrackID set to audio byte stream track ID and type set
997 // to "audio" and assign the value returned by the algorithm to
998 // audio language.
999 if (language.IsEmpty() || language == "und")
1000 language = DefaultTrackLanguage(TrackDefault::AudioKeyword(),
1001 byte_stream_track_id);
1002 // 5.2.4 Let audio label be a label specified in the initialization
1003 // segment for this track or an empty string if no label info is
1004 // present.
1005 WebString label = track_info.label;
1006 // 5.3.5 If audio label equals an empty string, then run the default track
1007 // label algorithm with byteStreamTrackID set to audio byte stream
1008 // track ID and type set to "audio" and assign the value returned by
1009 // the algorithm to audio label.
1010 if (label.IsEmpty())
1011 label = DefaultTrackLabel(TrackDefault::AudioKeyword(),
1012 byte_stream_track_id);
1013 // 5.2.6 Let audio kinds be an array of kind strings specified in the
1014 // initialization segment for this track or an empty array if no
1015 // kind information is provided.
1016 const auto& kind = track_info.kind;
1017 // 5.2.7 TODO(servolk): Implement track kind processing.
1018 // 5.2.8.2 Let new audio track be a new AudioTrack object.
1019 auto* audio_track = MakeGarbageCollected<AudioTrack>(
1020 track_info.id, kind, label, language, false);
1021 SourceBufferTrackBaseSupplement::SetSourceBuffer(*audio_track, this);
1022 // 5.2.8.7 If audioTracks.length equals 0, then run the following steps:
1023 if (audioTracks().length() == 0) {
1024 // 5.2.8.7.1 Set the enabled property on new audio track to true.
1025 audio_track->setEnabled(true);
1026 // 5.2.8.7.2 Set active track flag to true.
1027 active_track = true;
1028 }
1029 // 5.2.8.8 Add new audio track to the audioTracks attribute on this
1030 // SourceBuffer object.
1031 // 5.2.8.9 Queue a task to fire a trusted event named addtrack, that does
1032 // not bubble and is not cancelable, and that uses the TrackEvent
1033 // interface, at the AudioTrackList object referenced by the
1034 // audioTracks attribute on this SourceBuffer object.
1035 audioTracks().Add(audio_track);
1036 // 5.2.8.10 Add new audio track to the audioTracks attribute on the
1037 // HTMLMediaElement.
1038 // 5.2.8.11 Queue a task to fire a trusted event named addtrack, that does
1039 // not bubble and is not cancelable, and that uses the TrackEvent
1040 // interface, at the AudioTrackList object referenced by the
1041 // audioTracks attribute on the HTMLMediaElement.
1042 source_->MediaElement()->audioTracks().Add(audio_track);
1043 }
1044
1045 // 5.3. For each video track in the initialization segment, run following
1046 // steps:
1047 for (const MediaTrackInfo& track_info : new_video_tracks) {
1048 // 5.3.1 Let video byte stream track ID be the Track ID for the current
1049 // track being processed.
1050 const auto& byte_stream_track_id = track_info.byte_stream_track_id;
1051 // 5.3.2 Let video language be a BCP 47 language tag for the language
1052 // specified in the initialization segment for this track or an
1053 // empty string if no language info is present.
1054 WebString language = track_info.language;
1055 // 5.3.3 If video language equals an empty string or the 'und' BCP 47
1056 // value, then run the default track language algorithm with
1057 // byteStreamTrackID set to video byte stream track ID and type set
1058 // to "video" and assign the value returned by the algorithm to
1059 // video language.
1060 if (language.IsEmpty() || language == "und")
1061 language = DefaultTrackLanguage(TrackDefault::VideoKeyword(),
1062 byte_stream_track_id);
1063 // 5.3.4 Let video label be a label specified in the initialization
1064 // segment for this track or an empty string if no label info is
1065 // present.
1066 WebString label = track_info.label;
1067 // 5.3.5 If video label equals an empty string, then run the default track
1068 // label algorithm with byteStreamTrackID set to video byte stream
1069 // track ID and type set to "video" and assign the value returned by
1070 // the algorithm to video label.
1071 if (label.IsEmpty())
1072 label = DefaultTrackLabel(TrackDefault::VideoKeyword(),
1073 byte_stream_track_id);
1074 // 5.3.6 Let video kinds be an array of kind strings specified in the
1075 // initialization segment for this track or an empty array if no
1076 // kind information is provided.
1077 const auto& kind = track_info.kind;
1078 // 5.3.7 TODO(servolk): Implement track kind processing.
1079 // 5.3.8.2 Let new video track be a new VideoTrack object.
1080 auto* video_track = MakeGarbageCollected<VideoTrack>(
1081 track_info.id, kind, label, language, false);
1082 SourceBufferTrackBaseSupplement::SetSourceBuffer(*video_track, this);
1083 // 5.3.8.7 If videoTracks.length equals 0, then run the following steps:
1084 if (videoTracks().length() == 0) {
1085 // 5.3.8.7.1 Set the selected property on new audio track to true.
1086 video_track->setSelected(true);
1087 // 5.3.8.7.2 Set active track flag to true.
1088 active_track = true;
1089 }
1090 // 5.3.8.8 Add new video track to the videoTracks attribute on this
1091 // SourceBuffer object.
1092 // 5.3.8.9 Queue a task to fire a trusted event named addtrack, that does
1093 // not bubble and is not cancelable, and that uses the TrackEvent
1094 // interface, at the VideoTrackList object referenced by the
1095 // videoTracks attribute on this SourceBuffer object.
1096 videoTracks().Add(video_track);
1097 // 5.3.8.10 Add new video track to the videoTracks attribute on the
1098 // HTMLMediaElement.
1099 // 5.3.8.11 Queue a task to fire a trusted event named addtrack, that does
1100 // not bubble and is not cancelable, and that uses the TrackEvent
1101 // interface, at the VideoTrackList object referenced by the
1102 // videoTracks attribute on the HTMLMediaElement.
1103 source_->MediaElement()->videoTracks().Add(video_track);
1104 }
1105
1106 // 5.4 TODO(servolk): Add text track processing here.
1107
1108 // 5.5 If active track flag equals true, then run the following steps:
1109 // activesourcebuffers.
1110 if (active_track) {
1111 // 5.5.1 Add this SourceBuffer to activeSourceBuffers.
1112 // 5.5.2 Queue a task to fire a simple event named addsourcebuffer at
1113 // activeSourceBuffers
1114 source_->SetSourceBufferActive(this, true);
1115 }
1116
1117 // 5.6. Set first initialization segment received flag to true.
1118 first_initialization_segment_received_ = true;
1119 }
1120
1121 return true;
1122 }
1123
NotifyParseWarning(const ParseWarning warning)1124 void SourceBuffer::NotifyParseWarning(const ParseWarning warning) {
1125 switch (warning) {
1126 case WebSourceBufferClient::kKeyframeTimeGreaterThanDependant:
1127 // Report this problematic GOP structure to help inform follow-up work.
1128 // Media engine also records RAPPOR for these, up to once per track, at
1129 // Media.OriginUrl.MSE.KeyframeTimeGreaterThanDependant.
1130 // TODO(wolenetz): Use the data to scope additional work. See
1131 // https://crbug.com/739931.
1132 UseCounter::Count(
1133 source_->MediaElement()->GetDocument(),
1134 WebFeature::kMediaSourceKeyframeTimeGreaterThanDependant);
1135 break;
1136 case WebSourceBufferClient::kMuxedSequenceMode:
1137 // Report this problematic API usage to help inform follow-up work.
1138 // Media engine also records RAPPOR for these, up to once per
1139 // SourceBuffer, at Media.OriginUrl.MSE.MuxedSequenceModeSourceBuffer.
1140 // TODO(wolenetz): Use the data to scope additional work. See
1141 // https://crbug.com/737757.
1142 UseCounter::Count(source_->MediaElement()->GetDocument(),
1143 WebFeature::kMediaSourceMuxedSequenceMode);
1144 break;
1145 case WebSourceBufferClient::kGroupEndTimestampDecreaseWithinMediaSegment:
1146 // Report this problematic Media Segment structure usage to help inform
1147 // follow-up work.
1148 // TODO(wolenetz): Use the data to scope additional work. See
1149 // https://crbug.com/920853 and
1150 // https://github.com/w3c/media-source/issues/203.
1151 UseCounter::Count(
1152 source_->MediaElement()->GetDocument(),
1153 WebFeature::kMediaSourceGroupEndTimestampDecreaseWithinMediaSegment);
1154 break;
1155 }
1156 }
1157
HasPendingActivity() const1158 bool SourceBuffer::HasPendingActivity() const {
1159 return updating_ || append_buffer_async_task_handle_.IsActive() ||
1160 remove_async_task_handle_.IsActive() ||
1161 (async_event_queue_ && async_event_queue_->HasPendingEvents());
1162 }
1163
ContextDestroyed()1164 void SourceBuffer::ContextDestroyed() {
1165 append_buffer_async_task_handle_.Cancel();
1166 remove_async_task_handle_.Cancel();
1167 updating_ = false;
1168 }
1169
GetExecutionContext() const1170 ExecutionContext* SourceBuffer::GetExecutionContext() const {
1171 return ExecutionContextLifecycleObserver::GetExecutionContext();
1172 }
1173
InterfaceName() const1174 const AtomicString& SourceBuffer::InterfaceName() const {
1175 return event_target_names::kSourceBuffer;
1176 }
1177
IsRemoved() const1178 bool SourceBuffer::IsRemoved() const {
1179 return !source_;
1180 }
1181
ScheduleEvent(const AtomicString & event_name)1182 void SourceBuffer::ScheduleEvent(const AtomicString& event_name) {
1183 DCHECK(async_event_queue_);
1184
1185 Event* event = Event::Create(event_name);
1186 event->SetTarget(this);
1187
1188 async_event_queue_->EnqueueEvent(FROM_HERE, *event);
1189 }
1190
PrepareAppend(double media_time,size_t new_data_size,ExceptionState & exception_state)1191 bool SourceBuffer::PrepareAppend(double media_time,
1192 size_t new_data_size,
1193 ExceptionState& exception_state) {
1194 TRACE_EVENT_NESTABLE_ASYNC_BEGIN0("media", "SourceBuffer::prepareAppend",
1195 TRACE_ID_LOCAL(this));
1196 // http://w3c.github.io/media-source/#sourcebuffer-prepare-append
1197 // 3.5.4 Prepare Append Algorithm
1198 // 1. If the SourceBuffer has been removed from the sourceBuffers attribute of
1199 // the parent media source then throw an InvalidStateError exception and
1200 // abort these steps.
1201 // 2. If the updating attribute equals true, then throw an InvalidStateError
1202 // exception and abort these steps.
1203 if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
1204 exception_state)) {
1205 TRACE_EVENT_NESTABLE_ASYNC_END0("media", "SourceBuffer::prepareAppend",
1206 TRACE_ID_LOCAL(this));
1207 return false;
1208 }
1209
1210 // 3. If the HTMLMediaElement.error attribute is not null, then throw an
1211 // InvalidStateError exception and abort these steps.
1212 DCHECK(source_);
1213 DCHECK(source_->MediaElement());
1214 if (source_->MediaElement()->error()) {
1215 MediaSourceImpl::LogAndThrowDOMException(
1216 exception_state, DOMExceptionCode::kInvalidStateError,
1217 "The HTMLMediaElement.error attribute is not null.");
1218 TRACE_EVENT_NESTABLE_ASYNC_END0("media", "SourceBuffer::prepareAppend",
1219 TRACE_ID_LOCAL(this));
1220 return false;
1221 }
1222
1223 // 4. If the readyState attribute of the parent media source is in the "ended"
1224 // state then run the following steps:
1225 // 1. Set the readyState attribute of the parent media source to "open"
1226 // 2. Queue a task to fire a simple event named sourceopen at the parent
1227 // media source.
1228 source_->OpenIfInEndedState();
1229
1230 // 5. Run the coded frame eviction algorithm.
1231 if (!EvictCodedFrames(media_time, new_data_size) ||
1232 !base::CheckedNumeric<wtf_size_t>(new_data_size).IsValid()) {
1233 // 6. If the buffer full flag equals true, then throw a QUOTA_EXCEEDED_ERR
1234 // exception and abort these steps.
1235 // If the incoming data exceeds wtf_size_t::max, then our implementation
1236 // cannot deal with it, so we also throw a QuotaExceededError.
1237 DVLOG(3) << __func__ << " this=" << this << " -> throw QuotaExceededError";
1238 MediaSourceImpl::LogAndThrowDOMException(
1239 exception_state, DOMExceptionCode::kQuotaExceededError,
1240 "The SourceBuffer is full, and cannot free space to append additional "
1241 "buffers.");
1242 TRACE_EVENT_NESTABLE_ASYNC_END0("media", "SourceBuffer::prepareAppend",
1243 TRACE_ID_LOCAL(this));
1244 return false;
1245 }
1246
1247 TRACE_EVENT_NESTABLE_ASYNC_END0("media", "SourceBuffer::prepareAppend",
1248 TRACE_ID_LOCAL(this));
1249 return true;
1250 }
1251
EvictCodedFrames(double media_time,size_t new_data_size)1252 bool SourceBuffer::EvictCodedFrames(double media_time, size_t new_data_size) {
1253 DCHECK(source_);
1254 DCHECK(source_->MediaElement());
1255
1256 // Nothing to do if the mediaElement does not yet have frames to evict.
1257 if (source_->MediaElement()->getReadyState() <
1258 HTMLMediaElement::kHaveMetadata) {
1259 return true;
1260 }
1261
1262 bool result = web_source_buffer_->EvictCodedFrames(media_time, new_data_size);
1263 if (!result) {
1264 DVLOG(3) << __func__ << " this=" << this
1265 << " failed. newDataSize=" << new_data_size
1266 << " media_time=" << media_time << " buffered="
1267 << WebTimeRangesToString(web_source_buffer_->Buffered());
1268 }
1269 return result;
1270 }
1271
AppendBufferInternal(double media_time,const unsigned char * data,size_t size,ExceptionState & exception_state)1272 void SourceBuffer::AppendBufferInternal(double media_time,
1273 const unsigned char* data,
1274 size_t size,
1275 ExceptionState& exception_state) {
1276 TRACE_EVENT_NESTABLE_ASYNC_BEGIN1("media", "SourceBuffer::appendBuffer",
1277 TRACE_ID_LOCAL(this), "size", size);
1278 // Section 3.2 appendBuffer()
1279 // https://dvcs.w3.org/hg/html-media/raw-file/default/media-source/media-source.html#widl-SourceBuffer-appendBuffer-void-ArrayBufferView-data
1280
1281 // 1. Run the prepare append algorithm.
1282 if (!PrepareAppend(media_time, size, exception_state)) {
1283 TRACE_EVENT_NESTABLE_ASYNC_END0("media", "SourceBuffer::appendBuffer",
1284 TRACE_ID_LOCAL(this));
1285 return;
1286 }
1287 TRACE_EVENT_NESTABLE_ASYNC_BEGIN0("media", "prepareAppend",
1288 TRACE_ID_LOCAL(this));
1289
1290 // 2. Add data to the end of the input buffer.
1291 DCHECK(data || size == 0);
1292 if (data)
1293 pending_append_data_.Append(data, base::checked_cast<wtf_size_t>(size));
1294 pending_append_data_offset_ = 0;
1295
1296 // 3. Set the updating attribute to true.
1297 updating_ = true;
1298
1299 // 4. Queue a task to fire a simple event named updatestart at this
1300 // SourceBuffer object.
1301 ScheduleEvent(event_type_names::kUpdatestart);
1302
1303 // 5. Asynchronously run the buffer append algorithm.
1304 append_buffer_async_task_handle_ = PostCancellableTask(
1305 *GetExecutionContext()->GetTaskRunner(TaskType::kMediaElementEvent),
1306 FROM_HERE,
1307 WTF::Bind(&SourceBuffer::AppendBufferAsyncPart, WrapPersistent(this)));
1308
1309 TRACE_EVENT_NESTABLE_ASYNC_END0("media", "prepareAppend",
1310 TRACE_ID_LOCAL(this));
1311 TRACE_EVENT_NESTABLE_ASYNC_BEGIN1("media", "delay", TRACE_ID_LOCAL(this),
1312 "type", "initialDelay");
1313 }
1314
AppendBufferAsyncPart()1315 void SourceBuffer::AppendBufferAsyncPart() {
1316 DCHECK(updating_);
1317
1318 // Section 3.5.4 Buffer Append Algorithm
1319 // https://dvcs.w3.org/hg/html-media/raw-file/default/media-source/media-source.html#sourcebuffer-buffer-append
1320
1321 // 1. Run the segment parser loop algorithm.
1322 // Step 2 doesn't apply since we run Step 1 synchronously here.
1323 DCHECK_GE(pending_append_data_.size(), pending_append_data_offset_);
1324 wtf_size_t append_size =
1325 pending_append_data_.size() - pending_append_data_offset_;
1326
1327 // Impose an arbitrary max size for a single append() call so that an append
1328 // doesn't block the renderer event loop very long. This value was selected
1329 // by looking at YouTube SourceBuffer usage across a variety of bitrates.
1330 // This value allows relatively large appends while keeping append() call
1331 // duration in the ~5-15ms range.
1332 const wtf_size_t kMaxAppendSize = 128 * 1024;
1333 if (append_size > kMaxAppendSize)
1334 append_size = kMaxAppendSize;
1335
1336 TRACE_EVENT_NESTABLE_ASYNC_END0("media", "delay", TRACE_ID_LOCAL(this));
1337 TRACE_EVENT_NESTABLE_ASYNC_BEGIN1("media", "appending", TRACE_ID_LOCAL(this),
1338 "appendSize", append_size);
1339
1340 // |zero| is used for 0 byte appends so we always have a valid pointer.
1341 // We need to convey all appends, even 0 byte ones to |m_webSourceBuffer|
1342 // so that it can clear its end of stream state if necessary.
1343 unsigned char zero = 0;
1344 unsigned char* append_data = &zero;
1345 if (append_size)
1346 append_data = pending_append_data_.data() + pending_append_data_offset_;
1347
1348 bool append_success =
1349 web_source_buffer_->Append(append_data, append_size, ×tamp_offset_);
1350
1351 if (!append_success) {
1352 pending_append_data_.clear();
1353 pending_append_data_offset_ = 0;
1354 AppendError();
1355 } else {
1356 pending_append_data_offset_ += append_size;
1357
1358 if (pending_append_data_offset_ < pending_append_data_.size()) {
1359 append_buffer_async_task_handle_ = PostCancellableTask(
1360 *GetExecutionContext()->GetTaskRunner(TaskType::kMediaElementEvent),
1361 FROM_HERE,
1362 WTF::Bind(&SourceBuffer::AppendBufferAsyncPart,
1363 WrapPersistent(this)));
1364 TRACE_EVENT_NESTABLE_ASYNC_END0("media", "appending",
1365 TRACE_ID_LOCAL(this));
1366 TRACE_EVENT_NESTABLE_ASYNC_BEGIN1("media", "delay", TRACE_ID_LOCAL(this),
1367 "type", "nextPieceDelay");
1368 return;
1369 }
1370
1371 // 3. Set the updating attribute to false.
1372 updating_ = false;
1373 pending_append_data_.clear();
1374 pending_append_data_offset_ = 0;
1375
1376 // 4. Queue a task to fire a simple event named update at this SourceBuffer
1377 // object.
1378 ScheduleEvent(event_type_names::kUpdate);
1379
1380 // 5. Queue a task to fire a simple event named updateend at this
1381 // SourceBuffer object.
1382 ScheduleEvent(event_type_names::kUpdateend);
1383 }
1384
1385 TRACE_EVENT_NESTABLE_ASYNC_END0("media", "appending", TRACE_ID_LOCAL(this));
1386 TRACE_EVENT_NESTABLE_ASYNC_END0("media", "SourceBuffer::appendBuffer",
1387 TRACE_ID_LOCAL(this));
1388
1389 double media_time = GetMediaTime();
1390 DVLOG(3) << __func__ << " done. this=" << this << " media_time=" << media_time
1391 << " buffered="
1392 << WebTimeRangesToString(web_source_buffer_->Buffered());
1393 }
1394
RemoveAsyncPart()1395 void SourceBuffer::RemoveAsyncPart() {
1396 DCHECK(updating_);
1397 DCHECK_GE(pending_remove_start_, 0);
1398 DCHECK_LT(pending_remove_start_, pending_remove_end_);
1399
1400 // Section 3.2 remove() method steps
1401 // https://dvcs.w3.org/hg/html-media/raw-file/default/media-source/media-source.html#widl-SourceBuffer-remove-void-double-start-double-end
1402
1403 // 9. Run the coded frame removal algorithm with start and end as the start
1404 // and end of the removal range.
1405 web_source_buffer_->Remove(pending_remove_start_, pending_remove_end_);
1406
1407 // 10. Set the updating attribute to false.
1408 updating_ = false;
1409 pending_remove_start_ = -1;
1410 pending_remove_end_ = -1;
1411
1412 // 11. Queue a task to fire a simple event named update at this SourceBuffer
1413 // object.
1414 ScheduleEvent(event_type_names::kUpdate);
1415
1416 // 12. Queue a task to fire a simple event named updateend at this
1417 // SourceBuffer object.
1418 ScheduleEvent(event_type_names::kUpdateend);
1419 }
1420
AppendError()1421 void SourceBuffer::AppendError() {
1422 DVLOG(3) << __func__ << " this=" << this;
1423 // Section 3.5.3 Append Error Algorithm
1424 // https://dvcs.w3.org/hg/html-media/raw-file/default/media-source/media-source.html#sourcebuffer-append-error
1425
1426 // 1. Run the reset parser state algorithm.
1427 web_source_buffer_->ResetParserState();
1428
1429 // 2. Set the updating attribute to false.
1430 updating_ = false;
1431
1432 // 3. Queue a task to fire a simple event named error at this SourceBuffer
1433 // object.
1434 ScheduleEvent(event_type_names::kError);
1435
1436 // 4. Queue a task to fire a simple event named updateend at this SourceBuffer
1437 // object.
1438 ScheduleEvent(event_type_names::kUpdateend);
1439
1440 // 5. If decode error is true, then run the end of stream algorithm with the
1441 // error parameter set to "decode".
1442 source_->EndOfStreamAlgorithm(WebMediaSource::kEndOfStreamStatusDecodeError);
1443 }
1444
Trace(Visitor * visitor)1445 void SourceBuffer::Trace(Visitor* visitor) {
1446 visitor->Trace(source_);
1447 visitor->Trace(track_defaults_);
1448 visitor->Trace(async_event_queue_);
1449 visitor->Trace(audio_tracks_);
1450 visitor->Trace(video_tracks_);
1451 EventTargetWithInlineData::Trace(visitor);
1452 ExecutionContextLifecycleObserver::Trace(visitor);
1453 }
1454
1455 } // namespace blink
1456