1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef MEDIA_FILTERS_CHUNK_DEMUXER_H_ 6 #define MEDIA_FILTERS_CHUNK_DEMUXER_H_ 7 8 #include <stddef.h> 9 #include <stdint.h> 10 11 #include <map> 12 #include <set> 13 #include <string> 14 #include <utility> 15 #include <vector> 16 17 #include "base/containers/circular_deque.h" 18 #include "base/macros.h" 19 #include "base/memory/memory_pressure_listener.h" 20 #include "base/synchronization/lock.h" 21 #include "base/thread_annotations.h" 22 #include "media/base/byte_queue.h" 23 #include "media/base/demuxer.h" 24 #include "media/base/demuxer_stream.h" 25 #include "media/base/media_tracks.h" 26 #include "media/base/ranges.h" 27 #include "media/base/stream_parser.h" 28 #include "media/filters/source_buffer_parse_warnings.h" 29 #include "media/filters/source_buffer_state.h" 30 #include "media/filters/source_buffer_stream.h" 31 32 class MEDIA_EXPORT SourceBufferStream; 33 34 namespace media { 35 36 class MEDIA_EXPORT ChunkDemuxerStream : public DemuxerStream { 37 public: 38 using BufferQueue = base::circular_deque<scoped_refptr<StreamParserBuffer>>; 39 40 ChunkDemuxerStream(Type type, MediaTrack::Id media_track_id); 41 ~ChunkDemuxerStream() override; 42 43 // ChunkDemuxerStream control methods. 44 void StartReturningData(); 45 void AbortReads(); 46 void CompletePendingReadIfPossible(); 47 void Shutdown(); 48 49 // SourceBufferStream manipulation methods. 50 void Seek(base::TimeDelta time); 51 bool IsSeekWaitingForData() const; 52 53 // Add buffers to this stream. Buffers are stored in SourceBufferStreams, 54 // which handle ordering and overlap resolution. 55 // Returns true if buffers were successfully added. 56 bool Append(const StreamParser::BufferQueue& buffers); 57 58 // Removes buffers between |start| and |end| according to the steps 59 // in the "Coded Frame Removal Algorithm" in the Media Source 60 // Extensions Spec. 61 // https://dvcs.w3.org/hg/html-media/raw-file/default/media-source/media-source.html#sourcebuffer-coded-frame-removal 62 // 63 // |duration| is the current duration of the presentation. It is 64 // required by the computation outlined in the spec. 65 void Remove(base::TimeDelta start, base::TimeDelta end, 66 base::TimeDelta duration); 67 68 // If the buffer is full, attempts to try to free up space, as specified in 69 // the "Coded Frame Eviction Algorithm" in the Media Source Extensions Spec. 70 // Returns false iff buffer is still full after running eviction. 71 // https://w3c.github.io/media-source/#sourcebuffer-coded-frame-eviction 72 bool EvictCodedFrames(base::TimeDelta media_time, size_t newDataSize); 73 74 void OnMemoryPressure( 75 base::TimeDelta media_time, 76 base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level, 77 bool force_instant_gc); 78 79 // Signal to the stream that duration has changed to |duration|. 80 void OnSetDuration(base::TimeDelta duration); 81 82 // Returns the range of buffered data in this stream, capped at |duration|. 83 Ranges<base::TimeDelta> GetBufferedRanges(base::TimeDelta duration) const; 84 85 // Returns the highest PTS of the buffered data. 86 // Returns base::TimeDelta() if the stream has no buffered data. 87 base::TimeDelta GetHighestPresentationTimestamp() const; 88 89 // Returns the duration of the buffered data. 90 // Returns base::TimeDelta() if the stream has no buffered data. 91 base::TimeDelta GetBufferedDuration() const; 92 93 // Returns the size of the buffered data in bytes. 94 size_t GetBufferedSize() const; 95 96 // Signal to the stream that buffers handed in through subsequent calls to 97 // Append() belong to a coded frame group that starts at |start_pts|. 98 // |start_dts| is used only to help tests verify correctness of calls to this 99 // method. If |group_start_observer_cb_| is set, first invokes this test-only 100 // callback with |start_dts| and |start_pts| to assist test verification. 101 void OnStartOfCodedFrameGroup(DecodeTimestamp start_dts, 102 base::TimeDelta start_pts); 103 104 // Called when midstream config updates occur. 105 // For audio and video, if the codec is allowed to change, the caller should 106 // set |allow_codec_change| to true. 107 // Returns true if the new config is accepted. 108 // Returns false if the new config should trigger an error. 109 bool UpdateAudioConfig(const AudioDecoderConfig& config, 110 bool allow_codec_change, 111 MediaLog* media_log); 112 bool UpdateVideoConfig(const VideoDecoderConfig& config, 113 bool allow_codec_change, 114 MediaLog* media_log); 115 void UpdateTextConfig(const TextTrackConfig& config, MediaLog* media_log); 116 117 void MarkEndOfStream(); 118 void UnmarkEndOfStream(); 119 120 // DemuxerStream methods. 121 void Read(ReadCB read_cb) override; 122 bool IsReadPending() const override; 123 Type type() const override; 124 Liveness liveness() const override; 125 AudioDecoderConfig audio_decoder_config() override; 126 VideoDecoderConfig video_decoder_config() override; 127 bool SupportsConfigChanges() override; 128 129 bool IsEnabled() const; 130 void SetEnabled(bool enabled, base::TimeDelta timestamp); 131 132 // Returns the text track configuration. It is an error to call this method 133 // if type() != TEXT. 134 TextTrackConfig text_track_config(); 135 136 // Sets the memory limit, in bytes, on the SourceBufferStream. 137 void SetStreamMemoryLimit(size_t memory_limit); 138 139 void SetLiveness(Liveness liveness); 140 media_track_id()141 MediaTrack::Id media_track_id() const { return media_track_id_; } 142 143 // Allows tests to verify invocations of Append(). 144 using AppendObserverCB = base::RepeatingCallback<void(const BufferQueue*)>; set_append_observer_for_testing(AppendObserverCB append_observer_cb)145 void set_append_observer_for_testing(AppendObserverCB append_observer_cb) { 146 append_observer_cb_ = std::move(append_observer_cb); 147 } 148 149 // Allows tests to verify invocations of OnStartOfCodedFrameGroup(). 150 using GroupStartObserverCB = 151 base::RepeatingCallback<void(DecodeTimestamp, base::TimeDelta)>; set_group_start_observer_for_testing(GroupStartObserverCB group_start_observer_cb)152 void set_group_start_observer_for_testing( 153 GroupStartObserverCB group_start_observer_cb) { 154 group_start_observer_cb_ = std::move(group_start_observer_cb); 155 } 156 157 private: 158 enum State { 159 UNINITIALIZED, 160 RETURNING_DATA_FOR_READS, 161 RETURNING_ABORT_FOR_READS, 162 SHUTDOWN, 163 }; 164 165 // Assigns |state_| to |state| 166 void ChangeState_Locked(State state) EXCLUSIVE_LOCKS_REQUIRED(lock_); 167 168 void CompletePendingReadIfPossible_Locked() EXCLUSIVE_LOCKS_REQUIRED(lock_); 169 170 // Specifies the type of the stream. 171 const Type type_; 172 173 Liveness liveness_ GUARDED_BY(lock_); 174 175 std::unique_ptr<SourceBufferStream> stream_ GUARDED_BY(lock_); 176 177 const MediaTrack::Id media_track_id_; 178 179 // Test-only callbacks to assist verification of Append() and 180 // OnStartOfCodedFrameGroup() calls, respectively. 181 AppendObserverCB append_observer_cb_; 182 GroupStartObserverCB group_start_observer_cb_; 183 184 mutable base::Lock lock_; 185 State state_ GUARDED_BY(lock_); 186 ReadCB read_cb_ GUARDED_BY(lock_); 187 bool is_enabled_ GUARDED_BY(lock_); 188 189 DISALLOW_IMPLICIT_CONSTRUCTORS(ChunkDemuxerStream); 190 }; 191 192 // Demuxer implementation that allows chunks of media data to be passed 193 // from JavaScript to the media stack. 194 class MEDIA_EXPORT ChunkDemuxer : public Demuxer { 195 public: 196 enum Status { 197 kOk, // ID added w/o error. 198 kNotSupported, // Type specified is not supported. 199 kReachedIdLimit, // Reached ID limit. We can't handle any more IDs. 200 }; 201 202 // |open_cb| Run when Initialize() is called to signal that the demuxer 203 // is ready to receive media data via AppendData(). 204 // |progress_cb| Run each time data is appended. 205 // |encrypted_media_init_data_cb| Run when the demuxer determines that an 206 // encryption key is needed to decrypt the content. 207 // |media_log| Used to report content and engine debug messages. 208 ChunkDemuxer(base::OnceClosure open_cb, 209 base::RepeatingClosure progress_cb, 210 EncryptedMediaInitDataCB encrypted_media_init_data_cb, 211 MediaLog* media_log); 212 ~ChunkDemuxer() override; 213 214 // Demuxer implementation. 215 std::string GetDisplayName() const override; 216 217 // |enable_text| Process inband text tracks in the normal way when true, 218 // otherwise ignore them. 219 void Initialize(DemuxerHost* host, PipelineStatusCallback init_cb) override; 220 void Stop() override; 221 void Seek(base::TimeDelta time, PipelineStatusCallback cb) override; 222 base::Time GetTimelineOffset() const override; 223 std::vector<DemuxerStream*> GetAllStreams() override; 224 base::TimeDelta GetStartTime() const override; 225 int64_t GetMemoryUsage() const override; 226 void AbortPendingReads() override; 227 228 // ChunkDemuxer reads are abortable. StartWaitingForSeek() and 229 // CancelPendingSeek() always abort pending and future reads until the 230 // expected seek occurs, so that ChunkDemuxer can stay synchronized with the 231 // associated JS method calls. 232 void StartWaitingForSeek(base::TimeDelta seek_time) override; 233 void CancelPendingSeek(base::TimeDelta seek_time) override; 234 235 // Registers a new |id| to use for AppendData() calls. |content_type| 236 // indicates the MIME type's ContentType and |codecs| indicates the MIME 237 // type's "codecs" parameter string (if any) for the data that we intend to 238 // append for this ID. kOk is returned if the demuxer has enough resources to 239 // support another ID and supports the format indicated by |content_type| and 240 // |codecs|. kReachedIdLimit is returned if the demuxer cannot handle another 241 // ID right now. kNotSupported is returned if |content_type| and |codecs| is 242 // not a supported format. 243 Status AddId(const std::string& id, 244 const std::string& content_type, 245 const std::string& codecs); 246 247 // Notifies a caller via |tracks_updated_cb| that the set of media tracks 248 // for a given |id| has changed. 249 void SetTracksWatcher(const std::string& id, 250 const MediaTracksUpdatedCB& tracks_updated_cb); 251 252 // Notifies a caller via |parse_warning_cb| of a parse warning. 253 void SetParseWarningCallback(const std::string& id, 254 SourceBufferParseWarningCB parse_warning_cb); 255 256 // Removed an ID & associated resources that were previously added with 257 // AddId(). 258 void RemoveId(const std::string& id); 259 260 // Gets the currently buffered ranges for the specified ID. 261 Ranges<base::TimeDelta> GetBufferedRanges(const std::string& id) const; 262 263 // Gets the highest buffered PTS for the specified |id|. If there is nothing 264 // buffered, returns base::TimeDelta(). 265 base::TimeDelta GetHighestPresentationTimestamp(const std::string& id) const; 266 267 void OnEnabledAudioTracksChanged(const std::vector<MediaTrack::Id>& track_ids, 268 base::TimeDelta curr_time, 269 TrackChangeCB change_completed_cb) override; 270 271 void OnSelectedVideoTrackChanged(const std::vector<MediaTrack::Id>& track_ids, 272 base::TimeDelta curr_time, 273 TrackChangeCB change_completed_cb) override; 274 275 // Appends media data to the source buffer associated with |id|, applying 276 // and possibly updating |*timestamp_offset| during coded frame processing. 277 // |append_window_start| and |append_window_end| correspond to the MSE spec's 278 // similarly named source buffer attributes that are used in coded frame 279 // processing. Returns true on success, false if the caller needs to run the 280 // append error algorithm with decode error parameter set to true. 281 bool AppendData(const std::string& id, 282 const uint8_t* data, 283 size_t length, 284 base::TimeDelta append_window_start, 285 base::TimeDelta append_window_end, 286 base::TimeDelta* timestamp_offset); 287 288 // Aborts parsing the current segment and reset the parser to a state where 289 // it can accept a new segment. 290 // Some pending frames can be emitted during that process. These frames are 291 // applied |timestamp_offset|. 292 void ResetParserState(const std::string& id, 293 base::TimeDelta append_window_start, 294 base::TimeDelta append_window_end, 295 base::TimeDelta* timestamp_offset); 296 297 // Remove buffers between |start| and |end| for the source buffer 298 // associated with |id|. 299 void Remove(const std::string& id, base::TimeDelta start, 300 base::TimeDelta end); 301 302 // Returns whether or not the source buffer associated with |id| can change 303 // its parser type to one which parses |content_type| and |codecs|. 304 // |content_type| indicates the ContentType of the MIME type for the data that 305 // we intend to append for this |id|; |codecs| similarly indicates the MIME 306 // type's "codecs" parameter, if any. 307 bool CanChangeType(const std::string& id, 308 const std::string& content_type, 309 const std::string& codecs); 310 311 // For the source buffer associated with |id|, changes its parser type to one 312 // which parses |content_type| and |codecs|. |content_type| indicates the 313 // ContentType of the MIME type for the data that we intend to append for this 314 // |id|; |codecs| similarly indicates the MIME type's "codecs" parameter, if 315 // any. Caller must first ensure CanChangeType() returns true for the same 316 // parameters. Caller must also ensure that ResetParserState() is done before 317 // calling this, to flush any pending frames. 318 void ChangeType(const std::string& id, 319 const std::string& content_type, 320 const std::string& codecs); 321 322 // If the buffer is full, attempts to try to free up space, as specified in 323 // the "Coded Frame Eviction Algorithm" in the Media Source Extensions Spec. 324 // Returns false iff buffer is still full after running eviction. 325 // https://w3c.github.io/media-source/#sourcebuffer-coded-frame-eviction 326 bool EvictCodedFrames(const std::string& id, 327 base::TimeDelta currentMediaTime, 328 size_t newDataSize); 329 330 void OnMemoryPressure( 331 base::TimeDelta currentMediaTime, 332 base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level, 333 bool force_instant_gc); 334 335 // Returns the current presentation duration. 336 double GetDuration(); 337 double GetDuration_Locked(); 338 339 // Notifies the demuxer that the duration of the media has changed to 340 // |duration|. 341 void SetDuration(double duration); 342 343 // Returns true if the source buffer associated with |id| is currently parsing 344 // a media segment, or false otherwise. 345 bool IsParsingMediaSegment(const std::string& id); 346 347 // Returns the 'Generate Timestamps Flag', as described in the MSE Byte Stream 348 // Format Registry, for the source buffer associated with |id|. 349 bool GetGenerateTimestampsFlag(const std::string& id); 350 351 // Set the append mode to be applied to subsequent buffers appended to the 352 // source buffer associated with |id|. If |sequence_mode| is true, caller 353 // is requesting "sequence" mode. Otherwise, caller is requesting "segments" 354 // mode. 355 void SetSequenceMode(const std::string& id, bool sequence_mode); 356 357 // Signals the coded frame processor for the source buffer associated with 358 // |id| to update its group start timestamp to be |timestamp_offset| if it is 359 // in sequence append mode. 360 void SetGroupStartTimestampIfInSequenceMode(const std::string& id, 361 base::TimeDelta timestamp_offset); 362 363 // Called to signal changes in the "end of stream" 364 // state. UnmarkEndOfStream() must not be called if a matching 365 // MarkEndOfStream() has not come before it. 366 void MarkEndOfStream(PipelineStatus status); 367 void UnmarkEndOfStream(); 368 369 void Shutdown(); 370 371 // Sets the memory limit on each stream of a specific type. 372 // |memory_limit| is the maximum number of bytes each stream of type |type| 373 // is allowed to hold in its buffer. 374 void SetMemoryLimitsForTest(DemuxerStream::Type type, size_t memory_limit); 375 376 // Returns the ranges representing the buffered data in the demuxer. 377 // TODO(wolenetz): Remove this method once MediaSourceDelegate no longer 378 // requires it for doing hack browser seeks to I-frame on Android. See 379 // http://crbug.com/304234. 380 Ranges<base::TimeDelta> GetBufferedRanges() const; 381 382 private: 383 enum State { 384 WAITING_FOR_INIT = 0, 385 INITIALIZING, 386 INITIALIZED, 387 ENDED, 388 389 // Any State at or beyond PARSE_ERROR cannot be changed to a state before 390 // this. See ChangeState_Locked. 391 PARSE_ERROR, 392 SHUTDOWN, 393 }; 394 395 // Helper for vide and audio track changing. 396 void FindAndEnableProperTracks(const std::vector<MediaTrack::Id>& track_ids, 397 base::TimeDelta curr_time, 398 DemuxerStream::Type track_type, 399 TrackChangeCB change_completed_cb); 400 401 void ChangeState_Locked(State new_state); 402 403 // Reports an error and puts the demuxer in a state where it won't accept more 404 // data. 405 void ReportError_Locked(PipelineStatus error); 406 407 // Returns true if any stream has seeked to a time without buffered data. 408 bool IsSeekWaitingForData_Locked() const; 409 410 // Returns true if all streams can successfully call EndOfStream, 411 // false if any can not. 412 bool CanEndOfStream_Locked() const; 413 414 // SourceBufferState callbacks. 415 void OnSourceInitDone(const std::string& source_id, 416 const StreamParser::InitParameters& params); 417 418 // Creates a DemuxerStream of the specified |type| for the SourceBufferState 419 // with the given |source_id|. 420 // Returns a pointer to a new ChunkDemuxerStream instance, which is owned by 421 // ChunkDemuxer. 422 ChunkDemuxerStream* CreateDemuxerStream(const std::string& source_id, 423 DemuxerStream::Type type); 424 425 void OnNewTextTrack(ChunkDemuxerStream* text_stream, 426 const TextTrackConfig& config); 427 428 // Returns true if |source_id| is valid, false otherwise. 429 bool IsValidId(const std::string& source_id) const; 430 431 // Increases |duration_| to |new_duration|, if |new_duration| is higher. 432 void IncreaseDurationIfNecessary(base::TimeDelta new_duration); 433 434 // Decreases |duration_| if the buffered region is less than |duration_| when 435 // EndOfStream() is called. 436 void DecreaseDurationIfNecessary(); 437 438 // Sets |duration_| to |new_duration|, sets |user_specified_duration_| to -1 439 // and notifies |host_|. 440 void UpdateDuration(base::TimeDelta new_duration); 441 442 // Returns the ranges representing the buffered data in the demuxer. 443 Ranges<base::TimeDelta> GetBufferedRanges_Locked() const; 444 445 // Start returning data on all DemuxerStreams. 446 void StartReturningData(); 447 448 void AbortPendingReads_Locked(); 449 450 // Completes any pending reads if it is possible to do so. 451 void CompletePendingReadsIfPossible(); 452 453 // Seeks all SourceBufferStreams to |seek_time|. 454 void SeekAllSources(base::TimeDelta seek_time); 455 456 // Generates and returns a unique media track id. 457 static MediaTrack::Id GenerateMediaTrackId(); 458 459 // Shuts down all DemuxerStreams by calling Shutdown() on 460 // all objects in |source_state_map_|. 461 void ShutdownAllStreams(); 462 463 // Executes |init_cb_| with |status| and closes out the async trace. 464 void RunInitCB_Locked(PipelineStatus status); 465 466 // Executes |seek_cb_| with |status| and closes out the async trace. 467 void RunSeekCB_Locked(PipelineStatus status); 468 469 mutable base::Lock lock_; 470 State state_; 471 bool cancel_next_seek_; 472 473 DemuxerHost* host_; 474 base::OnceClosure open_cb_; 475 const base::RepeatingClosure progress_cb_; 476 EncryptedMediaInitDataCB encrypted_media_init_data_cb_; 477 478 // MediaLog for reporting messages and properties to debug content and engine. 479 MediaLog* media_log_; 480 481 PipelineStatusCallback init_cb_; 482 // Callback to execute upon seek completion. 483 // TODO(wolenetz/acolwell): Protect against possible double-locking by first 484 // releasing |lock_| before executing this callback. See 485 // http://crbug.com/308226 486 PipelineStatusCallback seek_cb_; 487 488 using OwnedChunkDemuxerStreamVector = 489 std::vector<std::unique_ptr<ChunkDemuxerStream>>; 490 OwnedChunkDemuxerStreamVector audio_streams_; 491 OwnedChunkDemuxerStreamVector video_streams_; 492 OwnedChunkDemuxerStreamVector text_streams_; 493 494 // Keep track of which ids still remain uninitialized so that we transition 495 // into the INITIALIZED only after all ids/SourceBuffers got init segment. 496 std::set<std::string> pending_source_init_ids_; 497 498 base::TimeDelta duration_; 499 500 // The duration passed to the last SetDuration(). If 501 // SetDuration() is never called or an AppendData() call or 502 // a EndOfStream() call changes |duration_|, then this 503 // variable is set to < 0 to indicate that the |duration_| represents 504 // the actual duration instead of a user specified value. 505 double user_specified_duration_; 506 507 base::Time timeline_offset_; 508 DemuxerStream::Liveness liveness_; 509 510 std::map<std::string, std::unique_ptr<SourceBufferState>> source_state_map_; 511 512 std::map<std::string, std::vector<ChunkDemuxerStream*>> id_to_streams_map_; 513 // Used to hold alive the demuxer streams that were created for removed / 514 // released SourceBufferState objects. Demuxer clients might still have 515 // references to these streams, so we need to keep them alive. But they'll be 516 // in a shut down state, so reading from them will return EOS. 517 std::vector<std::unique_ptr<ChunkDemuxerStream>> removed_streams_; 518 519 std::map<MediaTrack::Id, ChunkDemuxerStream*> track_id_to_demux_stream_map_; 520 521 DISALLOW_COPY_AND_ASSIGN(ChunkDemuxer); 522 }; 523 524 } // namespace media 525 526 #endif // MEDIA_FILTERS_CHUNK_DEMUXER_H_ 527