1 // Copyright 2018 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "media/gpu/test/video_player/video_decoder_client.h"
6
7 #include <string>
8 #include <utility>
9
10 #include "base/bind.h"
11 #include "base/memory/ptr_util.h"
12 #include "base/threading/thread_task_runner_handle.h"
13 #include "build/build_config.h"
14 #include "media/base/media_util.h"
15 #include "media/base/waiting.h"
16 #include "media/gpu/macros.h"
17 #include "media/gpu/test/video.h"
18 #include "media/gpu/test/video_player/frame_renderer.h"
19 #include "media/gpu/test/video_player/test_vda_video_decoder.h"
20 #include "media/gpu/test/video_test_helpers.h"
21 #include "media/media_buildflags.h"
22 #include "testing/gtest/include/gtest/gtest.h"
23
24 #if BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
25 #include "media/gpu/chromeos/chromeos_video_decoder_factory.h"
26 #include "media/gpu/chromeos/platform_video_frame_pool.h"
27 #include "media/gpu/chromeos/video_frame_converter.h"
28 #endif // BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
29
30 namespace media {
31 namespace test {
32
33 namespace {
34 // Callbacks can be called from any thread, but WeakPtrs are not thread-safe.
35 // This helper thunk wraps a WeakPtr into an 'Optional' value, so the WeakPtr is
36 // only dereferenced after rescheduling the task on the specified task runner.
37 template <typename F, typename... Args>
CallbackThunk(base::Optional<base::WeakPtr<VideoDecoderClient>> decoder_client,scoped_refptr<base::SequencedTaskRunner> task_runner,F f,Args...args)38 void CallbackThunk(
39 base::Optional<base::WeakPtr<VideoDecoderClient>> decoder_client,
40 scoped_refptr<base::SequencedTaskRunner> task_runner,
41 F f,
42 Args... args) {
43 DCHECK(decoder_client);
44 task_runner->PostTask(FROM_HERE, base::BindOnce(f, *decoder_client, args...));
45 }
46 } // namespace
47
VideoDecoderClient(const VideoPlayer::EventCallback & event_cb,gpu::GpuMemoryBufferFactory * gpu_memory_buffer_factory,std::unique_ptr<FrameRenderer> renderer,std::vector<std::unique_ptr<VideoFrameProcessor>> frame_processors,const VideoDecoderClientConfig & config)48 VideoDecoderClient::VideoDecoderClient(
49 const VideoPlayer::EventCallback& event_cb,
50 gpu::GpuMemoryBufferFactory* gpu_memory_buffer_factory,
51 std::unique_ptr<FrameRenderer> renderer,
52 std::vector<std::unique_ptr<VideoFrameProcessor>> frame_processors,
53 const VideoDecoderClientConfig& config)
54 : event_cb_(event_cb),
55 frame_renderer_(std::move(renderer)),
56 frame_processors_(std::move(frame_processors)),
57 decoder_client_config_(config),
58 decoder_client_thread_("VDAClientDecoderThread"),
59 decoder_client_state_(VideoDecoderClientState::kUninitialized),
60 gpu_memory_buffer_factory_(gpu_memory_buffer_factory) {
61 DETACH_FROM_SEQUENCE(decoder_client_sequence_checker_);
62
63 weak_this_ = weak_this_factory_.GetWeakPtr();
64 }
65
~VideoDecoderClient()66 VideoDecoderClient::~VideoDecoderClient() {
67 DCHECK_CALLED_ON_VALID_SEQUENCE(video_player_sequence_checker_);
68
69 DestroyDecoder();
70
71 // Wait until the renderer and frame processors are done before destroying
72 // them. This needs to be done after destroying the decoder so no new frames
73 // will be queued while waiting.
74 WaitForRenderer();
75 WaitForFrameProcessors();
76 frame_renderer_ = nullptr;
77 frame_processors_.clear();
78
79 decoder_client_thread_.Stop();
80 }
81
82 // static
Create(const VideoPlayer::EventCallback & event_cb,gpu::GpuMemoryBufferFactory * gpu_memory_buffer_factory,std::unique_ptr<FrameRenderer> frame_renderer,std::vector<std::unique_ptr<VideoFrameProcessor>> frame_processors,const VideoDecoderClientConfig & config)83 std::unique_ptr<VideoDecoderClient> VideoDecoderClient::Create(
84 const VideoPlayer::EventCallback& event_cb,
85 gpu::GpuMemoryBufferFactory* gpu_memory_buffer_factory,
86 std::unique_ptr<FrameRenderer> frame_renderer,
87 std::vector<std::unique_ptr<VideoFrameProcessor>> frame_processors,
88 const VideoDecoderClientConfig& config) {
89 auto decoder_client = base::WrapUnique(new VideoDecoderClient(
90 event_cb, gpu_memory_buffer_factory, std::move(frame_renderer),
91 std::move(frame_processors), config));
92 if (!decoder_client->CreateDecoder()) {
93 return nullptr;
94 }
95 return decoder_client;
96 }
97
CreateDecoder()98 bool VideoDecoderClient::CreateDecoder() {
99 DCHECK_CALLED_ON_VALID_SEQUENCE(video_player_sequence_checker_);
100 DCHECK(!decoder_client_thread_.IsRunning());
101 DCHECK(event_cb_ && frame_renderer_);
102
103 if (!decoder_client_thread_.Start()) {
104 VLOGF(1) << "Failed to start decoder thread";
105 return false;
106 }
107
108 bool success = false;
109 base::WaitableEvent done;
110 decoder_client_thread_.task_runner()->PostTask(
111 FROM_HERE, base::BindOnce(&VideoDecoderClient::CreateDecoderTask,
112 weak_this_, &success, &done));
113 done.Wait();
114 return success;
115 }
116
DestroyDecoder()117 void VideoDecoderClient::DestroyDecoder() {
118 DCHECK_CALLED_ON_VALID_SEQUENCE(video_player_sequence_checker_);
119
120 if (!decoder_client_thread_.IsRunning()) {
121 return;
122 }
123
124 base::WaitableEvent done;
125 decoder_client_thread_.task_runner()->PostTask(
126 FROM_HERE, base::BindOnce(&VideoDecoderClient::DestroyDecoderTask,
127 weak_this_, &done));
128 done.Wait();
129 }
130
WaitForFrameProcessors()131 bool VideoDecoderClient::WaitForFrameProcessors() {
132 bool success = true;
133 for (auto& frame_processor : frame_processors_)
134 success &= frame_processor->WaitUntilDone();
135 return success;
136 }
137
WaitForRenderer()138 void VideoDecoderClient::WaitForRenderer() {
139 ASSERT_TRUE(frame_renderer_);
140 frame_renderer_->WaitUntilRenderingDone();
141 }
142
GetFrameRenderer() const143 FrameRenderer* VideoDecoderClient::GetFrameRenderer() const {
144 return frame_renderer_.get();
145 }
146
Initialize(const Video * video)147 void VideoDecoderClient::Initialize(const Video* video) {
148 DCHECK_CALLED_ON_VALID_SEQUENCE(video_player_sequence_checker_);
149 DCHECK(video);
150
151 base::WaitableEvent done;
152 decoder_client_thread_.task_runner()->PostTask(
153 FROM_HERE, base::BindOnce(&VideoDecoderClient::InitializeDecoderTask,
154 weak_this_, video, &done));
155 done.Wait();
156 }
157
Play()158 void VideoDecoderClient::Play() {
159 DCHECK_CALLED_ON_VALID_SEQUENCE(video_player_sequence_checker_);
160
161 decoder_client_thread_.task_runner()->PostTask(
162 FROM_HERE, base::BindOnce(&VideoDecoderClient::PlayTask, weak_this_));
163 }
164
Flush()165 void VideoDecoderClient::Flush() {
166 DCHECK_CALLED_ON_VALID_SEQUENCE(video_player_sequence_checker_);
167
168 decoder_client_thread_.task_runner()->PostTask(
169 FROM_HERE, base::BindOnce(&VideoDecoderClient::FlushTask, weak_this_));
170 }
171
Reset()172 void VideoDecoderClient::Reset() {
173 DCHECK_CALLED_ON_VALID_SEQUENCE(video_player_sequence_checker_);
174
175 decoder_client_thread_.task_runner()->PostTask(
176 FROM_HERE, base::BindOnce(&VideoDecoderClient::ResetTask, weak_this_));
177 }
178
CreateDecoderTask(bool * success,base::WaitableEvent * done)179 void VideoDecoderClient::CreateDecoderTask(bool* success,
180 base::WaitableEvent* done) {
181 DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
182 DCHECK_EQ(decoder_client_state_, VideoDecoderClientState::kUninitialized);
183 ASSERT_TRUE(!decoder_) << "Can't create decoder: already created";
184
185 switch (decoder_client_config_.implementation) {
186 case DecoderImplementation::kVD:
187 #if BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
188 if (decoder_client_config_.allocation_mode == AllocationMode::kImport) {
189 decoder_ = ChromeosVideoDecoderFactory::Create(
190 base::ThreadTaskRunnerHandle::Get(),
191 std::make_unique<PlatformVideoFramePool>(
192 gpu_memory_buffer_factory_),
193 std::make_unique<VideoFrameConverter>(),
194 std::make_unique<NullMediaLog>());
195 } else {
196 LOG(ERROR) << "VD-based video decoders only support import mode";
197 }
198 #endif // BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
199 break;
200 case DecoderImplementation::kVDA:
201 case DecoderImplementation::kVDVDA:
202 // The video decoder client expects decoders to use the VD interface. We
203 // can use the TestVDAVideoDecoder wrapper here to test VDA-based video
204 // decoders.
205 decoder_ = std::make_unique<TestVDAVideoDecoder>(
206 decoder_client_config_.allocation_mode,
207 decoder_client_config_.implementation ==
208 DecoderImplementation::kVDVDA,
209 gfx::ColorSpace(), frame_renderer_.get(), gpu_memory_buffer_factory_);
210 break;
211 }
212
213 *success = (decoder_ != nullptr);
214 done->Signal();
215 }
216
InitializeDecoderTask(const Video * video,base::WaitableEvent * done)217 void VideoDecoderClient::InitializeDecoderTask(const Video* video,
218 base::WaitableEvent* done) {
219 DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
220 DCHECK(decoder_client_state_ == VideoDecoderClientState::kUninitialized ||
221 decoder_client_state_ == VideoDecoderClientState::kIdle);
222 ASSERT_TRUE(decoder_) << "Can't initialize decoder: not created yet";
223 ASSERT_TRUE(video);
224
225 video_ = video;
226 encoded_data_helper_ =
227 std::make_unique<EncodedDataHelper>(video_->Data(), video_->Profile());
228
229 // (Re-)initialize the decoder.
230 VideoDecoderConfig config(
231 video_->Codec(), video_->Profile(),
232 VideoDecoderConfig::AlphaMode::kIsOpaque, VideoColorSpace(),
233 kNoTransformation, video_->Resolution(), gfx::Rect(video_->Resolution()),
234 video_->Resolution(), std::vector<uint8_t>(0), EncryptionScheme());
235
236 VideoDecoder::InitCB init_cb = base::BindOnce(
237 CallbackThunk<decltype(&VideoDecoderClient::DecoderInitializedTask),
238 Status>,
239 weak_this_, decoder_client_thread_.task_runner(),
240 &VideoDecoderClient::DecoderInitializedTask);
241 VideoDecoder::OutputCB output_cb = base::BindRepeating(
242 CallbackThunk<decltype(&VideoDecoderClient::FrameReadyTask),
243 scoped_refptr<VideoFrame>>,
244 weak_this_, decoder_client_thread_.task_runner(),
245 &VideoDecoderClient::FrameReadyTask);
246
247 decoder_->Initialize(config, false, nullptr, std::move(init_cb), output_cb,
248 WaitingCB());
249
250 DCHECK_LE(decoder_client_config_.max_outstanding_decode_requests,
251 static_cast<size_t>(decoder_->GetMaxDecodeRequests()));
252
253 done->Signal();
254 }
255
DestroyDecoderTask(base::WaitableEvent * done)256 void VideoDecoderClient::DestroyDecoderTask(base::WaitableEvent* done) {
257 DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
258 DCHECK_EQ(0u, num_outstanding_decode_requests_);
259 DVLOGF(4);
260
261 // Invalidate all scheduled tasks.
262 weak_this_factory_.InvalidateWeakPtrs();
263
264 // Destroy the decoder. This will destroy all video frames.
265 if (decoder_) {
266 decoder_.reset();
267 }
268
269 decoder_client_state_ = VideoDecoderClientState::kUninitialized;
270 done->Signal();
271 }
272
PlayTask()273 void VideoDecoderClient::PlayTask() {
274 DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
275 DVLOGF(4);
276
277 // This method should only be called when the decoder client is idle. If
278 // called e.g. while flushing, the behavior is undefined.
279 ASSERT_EQ(decoder_client_state_, VideoDecoderClientState::kIdle);
280
281 // Start decoding the first fragments. While in the decoding state new
282 // fragments will automatically be fed to the decoder, when the decoder
283 // notifies us it reached the end of a bitstream buffer.
284 decoder_client_state_ = VideoDecoderClientState::kDecoding;
285 for (size_t i = 0; i < decoder_client_config_.max_outstanding_decode_requests;
286 ++i) {
287 DecodeNextFragmentTask();
288 }
289 }
290
DecodeNextFragmentTask()291 void VideoDecoderClient::DecodeNextFragmentTask() {
292 DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
293 DVLOGF(4);
294
295 // Stop decoding fragments if we're no longer in the decoding state.
296 if (decoder_client_state_ != VideoDecoderClientState::kDecoding)
297 return;
298
299 // Flush immediately when we reached the end of the stream. This changes the
300 // state to kFlushing so further decode tasks will be aborted.
301 if (encoded_data_helper_->ReachEndOfStream()) {
302 FlushTask();
303 return;
304 }
305
306 scoped_refptr<DecoderBuffer> bitstream_buffer =
307 encoded_data_helper_->GetNextBuffer();
308 if (!bitstream_buffer) {
309 LOG(ERROR) << "Failed to get next video stream data";
310 return;
311 }
312 bitstream_buffer->set_timestamp(base::TimeTicks::Now().since_origin());
313 bool has_config_info = media::test::EncodedDataHelper::HasConfigInfo(
314 bitstream_buffer->data(), bitstream_buffer->data_size(),
315 video_->Profile());
316
317 VideoDecoder::DecodeCB decode_cb = base::BindOnce(
318 CallbackThunk<decltype(&VideoDecoderClient::DecodeDoneTask),
319 media::Status>,
320 weak_this_, decoder_client_thread_.task_runner(),
321 &VideoDecoderClient::DecodeDoneTask);
322 decoder_->Decode(std::move(bitstream_buffer), std::move(decode_cb));
323
324 num_outstanding_decode_requests_++;
325
326 // Throw event when we encounter a config info in a H.264 stream.
327 if (has_config_info)
328 FireEvent(VideoPlayerEvent::kConfigInfo);
329 }
330
FlushTask()331 void VideoDecoderClient::FlushTask() {
332 DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
333 DVLOGF(4);
334
335 // Changing the state to flushing will abort any pending decodes.
336 decoder_client_state_ = VideoDecoderClientState::kFlushing;
337
338 VideoDecoder::DecodeCB flush_done_cb =
339 base::BindOnce(CallbackThunk<decltype(&VideoDecoderClient::FlushDoneTask),
340 media::Status>,
341 weak_this_, decoder_client_thread_.task_runner(),
342 &VideoDecoderClient::FlushDoneTask);
343 decoder_->Decode(DecoderBuffer::CreateEOSBuffer(), std::move(flush_done_cb));
344
345 FireEvent(VideoPlayerEvent::kFlushing);
346 }
347
ResetTask()348 void VideoDecoderClient::ResetTask() {
349 DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
350 DVLOGF(4);
351
352 // Changing the state to resetting will abort any pending decodes.
353 decoder_client_state_ = VideoDecoderClientState::kResetting;
354 // TODO(dstaessens@) Allow resetting to any point in the stream.
355 encoded_data_helper_->Rewind();
356
357 decoder_->Reset(base::BindOnce(
358 CallbackThunk<decltype(&VideoDecoderClient::ResetDoneTask)>, weak_this_,
359 decoder_client_thread_.task_runner(),
360 &VideoDecoderClient::ResetDoneTask));
361 FireEvent(VideoPlayerEvent::kResetting);
362 }
363
DecoderInitializedTask(Status status)364 void VideoDecoderClient::DecoderInitializedTask(Status status) {
365 DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
366 DCHECK(decoder_client_state_ == VideoDecoderClientState::kUninitialized ||
367 decoder_client_state_ == VideoDecoderClientState::kIdle);
368 ASSERT_TRUE(status.is_ok()) << "Initializing decoder failed";
369
370 decoder_client_state_ = VideoDecoderClientState::kIdle;
371 FireEvent(VideoPlayerEvent::kInitialized);
372 }
373
DecodeDoneTask(media::Status status)374 void VideoDecoderClient::DecodeDoneTask(media::Status status) {
375 DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
376 DCHECK_NE(VideoDecoderClientState::kIdle, decoder_client_state_);
377 ASSERT_TRUE(status.code() != media::StatusCode::kAborted ||
378 decoder_client_state_ == VideoDecoderClientState::kResetting);
379 DVLOGF(4);
380
381 num_outstanding_decode_requests_--;
382
383 // Queue the next fragment to be decoded.
384 decoder_client_thread_.task_runner()->PostTask(
385 FROM_HERE,
386 base::BindOnce(&VideoDecoderClient::DecodeNextFragmentTask, weak_this_));
387 }
388
FrameReadyTask(scoped_refptr<VideoFrame> video_frame)389 void VideoDecoderClient::FrameReadyTask(scoped_refptr<VideoFrame> video_frame) {
390 DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
391 DCHECK(video_frame->metadata()->power_efficient);
392
393 frame_renderer_->RenderFrame(video_frame);
394
395 for (auto& frame_processor : frame_processors_)
396 frame_processor->ProcessVideoFrame(video_frame, current_frame_index_);
397
398 // Notify the test a frame has been decoded. We should only do this after
399 // scheduling the frame to be processed, so calling WaitForFrameProcessors()
400 // after receiving this event will always guarantee the frame to be processed.
401 FireEvent(VideoPlayerEvent::kFrameDecoded);
402
403 current_frame_index_++;
404 }
405
FlushDoneTask(media::Status status)406 void VideoDecoderClient::FlushDoneTask(media::Status status) {
407 DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
408 DCHECK_EQ(0u, num_outstanding_decode_requests_);
409
410 // Send an EOS frame to the renderer, so it can reset any internal state it
411 // might keep in preparation of the next stream of video frames.
412 frame_renderer_->RenderFrame(VideoFrame::CreateEOSFrame());
413 decoder_client_state_ = VideoDecoderClientState::kIdle;
414 FireEvent(VideoPlayerEvent::kFlushDone);
415 }
416
ResetDoneTask()417 void VideoDecoderClient::ResetDoneTask() {
418 DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
419 DCHECK_EQ(0u, num_outstanding_decode_requests_);
420
421 // We finished resetting to a different point in the stream, so we should
422 // update the frame index. Currently only resetting to the start of the stream
423 // is supported, so we can set the frame index to zero here.
424 current_frame_index_ = 0;
425
426 frame_renderer_->RenderFrame(VideoFrame::CreateEOSFrame());
427 decoder_client_state_ = VideoDecoderClientState::kIdle;
428 FireEvent(VideoPlayerEvent::kResetDone);
429 }
430
FireEvent(VideoPlayerEvent event)431 void VideoDecoderClient::FireEvent(VideoPlayerEvent event) {
432 DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
433
434 bool continue_decoding = event_cb_.Run(event);
435 if (!continue_decoding) {
436 // Changing the state to idle will abort any pending decodes.
437 decoder_client_state_ = VideoDecoderClientState::kIdle;
438 }
439 }
440
441 } // namespace test
442 } // namespace media
443