1 /*************************************************************************/
2 /* video_stream_webm.cpp */
3 /*************************************************************************/
4 /* This file is part of: */
5 /* GODOT ENGINE */
6 /* https://godotengine.org */
7 /*************************************************************************/
8 /* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */
9 /* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */
10 /* */
11 /* Permission is hereby granted, free of charge, to any person obtaining */
12 /* a copy of this software and associated documentation files (the */
13 /* "Software"), to deal in the Software without restriction, including */
14 /* without limitation the rights to use, copy, modify, merge, publish, */
15 /* distribute, sublicense, and/or sell copies of the Software, and to */
16 /* permit persons to whom the Software is furnished to do so, subject to */
17 /* the following conditions: */
18 /* */
19 /* The above copyright notice and this permission notice shall be */
20 /* included in all copies or substantial portions of the Software. */
21 /* */
22 /* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
23 /* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
24 /* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
25 /* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
26 /* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
27 /* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
28 /* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
29 /*************************************************************************/
30
31 #include "video_stream_webm.h"
32
33 #include "core/os/file_access.h"
34 #include "core/os/os.h"
35 #include "core/project_settings.h"
36 #include "servers/audio_server.h"
37
38 #include "thirdparty/misc/yuv2rgb.h"
39
40 // libsimplewebm
41 #include <OpusVorbisDecoder.hpp>
42 #include <VPXDecoder.hpp>
43
44 // libvpx
45 #include <vpx/vpx_image.h>
46
47 // libwebm
48 #include <mkvparser/mkvparser.h>
49
50 class MkvReader : public mkvparser::IMkvReader {
51
52 public:
MkvReader(const String & p_file)53 MkvReader(const String &p_file) {
54
55 file = FileAccess::open(p_file, FileAccess::READ);
56
57 ERR_FAIL_COND_MSG(!file, "Failed loading resource: '" + p_file + "'.");
58 }
~MkvReader()59 ~MkvReader() {
60
61 if (file)
62 memdelete(file);
63 }
64
Read(long long pos,long len,unsigned char * buf)65 virtual int Read(long long pos, long len, unsigned char *buf) {
66
67 if (file) {
68
69 if (file->get_position() != (size_t)pos)
70 file->seek(pos);
71 if (file->get_buffer(buf, len) == len)
72 return 0;
73 }
74 return -1;
75 }
76
Length(long long * total,long long * available)77 virtual int Length(long long *total, long long *available) {
78
79 if (file) {
80
81 const size_t len = file->get_len();
82 if (total)
83 *total = len;
84 if (available)
85 *available = len;
86 return 0;
87 }
88 return -1;
89 }
90
91 private:
92 FileAccess *file;
93 };
94
95 /**/
96
VideoStreamPlaybackWebm()97 VideoStreamPlaybackWebm::VideoStreamPlaybackWebm() :
98 audio_track(0),
99 webm(NULL),
100 video(NULL),
101 audio(NULL),
102 video_frames(NULL),
103 audio_frame(NULL),
104 video_frames_pos(0),
105 video_frames_capacity(0),
106 num_decoded_samples(0),
107 samples_offset(-1),
108 mix_callback(NULL),
109 mix_udata(NULL),
110 playing(false),
111 paused(false),
112 delay_compensation(0.0),
113 time(0.0),
114 video_frame_delay(0.0),
115 video_pos(0.0),
116 texture(memnew(ImageTexture)),
117 pcm(NULL) {}
~VideoStreamPlaybackWebm()118 VideoStreamPlaybackWebm::~VideoStreamPlaybackWebm() {
119
120 delete_pointers();
121 }
122
open_file(const String & p_file)123 bool VideoStreamPlaybackWebm::open_file(const String &p_file) {
124
125 file_name = p_file;
126 webm = memnew(WebMDemuxer(new MkvReader(file_name), 0, audio_track));
127 if (webm->isOpen()) {
128
129 video = memnew(VPXDecoder(*webm, OS::get_singleton()->get_processor_count()));
130 if (video->isOpen()) {
131
132 audio = memnew(OpusVorbisDecoder(*webm));
133 if (audio->isOpen()) {
134
135 audio_frame = memnew(WebMFrame);
136 pcm = (float *)memalloc(sizeof(float) * audio->getBufferSamples() * webm->getChannels());
137 } else {
138
139 memdelete(audio);
140 audio = NULL;
141 }
142
143 frame_data.resize((webm->getWidth() * webm->getHeight()) << 2);
144 texture->create(webm->getWidth(), webm->getHeight(), Image::FORMAT_RGBA8, Texture::FLAG_FILTER | Texture::FLAG_VIDEO_SURFACE);
145
146 return true;
147 }
148 memdelete(video);
149 video = NULL;
150 }
151 memdelete(webm);
152 webm = NULL;
153 return false;
154 }
155
stop()156 void VideoStreamPlaybackWebm::stop() {
157
158 if (playing) {
159
160 delete_pointers();
161
162 pcm = NULL;
163
164 audio_frame = NULL;
165 video_frames = NULL;
166
167 video = NULL;
168 audio = NULL;
169
170 open_file(file_name); //Should not fail here...
171
172 video_frames_capacity = video_frames_pos = 0;
173 num_decoded_samples = 0;
174 samples_offset = -1;
175 video_frame_delay = video_pos = 0.0;
176 }
177 time = 0.0;
178 playing = false;
179 }
play()180 void VideoStreamPlaybackWebm::play() {
181
182 stop();
183
184 delay_compensation = ProjectSettings::get_singleton()->get("audio/video_delay_compensation_ms");
185 delay_compensation /= 1000.0;
186
187 playing = true;
188 }
189
is_playing() const190 bool VideoStreamPlaybackWebm::is_playing() const {
191
192 return playing;
193 }
194
set_paused(bool p_paused)195 void VideoStreamPlaybackWebm::set_paused(bool p_paused) {
196
197 paused = p_paused;
198 }
is_paused() const199 bool VideoStreamPlaybackWebm::is_paused() const {
200
201 return paused;
202 }
203
set_loop(bool p_enable)204 void VideoStreamPlaybackWebm::set_loop(bool p_enable) {
205
206 //Empty
207 }
has_loop() const208 bool VideoStreamPlaybackWebm::has_loop() const {
209
210 return false;
211 }
212
get_length() const213 float VideoStreamPlaybackWebm::get_length() const {
214
215 if (webm)
216 return webm->getLength();
217 return 0.0f;
218 }
219
get_playback_position() const220 float VideoStreamPlaybackWebm::get_playback_position() const {
221
222 return video_pos;
223 }
seek(float p_time)224 void VideoStreamPlaybackWebm::seek(float p_time) {
225
226 //Not implemented
227 }
228
set_audio_track(int p_idx)229 void VideoStreamPlaybackWebm::set_audio_track(int p_idx) {
230
231 audio_track = p_idx;
232 }
233
get_texture() const234 Ref<Texture> VideoStreamPlaybackWebm::get_texture() const {
235
236 return texture;
237 }
238
update(float p_delta)239 void VideoStreamPlaybackWebm::update(float p_delta) {
240
241 if ((!playing || paused) || !video)
242 return;
243
244 time += p_delta;
245
246 if (time < video_pos) {
247 return;
248 }
249
250 bool audio_buffer_full = false;
251
252 if (samples_offset > -1) {
253
254 //Mix remaining samples
255 const int to_read = num_decoded_samples - samples_offset;
256 const int mixed = mix_callback(mix_udata, pcm + samples_offset * webm->getChannels(), to_read);
257 if (mixed != to_read) {
258
259 samples_offset += mixed;
260 audio_buffer_full = true;
261 } else {
262
263 samples_offset = -1;
264 }
265 }
266
267 const bool hasAudio = (audio && mix_callback);
268 while ((hasAudio && !audio_buffer_full && !has_enough_video_frames()) ||
269 (!hasAudio && video_frames_pos == 0)) {
270
271 if (hasAudio && !audio_buffer_full && audio_frame->isValid() &&
272 audio->getPCMF(*audio_frame, pcm, num_decoded_samples) && num_decoded_samples > 0) {
273
274 const int mixed = mix_callback(mix_udata, pcm, num_decoded_samples);
275
276 if (mixed != num_decoded_samples) {
277 samples_offset = mixed;
278 audio_buffer_full = true;
279 }
280 }
281
282 WebMFrame *video_frame;
283 if (video_frames_pos >= video_frames_capacity) {
284
285 WebMFrame **video_frames_new = (WebMFrame **)memrealloc(video_frames, ++video_frames_capacity * sizeof(void *));
286 ERR_FAIL_COND(!video_frames_new); //Out of memory
287 (video_frames = video_frames_new)[video_frames_capacity - 1] = memnew(WebMFrame);
288 }
289 video_frame = video_frames[video_frames_pos];
290
291 if (!webm->readFrame(video_frame, audio_frame)) //This will invalidate frames
292 break; //Can't demux, EOS?
293
294 if (video_frame->isValid())
295 ++video_frames_pos;
296 };
297
298 bool video_frame_done = false;
299 while (video_frames_pos > 0 && !video_frame_done) {
300
301 WebMFrame *video_frame = video_frames[0];
302
303 // It seems VPXDecoder::decode has to be executed even though we might skip this frame
304 if (video->decode(*video_frame)) {
305
306 VPXDecoder::IMAGE_ERROR err;
307 VPXDecoder::Image image;
308
309 if (should_process(*video_frame)) {
310
311 if ((err = video->getImage(image)) != VPXDecoder::NO_FRAME) {
312
313 if (err == VPXDecoder::NO_ERROR && image.w == webm->getWidth() && image.h == webm->getHeight()) {
314
315 PoolVector<uint8_t>::Write w = frame_data.write();
316 bool converted = false;
317
318 if (image.chromaShiftW == 0 && image.chromaShiftH == 0 && image.cs == VPX_CS_SRGB) {
319
320 uint8_t *wp = w.ptr();
321 unsigned char *rRow = image.planes[2];
322 unsigned char *gRow = image.planes[0];
323 unsigned char *bRow = image.planes[1];
324 for (int i = 0; i < image.h; i++) {
325 for (int j = 0; j < image.w; j++) {
326 *wp++ = rRow[j];
327 *wp++ = gRow[j];
328 *wp++ = bRow[j];
329 *wp++ = 255;
330 }
331 rRow += image.linesize[2];
332 gRow += image.linesize[0];
333 bRow += image.linesize[1];
334 }
335 converted = true;
336 } else if (image.chromaShiftW == 1 && image.chromaShiftH == 1) {
337
338 yuv420_2_rgb8888(w.ptr(), image.planes[0], image.planes[1], image.planes[2], image.w, image.h, image.linesize[0], image.linesize[1], image.w << 2);
339 //libyuv::I420ToARGB(image.planes[0], image.linesize[0], image.planes[2], image.linesize[2], image.planes[1], image.linesize[1], w.ptr(), image.w << 2, image.w, image.h);
340 converted = true;
341 } else if (image.chromaShiftW == 1 && image.chromaShiftH == 0) {
342
343 yuv422_2_rgb8888(w.ptr(), image.planes[0], image.planes[1], image.planes[2], image.w, image.h, image.linesize[0], image.linesize[1], image.w << 2);
344 //libyuv::I422ToARGB(image.planes[0], image.linesize[0], image.planes[2], image.linesize[2], image.planes[1], image.linesize[1], w.ptr(), image.w << 2, image.w, image.h);
345 converted = true;
346 } else if (image.chromaShiftW == 0 && image.chromaShiftH == 0) {
347
348 yuv444_2_rgb8888(w.ptr(), image.planes[0], image.planes[1], image.planes[2], image.w, image.h, image.linesize[0], image.linesize[1], image.w << 2);
349 //libyuv::I444ToARGB(image.planes[0], image.linesize[0], image.planes[2], image.linesize[2], image.planes[1], image.linesize[1], w.ptr(), image.w << 2, image.w, image.h);
350 converted = true;
351 } else if (image.chromaShiftW == 2 && image.chromaShiftH == 0) {
352
353 //libyuv::I411ToARGB(image.planes[0], image.linesize[0], image.planes[2], image.linesize[2] image.planes[1], image.linesize[1], w.ptr(), image.w << 2, image.w, image.h);
354 //converted = true;
355 }
356
357 if (converted) {
358 Ref<Image> img = memnew(Image(image.w, image.h, 0, Image::FORMAT_RGBA8, frame_data));
359 texture->set_data(img); //Zero copy send to visual server
360 video_frame_done = true;
361 }
362 }
363 }
364 }
365 }
366
367 video_pos = video_frame->time;
368 memmove(video_frames, video_frames + 1, (--video_frames_pos) * sizeof(void *));
369 video_frames[video_frames_pos] = video_frame;
370 }
371
372 if (video_frames_pos == 0 && webm->isEOS())
373 stop();
374 }
375
set_mix_callback(VideoStreamPlayback::AudioMixCallback p_callback,void * p_userdata)376 void VideoStreamPlaybackWebm::set_mix_callback(VideoStreamPlayback::AudioMixCallback p_callback, void *p_userdata) {
377
378 mix_callback = p_callback;
379 mix_udata = p_userdata;
380 }
get_channels() const381 int VideoStreamPlaybackWebm::get_channels() const {
382
383 if (audio)
384 return webm->getChannels();
385 return 0;
386 }
get_mix_rate() const387 int VideoStreamPlaybackWebm::get_mix_rate() const {
388
389 if (audio)
390 return webm->getSampleRate();
391 return 0;
392 }
393
has_enough_video_frames() const394 inline bool VideoStreamPlaybackWebm::has_enough_video_frames() const {
395 if (video_frames_pos > 0) {
396 // FIXME: AudioServer output latency was fixed in af9bb0e, previously it used to
397 // systematically return 0. Now that it gives a proper latency, it broke this
398 // code where the delay compensation likely never really worked.
399 //const double audio_delay = AudioServer::get_singleton()->get_output_latency();
400 const double video_time = video_frames[video_frames_pos - 1]->time;
401 return video_time >= time + /* audio_delay + */ delay_compensation;
402 }
403 return false;
404 }
405
should_process(WebMFrame & video_frame)406 bool VideoStreamPlaybackWebm::should_process(WebMFrame &video_frame) {
407 // FIXME: AudioServer output latency was fixed in af9bb0e, previously it used to
408 // systematically return 0. Now that it gives a proper latency, it broke this
409 // code where the delay compensation likely never really worked.
410 //const double audio_delay = AudioServer::get_singleton()->get_output_latency();
411 return video_frame.time >= time + /* audio_delay + */ delay_compensation;
412 }
413
delete_pointers()414 void VideoStreamPlaybackWebm::delete_pointers() {
415
416 if (pcm)
417 memfree(pcm);
418
419 if (audio_frame)
420 memdelete(audio_frame);
421 if (video_frames) {
422 for (int i = 0; i < video_frames_capacity; ++i)
423 memdelete(video_frames[i]);
424 memfree(video_frames);
425 }
426
427 if (video)
428 memdelete(video);
429 if (audio)
430 memdelete(audio);
431
432 if (webm)
433 memdelete(webm);
434 }
435
436 /**/
437
VideoStreamWebm()438 VideoStreamWebm::VideoStreamWebm() :
439 audio_track(0) {}
440
instance_playback()441 Ref<VideoStreamPlayback> VideoStreamWebm::instance_playback() {
442
443 Ref<VideoStreamPlaybackWebm> pb = memnew(VideoStreamPlaybackWebm);
444 pb->set_audio_track(audio_track);
445 if (pb->open_file(file))
446 return pb;
447 return NULL;
448 }
449
set_file(const String & p_file)450 void VideoStreamWebm::set_file(const String &p_file) {
451
452 file = p_file;
453 }
get_file()454 String VideoStreamWebm::get_file() {
455
456 return file;
457 }
458
_bind_methods()459 void VideoStreamWebm::_bind_methods() {
460
461 ClassDB::bind_method(D_METHOD("set_file", "file"), &VideoStreamWebm::set_file);
462 ClassDB::bind_method(D_METHOD("get_file"), &VideoStreamWebm::get_file);
463
464 ADD_PROPERTY(PropertyInfo(Variant::STRING, "file", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NOEDITOR | PROPERTY_USAGE_INTERNAL), "set_file", "get_file");
465 }
466
set_audio_track(int p_track)467 void VideoStreamWebm::set_audio_track(int p_track) {
468
469 audio_track = p_track;
470 }
471
472 ////////////
473
load(const String & p_path,const String & p_original_path,Error * r_error)474 RES ResourceFormatLoaderWebm::load(const String &p_path, const String &p_original_path, Error *r_error) {
475
476 FileAccess *f = FileAccess::open(p_path, FileAccess::READ);
477 if (!f) {
478 if (r_error) {
479 *r_error = ERR_CANT_OPEN;
480 }
481 return RES();
482 }
483
484 VideoStreamWebm *stream = memnew(VideoStreamWebm);
485 stream->set_file(p_path);
486
487 Ref<VideoStreamWebm> webm_stream = Ref<VideoStreamWebm>(stream);
488
489 if (r_error) {
490 *r_error = OK;
491 }
492
493 f->close();
494 memdelete(f);
495 return webm_stream;
496 }
497
get_recognized_extensions(List<String> * p_extensions) const498 void ResourceFormatLoaderWebm::get_recognized_extensions(List<String> *p_extensions) const {
499
500 p_extensions->push_back("webm");
501 }
502
handles_type(const String & p_type) const503 bool ResourceFormatLoaderWebm::handles_type(const String &p_type) const {
504
505 return ClassDB::is_parent_class(p_type, "VideoStream");
506 }
507
get_resource_type(const String & p_path) const508 String ResourceFormatLoaderWebm::get_resource_type(const String &p_path) const {
509
510 String el = p_path.get_extension().to_lower();
511 if (el == "webm")
512 return "VideoStreamWebm";
513 return "";
514 }
515