1 /*****************************************************************************
2 * mediacodec.c: Video decoder module using the Android MediaCodec API
3 *****************************************************************************
4 * Copyright (C) 2012 Martin Storsjo
5 *
6 * Authors: Martin Storsjo <martin@martin.st>
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation; either version 2.1 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with this program; if not, write to the Free Software Foundation,
20 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
21 *****************************************************************************/
22
23 /*****************************************************************************
24 * Preamble
25 *****************************************************************************/
26 #ifdef HAVE_CONFIG_H
27 # include "config.h"
28 #endif
29
30 #include <stdint.h>
31 #include <assert.h>
32
33 #include <vlc_common.h>
34 #include <vlc_aout.h>
35 #include <vlc_plugin.h>
36 #include <vlc_codec.h>
37 #include <vlc_block_helper.h>
38 #include <vlc_memory.h>
39 #include <vlc_timestamp_helper.h>
40 #include <vlc_threads.h>
41 #include <vlc_bits.h>
42
43 #include "mediacodec.h"
44 #include "../codec/hxxx_helper.h"
45 #include <OMX_Core.h>
46 #include <OMX_Component.h>
47 #include "omxil_utils.h"
48 #include "../../video_output/android/display.h"
49
50 #define BLOCK_FLAG_CSD (0x01 << BLOCK_FLAG_PRIVATE_SHIFT)
51
52 #define DECODE_FLAG_RESTART (0x01)
53 #define DECODE_FLAG_DRAIN (0x02)
54 /**
55 * Callback called when a new block is processed from DecodeBlock.
56 * It returns -1 in case of error, 0 if block should be dropped, 1 otherwise.
57 */
58 typedef int (*dec_on_new_block_cb)(decoder_t *, block_t **);
59
60 /**
61 * Callback called when decoder is flushing.
62 */
63 typedef void (*dec_on_flush_cb)(decoder_t *);
64
65 /**
66 * Callback called when DecodeBlock try to get an output buffer (pic or block).
67 * It returns -1 in case of error, or the number of output buffer returned.
68 */
69 typedef int (*dec_process_output_cb)(decoder_t *, mc_api_out *, picture_t **,
70 block_t **);
71
72 struct decoder_sys_t
73 {
74 mc_api api;
75
76 /* Codec Specific Data buffer: sent in DecodeBlock after a start or a flush
77 * with the BUFFER_FLAG_CODEC_CONFIG flag.*/
78 #define MAX_CSD_COUNT 3
79 block_t *pp_csd[MAX_CSD_COUNT];
80 size_t i_csd_count;
81 size_t i_csd_send;
82
83 bool b_has_format;
84
85 int64_t i_preroll_end;
86 int i_quirks;
87
88 /* Specific Audio/Video callbacks */
89 dec_on_new_block_cb pf_on_new_block;
90 dec_on_flush_cb pf_on_flush;
91 dec_process_output_cb pf_process_output;
92
93 vlc_mutex_t lock;
94 vlc_thread_t out_thread;
95 /* Cond used to signal the output thread */
96 vlc_cond_t cond;
97 /* Cond used to signal the decoder thread */
98 vlc_cond_t dec_cond;
99 /* Set to true by pf_flush to signal the output thread to flush */
100 bool b_flush_out;
101 /* If true, the output thread will start to dequeue output pictures */
102 bool b_output_ready;
103 /* If true, the first input block was successfully dequeued */
104 bool b_input_dequeued;
105 bool b_aborted;
106 bool b_drained;
107 bool b_adaptive;
108 int i_decode_flags;
109
110 union
111 {
112 struct
113 {
114 void *p_surface, *p_jsurface;
115 unsigned i_angle;
116 unsigned i_input_width, i_input_height;
117 unsigned int i_stride, i_slice_height;
118 int i_pixel_format;
119 struct hxxx_helper hh;
120 /* stores the inflight picture for each output buffer or NULL */
121 picture_sys_t** pp_inflight_pictures;
122 unsigned int i_inflight_pictures;
123 timestamp_fifo_t *timestamp_fifo;
124 int i_mpeg_dar_num, i_mpeg_dar_den;
125 } video;
126 struct {
127 date_t i_end_date;
128 int i_channels;
129 bool b_extract;
130 /* Some audio decoders need a valid channel count */
131 bool b_need_channels;
132 int pi_extraction[AOUT_CHAN_MAX];
133 } audio;
134 };
135 };
136
137 /*****************************************************************************
138 * Local prototypes
139 *****************************************************************************/
140 static int OpenDecoderJni(vlc_object_t *);
141 static int OpenDecoderNdk(vlc_object_t *);
142 static void CleanDecoder(decoder_t *);
143 static void CloseDecoder(vlc_object_t *);
144
145 static int Video_OnNewBlock(decoder_t *, block_t **);
146 static int VideoHXXX_OnNewBlock(decoder_t *, block_t **);
147 static int VideoMPEG2_OnNewBlock(decoder_t *, block_t **);
148 static int VideoVC1_OnNewBlock(decoder_t *, block_t **);
149 static void Video_OnFlush(decoder_t *);
150 static int Video_ProcessOutput(decoder_t *, mc_api_out *, picture_t **,
151 block_t **);
152 static int DecodeBlock(decoder_t *, block_t *);
153
154 static int Audio_OnNewBlock(decoder_t *, block_t **);
155 static void Audio_OnFlush(decoder_t *);
156 static int Audio_ProcessOutput(decoder_t *, mc_api_out *, picture_t **,
157 block_t **);
158
159 static void DecodeFlushLocked(decoder_t *);
160 static void DecodeFlush(decoder_t *);
161 static void StopMediaCodec(decoder_t *);
162 static void *OutThread(void *);
163
164 static void InvalidateAllPictures(decoder_t *);
165 static void RemoveInflightPictures(decoder_t *);
166
167 /*****************************************************************************
168 * Module descriptor
169 *****************************************************************************/
170 #define DIRECTRENDERING_TEXT "Android direct rendering"
171 #define DIRECTRENDERING_LONGTEXT \
172 "Enable Android direct rendering using opaque buffers."
173
174 #define MEDIACODEC_AUDIO_TEXT "Use MediaCodec for audio decoding"
175 #define MEDIACODEC_AUDIO_LONGTEXT "Still experimental."
176
177 #define MEDIACODEC_TUNNELEDPLAYBACK_TEXT "Use a tunneled surface for playback"
178
179 #define CFG_PREFIX "mediacodec-"
180
181 vlc_module_begin ()
182 set_description("Video decoder using Android MediaCodec via NDK")
set_category(CAT_INPUT)183 set_category(CAT_INPUT)
184 set_subcategory(SUBCAT_INPUT_VCODEC)
185 set_section(N_("Decoding"), NULL)
186 set_capability("video decoder", 0) /* Only enabled via commandline arguments */
187 add_bool(CFG_PREFIX "dr", true,
188 DIRECTRENDERING_TEXT, DIRECTRENDERING_LONGTEXT, true)
189 add_bool(CFG_PREFIX "audio", false,
190 MEDIACODEC_AUDIO_TEXT, MEDIACODEC_AUDIO_LONGTEXT, true)
191 add_bool(CFG_PREFIX "tunneled-playback", false,
192 MEDIACODEC_TUNNELEDPLAYBACK_TEXT, NULL, true)
193 set_callbacks(OpenDecoderNdk, CloseDecoder)
194 add_shortcut("mediacodec_ndk")
195 add_submodule ()
196 set_capability("audio decoder", 0)
197 set_callbacks(OpenDecoderNdk, CloseDecoder)
198 add_shortcut("mediacodec_ndk")
199 add_submodule ()
200 set_description("Video decoder using Android MediaCodec via JNI")
201 set_capability("video decoder", 0)
202 set_callbacks(OpenDecoderJni, CloseDecoder)
203 add_shortcut("mediacodec_jni")
204 add_submodule ()
205 set_capability("audio decoder", 0)
206 set_callbacks(OpenDecoderJni, CloseDecoder)
207 add_shortcut("mediacodec_jni")
208 vlc_module_end ()
209
210 static void CSDFree(decoder_t *p_dec)
211 {
212 decoder_sys_t *p_sys = p_dec->p_sys;
213
214 for (unsigned int i = 0; i < p_sys->i_csd_count; ++i)
215 block_Release(p_sys->pp_csd[i]);
216 p_sys->i_csd_count = 0;
217 }
218
219 /* Init the p_sys->p_csd that will be sent from DecodeBlock */
CSDInit(decoder_t * p_dec,block_t * p_blocks,size_t i_count)220 static void CSDInit(decoder_t *p_dec, block_t *p_blocks, size_t i_count)
221 {
222 decoder_sys_t *p_sys = p_dec->p_sys;
223 assert(i_count <= MAX_CSD_COUNT);
224
225 CSDFree(p_dec);
226
227 for (size_t i = 0; i < i_count; ++i)
228 {
229 assert(p_blocks != NULL);
230 p_sys->pp_csd[i] = p_blocks;
231 p_sys->pp_csd[i]->i_flags = BLOCK_FLAG_CSD;
232 p_blocks = p_blocks->p_next;
233 p_sys->pp_csd[i]->p_next = NULL;
234 }
235
236 p_sys->i_csd_count = i_count;
237 p_sys->i_csd_send = 0;
238 }
239
CSDDup(decoder_t * p_dec,const void * p_buf,size_t i_buf)240 static int CSDDup(decoder_t *p_dec, const void *p_buf, size_t i_buf)
241 {
242 block_t *p_block = block_Alloc(i_buf);
243 if (!p_block)
244 return VLC_ENOMEM;
245 memcpy(p_block->p_buffer, p_buf, i_buf);
246
247 CSDInit(p_dec, p_block, 1);
248 return VLC_SUCCESS;
249 }
250
HXXXInitSize(decoder_t * p_dec,bool * p_size_changed)251 static void HXXXInitSize(decoder_t *p_dec, bool *p_size_changed)
252 {
253 if (p_size_changed)
254 {
255 decoder_sys_t *p_sys = p_dec->p_sys;
256 struct hxxx_helper *hh = &p_sys->video.hh;
257 unsigned i_w, i_h, i_vw, i_vh;
258 hxxx_helper_get_current_picture_size(hh, &i_w, &i_h, &i_vw, &i_vh);
259
260 *p_size_changed = (i_w != p_sys->video.i_input_width
261 || i_h != p_sys->video.i_input_height);
262 p_sys->video.i_input_width = i_w;
263 p_sys->video.i_input_height = i_h;
264 /* fmt_out video size will be updated by mediacodec output callback */
265 }
266 }
267
268 /* Fill the p_sys->p_csd struct with H264 Parameter Sets */
H264SetCSD(decoder_t * p_dec,bool * p_size_changed)269 static int H264SetCSD(decoder_t *p_dec, bool *p_size_changed)
270 {
271 decoder_sys_t *p_sys = p_dec->p_sys;
272 struct hxxx_helper *hh = &p_sys->video.hh;
273 assert(hh->h264.i_sps_count > 0 || hh->h264.i_pps_count > 0);
274
275 block_t *p_spspps_blocks = h264_helper_get_annexb_config(hh);
276
277 if (p_spspps_blocks != NULL)
278 CSDInit(p_dec, p_spspps_blocks, 2);
279
280 HXXXInitSize(p_dec, p_size_changed);
281
282 return VLC_SUCCESS;
283 }
284
285 /* Fill the p_sys->p_csd struct with HEVC Parameter Sets */
HEVCSetCSD(decoder_t * p_dec,bool * p_size_changed)286 static int HEVCSetCSD(decoder_t *p_dec, bool *p_size_changed)
287 {
288 decoder_sys_t *p_sys = p_dec->p_sys;
289 struct hxxx_helper *hh = &p_sys->video.hh;
290
291 assert(hh->hevc.i_vps_count > 0 || hh->hevc.i_sps_count > 0 ||
292 hh->hevc.i_pps_count > 0 );
293
294 block_t *p_xps_blocks = hevc_helper_get_annexb_config(hh);
295 if (p_xps_blocks != NULL)
296 {
297 block_t *p_monolith = block_ChainGather(p_xps_blocks);
298 if (p_monolith == NULL)
299 {
300 block_ChainRelease(p_xps_blocks);
301 return VLC_ENOMEM;
302 }
303 CSDInit(p_dec, p_monolith, 1);
304 }
305
306 HXXXInitSize(p_dec, p_size_changed);
307 return VLC_SUCCESS;
308 }
309
ParseVideoExtraH264(decoder_t * p_dec,uint8_t * p_extra,int i_extra)310 static int ParseVideoExtraH264(decoder_t *p_dec, uint8_t *p_extra, int i_extra)
311 {
312 decoder_sys_t *p_sys = p_dec->p_sys;
313 struct hxxx_helper *hh = &p_sys->video.hh;
314
315 int i_ret = hxxx_helper_set_extra(hh, p_extra, i_extra);
316 if (i_ret != VLC_SUCCESS)
317 return i_ret;
318 assert(hh->pf_process_block != NULL);
319
320 if (p_sys->api.i_quirks & MC_API_VIDEO_QUIRKS_ADAPTIVE)
321 p_sys->b_adaptive = true;
322
323 p_sys->pf_on_new_block = VideoHXXX_OnNewBlock;
324
325 if (hh->h264.i_sps_count > 0 || hh->h264.i_pps_count > 0)
326 return H264SetCSD(p_dec, NULL);
327 return VLC_SUCCESS;
328 }
329
ParseVideoExtraHEVC(decoder_t * p_dec,uint8_t * p_extra,int i_extra)330 static int ParseVideoExtraHEVC(decoder_t *p_dec, uint8_t *p_extra, int i_extra)
331 {
332 decoder_sys_t *p_sys = p_dec->p_sys;
333 struct hxxx_helper *hh = &p_sys->video.hh;
334
335 int i_ret = hxxx_helper_set_extra(hh, p_extra, i_extra);
336 if (i_ret != VLC_SUCCESS)
337 return i_ret;
338 assert(hh->pf_process_block != NULL);
339
340 if (p_sys->api.i_quirks & MC_API_VIDEO_QUIRKS_ADAPTIVE)
341 p_sys->b_adaptive = true;
342
343 p_sys->pf_on_new_block = VideoHXXX_OnNewBlock;
344
345 if (hh->hevc.i_vps_count > 0 || hh->hevc.i_sps_count > 0 ||
346 hh->hevc.i_pps_count > 0 )
347 return HEVCSetCSD(p_dec, NULL);
348 return VLC_SUCCESS;
349 }
350
ParseVideoExtraVc1(decoder_t * p_dec,uint8_t * p_extra,int i_extra)351 static int ParseVideoExtraVc1(decoder_t *p_dec, uint8_t *p_extra, int i_extra)
352 {
353 int offset = 0;
354
355 if (i_extra < 4)
356 return VLC_EGENERIC;
357
358 /* Initialisation data starts with : 0x00 0x00 0x01 0x0f */
359 /* Skipping unecessary data */
360 static const uint8_t vc1_start_code[4] = {0x00, 0x00, 0x01, 0x0f};
361 for (; offset < i_extra - 4 ; ++offset)
362 {
363 if (!memcmp(&p_extra[offset], vc1_start_code, 4))
364 break;
365 }
366
367 /* Could not find the sequence header start code */
368 if (offset >= i_extra - 4)
369 return VLC_EGENERIC;
370
371 p_dec->p_sys->pf_on_new_block = VideoVC1_OnNewBlock;
372 return CSDDup(p_dec, p_extra + offset, i_extra - offset);
373 }
374
ParseVideoExtraWmv3(decoder_t * p_dec,uint8_t * p_extra,int i_extra)375 static int ParseVideoExtraWmv3(decoder_t *p_dec, uint8_t *p_extra, int i_extra)
376 {
377 /* WMV3 initialisation data :
378 * 8 fixed bytes
379 * 4 extradata bytes
380 * 4 height bytes (little endian)
381 * 4 width bytes (little endian)
382 * 16 fixed bytes */
383
384 if (i_extra < 4)
385 return VLC_EGENERIC;
386
387 uint8_t p_data[36] = {
388 0x8e, 0x01, 0x00, 0xc5, /* Fixed bytes values */
389 0x04, 0x00, 0x00, 0x00, /* Same */
390 0x00, 0x00, 0x00, 0x00, /* extradata emplacement */
391 0x00, 0x00, 0x00, 0x00, /* height emplacement (little endian) */
392 0x00, 0x00, 0x00, 0x00, /* width emplacement (little endian) */
393 0x0c, 0x00, 0x00, 0x00, /* Fixed byte pattern */
394 0x00, 0x00, 0x00, 0x00,
395 0x00, 0x00, 0x00, 0x00,
396 0x00, 0x00, 0x00, 0x00
397 };
398
399 /* Adding extradata */
400 memcpy(&p_data[8], p_extra, 4);
401 /* Adding height and width, little endian */
402 SetDWLE(&(p_data[12]), p_dec->fmt_in.video.i_height);
403 SetDWLE(&(p_data[16]), p_dec->fmt_in.video.i_width);
404
405 return CSDDup(p_dec, p_data, sizeof(p_data));
406 }
407
ParseExtra(decoder_t * p_dec)408 static int ParseExtra(decoder_t *p_dec)
409 {
410 decoder_sys_t *p_sys = p_dec->p_sys;
411 uint8_t *p_extra = p_dec->fmt_in.p_extra;
412 int i_extra = p_dec->fmt_in.i_extra;
413
414 switch (p_dec->fmt_in.i_codec)
415 {
416 case VLC_CODEC_H264:
417 return ParseVideoExtraH264(p_dec, p_extra, i_extra);
418 case VLC_CODEC_HEVC:
419 return ParseVideoExtraHEVC(p_dec, p_extra, i_extra);
420 case VLC_CODEC_WMV3:
421 return ParseVideoExtraWmv3(p_dec, p_extra, i_extra);
422 case VLC_CODEC_VC1:
423 return ParseVideoExtraVc1(p_dec, p_extra, i_extra);
424 case VLC_CODEC_MP4V:
425 if (!i_extra && p_sys->api.i_quirks & MC_API_VIDEO_QUIRKS_ADAPTIVE)
426 p_sys->b_adaptive = true;
427 break;
428 case VLC_CODEC_MPGV:
429 case VLC_CODEC_MP2V:
430 p_dec->p_sys->pf_on_new_block = VideoMPEG2_OnNewBlock;
431 break;
432 }
433 /* Set default CSD */
434 if (p_dec->fmt_in.i_extra)
435 return CSDDup(p_dec, p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra);
436 else
437 return VLC_SUCCESS;
438 }
439
UpdateVout(decoder_t * p_dec)440 static int UpdateVout(decoder_t *p_dec)
441 {
442 decoder_sys_t *p_sys = p_dec->p_sys;
443
444 if ((p_dec->fmt_in.i_codec == VLC_CODEC_MPGV ||
445 p_dec->fmt_in.i_codec == VLC_CODEC_MP2V) &&
446 (p_sys->video.i_mpeg_dar_num * p_sys->video.i_mpeg_dar_den != 0))
447 {
448 p_dec->fmt_out.video.i_sar_num =
449 p_sys->video.i_mpeg_dar_num * p_dec->fmt_out.video.i_height;
450 p_dec->fmt_out.video.i_sar_den =
451 p_sys->video.i_mpeg_dar_den * p_dec->fmt_out.video.i_width;
452 }
453
454 /* If MediaCodec can handle the rotation, reset the orientation to
455 * Normal in order to ask the vout not to rotate. */
456 p_dec->fmt_out.video.orientation = p_dec->fmt_in.video.orientation;
457 if (p_sys->video.i_angle != 0)
458 {
459 assert(p_dec->fmt_out.i_codec == VLC_CODEC_ANDROID_OPAQUE);
460 video_format_TransformTo(&p_dec->fmt_out.video, ORIENT_NORMAL);
461 }
462
463 if (decoder_UpdateVideoFormat(p_dec) != 0)
464 return VLC_EGENERIC;
465
466 if (p_dec->fmt_out.i_codec != VLC_CODEC_ANDROID_OPAQUE)
467 return VLC_SUCCESS;
468
469 /* Direct rendering: get the surface attached to the VOUT */
470 picture_t *p_dummy_hwpic = decoder_NewPicture(p_dec);
471 if (p_dummy_hwpic == NULL)
472 return VLC_EGENERIC;
473
474 assert(p_dummy_hwpic->p_sys);
475 assert(p_dummy_hwpic->p_sys->hw.p_surface);
476 assert(p_dummy_hwpic->p_sys->hw.p_jsurface);
477
478 p_sys->video.p_surface = p_dummy_hwpic->p_sys->hw.p_surface;
479 p_sys->video.p_jsurface = p_dummy_hwpic->p_sys->hw.p_jsurface;
480 picture_Release(p_dummy_hwpic);
481 return VLC_SUCCESS;
482 }
483
484 /*****************************************************************************
485 * StartMediaCodec: Create the mediacodec instance
486 *****************************************************************************/
StartMediaCodec(decoder_t * p_dec)487 static int StartMediaCodec(decoder_t *p_dec)
488 {
489 decoder_sys_t *p_sys = p_dec->p_sys;
490 union mc_api_args args;
491
492 if (p_dec->fmt_in.i_cat == VIDEO_ES)
493 {
494 args.video.i_width = p_dec->fmt_out.video.i_width;
495 args.video.i_height = p_dec->fmt_out.video.i_height;
496 args.video.i_angle = p_sys->video.i_angle;
497
498 args.video.p_surface = p_sys->video.p_surface;
499 args.video.p_jsurface = p_sys->video.p_jsurface;
500 args.video.b_tunneled_playback = args.video.p_surface ?
501 var_InheritBool(p_dec, CFG_PREFIX "tunneled-playback") : false;
502 if (p_sys->b_adaptive)
503 msg_Dbg(p_dec, "mediacodec configured for adaptative playback");
504 args.video.b_adaptive_playback = p_sys->b_adaptive;
505 }
506 else
507 {
508 date_Set(&p_sys->audio.i_end_date, VLC_TS_INVALID);
509
510 args.audio.i_sample_rate = p_dec->fmt_in.audio.i_rate;
511 args.audio.i_channel_count = p_dec->p_sys->audio.i_channels;
512 }
513
514 return p_sys->api.start(&p_sys->api, &args);
515 }
516
517 /*****************************************************************************
518 * StopMediaCodec: Close the mediacodec instance
519 *****************************************************************************/
StopMediaCodec(decoder_t * p_dec)520 static void StopMediaCodec(decoder_t *p_dec)
521 {
522 decoder_sys_t *p_sys = p_dec->p_sys;
523
524 /* Remove all pictures that are currently in flight in order
525 * to prevent the vout from using destroyed output buffers. */
526 if (p_sys->api.b_direct_rendering)
527 RemoveInflightPictures(p_dec);
528
529 p_sys->api.stop(&p_sys->api);
530 }
531
532 /*****************************************************************************
533 * OpenDecoder: Create the decoder instance
534 *****************************************************************************/
OpenDecoder(vlc_object_t * p_this,pf_MediaCodecApi_init pf_init)535 static int OpenDecoder(vlc_object_t *p_this, pf_MediaCodecApi_init pf_init)
536 {
537 decoder_t *p_dec = (decoder_t *)p_this;
538 decoder_sys_t *p_sys;
539 int i_ret;
540 int i_profile = p_dec->fmt_in.i_profile;
541 const char *mime = NULL;
542
543 /* Video or Audio if "mediacodec-audio" bool is true */
544 if (p_dec->fmt_in.i_cat != VIDEO_ES && (p_dec->fmt_in.i_cat != AUDIO_ES
545 || !var_InheritBool(p_dec, CFG_PREFIX "audio")))
546 return VLC_EGENERIC;
547
548 /* Fail if this module already failed to decode this ES */
549 if (var_Type(p_dec, "mediacodec-failed") != 0)
550 return VLC_EGENERIC;
551
552 if (p_dec->fmt_in.i_cat == VIDEO_ES)
553 {
554 /* Not all mediacodec versions can handle a size of 0. Hopefully, the
555 * packetizer will trigger a decoder restart when a new video size is
556 * found. */
557 if (!p_dec->fmt_in.video.i_width || !p_dec->fmt_in.video.i_height)
558 return VLC_EGENERIC;
559
560 switch (p_dec->fmt_in.i_codec) {
561 case VLC_CODEC_HEVC:
562 if (i_profile == -1)
563 {
564 uint8_t i_hevc_profile;
565 if (hevc_get_profile_level(&p_dec->fmt_in, &i_hevc_profile, NULL, NULL))
566 i_profile = i_hevc_profile;
567 }
568 mime = "video/hevc";
569 break;
570 case VLC_CODEC_H264:
571 if (i_profile == -1)
572 {
573 uint8_t i_h264_profile;
574 if (h264_get_profile_level(&p_dec->fmt_in, &i_h264_profile, NULL, NULL))
575 i_profile = i_h264_profile;
576 }
577 mime = "video/avc";
578 break;
579 case VLC_CODEC_H263: mime = "video/3gpp"; break;
580 case VLC_CODEC_MP4V: mime = "video/mp4v-es"; break;
581 case VLC_CODEC_MPGV:
582 case VLC_CODEC_MP2V:
583 mime = "video/mpeg2";
584 break;
585 case VLC_CODEC_WMV3: mime = "video/x-ms-wmv"; break;
586 case VLC_CODEC_VC1: mime = "video/wvc1"; break;
587 case VLC_CODEC_VP8: mime = "video/x-vnd.on2.vp8"; break;
588 case VLC_CODEC_VP9: mime = "video/x-vnd.on2.vp9"; break;
589 }
590 }
591 else
592 {
593 switch (p_dec->fmt_in.i_codec) {
594 case VLC_CODEC_AMR_NB: mime = "audio/3gpp"; break;
595 case VLC_CODEC_AMR_WB: mime = "audio/amr-wb"; break;
596 case VLC_CODEC_MPGA:
597 case VLC_CODEC_MP3: mime = "audio/mpeg"; break;
598 case VLC_CODEC_MP2: mime = "audio/mpeg-L2"; break;
599 case VLC_CODEC_MP4A: mime = "audio/mp4a-latm"; break;
600 case VLC_CODEC_QCELP: mime = "audio/qcelp"; break;
601 case VLC_CODEC_VORBIS: mime = "audio/vorbis"; break;
602 case VLC_CODEC_OPUS: mime = "audio/opus"; break;
603 case VLC_CODEC_ALAW: mime = "audio/g711-alaw"; break;
604 case VLC_CODEC_MULAW: mime = "audio/g711-mlaw"; break;
605 case VLC_CODEC_FLAC: mime = "audio/flac"; break;
606 case VLC_CODEC_GSM: mime = "audio/gsm"; break;
607 case VLC_CODEC_A52: mime = "audio/ac3"; break;
608 case VLC_CODEC_EAC3: mime = "audio/eac3"; break;
609 case VLC_CODEC_ALAC: mime = "audio/alac"; break;
610 case VLC_CODEC_DTS: mime = "audio/vnd.dts"; break;
611 /* case VLC_CODEC_: mime = "audio/mpeg-L1"; break; */
612 /* case VLC_CODEC_: mime = "audio/aac-adts"; break; */
613 }
614 }
615 if (!mime)
616 {
617 msg_Dbg(p_dec, "codec %4.4s not supported",
618 (char *)&p_dec->fmt_in.i_codec);
619 return VLC_EGENERIC;
620 }
621
622 /* Allocate the memory needed to store the decoder's structure */
623 if ((p_sys = calloc(1, sizeof(*p_sys))) == NULL)
624 return VLC_ENOMEM;
625
626 p_sys->api.p_obj = p_this;
627 p_sys->api.i_codec = p_dec->fmt_in.i_codec;
628 p_sys->api.i_cat = p_dec->fmt_in.i_cat;
629 p_sys->api.psz_mime = mime;
630 p_sys->video.i_mpeg_dar_num = 0;
631 p_sys->video.i_mpeg_dar_den = 0;
632
633 if (pf_init(&p_sys->api) != 0)
634 {
635 free(p_sys);
636 return VLC_EGENERIC;
637 }
638 if (p_sys->api.configure(&p_sys->api, i_profile) != 0)
639 {
640 /* If the device can't handle video/wvc1,
641 * it can probably handle video/x-ms-wmv */
642 if (!strcmp(mime, "video/wvc1") && p_dec->fmt_in.i_codec == VLC_CODEC_VC1)
643 {
644 p_sys->api.psz_mime = "video/x-ms-wmv";
645 if (p_sys->api.configure(&p_sys->api, i_profile) != 0)
646 {
647 p_sys->api.clean(&p_sys->api);
648 free(p_sys);
649 return (VLC_EGENERIC);
650 }
651 }
652 else
653 {
654 p_sys->api.clean(&p_sys->api);
655 free(p_sys);
656 return VLC_EGENERIC;
657 }
658 }
659
660 p_dec->p_sys = p_sys;
661
662 vlc_mutex_init(&p_sys->lock);
663 vlc_cond_init(&p_sys->cond);
664 vlc_cond_init(&p_sys->dec_cond);
665
666 if (p_dec->fmt_in.i_cat == VIDEO_ES)
667 {
668 switch (p_dec->fmt_in.i_codec)
669 {
670 case VLC_CODEC_H264:
671 case VLC_CODEC_HEVC:
672 hxxx_helper_init(&p_sys->video.hh, VLC_OBJECT(p_dec),
673 p_dec->fmt_in.i_codec, false);
674 break;
675 }
676 p_sys->pf_on_new_block = Video_OnNewBlock;
677 p_sys->pf_on_flush = Video_OnFlush;
678 p_sys->pf_process_output = Video_ProcessOutput;
679
680 p_sys->video.timestamp_fifo = timestamp_FifoNew(32);
681 if (!p_sys->video.timestamp_fifo)
682 goto bailout;
683
684 TAB_INIT(p_sys->video.i_inflight_pictures,
685 p_sys->video.pp_inflight_pictures);
686
687 if (var_InheritBool(p_dec, CFG_PREFIX "dr"))
688 {
689 /* Direct rendering: Request a valid OPAQUE Vout in order to get
690 * the surface attached to it */
691 p_dec->fmt_out.i_codec = VLC_CODEC_ANDROID_OPAQUE;
692
693 if (p_sys->api.b_support_rotation)
694 {
695 switch (p_dec->fmt_in.video.orientation)
696 {
697 case ORIENT_ROTATED_90:
698 p_sys->video.i_angle = 90;
699 break;
700 case ORIENT_ROTATED_180:
701 p_sys->video.i_angle = 180;
702 break;
703 case ORIENT_ROTATED_270:
704 p_sys->video.i_angle = 270;
705 break;
706 default:
707 p_sys->video.i_angle = 0;
708 break;
709 }
710 }
711 else
712 p_sys->video.i_angle = 0;
713
714 p_dec->fmt_out.video = p_dec->fmt_in.video;
715 if (p_dec->fmt_out.video.i_sar_num * p_dec->fmt_out.video.i_sar_den == 0)
716 {
717 p_dec->fmt_out.video.i_sar_num = 1;
718 p_dec->fmt_out.video.i_sar_den = 1;
719 }
720
721 p_sys->video.i_input_width =
722 p_dec->fmt_out.video.i_visible_width = p_dec->fmt_out.video.i_width;
723 p_sys->video.i_input_height =
724 p_dec->fmt_out.video.i_visible_height = p_dec->fmt_out.video.i_height;
725
726 if (UpdateVout(p_dec) != VLC_SUCCESS)
727 {
728 msg_Err(p_dec, "Opaque Vout request failed");
729 goto bailout;
730 }
731 }
732 }
733 else
734 {
735 p_sys->pf_on_new_block = Audio_OnNewBlock;
736 p_sys->pf_on_flush = Audio_OnFlush;
737 p_sys->pf_process_output = Audio_ProcessOutput;
738 p_sys->audio.i_channels = p_dec->fmt_in.audio.i_channels;
739
740 if ((p_sys->api.i_quirks & MC_API_AUDIO_QUIRKS_NEED_CHANNELS)
741 && !p_sys->audio.i_channels)
742 {
743 msg_Warn(p_dec, "codec need a valid channel count");
744 goto bailout;
745 }
746
747 p_dec->fmt_out.audio = p_dec->fmt_in.audio;
748 }
749
750 /* Try first to configure CSD */
751 if (ParseExtra(p_dec) != VLC_SUCCESS)
752 goto bailout;
753
754 if ((p_sys->api.i_quirks & MC_API_QUIRKS_NEED_CSD) && !p_sys->i_csd_count
755 && !p_sys->b_adaptive)
756 {
757 switch (p_dec->fmt_in.i_codec)
758 {
759 case VLC_CODEC_H264:
760 case VLC_CODEC_HEVC:
761 break; /* CSDs will come from hxxx_helper */
762 default:
763 msg_Warn(p_dec, "Not CSD found for %4.4s",
764 (const char *) &p_dec->fmt_in.i_codec);
765 goto bailout;
766 }
767 }
768
769 i_ret = StartMediaCodec(p_dec);
770 if (i_ret != VLC_SUCCESS)
771 {
772 msg_Err(p_dec, "StartMediaCodec failed");
773 goto bailout;
774 }
775
776 if (vlc_clone(&p_sys->out_thread, OutThread, p_dec,
777 VLC_THREAD_PRIORITY_LOW))
778 {
779 msg_Err(p_dec, "vlc_clone failed");
780 vlc_mutex_unlock(&p_sys->lock);
781 goto bailout;
782 }
783
784 p_dec->pf_decode = DecodeBlock;
785 p_dec->pf_flush = DecodeFlush;
786
787 return VLC_SUCCESS;
788
789 bailout:
790 CleanDecoder(p_dec);
791 return VLC_EGENERIC;
792 }
793
OpenDecoderNdk(vlc_object_t * p_this)794 static int OpenDecoderNdk(vlc_object_t *p_this)
795 {
796 return OpenDecoder(p_this, MediaCodecNdk_Init);
797 }
798
OpenDecoderJni(vlc_object_t * p_this)799 static int OpenDecoderJni(vlc_object_t *p_this)
800 {
801 return OpenDecoder(p_this, MediaCodecJni_Init);
802 }
803
AbortDecoderLocked(decoder_t * p_dec)804 static void AbortDecoderLocked(decoder_t *p_dec)
805 {
806 decoder_sys_t *p_sys = p_dec->p_sys;
807
808 if (!p_sys->b_aborted)
809 {
810 p_sys->b_aborted = true;
811 vlc_cancel(p_sys->out_thread);
812 }
813 }
814
CleanDecoder(decoder_t * p_dec)815 static void CleanDecoder(decoder_t *p_dec)
816 {
817 decoder_sys_t *p_sys = p_dec->p_sys;
818
819 vlc_mutex_destroy(&p_sys->lock);
820 vlc_cond_destroy(&p_sys->cond);
821 vlc_cond_destroy(&p_sys->dec_cond);
822
823 StopMediaCodec(p_dec);
824
825 CSDFree(p_dec);
826 p_sys->api.clean(&p_sys->api);
827
828 if (p_dec->fmt_in.i_cat == VIDEO_ES)
829 {
830 if (p_dec->fmt_in.i_codec == VLC_CODEC_H264
831 || p_dec->fmt_in.i_codec == VLC_CODEC_HEVC)
832 hxxx_helper_clean(&p_sys->video.hh);
833
834 if (p_sys->video.timestamp_fifo)
835 timestamp_FifoRelease(p_sys->video.timestamp_fifo);
836 }
837 free(p_sys);
838 }
839
840 /*****************************************************************************
841 * CloseDecoder: Close the decoder instance
842 *****************************************************************************/
CloseDecoder(vlc_object_t * p_this)843 static void CloseDecoder(vlc_object_t *p_this)
844 {
845 decoder_t *p_dec = (decoder_t *)p_this;
846 decoder_sys_t *p_sys = p_dec->p_sys;
847
848 vlc_mutex_lock(&p_sys->lock);
849 /* Unblock output thread waiting in dequeue_out */
850 DecodeFlushLocked(p_dec);
851 /* Cancel the output thread */
852 AbortDecoderLocked(p_dec);
853 vlc_mutex_unlock(&p_sys->lock);
854
855 vlc_join(p_sys->out_thread, NULL);
856
857 CleanDecoder(p_dec);
858 }
859
860 /*****************************************************************************
861 * vout callbacks
862 *****************************************************************************/
ReleasePicture(decoder_t * p_dec,unsigned i_index,bool b_render)863 static void ReleasePicture(decoder_t *p_dec, unsigned i_index, bool b_render)
864 {
865 decoder_sys_t *p_sys = p_dec->p_sys;
866
867 p_sys->api.release_out(&p_sys->api, i_index, b_render);
868 }
869
ReleasePictureTs(decoder_t * p_dec,unsigned i_index,mtime_t i_ts)870 static void ReleasePictureTs(decoder_t *p_dec, unsigned i_index, mtime_t i_ts)
871 {
872 decoder_sys_t *p_sys = p_dec->p_sys;
873 assert(p_sys->api.release_out_ts);
874
875 p_sys->api.release_out_ts(&p_sys->api, i_index, i_ts * INT64_C(1000));
876 }
877
InvalidateAllPictures(decoder_t * p_dec)878 static void InvalidateAllPictures(decoder_t *p_dec)
879 {
880 decoder_sys_t *p_sys = p_dec->p_sys;
881
882 for (unsigned int i = 0; i < p_sys->video.i_inflight_pictures; ++i)
883 AndroidOpaquePicture_Release(p_sys->video.pp_inflight_pictures[i],
884 false);
885 }
886
InsertInflightPicture(decoder_t * p_dec,picture_sys_t * p_picsys)887 static int InsertInflightPicture(decoder_t *p_dec, picture_sys_t *p_picsys)
888 {
889 decoder_sys_t *p_sys = p_dec->p_sys;
890
891 if (!p_picsys->hw.p_dec)
892 {
893 p_picsys->hw.p_dec = p_dec;
894 p_picsys->hw.pf_release = ReleasePicture;
895 if (p_sys->api.release_out_ts)
896 p_picsys->hw.pf_release_ts = ReleasePictureTs;
897 TAB_APPEND_CAST((picture_sys_t **),
898 p_sys->video.i_inflight_pictures,
899 p_sys->video.pp_inflight_pictures,
900 p_picsys);
901 } /* else already attached */
902 return 0;
903 }
904
RemoveInflightPictures(decoder_t * p_dec)905 static void RemoveInflightPictures(decoder_t *p_dec)
906 {
907 decoder_sys_t *p_sys = p_dec->p_sys;
908
909 for (unsigned int i = 0; i < p_sys->video.i_inflight_pictures; ++i)
910 AndroidOpaquePicture_DetachDecoder(p_sys->video.pp_inflight_pictures[i]);
911 TAB_CLEAN(p_sys->video.i_inflight_pictures,
912 p_sys->video.pp_inflight_pictures);
913 }
914
Video_ProcessOutput(decoder_t * p_dec,mc_api_out * p_out,picture_t ** pp_out_pic,block_t ** pp_out_block)915 static int Video_ProcessOutput(decoder_t *p_dec, mc_api_out *p_out,
916 picture_t **pp_out_pic, block_t **pp_out_block)
917 {
918 decoder_sys_t *p_sys = p_dec->p_sys;
919 (void) pp_out_block;
920 assert(pp_out_pic);
921
922 if (p_out->type == MC_OUT_TYPE_BUF)
923 {
924 picture_t *p_pic = NULL;
925
926 /* If the oldest input block had no PTS, the timestamp of
927 * the frame returned by MediaCodec might be wrong so we
928 * overwrite it with the corresponding dts. Call FifoGet
929 * first in order to avoid a gap if buffers are released
930 * due to an invalid format or a preroll */
931 int64_t forced_ts = timestamp_FifoGet(p_sys->video.timestamp_fifo);
932
933 if (!p_sys->b_has_format) {
934 msg_Warn(p_dec, "Buffers returned before output format is set, dropping frame");
935 return p_sys->api.release_out(&p_sys->api, p_out->buf.i_index, false);
936 }
937
938 if (p_out->buf.i_ts <= p_sys->i_preroll_end)
939 return p_sys->api.release_out(&p_sys->api, p_out->buf.i_index, false);
940
941 if (!p_sys->api.b_direct_rendering && p_out->buf.p_ptr == NULL)
942 {
943 /* This can happen when receiving an EOS buffer */
944 msg_Warn(p_dec, "Invalid buffer, dropping frame");
945 return p_sys->api.release_out(&p_sys->api, p_out->buf.i_index, false);
946 }
947
948 p_pic = decoder_NewPicture(p_dec);
949 if (!p_pic) {
950 msg_Warn(p_dec, "NewPicture failed");
951 return p_sys->api.release_out(&p_sys->api, p_out->buf.i_index, false);
952 }
953
954 if (forced_ts == VLC_TS_INVALID)
955 p_pic->date = p_out->buf.i_ts;
956 else
957 p_pic->date = forced_ts;
958 p_pic->b_progressive = true;
959
960 if (p_sys->api.b_direct_rendering)
961 {
962 p_pic->p_sys->hw.i_index = p_out->buf.i_index;
963 InsertInflightPicture(p_dec, p_pic->p_sys);
964 } else {
965 unsigned int chroma_div;
966 GetVlcChromaSizes(p_dec->fmt_out.i_codec,
967 p_dec->fmt_out.video.i_width,
968 p_dec->fmt_out.video.i_height,
969 NULL, NULL, &chroma_div);
970 CopyOmxPicture(p_sys->video.i_pixel_format, p_pic,
971 p_sys->video.i_slice_height, p_sys->video.i_stride,
972 (uint8_t *)p_out->buf.p_ptr, chroma_div, NULL);
973
974 if (p_sys->api.release_out(&p_sys->api, p_out->buf.i_index, false))
975 {
976 picture_Release(p_pic);
977 return -1;
978 }
979 }
980 assert(!(*pp_out_pic));
981 *pp_out_pic = p_pic;
982 return 1;
983 } else {
984 assert(p_out->type == MC_OUT_TYPE_CONF);
985 p_sys->video.i_pixel_format = p_out->conf.video.pixel_format;
986
987 const char *name = "unknown";
988 if (!p_sys->api.b_direct_rendering
989 && !GetVlcChromaFormat(p_sys->video.i_pixel_format,
990 &p_dec->fmt_out.i_codec, &name))
991 {
992 msg_Err(p_dec, "color-format not recognized");
993 return -1;
994 }
995
996 msg_Err(p_dec, "output: %d %s, %dx%d stride %d %d, crop %d %d %d %d",
997 p_sys->video.i_pixel_format, name,
998 p_out->conf.video.width, p_out->conf.video.height,
999 p_out->conf.video.stride, p_out->conf.video.slice_height,
1000 p_out->conf.video.crop_left, p_out->conf.video.crop_top,
1001 p_out->conf.video.crop_right, p_out->conf.video.crop_bottom);
1002
1003 int i_width = p_out->conf.video.crop_right + 1
1004 - p_out->conf.video.crop_left;
1005 int i_height = p_out->conf.video.crop_bottom + 1
1006 - p_out->conf.video.crop_top;
1007 if (i_width <= 1 || i_height <= 1)
1008 {
1009 i_width = p_out->conf.video.width;
1010 i_height = p_out->conf.video.height;
1011 }
1012
1013 if (!(p_sys->api.i_quirks & MC_API_VIDEO_QUIRKS_IGNORE_SIZE))
1014 {
1015 p_dec->fmt_out.video.i_visible_width =
1016 p_dec->fmt_out.video.i_width = i_width;
1017 p_dec->fmt_out.video.i_visible_height =
1018 p_dec->fmt_out.video.i_height = i_height;
1019 }
1020 else
1021 {
1022 p_dec->fmt_out.video.i_visible_width =
1023 p_dec->fmt_out.video.i_width = p_sys->video.i_input_width;
1024 p_dec->fmt_out.video.i_visible_height =
1025 p_dec->fmt_out.video.i_height = p_sys->video.i_input_height;
1026 msg_Dbg(p_dec, "video size ignored from MediaCodec");
1027 }
1028
1029 p_sys->video.i_stride = p_out->conf.video.stride;
1030 p_sys->video.i_slice_height = p_out->conf.video.slice_height;
1031 if (p_sys->video.i_stride <= 0)
1032 p_sys->video.i_stride = p_out->conf.video.width;
1033 if (p_sys->video.i_slice_height <= 0)
1034 p_sys->video.i_slice_height = p_out->conf.video.height;
1035
1036 if (p_sys->video.i_pixel_format == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar)
1037 p_sys->video.i_slice_height -= p_out->conf.video.crop_top/2;
1038 if ((p_sys->api.i_quirks & MC_API_VIDEO_QUIRKS_IGNORE_PADDING))
1039 {
1040 p_sys->video.i_slice_height = 0;
1041 p_sys->video.i_stride = p_dec->fmt_out.video.i_width;
1042 }
1043
1044 if (UpdateVout(p_dec) != VLC_SUCCESS)
1045 {
1046 msg_Err(p_dec, "UpdateVout failed");
1047 return -1;
1048 }
1049
1050 p_sys->b_has_format = true;
1051 return 0;
1052 }
1053 }
1054
1055 /* samples will be in the following order: FL FR FC LFE BL BR BC SL SR */
1056 static uint32_t pi_audio_order_src[] =
1057 {
1058 AOUT_CHAN_LEFT, AOUT_CHAN_RIGHT, AOUT_CHAN_CENTER, AOUT_CHAN_LFE,
1059 AOUT_CHAN_REARLEFT, AOUT_CHAN_REARRIGHT, AOUT_CHAN_REARCENTER,
1060 AOUT_CHAN_MIDDLELEFT, AOUT_CHAN_MIDDLERIGHT,
1061 };
1062
Audio_ProcessOutput(decoder_t * p_dec,mc_api_out * p_out,picture_t ** pp_out_pic,block_t ** pp_out_block)1063 static int Audio_ProcessOutput(decoder_t *p_dec, mc_api_out *p_out,
1064 picture_t **pp_out_pic, block_t **pp_out_block)
1065 {
1066 decoder_sys_t *p_sys = p_dec->p_sys;
1067 (void) pp_out_pic;
1068 assert(pp_out_block);
1069
1070 if (p_out->type == MC_OUT_TYPE_BUF)
1071 {
1072 block_t *p_block = NULL;
1073 if (p_out->buf.p_ptr == NULL)
1074 {
1075 /* This can happen when receiving an EOS buffer */
1076 msg_Warn(p_dec, "Invalid buffer, dropping frame");
1077 return p_sys->api.release_out(&p_sys->api, p_out->buf.i_index, false);
1078 }
1079
1080 if (!p_sys->b_has_format) {
1081 msg_Warn(p_dec, "Buffers returned before output format is set, dropping frame");
1082 return p_sys->api.release_out(&p_sys->api, p_out->buf.i_index, false);
1083 }
1084
1085 p_block = block_Alloc(p_out->buf.i_size);
1086 if (!p_block)
1087 return -1;
1088 p_block->i_nb_samples = p_out->buf.i_size
1089 / p_dec->fmt_out.audio.i_bytes_per_frame;
1090
1091 if (p_sys->audio.b_extract)
1092 {
1093 aout_ChannelExtract(p_block->p_buffer,
1094 p_dec->fmt_out.audio.i_channels,
1095 p_out->buf.p_ptr, p_sys->audio.i_channels,
1096 p_block->i_nb_samples, p_sys->audio.pi_extraction,
1097 p_dec->fmt_out.audio.i_bitspersample);
1098 }
1099 else
1100 memcpy(p_block->p_buffer, p_out->buf.p_ptr, p_out->buf.i_size);
1101
1102 if (p_out->buf.i_ts != 0
1103 && p_out->buf.i_ts != date_Get(&p_sys->audio.i_end_date))
1104 date_Set(&p_sys->audio.i_end_date, p_out->buf.i_ts);
1105
1106 p_block->i_pts = date_Get(&p_sys->audio.i_end_date);
1107 p_block->i_length = date_Increment(&p_sys->audio.i_end_date,
1108 p_block->i_nb_samples)
1109 - p_block->i_pts;
1110
1111 if (p_sys->api.release_out(&p_sys->api, p_out->buf.i_index, false))
1112 {
1113 block_Release(p_block);
1114 return -1;
1115 }
1116 *pp_out_block = p_block;
1117 return 1;
1118 } else {
1119 uint32_t i_layout_dst;
1120 int i_channels_dst;
1121
1122 assert(p_out->type == MC_OUT_TYPE_CONF);
1123
1124 if (p_out->conf.audio.channel_count <= 0
1125 || p_out->conf.audio.channel_count > 8
1126 || p_out->conf.audio.sample_rate <= 0)
1127 {
1128 msg_Warn(p_dec, "invalid audio properties channels count %d, sample rate %d",
1129 p_out->conf.audio.channel_count,
1130 p_out->conf.audio.sample_rate);
1131 return -1;
1132 }
1133
1134 msg_Err(p_dec, "output: channel_count: %d, channel_mask: 0x%X, rate: %d",
1135 p_out->conf.audio.channel_count, p_out->conf.audio.channel_mask,
1136 p_out->conf.audio.sample_rate);
1137
1138 p_dec->fmt_out.i_codec = VLC_CODEC_S16N;
1139 p_dec->fmt_out.audio.i_format = p_dec->fmt_out.i_codec;
1140
1141 p_dec->fmt_out.audio.i_rate = p_out->conf.audio.sample_rate;
1142 date_Init(&p_sys->audio.i_end_date, p_out->conf.audio.sample_rate, 1);
1143
1144 p_sys->audio.i_channels = p_out->conf.audio.channel_count;
1145 p_sys->audio.b_extract =
1146 aout_CheckChannelExtraction(p_sys->audio.pi_extraction,
1147 &i_layout_dst, &i_channels_dst,
1148 NULL, pi_audio_order_src,
1149 p_sys->audio.i_channels);
1150
1151 if (p_sys->audio.b_extract)
1152 msg_Warn(p_dec, "need channel extraction: %d -> %d",
1153 p_sys->audio.i_channels, i_channels_dst);
1154
1155 p_dec->fmt_out.audio.i_physical_channels = i_layout_dst;
1156 aout_FormatPrepare(&p_dec->fmt_out.audio);
1157
1158 if (decoder_UpdateAudioFormat(p_dec))
1159 return -1;
1160
1161 p_sys->b_has_format = true;
1162 return 0;
1163 }
1164 }
1165
DecodeFlushLocked(decoder_t * p_dec)1166 static void DecodeFlushLocked(decoder_t *p_dec)
1167 {
1168 decoder_sys_t *p_sys = p_dec->p_sys;
1169 bool b_had_input = p_sys->b_input_dequeued;
1170
1171 p_sys->b_input_dequeued = false;
1172 p_sys->b_flush_out = true;
1173 p_sys->i_preroll_end = 0;
1174 p_sys->b_output_ready = false;
1175 /* Resend CODEC_CONFIG buffer after a flush */
1176 p_sys->i_csd_send = 0;
1177
1178 p_sys->pf_on_flush(p_dec);
1179
1180 if (b_had_input && p_sys->api.flush(&p_sys->api) != VLC_SUCCESS)
1181 {
1182 AbortDecoderLocked(p_dec);
1183 return;
1184 }
1185
1186 vlc_cond_broadcast(&p_sys->cond);
1187
1188 while (!p_sys->b_aborted && p_sys->b_flush_out)
1189 vlc_cond_wait(&p_sys->dec_cond, &p_sys->lock);
1190 }
1191
DecodeFlush(decoder_t * p_dec)1192 static void DecodeFlush(decoder_t *p_dec)
1193 {
1194 decoder_sys_t *p_sys = p_dec->p_sys;
1195
1196 vlc_mutex_lock(&p_sys->lock);
1197 DecodeFlushLocked(p_dec);
1198 vlc_mutex_unlock(&p_sys->lock);
1199 }
1200
OutThread(void * data)1201 static void *OutThread(void *data)
1202 {
1203 decoder_t *p_dec = data;
1204 decoder_sys_t *p_sys = p_dec->p_sys;
1205
1206 vlc_mutex_lock(&p_sys->lock);
1207 mutex_cleanup_push(&p_sys->lock);
1208 for (;;)
1209 {
1210 int i_index;
1211
1212 /* Wait for output ready */
1213 while (!p_sys->b_flush_out && !p_sys->b_output_ready)
1214 vlc_cond_wait(&p_sys->cond, &p_sys->lock);
1215
1216 if (p_sys->b_flush_out)
1217 {
1218 /* Acknowledge flushed state */
1219 p_sys->b_flush_out = false;
1220 vlc_cond_broadcast(&p_sys->dec_cond);
1221 continue;
1222 }
1223
1224 int canc = vlc_savecancel();
1225
1226 vlc_mutex_unlock(&p_sys->lock);
1227
1228 /* Wait for an output buffer. This function returns when a new output
1229 * is available or if output is flushed. */
1230 i_index = p_sys->api.dequeue_out(&p_sys->api, -1);
1231
1232 vlc_mutex_lock(&p_sys->lock);
1233
1234 /* Ignore dequeue_out errors caused by flush */
1235 if (p_sys->b_flush_out)
1236 {
1237 /* If i_index >= 0, Release it. There is no way to know if i_index
1238 * is owned by us, so don't check the error. */
1239 if (i_index >= 0)
1240 p_sys->api.release_out(&p_sys->api, i_index, false);
1241
1242 /* Parse output format/buffers even when we are flushing */
1243 if (i_index != MC_API_INFO_OUTPUT_FORMAT_CHANGED
1244 && i_index != MC_API_INFO_OUTPUT_BUFFERS_CHANGED)
1245 {
1246 vlc_restorecancel(canc);
1247 continue;
1248 }
1249 }
1250
1251 /* Process output returned by dequeue_out */
1252 if (i_index >= 0 || i_index == MC_API_INFO_OUTPUT_FORMAT_CHANGED
1253 || i_index == MC_API_INFO_OUTPUT_BUFFERS_CHANGED)
1254 {
1255 struct mc_api_out out;
1256 int i_ret = p_sys->api.get_out(&p_sys->api, i_index, &out);
1257
1258 if (i_ret == 1)
1259 {
1260 picture_t *p_pic = NULL;
1261 block_t *p_block = NULL;
1262
1263 if (p_sys->pf_process_output(p_dec, &out, &p_pic,
1264 &p_block) == -1 && !out.b_eos)
1265 {
1266 msg_Err(p_dec, "pf_process_output failed");
1267 vlc_restorecancel(canc);
1268 break;
1269 }
1270 if (p_pic)
1271 decoder_QueueVideo(p_dec, p_pic);
1272 else if (p_block)
1273 decoder_QueueAudio(p_dec, p_block);
1274
1275 if (out.b_eos)
1276 {
1277 msg_Warn(p_dec, "EOS received");
1278 p_sys->b_drained = true;
1279 vlc_cond_signal(&p_sys->dec_cond);
1280 }
1281 } else if (i_ret != 0)
1282 {
1283 msg_Err(p_dec, "get_out failed");
1284 vlc_restorecancel(canc);
1285 break;
1286 }
1287 }
1288 else
1289 {
1290 vlc_restorecancel(canc);
1291 break;
1292 }
1293 vlc_restorecancel(canc);
1294 }
1295 msg_Warn(p_dec, "OutThread stopped");
1296
1297 /* Signal DecoderFlush that the output thread aborted */
1298 p_sys->b_aborted = true;
1299 vlc_cond_signal(&p_sys->dec_cond);
1300
1301 vlc_cleanup_pop();
1302 vlc_mutex_unlock(&p_sys->lock);
1303
1304 return NULL;
1305 }
1306
GetNextBlock(decoder_sys_t * p_sys,block_t * p_block)1307 static block_t *GetNextBlock(decoder_sys_t *p_sys, block_t *p_block)
1308 {
1309 if (p_sys->i_csd_send < p_sys->i_csd_count)
1310 return p_sys->pp_csd[p_sys->i_csd_send++];
1311 else
1312 return p_block;
1313 }
1314
QueueBlockLocked(decoder_t * p_dec,block_t * p_in_block,bool b_drain)1315 static int QueueBlockLocked(decoder_t *p_dec, block_t *p_in_block,
1316 bool b_drain)
1317 {
1318 decoder_sys_t *p_sys = p_dec->p_sys;
1319 block_t *p_block = NULL;
1320 bool b_dequeue_timeout = false;
1321
1322 assert(p_sys->api.b_started);
1323
1324 if ((p_sys->api.i_quirks & MC_API_QUIRKS_NEED_CSD) && !p_sys->i_csd_count
1325 && !p_sys->b_adaptive)
1326 return VLC_EGENERIC; /* Wait for CSDs */
1327
1328 /* Queue CSD blocks and input blocks */
1329 while (b_drain || (p_block = GetNextBlock(p_sys, p_in_block)))
1330 {
1331 int i_index;
1332
1333 vlc_mutex_unlock(&p_sys->lock);
1334 /* Wait for an input buffer. This function returns when a new input
1335 * buffer is available or after 2secs of timeout. */
1336 i_index = p_sys->api.dequeue_in(&p_sys->api,
1337 p_sys->api.b_direct_rendering ?
1338 INT64_C(2000000) : -1);
1339 vlc_mutex_lock(&p_sys->lock);
1340
1341 if (p_sys->b_aborted)
1342 return VLC_EGENERIC;
1343
1344 bool b_config = false;
1345 mtime_t i_ts = 0;
1346 p_sys->b_input_dequeued = true;
1347 const void *p_buf = NULL;
1348 size_t i_size = 0;
1349
1350 if (i_index >= 0)
1351 {
1352 assert(b_drain || p_block != NULL);
1353 if (p_block != NULL)
1354 {
1355 b_config = (p_block->i_flags & BLOCK_FLAG_CSD);
1356 if (!b_config)
1357 {
1358 i_ts = p_block->i_pts;
1359 if (!i_ts && p_block->i_dts)
1360 i_ts = p_block->i_dts;
1361 }
1362 p_buf = p_block->p_buffer;
1363 i_size = p_block->i_buffer;
1364 }
1365
1366 if (p_sys->api.queue_in(&p_sys->api, i_index, p_buf, i_size,
1367 i_ts, b_config) == 0)
1368 {
1369 if (!b_config && p_block != NULL)
1370 {
1371 if (p_block->i_flags & BLOCK_FLAG_PREROLL)
1372 p_sys->i_preroll_end = i_ts;
1373
1374 /* One input buffer is queued, signal OutThread that will
1375 * fetch output buffers */
1376 p_sys->b_output_ready = true;
1377 vlc_cond_broadcast(&p_sys->cond);
1378
1379 assert(p_block == p_in_block),
1380 p_in_block = NULL;
1381 }
1382 b_dequeue_timeout = false;
1383 if (b_drain)
1384 break;
1385 } else
1386 {
1387 msg_Err(p_dec, "queue_in failed");
1388 goto error;
1389 }
1390 }
1391 else if (i_index == MC_API_INFO_TRYAGAIN)
1392 {
1393 /* HACK: When direct rendering is enabled, there is a possible
1394 * deadlock between the Decoder and the Vout. It happens when the
1395 * Vout is paused and when the Decoder is flushing. In that case,
1396 * the Vout won't release any output buffers, therefore MediaCodec
1397 * won't dequeue any input buffers. To work around this issue,
1398 * release all output buffers if DecodeBlock is waiting more than
1399 * 2secs for a new input buffer. */
1400 if (!b_dequeue_timeout)
1401 {
1402 msg_Warn(p_dec, "Decoder stuck: invalidate all buffers");
1403 InvalidateAllPictures(p_dec);
1404 b_dequeue_timeout = true;
1405 continue;
1406 }
1407 else
1408 {
1409 msg_Err(p_dec, "dequeue_in timeout: no input available for 2secs");
1410 goto error;
1411 }
1412 }
1413 else
1414 {
1415 msg_Err(p_dec, "dequeue_in failed");
1416 goto error;
1417 }
1418 }
1419
1420 if (b_drain)
1421 {
1422 msg_Warn(p_dec, "EOS sent, waiting for OutThread");
1423
1424 /* Wait for the OutThread to stop (and process all remaining output
1425 * frames. Use a timeout here since we can't know if all decoders will
1426 * behave correctly. */
1427 mtime_t deadline = mdate() + INT64_C(3000000);
1428 while (!p_sys->b_aborted && !p_sys->b_drained
1429 && vlc_cond_timedwait(&p_sys->dec_cond, &p_sys->lock, deadline) == 0);
1430
1431 if (!p_sys->b_drained)
1432 {
1433 msg_Err(p_dec, "OutThread timed out");
1434 AbortDecoderLocked(p_dec);
1435 }
1436 p_sys->b_drained = false;
1437 }
1438
1439 return VLC_SUCCESS;
1440
1441 error:
1442 AbortDecoderLocked(p_dec);
1443 return VLC_EGENERIC;
1444 }
1445
DecodeBlock(decoder_t * p_dec,block_t * p_in_block)1446 static int DecodeBlock(decoder_t *p_dec, block_t *p_in_block)
1447 {
1448 decoder_sys_t *p_sys = p_dec->p_sys;
1449 int i_ret;
1450
1451 vlc_mutex_lock(&p_sys->lock);
1452
1453 if (p_sys->b_aborted)
1454 {
1455 if (p_sys->b_has_format)
1456 goto end;
1457 else
1458 goto reload;
1459 }
1460
1461 if (p_in_block == NULL)
1462 {
1463 /* No input block, decoder is draining */
1464 msg_Err(p_dec, "Decoder is draining");
1465
1466 if (p_sys->b_output_ready)
1467 QueueBlockLocked(p_dec, NULL, true);
1468 goto end;
1469 }
1470
1471 if (p_in_block->i_flags & (BLOCK_FLAG_DISCONTINUITY|BLOCK_FLAG_CORRUPTED))
1472 {
1473 if (p_sys->b_output_ready)
1474 QueueBlockLocked(p_dec, NULL, true);
1475 DecodeFlushLocked(p_dec);
1476 if (p_sys->b_aborted)
1477 goto end;
1478 if (p_in_block->i_flags & BLOCK_FLAG_CORRUPTED)
1479 goto end;
1480 }
1481
1482 if (p_in_block->i_flags & BLOCK_FLAG_INTERLACED_MASK
1483 && !(p_sys->api.i_quirks & MC_API_VIDEO_QUIRKS_SUPPORT_INTERLACED))
1484 {
1485 /* Before Android 21 and depending on the vendor, MediaCodec can
1486 * crash or be in an inconsistent state when decoding interlaced
1487 * videos. See OMXCodec_GetQuirks() for a white list of decoders
1488 * that supported interlaced videos before Android 21. */
1489 msg_Warn(p_dec, "codec doesn't support interlaced videos");
1490 goto reload;
1491 }
1492
1493 /* Parse input block */
1494 if ((i_ret = p_sys->pf_on_new_block(p_dec, &p_in_block)) != 1)
1495 {
1496 if (i_ret != 0)
1497 {
1498 AbortDecoderLocked(p_dec);
1499 msg_Err(p_dec, "pf_on_new_block failed");
1500 }
1501 goto end;
1502 }
1503 if (p_sys->i_decode_flags & (DECODE_FLAG_DRAIN|DECODE_FLAG_RESTART))
1504 {
1505 msg_Warn(p_dec, "Draining from DecodeBlock");
1506 const bool b_restart = p_sys->i_decode_flags & DECODE_FLAG_RESTART;
1507 p_sys->i_decode_flags = 0;
1508
1509 /* Drain and flush before restart to unblock OutThread */
1510 if (p_sys->b_output_ready)
1511 QueueBlockLocked(p_dec, NULL, true);
1512 DecodeFlushLocked(p_dec);
1513 if (p_sys->b_aborted)
1514 goto end;
1515
1516 if (b_restart)
1517 {
1518 StopMediaCodec(p_dec);
1519
1520 int i_ret = StartMediaCodec(p_dec);
1521 switch (i_ret)
1522 {
1523 case VLC_SUCCESS:
1524 msg_Warn(p_dec, "Restarted from DecodeBlock");
1525 break;
1526 case VLC_ENOOBJ:
1527 break;
1528 default:
1529 msg_Err(p_dec, "StartMediaCodec failed");
1530 AbortDecoderLocked(p_dec);
1531 goto end;
1532 }
1533 }
1534 }
1535
1536 /* Abort if MediaCodec is not yet started */
1537 if (p_sys->api.b_started)
1538 QueueBlockLocked(p_dec, p_in_block, false);
1539
1540 end:
1541 if (p_in_block)
1542 block_Release(p_in_block);
1543 /* Too late to reload here, we already modified/released the input block,
1544 * do it next time. */
1545 int ret = p_sys->b_aborted && p_sys->b_has_format ? VLCDEC_ECRITICAL
1546 : VLCDEC_SUCCESS;
1547 vlc_mutex_unlock(&p_sys->lock);
1548 return ret;
1549
1550 reload:
1551 vlc_mutex_unlock(&p_sys->lock);
1552 /* Add an empty variable so that mediacodec won't be loaded again
1553 * for this ES */
1554 var_Create(p_dec, "mediacodec-failed", VLC_VAR_VOID);
1555 return VLCDEC_RELOAD;
1556 }
1557
Video_OnNewBlock(decoder_t * p_dec,block_t ** pp_block)1558 static int Video_OnNewBlock(decoder_t *p_dec, block_t **pp_block)
1559 {
1560 decoder_sys_t *p_sys = p_dec->p_sys;
1561 block_t *p_block = *pp_block;
1562
1563 timestamp_FifoPut(p_sys->video.timestamp_fifo,
1564 p_block->i_pts ? VLC_TS_INVALID : p_block->i_dts);
1565
1566 return 1;
1567 }
1568
VideoHXXX_OnNewBlock(decoder_t * p_dec,block_t ** pp_block)1569 static int VideoHXXX_OnNewBlock(decoder_t *p_dec, block_t **pp_block)
1570 {
1571 decoder_sys_t *p_sys = p_dec->p_sys;
1572 struct hxxx_helper *hh = &p_sys->video.hh;
1573 bool b_config_changed = false;
1574 bool *p_config_changed = p_sys->b_adaptive ? NULL : &b_config_changed;
1575
1576 *pp_block = hh->pf_process_block(hh, *pp_block, p_config_changed);
1577 if (!*pp_block)
1578 return 0;
1579 if (b_config_changed)
1580 {
1581 bool b_size_changed;
1582 int i_ret;
1583 switch (p_dec->fmt_in.i_codec)
1584 {
1585 case VLC_CODEC_H264:
1586 if (hh->h264.i_sps_count > 0 || hh->h264.i_pps_count > 0)
1587 i_ret = H264SetCSD(p_dec, &b_size_changed);
1588 else
1589 i_ret = VLC_EGENERIC;
1590 break;
1591 case VLC_CODEC_HEVC:
1592 if (hh->hevc.i_vps_count > 0 || hh->hevc.i_sps_count > 0 ||
1593 hh->hevc.i_pps_count > 0 )
1594 i_ret = HEVCSetCSD(p_dec, &b_size_changed);
1595 else
1596 i_ret = VLC_EGENERIC;
1597 break;
1598 }
1599 if (i_ret != VLC_SUCCESS)
1600 return i_ret;
1601 if (b_size_changed || !p_sys->api.b_started)
1602 {
1603 if (p_sys->api.b_started)
1604 msg_Err(p_dec, "SPS/PPS changed during playback and "
1605 "video size are different. Restart it !");
1606 p_sys->i_decode_flags |= DECODE_FLAG_RESTART;
1607 } else
1608 {
1609 msg_Err(p_dec, "SPS/PPS changed during playback. Drain it");
1610 p_sys->i_decode_flags |= DECODE_FLAG_DRAIN;
1611 }
1612 }
1613
1614 return Video_OnNewBlock(p_dec, pp_block);
1615 }
1616
VideoMPEG2_OnNewBlock(decoder_t * p_dec,block_t ** pp_block)1617 static int VideoMPEG2_OnNewBlock(decoder_t *p_dec, block_t **pp_block)
1618 {
1619 if (pp_block == NULL || (*pp_block)->i_buffer <= 7)
1620 return 1;
1621
1622 decoder_sys_t *p_sys = p_dec->p_sys;
1623 const int startcode = (*pp_block)->p_buffer[3];
1624
1625 /* DAR aspect ratio from the DVD MPEG2 standard */
1626 static const int mpeg2_aspect[16][2] =
1627 {
1628 {0,0}, /* reserved */
1629 {0,0}, /* DAR = 0:0 will result in SAR = 1:1 */
1630 {4,3}, {16,9}, {221,100},
1631 /* reserved */
1632 {0,0}, {0,0}, {0,0}, {0,0}, {0,0}, {0,0}, {0,0}, {0,0}, {0,0},
1633 {0,0}, {0,0}
1634 };
1635
1636 if (startcode == 0xB3 /* SEQUENCE_HEADER_STARTCODE */)
1637 {
1638 int mpeg_dar_code = (*pp_block)->p_buffer[7] >> 4;
1639
1640 if (mpeg_dar_code >= 16)
1641 return 0;
1642
1643 p_sys->video.i_mpeg_dar_num = mpeg2_aspect[mpeg_dar_code][0];
1644 p_sys->video.i_mpeg_dar_den = mpeg2_aspect[mpeg_dar_code][1];
1645 }
1646
1647 return 1;
1648 }
1649
VideoVC1_OnNewBlock(decoder_t * p_dec,block_t ** pp_block)1650 static int VideoVC1_OnNewBlock(decoder_t *p_dec, block_t **pp_block)
1651 {
1652 block_t *p_block = *pp_block;
1653
1654 /* Adding frame start code */
1655 p_block = *pp_block = block_Realloc(p_block, 4, p_block->i_buffer);
1656 if (p_block == NULL)
1657 return VLC_ENOMEM;
1658 p_block->p_buffer[0] = 0x00;
1659 p_block->p_buffer[1] = 0x00;
1660 p_block->p_buffer[2] = 0x01;
1661 p_block->p_buffer[3] = 0x0d;
1662
1663 return Video_OnNewBlock(p_dec, pp_block);
1664 }
1665
Video_OnFlush(decoder_t * p_dec)1666 static void Video_OnFlush(decoder_t *p_dec)
1667 {
1668 decoder_sys_t *p_sys = p_dec->p_sys;
1669
1670 timestamp_FifoEmpty(p_sys->video.timestamp_fifo);
1671 /* Invalidate all pictures that are currently in flight
1672 * since flushing make all previous indices returned by
1673 * MediaCodec invalid. */
1674 if (p_sys->api.b_direct_rendering)
1675 InvalidateAllPictures(p_dec);
1676 }
1677
Audio_OnNewBlock(decoder_t * p_dec,block_t ** pp_block)1678 static int Audio_OnNewBlock(decoder_t *p_dec, block_t **pp_block)
1679 {
1680 decoder_sys_t *p_sys = p_dec->p_sys;
1681 block_t *p_block = *pp_block;
1682
1683 /* We've just started the stream, wait for the first PTS. */
1684 if (!date_Get(&p_sys->audio.i_end_date))
1685 {
1686 if (p_block->i_pts <= VLC_TS_INVALID)
1687 return 0;
1688 date_Set(&p_sys->audio.i_end_date, p_block->i_pts);
1689 }
1690
1691 return 1;
1692 }
1693
Audio_OnFlush(decoder_t * p_dec)1694 static void Audio_OnFlush(decoder_t *p_dec)
1695 {
1696 decoder_sys_t *p_sys = p_dec->p_sys;
1697
1698 date_Set(&p_sys->audio.i_end_date, VLC_TS_INVALID);
1699 }
1700