1/*****************************************************************************
2 * avcapture.m: AVFoundation (Mac OS X) based video capture module
3 *****************************************************************************
4 * Copyright © 2008-2013 VLC authors and VideoLAN
5 *
6 * Authors: Michael Feurstein <michael.feurstein@gmail.com>
7 *
8 ****************************************************************************
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms of the GNU Lesser General Public License as published by
11 * the Free Software Foundation; either version 2.1 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU Lesser General Public License for more details.
18 *
19 * You should have received a copy of the GNU Lesser General Public License
20 * along with this program; if not, write to the Free Software Foundation,
21 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
22 *****************************************************************************/
23
24/*****************************************************************************
25 * Preamble
26 *****************************************************************************/
27
28#define OS_OBJECT_USE_OBJC 0
29
30#ifdef HAVE_CONFIG_H
31# include "config.h"
32#endif
33
34#include <vlc_common.h>
35#include <vlc_plugin.h>
36#include <vlc_input.h>
37#include <vlc_demux.h>
38#include <vlc_interface.h>
39#include <vlc_dialog.h>
40#include <vlc_access.h>
41
42#import <AvailabilityMacros.h>
43#import <AVFoundation/AVFoundation.h>
44#import <CoreMedia/CoreMedia.h>
45
46#ifndef MAC_OS_X_VERSION_10_14
47@interface AVCaptureDevice (AVCaptureDeviceAuthorizationSince10_14)
48
49+ (void)requestAccessForMediaType:(AVMediaType)mediaType completionHandler:(void (^)(BOOL granted))handler API_AVAILABLE(macos(10.14), ios(7.0));
50
51@end
52#endif
53
54/*****************************************************************************
55* Local prototypes
56*****************************************************************************/
57static int Open(vlc_object_t *p_this);
58static void Close(vlc_object_t *p_this);
59static int Demux(demux_t *p_demux);
60static int Control(demux_t *, int, va_list);
61
62/*****************************************************************************
63* Module descriptor
64*****************************************************************************/
65vlc_module_begin ()
66   set_shortname(N_("AVFoundation Video Capture"))
67   set_description(N_("AVFoundation video capture module."))
68   set_category(CAT_INPUT)
69   set_subcategory(SUBCAT_INPUT_ACCESS)
70   add_shortcut("avcapture")
71   set_capability("access_demux", 10)
72   set_callbacks(Open, Close)
73vlc_module_end ()
74
75
76/*****************************************************************************
77* AVFoundation Bridge
78*****************************************************************************/
79@interface VLCAVDecompressedVideoOutput : AVCaptureVideoDataOutput
80{
81    demux_t             *p_avcapture;
82
83    CVImageBufferRef    currentImageBuffer;
84
85    mtime_t             currentPts;
86    mtime_t             previousPts;
87    size_t              bytesPerRow;
88
89    long                timeScale;
90    BOOL                videoDimensionsReady;
91}
92
93@property (readwrite) CMVideoDimensions videoDimensions;
94
95- (id)initWithDemux:(demux_t *)p_demux;
96- (int)width;
97- (int)height;
98- (void)getVideoDimensions:(CMSampleBufferRef)sampleBuffer;
99- (mtime_t)currentPts;
100- (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection;
101- (mtime_t)copyCurrentFrameToBuffer:(void *)buffer;
102@end
103
104@implementation VLCAVDecompressedVideoOutput : AVCaptureVideoDataOutput
105
106- (id)initWithDemux:(demux_t *)p_demux
107{
108    if (self = [super init])
109    {
110        p_avcapture = p_demux;
111        currentImageBuffer = nil;
112        currentPts = 0;
113        previousPts = 0;
114        bytesPerRow = 0;
115        timeScale = 0;
116        videoDimensionsReady = NO;
117    }
118    return self;
119}
120
121- (void)dealloc
122{
123    @synchronized (self)
124    {
125        CVBufferRelease(currentImageBuffer);
126        currentImageBuffer = nil;
127        bytesPerRow = 0;
128        videoDimensionsReady = NO;
129    }
130}
131
132- (long)timeScale
133{
134    return timeScale;
135}
136
137- (int)width
138{
139    return self.videoDimensions.width;
140}
141
142- (int)height
143{
144    return self.videoDimensions.height;
145}
146
147- (size_t)bytesPerRow
148{
149    return bytesPerRow;
150}
151
152- (void)getVideoDimensions:(CMSampleBufferRef)sampleBuffer
153{
154    if (!videoDimensionsReady)
155    {
156        CMFormatDescriptionRef formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer);
157        self.videoDimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
158        bytesPerRow = CVPixelBufferGetBytesPerRow(CMSampleBufferGetImageBuffer(sampleBuffer));
159        videoDimensionsReady = YES;
160        msg_Dbg(p_avcapture, "Dimensionns obtained height:%i width:%i bytesPerRow:%lu", [self height], [self width], bytesPerRow);
161    }
162}
163
164-(mtime_t)currentPts
165{
166    mtime_t pts;
167
168    if ( !currentImageBuffer || currentPts == previousPts )
169        return 0;
170
171    @synchronized (self)
172    {
173        pts = previousPts = currentPts;
174    }
175
176    return currentPts;
177}
178
179- (void)captureOutput:(AVCaptureOutput *)captureOutput
180didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
181       fromConnection:(AVCaptureConnection *)connection
182{
183    @autoreleasepool {
184        CVImageBufferRef imageBufferToRelease;
185        CMTime presentationtimestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
186        CVImageBufferRef videoFrame = CMSampleBufferGetImageBuffer(sampleBuffer);
187        CVBufferRetain(videoFrame);
188        [self getVideoDimensions:sampleBuffer];
189
190        @synchronized (self) {
191            imageBufferToRelease = currentImageBuffer;
192            currentImageBuffer = videoFrame;
193            currentPts = (mtime_t)presentationtimestamp.value;
194            timeScale = (long)presentationtimestamp.timescale;
195        }
196
197        CVBufferRelease(imageBufferToRelease);
198    }
199}
200
201- (mtime_t)copyCurrentFrameToBuffer:(void *)buffer
202{
203    CVImageBufferRef imageBuffer;
204    mtime_t pts;
205
206    void *pixels;
207
208    if ( !currentImageBuffer || currentPts == previousPts )
209        return 0;
210
211    @synchronized (self)
212    {
213        imageBuffer = CVBufferRetain(currentImageBuffer);
214        if (imageBuffer)
215        {
216            pts = previousPts = currentPts;
217            CVPixelBufferLockBaseAddress(imageBuffer, 0);
218            pixels = CVPixelBufferGetBaseAddress(imageBuffer);
219            if (pixels)
220            {
221                memcpy(buffer, pixels, CVPixelBufferGetHeight(imageBuffer) * CVPixelBufferGetBytesPerRow(imageBuffer));
222            }
223            CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
224        }
225    }
226    CVBufferRelease(imageBuffer);
227
228    if (pixels)
229        return currentPts;
230    else
231        return 0;
232}
233
234@end
235
236/*****************************************************************************
237* Struct
238*****************************************************************************/
239
240struct demux_sys_t
241{
242    CFTypeRef _Nullable             session;       // AVCaptureSession
243    CFTypeRef _Nullable             device;        // AVCaptureDevice
244    CFTypeRef _Nullable             output;        // VLCAVDecompressedVideoOutput
245    es_out_id_t                     *p_es_video;
246    es_format_t                     fmt;
247    int                             height, width;
248    BOOL                            b_es_setup;
249};
250
251/*****************************************************************************
252* Open:
253*****************************************************************************/
254static int Open(vlc_object_t *p_this)
255{
256    demux_t                 *p_demux = (demux_t*)p_this;
257    demux_sys_t             *p_sys = NULL;
258
259    NSString                *avf_currdevice_uid;
260    NSArray                 *myVideoDevices;
261    NSError                 *o_returnedError;
262
263    AVCaptureDeviceInput    *input = nil;
264
265    int                     i, i_width, i_height, deviceCount, ivideo;
266
267    char                    *psz_uid = NULL;
268
269    /* Only when selected */
270    if ( *p_demux->psz_access == '\0' )
271        return VLC_EGENERIC;
272
273    @autoreleasepool {
274        if (p_demux->psz_location && *p_demux->psz_location)
275            psz_uid = strdup(p_demux->psz_location);
276
277        msg_Dbg(p_demux, "avcapture uid = %s", psz_uid);
278        avf_currdevice_uid = [[NSString alloc] initWithFormat:@"%s", psz_uid];
279
280        /* Set up p_demux */
281        p_demux->pf_demux = Demux;
282        p_demux->pf_control = Control;
283        p_demux->info.i_update = 0;
284        p_demux->info.i_title = 0;
285        p_demux->info.i_seekpoint = 0;
286
287        p_demux->p_sys = p_sys = calloc(1, sizeof(demux_sys_t));
288        if ( !p_sys )
289            return VLC_ENOMEM;
290
291        myVideoDevices = [[AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]
292                           arrayByAddingObjectsFromArray:[AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed]];
293        if ( [myVideoDevices count] == 0 )
294        {
295            vlc_dialog_display_error(p_demux, _("No video devices found"),
296                _("Your Mac does not seem to be equipped with a suitable video input device. "
297                "Please check your connectors and drivers."));
298            msg_Err(p_demux, "Can't find any suitable video device");
299            goto error;
300        }
301
302        deviceCount = [myVideoDevices count];
303        for ( ivideo = 0; ivideo < deviceCount; ivideo++ )
304        {
305            AVCaptureDevice *avf_device;
306            avf_device = [myVideoDevices objectAtIndex:ivideo];
307            msg_Dbg(p_demux, "avcapture %i/%i %s %s", ivideo, deviceCount, [[avf_device modelID] UTF8String], [[avf_device uniqueID] UTF8String]);
308            if ([[[avf_device uniqueID]stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]] isEqualToString:avf_currdevice_uid]) {
309                break;
310            }
311        }
312
313        if ( ivideo < [myVideoDevices count] )
314        {
315            p_sys->device = CFBridgingRetain([myVideoDevices objectAtIndex:ivideo]);
316        }
317        else
318        {
319            msg_Dbg(p_demux, "Cannot find designated device as %s, falling back to default.", [avf_currdevice_uid UTF8String]);
320            p_sys->device = CFBridgingRetain([AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]);
321        }
322        if ( !p_sys->device )
323        {
324            vlc_dialog_display_error(p_demux, _("No video devices found"),
325                _("Your Mac does not seem to be equipped with a suitable input device. "
326                "Please check your connectors and drivers."));
327            msg_Err(p_demux, "Can't find any suitable video device");
328            goto error;
329        }
330
331        if ( [(__bridge AVCaptureDevice *)p_sys->device isInUseByAnotherApplication] == YES )
332        {
333            msg_Err(p_demux, "default capture device is exclusively in use by another application");
334            goto error;
335        }
336
337        if (@available(macOS 10.14, *)) {
338            msg_Dbg(p_demux, "Check user consent for access to the video device");
339
340            dispatch_semaphore_t sema = dispatch_semaphore_create(0);
341            __block bool accessGranted = NO;
342            [AVCaptureDevice requestAccessForMediaType: AVMediaTypeVideo completionHandler:^(BOOL granted) {
343                accessGranted = granted;
344                dispatch_semaphore_signal(sema);
345            } ];
346            dispatch_semaphore_wait(sema, DISPATCH_TIME_FOREVER);
347            dispatch_release(sema);
348            if (!accessGranted) {
349                msg_Err(p_demux, "Can't use the video device as access has not been granted by the user");
350                vlc_dialog_display_error(p_demux, _("Problem accessing a system resource"),
351                    _("Please open \"System Preferences\" -> \"Security & Privacy\" "
352                      "and allow VLC to access your camera."));
353
354                goto error;
355            }
356        }
357
358        input = [AVCaptureDeviceInput deviceInputWithDevice:(__bridge AVCaptureDevice *)p_sys->device error:&o_returnedError];
359
360        if ( !input )
361        {
362            msg_Err(p_demux, "can't create a valid capture input facility: %s (%ld)",[[o_returnedError localizedDescription] UTF8String], [o_returnedError code]);
363            goto error;
364        }
365
366
367        int chroma = VLC_CODEC_RGB32;
368
369        memset(&p_sys->fmt, 0, sizeof(es_format_t));
370        es_format_Init(&p_sys->fmt, VIDEO_ES, chroma);
371
372        p_sys->session = CFBridgingRetain([[AVCaptureSession alloc] init]);
373        [(__bridge AVCaptureSession *)p_sys->session addInput:input];
374
375        p_sys->output = CFBridgingRetain([[VLCAVDecompressedVideoOutput alloc] initWithDemux:p_demux]);
376        [(__bridge AVCaptureSession *)p_sys->session addOutput:(__bridge VLCAVDecompressedVideoOutput *)p_sys->output];
377
378        dispatch_queue_t queue = dispatch_queue_create("avCaptureQueue", NULL);
379        [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output setSampleBufferDelegate:(__bridge id)p_sys->output queue:queue];
380        dispatch_release(queue);
381
382        [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_32BGRA] forKey:(id)kCVPixelBufferPixelFormatTypeKey]];
383        [(__bridge AVCaptureSession *)p_sys->session startRunning];
384
385        input = nil;
386
387        msg_Dbg(p_demux, "AVCapture: Video device ready!");
388
389        return VLC_SUCCESS;
390    error:
391        msg_Err(p_demux, "Error");
392        input = nil;
393
394        free(p_sys);
395
396        return VLC_EGENERIC;
397    }
398}
399
400/*****************************************************************************
401* Close:
402*****************************************************************************/
403static void Close(vlc_object_t *p_this)
404{
405    demux_t             *p_demux = (demux_t*)p_this;
406    demux_sys_t         *p_sys = p_demux->p_sys;
407
408    @autoreleasepool {
409        msg_Dbg(p_demux,"Close AVCapture");
410
411        // Perform this on main thread, as the framework itself will sometimes try to synchronously
412        // work on main thread. And this will create a dead lock.
413        [(__bridge AVCaptureSession *)p_sys->session performSelectorOnMainThread:@selector(stopRunning) withObject:nil waitUntilDone:NO];
414        CFBridgingRelease(p_sys->output);
415        CFBridgingRelease(p_sys->session);
416
417        free(p_sys);
418    }
419}
420
421/*****************************************************************************
422* Demux:
423*****************************************************************************/
424static int Demux(demux_t *p_demux)
425{
426    demux_sys_t *p_sys = p_demux->p_sys;
427    block_t     *p_block;
428
429    @autoreleasepool {
430        @synchronized ( p_sys->output )
431        {
432            p_block = block_Alloc([(__bridge VLCAVDecompressedVideoOutput *)p_sys->output width] * [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output bytesPerRow]);
433
434            if ( !p_block )
435            {
436                msg_Err(p_demux, "cannot get block");
437                return 0;
438            }
439
440            p_block->i_pts = [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output copyCurrentFrameToBuffer: p_block->p_buffer];
441
442            if ( !p_block->i_pts )
443            {
444                /* Nothing to display yet, just forget */
445                block_Release(p_block);
446                msleep(10000);
447                return 1;
448            }
449            else if ( !p_sys->b_es_setup )
450            {
451                p_sys->fmt.video.i_frame_rate_base = [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output timeScale];
452                msg_Dbg(p_demux, "using frame rate base: %i", p_sys->fmt.video.i_frame_rate_base);
453                p_sys->width = p_sys->fmt.video.i_width = [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output width];
454                p_sys->height = p_sys->fmt.video.i_height = [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output height];
455                p_sys->p_es_video = es_out_Add(p_demux->out, &p_sys->fmt);
456                msg_Dbg(p_demux, "added new video es %4.4s %dx%d", (char*)&p_sys->fmt.i_codec, p_sys->width, p_sys->height);
457                p_sys->b_es_setup = YES;
458            }
459        }
460
461        es_out_SetPCR(p_demux->out, p_block->i_pts);
462        es_out_Send(p_demux->out, p_sys->p_es_video, p_block);
463
464    }
465    return 1;
466}
467
468/*****************************************************************************
469* Control:
470*****************************************************************************/
471static int Control(demux_t *p_demux, int i_query, va_list args)
472{
473    bool        *pb;
474    int64_t     *pi64;
475
476    switch( i_query )
477    {
478        /* Special for access_demux */
479        case DEMUX_CAN_PAUSE:
480        case DEMUX_CAN_SEEK:
481        case DEMUX_SET_PAUSE_STATE:
482        case DEMUX_CAN_CONTROL_PACE:
483           pb = va_arg(args, bool *);
484           *pb = false;
485           return VLC_SUCCESS;
486
487        case DEMUX_GET_PTS_DELAY:
488           pi64 = va_arg(args, int64_t *);
489           *pi64 = INT64_C(1000) * var_InheritInteger(p_demux, "live-caching");
490           return VLC_SUCCESS;
491
492        case DEMUX_GET_TIME:
493            pi64 = va_arg(args, int64_t *);
494            *pi64 = mdate();
495            return VLC_SUCCESS;
496
497        default:
498           return VLC_EGENERIC;
499    }
500    return VLC_EGENERIC;
501}
502