1/***************************************************************************** 2 * avcapture.m: AVFoundation (Mac OS X) based video capture module 3 ***************************************************************************** 4 * Copyright © 2008-2013 VLC authors and VideoLAN 5 * 6 * Authors: Michael Feurstein <michael.feurstein@gmail.com> 7 * 8 **************************************************************************** 9 * This program is free software; you can redistribute it and/or modify it 10 * under the terms of the GNU Lesser General Public License as published by 11 * the Free Software Foundation; either version 2.1 of the License, or 12 * (at your option) any later version. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU Lesser General Public License for more details. 18 * 19 * You should have received a copy of the GNU Lesser General Public License 20 * along with this program; if not, write to the Free Software Foundation, 21 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA. 22 *****************************************************************************/ 23 24/***************************************************************************** 25 * Preamble 26 *****************************************************************************/ 27 28#define OS_OBJECT_USE_OBJC 0 29 30#ifdef HAVE_CONFIG_H 31# include "config.h" 32#endif 33 34#include <vlc_common.h> 35#include <vlc_plugin.h> 36#include <vlc_input.h> 37#include <vlc_demux.h> 38#include <vlc_interface.h> 39#include <vlc_dialog.h> 40#include <vlc_access.h> 41 42#import <AvailabilityMacros.h> 43#import <AVFoundation/AVFoundation.h> 44#import <CoreMedia/CoreMedia.h> 45 46#ifndef MAC_OS_X_VERSION_10_14 47@interface AVCaptureDevice (AVCaptureDeviceAuthorizationSince10_14) 48 49+ (void)requestAccessForMediaType:(AVMediaType)mediaType completionHandler:(void (^)(BOOL granted))handler API_AVAILABLE(macos(10.14), ios(7.0)); 50 51@end 52#endif 53 54/***************************************************************************** 55* Local prototypes 56*****************************************************************************/ 57static int Open(vlc_object_t *p_this); 58static void Close(vlc_object_t *p_this); 59static int Demux(demux_t *p_demux); 60static int Control(demux_t *, int, va_list); 61 62/***************************************************************************** 63* Module descriptor 64*****************************************************************************/ 65vlc_module_begin () 66 set_shortname(N_("AVFoundation Video Capture")) 67 set_description(N_("AVFoundation video capture module.")) 68 set_category(CAT_INPUT) 69 set_subcategory(SUBCAT_INPUT_ACCESS) 70 add_shortcut("avcapture") 71 set_capability("access_demux", 10) 72 set_callbacks(Open, Close) 73vlc_module_end () 74 75 76/***************************************************************************** 77* AVFoundation Bridge 78*****************************************************************************/ 79@interface VLCAVDecompressedVideoOutput : AVCaptureVideoDataOutput 80{ 81 demux_t *p_avcapture; 82 83 CVImageBufferRef currentImageBuffer; 84 85 mtime_t currentPts; 86 mtime_t previousPts; 87 size_t bytesPerRow; 88 89 long timeScale; 90 BOOL videoDimensionsReady; 91} 92 93@property (readwrite) CMVideoDimensions videoDimensions; 94 95- (id)initWithDemux:(demux_t *)p_demux; 96- (int)width; 97- (int)height; 98- (void)getVideoDimensions:(CMSampleBufferRef)sampleBuffer; 99- (mtime_t)currentPts; 100- (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection; 101- (mtime_t)copyCurrentFrameToBuffer:(void *)buffer; 102@end 103 104@implementation VLCAVDecompressedVideoOutput : AVCaptureVideoDataOutput 105 106- (id)initWithDemux:(demux_t *)p_demux 107{ 108 if (self = [super init]) 109 { 110 p_avcapture = p_demux; 111 currentImageBuffer = nil; 112 currentPts = 0; 113 previousPts = 0; 114 bytesPerRow = 0; 115 timeScale = 0; 116 videoDimensionsReady = NO; 117 } 118 return self; 119} 120 121- (void)dealloc 122{ 123 @synchronized (self) 124 { 125 CVBufferRelease(currentImageBuffer); 126 currentImageBuffer = nil; 127 bytesPerRow = 0; 128 videoDimensionsReady = NO; 129 } 130} 131 132- (long)timeScale 133{ 134 return timeScale; 135} 136 137- (int)width 138{ 139 return self.videoDimensions.width; 140} 141 142- (int)height 143{ 144 return self.videoDimensions.height; 145} 146 147- (size_t)bytesPerRow 148{ 149 return bytesPerRow; 150} 151 152- (void)getVideoDimensions:(CMSampleBufferRef)sampleBuffer 153{ 154 if (!videoDimensionsReady) 155 { 156 CMFormatDescriptionRef formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer); 157 self.videoDimensions = CMVideoFormatDescriptionGetDimensions(formatDescription); 158 bytesPerRow = CVPixelBufferGetBytesPerRow(CMSampleBufferGetImageBuffer(sampleBuffer)); 159 videoDimensionsReady = YES; 160 msg_Dbg(p_avcapture, "Dimensionns obtained height:%i width:%i bytesPerRow:%lu", [self height], [self width], bytesPerRow); 161 } 162} 163 164-(mtime_t)currentPts 165{ 166 mtime_t pts; 167 168 if ( !currentImageBuffer || currentPts == previousPts ) 169 return 0; 170 171 @synchronized (self) 172 { 173 pts = previousPts = currentPts; 174 } 175 176 return currentPts; 177} 178 179- (void)captureOutput:(AVCaptureOutput *)captureOutput 180didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer 181 fromConnection:(AVCaptureConnection *)connection 182{ 183 @autoreleasepool { 184 CVImageBufferRef imageBufferToRelease; 185 CMTime presentationtimestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); 186 CVImageBufferRef videoFrame = CMSampleBufferGetImageBuffer(sampleBuffer); 187 CVBufferRetain(videoFrame); 188 [self getVideoDimensions:sampleBuffer]; 189 190 @synchronized (self) { 191 imageBufferToRelease = currentImageBuffer; 192 currentImageBuffer = videoFrame; 193 currentPts = (mtime_t)presentationtimestamp.value; 194 timeScale = (long)presentationtimestamp.timescale; 195 } 196 197 CVBufferRelease(imageBufferToRelease); 198 } 199} 200 201- (mtime_t)copyCurrentFrameToBuffer:(void *)buffer 202{ 203 CVImageBufferRef imageBuffer; 204 mtime_t pts; 205 206 void *pixels; 207 208 if ( !currentImageBuffer || currentPts == previousPts ) 209 return 0; 210 211 @synchronized (self) 212 { 213 imageBuffer = CVBufferRetain(currentImageBuffer); 214 if (imageBuffer) 215 { 216 pts = previousPts = currentPts; 217 CVPixelBufferLockBaseAddress(imageBuffer, 0); 218 pixels = CVPixelBufferGetBaseAddress(imageBuffer); 219 if (pixels) 220 { 221 memcpy(buffer, pixels, CVPixelBufferGetHeight(imageBuffer) * CVPixelBufferGetBytesPerRow(imageBuffer)); 222 } 223 CVPixelBufferUnlockBaseAddress(imageBuffer, 0); 224 } 225 } 226 CVBufferRelease(imageBuffer); 227 228 if (pixels) 229 return currentPts; 230 else 231 return 0; 232} 233 234@end 235 236/***************************************************************************** 237* Struct 238*****************************************************************************/ 239 240struct demux_sys_t 241{ 242 CFTypeRef _Nullable session; // AVCaptureSession 243 CFTypeRef _Nullable device; // AVCaptureDevice 244 CFTypeRef _Nullable output; // VLCAVDecompressedVideoOutput 245 es_out_id_t *p_es_video; 246 es_format_t fmt; 247 int height, width; 248 BOOL b_es_setup; 249}; 250 251/***************************************************************************** 252* Open: 253*****************************************************************************/ 254static int Open(vlc_object_t *p_this) 255{ 256 demux_t *p_demux = (demux_t*)p_this; 257 demux_sys_t *p_sys = NULL; 258 259 NSString *avf_currdevice_uid; 260 NSArray *myVideoDevices; 261 NSError *o_returnedError; 262 263 AVCaptureDeviceInput *input = nil; 264 265 int i, i_width, i_height, deviceCount, ivideo; 266 267 char *psz_uid = NULL; 268 269 /* Only when selected */ 270 if ( *p_demux->psz_access == '\0' ) 271 return VLC_EGENERIC; 272 273 @autoreleasepool { 274 if (p_demux->psz_location && *p_demux->psz_location) 275 psz_uid = strdup(p_demux->psz_location); 276 277 msg_Dbg(p_demux, "avcapture uid = %s", psz_uid); 278 avf_currdevice_uid = [[NSString alloc] initWithFormat:@"%s", psz_uid]; 279 280 /* Set up p_demux */ 281 p_demux->pf_demux = Demux; 282 p_demux->pf_control = Control; 283 p_demux->info.i_update = 0; 284 p_demux->info.i_title = 0; 285 p_demux->info.i_seekpoint = 0; 286 287 p_demux->p_sys = p_sys = calloc(1, sizeof(demux_sys_t)); 288 if ( !p_sys ) 289 return VLC_ENOMEM; 290 291 myVideoDevices = [[AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo] 292 arrayByAddingObjectsFromArray:[AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed]]; 293 if ( [myVideoDevices count] == 0 ) 294 { 295 vlc_dialog_display_error(p_demux, _("No video devices found"), 296 _("Your Mac does not seem to be equipped with a suitable video input device. " 297 "Please check your connectors and drivers.")); 298 msg_Err(p_demux, "Can't find any suitable video device"); 299 goto error; 300 } 301 302 deviceCount = [myVideoDevices count]; 303 for ( ivideo = 0; ivideo < deviceCount; ivideo++ ) 304 { 305 AVCaptureDevice *avf_device; 306 avf_device = [myVideoDevices objectAtIndex:ivideo]; 307 msg_Dbg(p_demux, "avcapture %i/%i %s %s", ivideo, deviceCount, [[avf_device modelID] UTF8String], [[avf_device uniqueID] UTF8String]); 308 if ([[[avf_device uniqueID]stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]] isEqualToString:avf_currdevice_uid]) { 309 break; 310 } 311 } 312 313 if ( ivideo < [myVideoDevices count] ) 314 { 315 p_sys->device = CFBridgingRetain([myVideoDevices objectAtIndex:ivideo]); 316 } 317 else 318 { 319 msg_Dbg(p_demux, "Cannot find designated device as %s, falling back to default.", [avf_currdevice_uid UTF8String]); 320 p_sys->device = CFBridgingRetain([AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]); 321 } 322 if ( !p_sys->device ) 323 { 324 vlc_dialog_display_error(p_demux, _("No video devices found"), 325 _("Your Mac does not seem to be equipped with a suitable input device. " 326 "Please check your connectors and drivers.")); 327 msg_Err(p_demux, "Can't find any suitable video device"); 328 goto error; 329 } 330 331 if ( [(__bridge AVCaptureDevice *)p_sys->device isInUseByAnotherApplication] == YES ) 332 { 333 msg_Err(p_demux, "default capture device is exclusively in use by another application"); 334 goto error; 335 } 336 337 if (@available(macOS 10.14, *)) { 338 msg_Dbg(p_demux, "Check user consent for access to the video device"); 339 340 dispatch_semaphore_t sema = dispatch_semaphore_create(0); 341 __block bool accessGranted = NO; 342 [AVCaptureDevice requestAccessForMediaType: AVMediaTypeVideo completionHandler:^(BOOL granted) { 343 accessGranted = granted; 344 dispatch_semaphore_signal(sema); 345 } ]; 346 dispatch_semaphore_wait(sema, DISPATCH_TIME_FOREVER); 347 dispatch_release(sema); 348 if (!accessGranted) { 349 msg_Err(p_demux, "Can't use the video device as access has not been granted by the user"); 350 vlc_dialog_display_error(p_demux, _("Problem accessing a system resource"), 351 _("Please open \"System Preferences\" -> \"Security & Privacy\" " 352 "and allow VLC to access your camera.")); 353 354 goto error; 355 } 356 } 357 358 input = [AVCaptureDeviceInput deviceInputWithDevice:(__bridge AVCaptureDevice *)p_sys->device error:&o_returnedError]; 359 360 if ( !input ) 361 { 362 msg_Err(p_demux, "can't create a valid capture input facility: %s (%ld)",[[o_returnedError localizedDescription] UTF8String], [o_returnedError code]); 363 goto error; 364 } 365 366 367 int chroma = VLC_CODEC_RGB32; 368 369 memset(&p_sys->fmt, 0, sizeof(es_format_t)); 370 es_format_Init(&p_sys->fmt, VIDEO_ES, chroma); 371 372 p_sys->session = CFBridgingRetain([[AVCaptureSession alloc] init]); 373 [(__bridge AVCaptureSession *)p_sys->session addInput:input]; 374 375 p_sys->output = CFBridgingRetain([[VLCAVDecompressedVideoOutput alloc] initWithDemux:p_demux]); 376 [(__bridge AVCaptureSession *)p_sys->session addOutput:(__bridge VLCAVDecompressedVideoOutput *)p_sys->output]; 377 378 dispatch_queue_t queue = dispatch_queue_create("avCaptureQueue", NULL); 379 [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output setSampleBufferDelegate:(__bridge id)p_sys->output queue:queue]; 380 dispatch_release(queue); 381 382 [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_32BGRA] forKey:(id)kCVPixelBufferPixelFormatTypeKey]]; 383 [(__bridge AVCaptureSession *)p_sys->session startRunning]; 384 385 input = nil; 386 387 msg_Dbg(p_demux, "AVCapture: Video device ready!"); 388 389 return VLC_SUCCESS; 390 error: 391 msg_Err(p_demux, "Error"); 392 input = nil; 393 394 free(p_sys); 395 396 return VLC_EGENERIC; 397 } 398} 399 400/***************************************************************************** 401* Close: 402*****************************************************************************/ 403static void Close(vlc_object_t *p_this) 404{ 405 demux_t *p_demux = (demux_t*)p_this; 406 demux_sys_t *p_sys = p_demux->p_sys; 407 408 @autoreleasepool { 409 msg_Dbg(p_demux,"Close AVCapture"); 410 411 // Perform this on main thread, as the framework itself will sometimes try to synchronously 412 // work on main thread. And this will create a dead lock. 413 [(__bridge AVCaptureSession *)p_sys->session performSelectorOnMainThread:@selector(stopRunning) withObject:nil waitUntilDone:NO]; 414 CFBridgingRelease(p_sys->output); 415 CFBridgingRelease(p_sys->session); 416 417 free(p_sys); 418 } 419} 420 421/***************************************************************************** 422* Demux: 423*****************************************************************************/ 424static int Demux(demux_t *p_demux) 425{ 426 demux_sys_t *p_sys = p_demux->p_sys; 427 block_t *p_block; 428 429 @autoreleasepool { 430 @synchronized ( p_sys->output ) 431 { 432 p_block = block_Alloc([(__bridge VLCAVDecompressedVideoOutput *)p_sys->output width] * [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output bytesPerRow]); 433 434 if ( !p_block ) 435 { 436 msg_Err(p_demux, "cannot get block"); 437 return 0; 438 } 439 440 p_block->i_pts = [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output copyCurrentFrameToBuffer: p_block->p_buffer]; 441 442 if ( !p_block->i_pts ) 443 { 444 /* Nothing to display yet, just forget */ 445 block_Release(p_block); 446 msleep(10000); 447 return 1; 448 } 449 else if ( !p_sys->b_es_setup ) 450 { 451 p_sys->fmt.video.i_frame_rate_base = [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output timeScale]; 452 msg_Dbg(p_demux, "using frame rate base: %i", p_sys->fmt.video.i_frame_rate_base); 453 p_sys->width = p_sys->fmt.video.i_width = [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output width]; 454 p_sys->height = p_sys->fmt.video.i_height = [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output height]; 455 p_sys->p_es_video = es_out_Add(p_demux->out, &p_sys->fmt); 456 msg_Dbg(p_demux, "added new video es %4.4s %dx%d", (char*)&p_sys->fmt.i_codec, p_sys->width, p_sys->height); 457 p_sys->b_es_setup = YES; 458 } 459 } 460 461 es_out_SetPCR(p_demux->out, p_block->i_pts); 462 es_out_Send(p_demux->out, p_sys->p_es_video, p_block); 463 464 } 465 return 1; 466} 467 468/***************************************************************************** 469* Control: 470*****************************************************************************/ 471static int Control(demux_t *p_demux, int i_query, va_list args) 472{ 473 bool *pb; 474 int64_t *pi64; 475 476 switch( i_query ) 477 { 478 /* Special for access_demux */ 479 case DEMUX_CAN_PAUSE: 480 case DEMUX_CAN_SEEK: 481 case DEMUX_SET_PAUSE_STATE: 482 case DEMUX_CAN_CONTROL_PACE: 483 pb = va_arg(args, bool *); 484 *pb = false; 485 return VLC_SUCCESS; 486 487 case DEMUX_GET_PTS_DELAY: 488 pi64 = va_arg(args, int64_t *); 489 *pi64 = INT64_C(1000) * var_InheritInteger(p_demux, "live-caching"); 490 return VLC_SUCCESS; 491 492 case DEMUX_GET_TIME: 493 pi64 = va_arg(args, int64_t *); 494 *pi64 = mdate(); 495 return VLC_SUCCESS; 496 497 default: 498 return VLC_EGENERIC; 499 } 500 return VLC_EGENERIC; 501} 502