1 /*
2 ==============================================================================
3
4 This file is part of the JUCE library.
5 Copyright (c) 2020 - Raw Material Software Limited
6
7 JUCE is an open source library subject to commercial or open-source
8 licensing.
9
10 By using JUCE, you agree to the terms of both the JUCE 6 End-User License
11 Agreement and JUCE Privacy Policy (both effective as of the 16th June 2020).
12
13 End User License Agreement: www.juce.com/juce-6-licence
14 Privacy Policy: www.juce.com/juce-privacy-policy
15
16 Or: You may also use this code under the terms of the GPL v3 (see
17 www.gnu.org/licenses).
18
19 JUCE IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER
20 EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE
21 DISCLAIMED.
22
23 ==============================================================================
24 */
25
26 struct CameraDevice::Pimpl
27 {
28 #if defined (MAC_OS_X_VERSION_10_15) && MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_15
29 #define JUCE_USE_NEW_APPLE_CAMERA_API 1
30 #else
31 #define JUCE_USE_NEW_APPLE_CAMERA_API 0
32 #endif
33
34 #if JUCE_USE_NEW_APPLE_CAMERA_API
35 class PostCatalinaPhotoOutput
36 {
37 public:
PostCatalinaPhotoOutputPimpl38 PostCatalinaPhotoOutput()
39 {
40 static PhotoOutputDelegateClass cls;
41 delegate.reset ([cls.createInstance() init]);
42 }
43
addImageCapturePimpl44 void addImageCapture (AVCaptureSession* s)
45 {
46 if (imageOutput != nil)
47 return;
48
49 imageOutput = [[AVCapturePhotoOutput alloc] init];
50 [s addOutput: imageOutput];
51 }
52
removeImageCapturePimpl53 void removeImageCapture (AVCaptureSession* s)
54 {
55 if (imageOutput == nil)
56 return;
57
58 [s removeOutput: imageOutput];
59 [imageOutput release];
60 imageOutput = nil;
61 }
62
getConnectionsPimpl63 NSArray<AVCaptureConnection*>* getConnections() const
64 {
65 if (imageOutput != nil)
66 return imageOutput.connections;
67
68 return nil;
69 }
70
triggerImageCapturePimpl71 void triggerImageCapture (Pimpl& p)
72 {
73 if (imageOutput == nil)
74 return;
75
76 PhotoOutputDelegateClass::setOwner (delegate.get(), &p);
77
78 [imageOutput capturePhotoWithSettings: [AVCapturePhotoSettings photoSettings]
79 delegate: id<AVCapturePhotoCaptureDelegate> (delegate.get())];
80 }
81
getAvailableDevicesPimpl82 static NSArray* getAvailableDevices()
83 {
84 auto* discovery = [AVCaptureDeviceDiscoverySession discoverySessionWithDeviceTypes: @[AVCaptureDeviceTypeBuiltInWideAngleCamera,
85 AVCaptureDeviceTypeExternalUnknown]
86 mediaType: AVMediaTypeVideo
87 position: AVCaptureDevicePositionUnspecified];
88 return [discovery devices];
89 }
90
91 private:
92 class PhotoOutputDelegateClass : public ObjCClass<NSObject>
93 {
94 public:
PhotoOutputDelegateClassPimpl95 PhotoOutputDelegateClass() : ObjCClass<NSObject> ("PhotoOutputDelegateClass_")
96 {
97 addMethod (@selector (captureOutput:didFinishProcessingPhoto:error:), didFinishProcessingPhoto, "v@:@@@");
98 addIvar<Pimpl*> ("owner");
99 registerClass();
100 }
101
didFinishProcessingPhotoPimpl102 static void didFinishProcessingPhoto (id self, SEL, AVCapturePhotoOutput*, AVCapturePhoto* photo, NSError* error)
103 {
104 if (error != nil)
105 {
106 String errorString = error != nil ? nsStringToJuce (error.localizedDescription) : String();
107 ignoreUnused (errorString);
108
109 JUCE_CAMERA_LOG ("Still picture capture failed, error: " + errorString);
110 jassertfalse;
111
112 return;
113 }
114
115 auto* imageData = [photo fileDataRepresentation];
116 auto image = ImageFileFormat::loadFrom (imageData.bytes, (size_t) imageData.length);
117
118 getOwner (self).imageCaptureFinished (image);
119 }
120
getOwnerPimpl121 static Pimpl& getOwner (id self) { return *getIvar<Pimpl*> (self, "owner"); }
setOwnerPimpl122 static void setOwner (id self, Pimpl* t) { object_setInstanceVariable (self, "owner", t); }
123 };
124
125 AVCapturePhotoOutput* imageOutput = nil;
126 std::unique_ptr<NSObject, NSObjectDeleter> delegate;
127 };
128 #else
129 struct PreCatalinaStillImageOutput
130 {
131 public:
132 void addImageCapture (AVCaptureSession* s)
133 {
134 if (imageOutput != nil)
135 return;
136
137 const auto codecType =
138 #if defined (MAC_OS_X_VERSION_10_13) && MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_13
139 AVVideoCodecTypeJPEG;
140 #else
141 AVVideoCodecJPEG;
142 #endif
143
144 imageOutput = [[AVCaptureStillImageOutput alloc] init];
145 auto imageSettings = [[NSDictionary alloc] initWithObjectsAndKeys: codecType, AVVideoCodecKey, nil];
146 [imageOutput setOutputSettings: imageSettings];
147 [imageSettings release];
148 [s addOutput: imageOutput];
149 }
150
151 void removeImageCapture (AVCaptureSession* s)
152 {
153 if (imageOutput == nil)
154 return;
155
156 [s removeOutput: imageOutput];
157 [imageOutput release];
158 imageOutput = nil;
159 }
160
161 NSArray<AVCaptureConnection*>* getConnections() const
162 {
163 if (imageOutput != nil)
164 return imageOutput.connections;
165
166 return nil;
167 }
168
169 void triggerImageCapture (Pimpl& p)
170 {
171 if (auto* videoConnection = p.getVideoConnection())
172 {
173 [imageOutput captureStillImageAsynchronouslyFromConnection: videoConnection
174 completionHandler: ^(CMSampleBufferRef sampleBuffer, NSError* error)
175 {
176 if (error != nil)
177 {
178 JUCE_CAMERA_LOG ("Still picture capture failed, error: " + nsStringToJuce (error.localizedDescription));
179 jassertfalse;
180 return;
181 }
182
183 auto* imageData = [AVCaptureStillImageOutput jpegStillImageNSDataRepresentation: sampleBuffer];
184 auto image = ImageFileFormat::loadFrom (imageData.bytes, (size_t) imageData.length);
185 p.imageCaptureFinished (image);
186 }];
187 }
188 }
189
190 static NSArray* getAvailableDevices()
191 {
192 return [AVCaptureDevice devicesWithMediaType: AVMediaTypeVideo];
193 }
194
195 private:
196 AVCaptureStillImageOutput* imageOutput = nil;
197 };
198 #endif
199
PimplPimpl200 Pimpl (CameraDevice& ownerToUse, const String& deviceNameToUse, int /*index*/,
201 int /*minWidth*/, int /*minHeight*/,
202 int /*maxWidth*/, int /*maxHeight*/,
203 bool useHighQuality)
204 : owner (ownerToUse),
205 deviceName (deviceNameToUse)
206 {
207 session = [[AVCaptureSession alloc] init];
208
209 session.sessionPreset = useHighQuality ? AVCaptureSessionPresetHigh
210 : AVCaptureSessionPresetMedium;
211
212 refreshConnections();
213
214 static DelegateClass cls;
215 callbackDelegate = (id<AVCaptureFileOutputRecordingDelegate>) [cls.createInstance() init];
216 DelegateClass::setOwner (callbackDelegate, this);
217
218 JUCE_BEGIN_IGNORE_WARNINGS_GCC_LIKE ("-Wundeclared-selector")
219 [[NSNotificationCenter defaultCenter] addObserver: callbackDelegate
220 selector: @selector (captureSessionRuntimeError:)
221 name: AVCaptureSessionRuntimeErrorNotification
222 object: session];
223 JUCE_END_IGNORE_WARNINGS_GCC_LIKE
224 }
225
~PimplPimpl226 ~Pimpl()
227 {
228 [[NSNotificationCenter defaultCenter] removeObserver: callbackDelegate];
229
230 [session stopRunning];
231 removeInput();
232 removeImageCapture();
233 removeMovieCapture();
234 [session release];
235 [callbackDelegate release];
236 }
237
238 //==============================================================================
openedOkPimpl239 bool openedOk() const noexcept { return openingError.isEmpty(); }
240
startSessionPimpl241 void startSession()
242 {
243 if (! [session isRunning])
244 [session startRunning];
245 }
246
takeStillPicturePimpl247 void takeStillPicture (std::function<void (const Image&)> pictureTakenCallbackToUse)
248 {
249 if (pictureTakenCallbackToUse == nullptr)
250 {
251 jassertfalse;
252 return;
253 }
254
255 pictureTakenCallback = std::move (pictureTakenCallbackToUse);
256
257 triggerImageCapture();
258 }
259
startRecordingToFilePimpl260 void startRecordingToFile (const File& file, int /*quality*/)
261 {
262 stopRecording();
263 refreshIfNeeded();
264 firstPresentationTime = Time::getCurrentTime();
265 file.deleteFile();
266
267 startSession();
268 isRecording = true;
269 [fileOutput startRecordingToOutputFileURL: createNSURLFromFile (file)
270 recordingDelegate: callbackDelegate];
271 }
272
stopRecordingPimpl273 void stopRecording()
274 {
275 if (isRecording)
276 {
277 [fileOutput stopRecording];
278 isRecording = false;
279 }
280 }
281
getTimeOfFirstRecordedFramePimpl282 Time getTimeOfFirstRecordedFrame() const
283 {
284 return firstPresentationTime;
285 }
286
addListenerPimpl287 void addListener (CameraDevice::Listener* listenerToAdd)
288 {
289 const ScopedLock sl (listenerLock);
290 listeners.add (listenerToAdd);
291
292 if (listeners.size() == 1)
293 triggerImageCapture();
294 }
295
removeListenerPimpl296 void removeListener (CameraDevice::Listener* listenerToRemove)
297 {
298 const ScopedLock sl (listenerLock);
299 listeners.remove (listenerToRemove);
300 }
301
getAvailableDevicesPimpl302 static StringArray getAvailableDevices()
303 {
304 auto* devices = decltype (imageOutput)::getAvailableDevices();
305
306 StringArray results;
307
308 for (AVCaptureDevice* device : devices)
309 results.add (nsStringToJuce ([device localizedName]));
310
311 return results;
312 }
313
getCaptureSessionPimpl314 AVCaptureSession* getCaptureSession()
315 {
316 return session;
317 }
318
createVideoCapturePreviewPimpl319 NSView* createVideoCapturePreview()
320 {
321 // The video preview must be created before the capture session is
322 // started. Make sure you haven't called `addListener`,
323 // `startRecordingToFile`, or `takeStillPicture` before calling this
324 // function.
325 jassert (! [session isRunning]);
326 startSession();
327
328 JUCE_AUTORELEASEPOOL
329 {
330 NSView* view = [[NSView alloc] init];
331 [view setLayer: [AVCaptureVideoPreviewLayer layerWithSession: getCaptureSession()]];
332 return view;
333 }
334 }
335
336 private:
337 //==============================================================================
338 struct DelegateClass : public ObjCClass<NSObject>
339 {
DelegateClassPimpl::DelegateClass340 DelegateClass() : ObjCClass<NSObject> ("JUCECameraDelegate_")
341 {
342 addIvar<Pimpl*> ("owner");
343 addProtocol (@protocol (AVCaptureFileOutputRecordingDelegate));
344
345 addMethod (@selector (captureOutput:didStartRecordingToOutputFileAtURL: fromConnections:), didStartRecordingToOutputFileAtURL, "v@:@@@");
346 addMethod (@selector (captureOutput:didPauseRecordingToOutputFileAtURL: fromConnections:), didPauseRecordingToOutputFileAtURL, "v@:@@@");
347 addMethod (@selector (captureOutput:didResumeRecordingToOutputFileAtURL: fromConnections:), didResumeRecordingToOutputFileAtURL, "v@:@@@");
348 addMethod (@selector (captureOutput:willFinishRecordingToOutputFileAtURL:fromConnections:error:), willFinishRecordingToOutputFileAtURL, "v@:@@@@");
349
350 JUCE_BEGIN_IGNORE_WARNINGS_GCC_LIKE ("-Wundeclared-selector")
351 addMethod (@selector (captureSessionRuntimeError:), sessionRuntimeError, "v@:@");
352 JUCE_END_IGNORE_WARNINGS_GCC_LIKE
353
354 registerClass();
355 }
356
setOwnerPimpl::DelegateClass357 static void setOwner (id self, Pimpl* owner) { object_setInstanceVariable (self, "owner", owner); }
getOwnerPimpl::DelegateClass358 static Pimpl& getOwner (id self) { return *getIvar<Pimpl*> (self, "owner"); }
359
360 private:
didStartRecordingToOutputFileAtURLPimpl::DelegateClass361 static void didStartRecordingToOutputFileAtURL (id, SEL, AVCaptureFileOutput*, NSURL*, NSArray*) {}
didPauseRecordingToOutputFileAtURLPimpl::DelegateClass362 static void didPauseRecordingToOutputFileAtURL (id, SEL, AVCaptureFileOutput*, NSURL*, NSArray*) {}
didResumeRecordingToOutputFileAtURLPimpl::DelegateClass363 static void didResumeRecordingToOutputFileAtURL (id, SEL, AVCaptureFileOutput*, NSURL*, NSArray*) {}
willFinishRecordingToOutputFileAtURLPimpl::DelegateClass364 static void willFinishRecordingToOutputFileAtURL (id, SEL, AVCaptureFileOutput*, NSURL*, NSArray*, NSError*) {}
365
sessionRuntimeErrorPimpl::DelegateClass366 static void sessionRuntimeError (id self, SEL, NSNotification* notification)
367 {
368 JUCE_CAMERA_LOG (nsStringToJuce ([notification description]));
369
370 NSError* error = notification.userInfo[AVCaptureSessionErrorKey];
371 auto errorString = error != nil ? nsStringToJuce (error.localizedDescription) : String();
372 getOwner (self).cameraSessionRuntimeError (errorString);
373 }
374 };
375
376 //==============================================================================
addImageCapturePimpl377 void addImageCapture()
378 {
379 imageOutput.addImageCapture (session);
380 }
381
addMovieCapturePimpl382 void addMovieCapture()
383 {
384 if (fileOutput == nil)
385 {
386 fileOutput = [[AVCaptureMovieFileOutput alloc] init];
387 [session addOutput: fileOutput];
388 }
389 }
390
removeImageCapturePimpl391 void removeImageCapture()
392 {
393 imageOutput.removeImageCapture (session);
394 }
395
removeMovieCapturePimpl396 void removeMovieCapture()
397 {
398 if (fileOutput != nil)
399 {
400 [session removeOutput: fileOutput];
401 [fileOutput release];
402 fileOutput = nil;
403 }
404 }
405
removeCurrentSessionVideoInputsPimpl406 void removeCurrentSessionVideoInputs()
407 {
408 if (session != nil)
409 {
410 NSArray<AVCaptureDeviceInput*>* inputs = session.inputs;
411
412 for (AVCaptureDeviceInput* input : inputs)
413 if ([input.device hasMediaType: AVMediaTypeVideo])
414 [session removeInput:input];
415 }
416 }
417
addInputPimpl418 void addInput()
419 {
420 if (currentInput == nil)
421 {
422 auto* availableDevices = decltype (imageOutput)::getAvailableDevices();
423
424 for (AVCaptureDevice* device : availableDevices)
425 {
426 if (deviceName == nsStringToJuce ([device localizedName]))
427 {
428 removeCurrentSessionVideoInputs();
429
430 NSError* err = nil;
431 AVCaptureDeviceInput* inputDevice = [[AVCaptureDeviceInput alloc] initWithDevice: device
432 error: &err];
433
434 jassert (err == nil);
435
436 if ([session canAddInput: inputDevice])
437 {
438 [session addInput: inputDevice];
439 currentInput = inputDevice;
440 }
441 else
442 {
443 jassertfalse;
444 [inputDevice release];
445 }
446
447 return;
448 }
449 }
450 }
451 }
452
removeInputPimpl453 void removeInput()
454 {
455 if (currentInput != nil)
456 {
457 [session removeInput: currentInput];
458 [currentInput release];
459 currentInput = nil;
460 }
461 }
462
refreshConnectionsPimpl463 void refreshConnections()
464 {
465 [session beginConfiguration];
466 removeInput();
467 removeImageCapture();
468 removeMovieCapture();
469 addInput();
470 addImageCapture();
471 addMovieCapture();
472 [session commitConfiguration];
473 }
474
refreshIfNeededPimpl475 void refreshIfNeeded()
476 {
477 if (getVideoConnection() == nullptr)
478 refreshConnections();
479 }
480
getVideoConnectionPimpl481 AVCaptureConnection* getVideoConnection() const
482 {
483 auto* connections = imageOutput.getConnections();
484
485 if (connections != nil)
486 for (AVCaptureConnection* connection in connections)
487 if ([connection isActive] && [connection isEnabled])
488 for (AVCaptureInputPort* port in [connection inputPorts])
489 if ([[port mediaType] isEqual: AVMediaTypeVideo])
490 return connection;
491
492 return nil;
493 }
494
imageCaptureFinishedPimpl495 void imageCaptureFinished (const Image& image)
496 {
497 handleImageCapture (image);
498
499 WeakReference<Pimpl> weakRef (this);
500 MessageManager::callAsync ([weakRef, image]() mutable
501 {
502 if (weakRef != nullptr && weakRef->pictureTakenCallback != nullptr)
503 weakRef->pictureTakenCallback (image);
504 });
505 }
506
handleImageCapturePimpl507 void handleImageCapture (const Image& image)
508 {
509 const ScopedLock sl (listenerLock);
510 listeners.call ([=] (Listener& l) { l.imageReceived (image); });
511
512 if (! listeners.isEmpty())
513 triggerImageCapture();
514 }
515
triggerImageCapturePimpl516 void triggerImageCapture()
517 {
518 refreshIfNeeded();
519
520 startSession();
521
522 if (auto* videoConnection = getVideoConnection())
523 imageOutput.triggerImageCapture (*this);
524 }
525
cameraSessionRuntimeErrorPimpl526 void cameraSessionRuntimeError (const String& error)
527 {
528 JUCE_CAMERA_LOG ("cameraSessionRuntimeError(), error = " + error);
529
530 if (owner.onErrorOccurred != nullptr)
531 owner.onErrorOccurred (error);
532 }
533
534 //==============================================================================
535 CameraDevice& owner;
536 String deviceName;
537
538 AVCaptureSession* session = nil;
539 AVCaptureMovieFileOutput* fileOutput = nil;
540 #if JUCE_USE_NEW_APPLE_CAMERA_API
541 PostCatalinaPhotoOutput imageOutput;
542 #else
543 PreCatalinaStillImageOutput imageOutput;
544 #endif
545 AVCaptureDeviceInput* currentInput = nil;
546
547 id<AVCaptureFileOutputRecordingDelegate> callbackDelegate = nil;
548 String openingError;
549 Time firstPresentationTime;
550 bool isRecording = false;
551
552 CriticalSection listenerLock;
553 ListenerList<Listener> listeners;
554
555 std::function<void (const Image&)> pictureTakenCallback = nullptr;
556
557 //==============================================================================
558 JUCE_DECLARE_WEAK_REFERENCEABLE (Pimpl)
559 JUCE_DECLARE_NON_COPYABLE (Pimpl)
560 };
561
562 //==============================================================================
563 struct CameraDevice::ViewerComponent : public NSViewComponent
564 {
ViewerComponentViewerComponent565 ViewerComponent (CameraDevice& device)
566 {
567 setView (device.pimpl->createVideoCapturePreview());
568 }
569
~ViewerComponentViewerComponent570 ~ViewerComponent()
571 {
572 setView (nil);
573 }
574
575 JUCE_DECLARE_NON_COPYABLE (ViewerComponent)
576 };
577
getFileExtension()578 String CameraDevice::getFileExtension()
579 {
580 return ".mov";
581 }
582
583 #undef JUCE_USE_NEW_APPLE_CAMERA_API
584