1#import "VideoMetalViewMac.h"
2#import <Metal/Metal.h>
3#import <MetalKit/MetalKit.h>
4#import "TGRTCCVPixelBuffer.h"
5#import "base/RTCLogging.h"
6#import "base/RTCVideoFrame.h"
7#import "base/RTCVideoFrameBuffer.h"
8#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
9#include "sdk/objc/native/api/video_frame.h"
10#include "sdk/objc/native/src/objc_frame_buffer.h"
11
12#import "api/video/video_sink_interface.h"
13#import "api/media_stream_interface.h"
14#import "rtc_base/time_utils.h"
15
16#import <SSignalKit/SSignalKit.h>
17
18
19#import "api/video/video_sink_interface.h"
20#import "api/media_stream_interface.h"
21
22#import "TGRTCMTLI420Renderer.h"
23
24#define MTKViewClass NSClassFromString(@"MTKView")
25#define TGRTCMTLI420RendererClass NSClassFromString(@"TGRTCMTLI420Renderer")
26
27SQueue *renderQueue = [[SQueue alloc] init];
28
29namespace {
30
31static RTCVideoFrame *customToObjCVideoFrame(const webrtc::VideoFrame &frame, RTCVideoRotation &rotation) {
32    rotation = RTCVideoRotation(frame.rotation());
33    RTCVideoFrame *videoFrame =
34    [[RTCVideoFrame alloc] initWithBuffer:webrtc::ToObjCVideoFrameBuffer(frame.video_frame_buffer())
35                                 rotation:rotation
36                              timeStampNs:frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec];
37    videoFrame.timeStamp = frame.timestamp();
38
39    return videoFrame;
40}
41
42class VideoRendererAdapterImpl : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
43public:
44    VideoRendererAdapterImpl(void (^frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation)) {
45        _frameReceived = [frameReceived copy];
46    }
47
48    void OnFrame(const webrtc::VideoFrame& nativeVideoFrame) override {
49        RTCVideoRotation rotation = RTCVideoRotation_0;
50        RTCVideoFrame* videoFrame = customToObjCVideoFrame(nativeVideoFrame, rotation);
51
52        CGSize currentSize = (videoFrame.rotation % 180 == 0) ? CGSizeMake(videoFrame.width, videoFrame.height) : CGSizeMake(videoFrame.height, videoFrame.width);
53
54        if (_frameReceived) {
55            _frameReceived(currentSize, videoFrame, rotation);
56        }
57    }
58
59private:
60    void (^_frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation);
61};
62
63}
64
65@interface TGCAMetalLayer : CAMetalLayer
66
67@end
68
69@implementation TGCAMetalLayer
70
71
72-(void)dealloc {
73    int bp = 0;
74    bp += 1;
75}
76@end
77
78@interface VideoMetalView () {
79    SQueueLocalObject *_rendererI420;
80
81    CAMetalLayer *_metalView;
82    NSView *_foregroundView;
83
84    CGSize _videoFrameSize;
85    RTCVideoRotation _rotation;
86    int64_t _lastFrameTimeNs;
87
88    CGSize _currentSize;
89    std::shared_ptr<VideoRendererAdapterImpl> _sink;
90
91    void (^_onFirstFrameReceived)(float);
92    bool _firstFrameReceivedReported;
93    void (^_onOrientationUpdated)(int, CGFloat);
94    void (^_onIsMirroredUpdated)(bool);
95
96    bool _didSetShouldBeMirrored;
97    bool _shouldBeMirrored;
98    bool _forceMirrored;
99
100    bool _isPaused;
101
102    NSMutableArray<RTCVideoFrame *> *_frames;
103    RTCVideoFrame *_videoFrame;
104    BOOL _drawing;
105
106    BOOL _deleteForegroundOnNextDrawing;
107}
108
109@end
110
111@implementation VideoMetalView
112
113+ (bool)isSupported {
114    return [VideoMetalView isMetalAvailable];
115}
116
117- (instancetype)initWithFrame:(CGRect)frameRect {
118    self = [super initWithFrame:frameRect];
119    if (self) {
120        [self configure];
121        _lastFrameTimeNs = INT32_MAX;
122        _currentSize = CGSizeZero;
123        _frames = [[NSMutableArray alloc] init];
124        _drawing = false;
125        _isPaused = false;
126        _deleteForegroundOnNextDrawing = false;
127        __weak VideoMetalView *weakSelf = self;
128        _sink.reset(new VideoRendererAdapterImpl(^(CGSize size, RTCVideoFrame *videoFrame, RTCVideoRotation rotation) {
129            dispatch_async(dispatch_get_main_queue(), ^{
130                __strong VideoMetalView *strongSelf = weakSelf;
131                if (strongSelf == nil) {
132                    return;
133                }
134                strongSelf->_rotation = videoFrame.rotation;
135                if (!CGSizeEqualToSize(size, strongSelf->_currentSize)) {
136                    strongSelf->_currentSize = size;
137                    [strongSelf setSize:size];
138                }
139
140                int mappedValue = 0;
141                switch (rotation) {
142                    case RTCVideoRotation_90:
143                        mappedValue = 0;
144                        break;
145                    case RTCVideoRotation_180:
146                        mappedValue = 1;
147                        break;
148                    case RTCVideoRotation_270:
149                        mappedValue = 2;
150                        break;
151                    default:
152                        mappedValue = 0;
153                        break;
154                }
155                [strongSelf setInternalOrientation:mappedValue];
156
157                [strongSelf renderFrame:videoFrame];
158
159                if ([videoFrame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
160                    RTCCVPixelBuffer *buffer = (RTCCVPixelBuffer*)videoFrame.buffer;
161
162                    if ([buffer isKindOfClass:[TGRTCCVPixelBuffer class]]) {
163                        bool shouldBeMirrored = ((TGRTCCVPixelBuffer *)buffer).shouldBeMirrored;
164                        if (shouldBeMirrored != strongSelf->_shouldBeMirrored) {
165                            strongSelf->_shouldBeMirrored = shouldBeMirrored;
166                            if (strongSelf->_onIsMirroredUpdated) {
167                                strongSelf->_onIsMirroredUpdated(strongSelf->_shouldBeMirrored);
168                            }
169                        }
170                    }
171                }
172
173                if (!strongSelf->_firstFrameReceivedReported && strongSelf->_onFirstFrameReceived) {
174                    strongSelf->_firstFrameReceivedReported = true;
175                    strongSelf->_onFirstFrameReceived((float)videoFrame.width / (float)videoFrame.height);
176                }
177            });
178        }));
179
180    }
181    return self;
182}
183
184- (BOOL)isEnabled {
185    return !_isPaused;
186}
187
188- (void)setEnabled:(BOOL)enabled {
189    _isPaused = enabled;
190}
191-(void)setIsPaused:(bool)paused {
192    _isPaused = paused;
193    [self updateDrawingSize:self.frame.size];
194}
195-(void)renderToSize:(NSSize)size animated: (bool)animated {
196    [CATransaction begin];
197    [CATransaction setDisableActions:YES];
198    if (animated) {
199        if (!_foregroundView) {
200            _foregroundView = [[NSView alloc] initWithFrame:self.bounds];
201            _foregroundView.wantsLayer = YES;
202            _foregroundView.autoresizingMask = 0;
203            _foregroundView.layer = [VideoMetalView createMetalView:self.bounds];
204            _foregroundView.layer.contentsGravity = kCAGravityResizeAspect;
205            [self addSubview:_foregroundView];
206        }
207
208        CAMetalLayer *layer = _metalView;
209        CAMetalLayer *foreground = (CAMetalLayer *)_foregroundView.layer;
210
211        [_rendererI420 with:^(TGRTCMTLI420Renderer * renderer) {
212            [renderer setDoubleRendering:layer foreground:foreground];
213        }];
214        _deleteForegroundOnNextDrawing = false;
215    } else {
216        _deleteForegroundOnNextDrawing = true;
217        CAMetalLayer *layer = _metalView;
218
219        [_rendererI420 with:^(TGRTCMTLI420Renderer * renderer) {
220            [renderer setSingleRendering:layer];
221        }];
222    }
223    [self updateDrawingSize:size];
224    [CATransaction commit];
225}
226
227- (CALayerContentsGravity)videoContentMode {
228    return _metalView.contentsGravity;
229}
230
231- (void)setVideoContentMode:(CALayerContentsGravity)mode {
232   // _metalView.contentsGravity = mode;
233}
234
235#pragma mark - Private
236
237+ (BOOL)isMetalAvailable {
238    return initMetal();
239}
240
241+ (CAMetalLayer *)createMetalView:(CGRect)frame {
242    CAMetalLayer *layer = [[TGCAMetalLayer alloc] init];
243    [CATransaction begin];
244    [CATransaction setDisableActions:true];
245    layer.framebufferOnly = true;
246    layer.opaque = false;
247//    layer.cornerRadius = 4;
248    if (@available(macOS 10.13, *)) {
249        layer.displaySyncEnabled = NO;
250    }
251//    layer.presentsWithTransaction = YES;
252    layer.backgroundColor = [NSColor clearColor].CGColor;
253    layer.contentsGravity = kCAGravityResizeAspectFill;
254    layer.frame = frame;
255
256    [CATransaction commit];
257    return layer;
258}
259
260+ (TGRTCMTLI420Renderer *)createI420Renderer {
261    return [[TGRTCMTLI420RendererClass alloc] init];
262}
263
264
265- (void)configure {
266    NSAssert([VideoMetalView isMetalAvailable], @"Metal not availiable on this device");
267    self.wantsLayer = YES;
268    self.layerContentsRedrawPolicy = NSViewLayerContentsRedrawOnSetNeedsDisplay;
269    _metalView = [VideoMetalView createMetalView:self.bounds];
270
271
272    self.layer = _metalView;
273    _videoFrameSize = CGSizeZero;
274
275    CAMetalLayer *layer = _metalView;
276
277    _rendererI420 = [[SQueueLocalObject alloc] initWithQueue:renderQueue generate: ^{
278        TGRTCMTLI420Renderer *renderer = [VideoMetalView createI420Renderer];
279        [renderer setSingleRendering:layer];
280        return renderer;
281    }];
282}
283
284
285-(void)setFrameSize:(NSSize)newSize {
286    [super setFrameSize:newSize];
287
288    [self updateDrawingSize: newSize];
289
290}
291- (void)layout {
292    [super layout];
293}
294
295-(void)updateDrawingSize:(NSSize)size {
296    if (_isPaused) {
297        return;
298    }
299    _metalView.frame = CGRectMake(0, 0, size.width, size.height);
300    _foregroundView.frame = self.bounds;
301    if (!CGSizeEqualToSize(_videoFrameSize, CGSizeZero)) {
302        _metalView.drawableSize = [self drawableSize:size];
303        ((CAMetalLayer *)_foregroundView.layer).drawableSize = _videoFrameSize;
304    } else {
305        _metalView.drawableSize = size;
306        ((CAMetalLayer *)_foregroundView.layer).drawableSize = size;
307    }
308
309    if(!_isPaused) {
310        RTCVideoFrame *frame = [_frames lastObject];
311        if (frame == nil) {
312            frame = _videoFrame;
313        }
314        if (frame) {
315            [self renderFrame:frame];
316        }
317    }
318}
319
320
321-(void)dealloc {
322    int bp = 0;
323    bp += 1;
324}
325
326#pragma mark -
327
328- (void)setRotationOverride:(NSValue *)rotationOverride {
329    _rotationOverride = rotationOverride;
330
331    [self setNeedsLayout:YES];
332}
333
334- (RTCVideoRotation)rtcFrameRotation {
335    if (_rotationOverride) {
336        RTCVideoRotation rotation;
337        if (@available(macOS 10.13, *)) {
338            [_rotationOverride getValue:&rotation size:sizeof(rotation)];
339        } else {
340            [_rotationOverride getValue:&rotation];
341        }
342        return rotation;
343    }
344
345    return _rotation;
346}
347
348- (CGSize)drawableSize:(NSSize)forSize {
349
350    MTLFrameSize from;
351    MTLFrameSize to;
352
353    from.width = _videoFrameSize.width;
354    from.height = _videoFrameSize.height;
355
356    if (CGSizeEqualToSize(CGSizeZero, forSize)) {
357        to.width = _videoFrameSize.width;
358        to.height = _videoFrameSize.height;
359    } else {
360        to.width = forSize.width;
361        to.height = forSize.height;
362    }
363
364
365    MTLFrameSize size = MTLAspectFilled(to, from);
366
367    return CGSizeMake(size.width, size.height);
368}
369
370#pragma mark - RTCVideoRenderer
371
372- (void)setSize:(CGSize)size {
373    assert([NSThread isMainThread]);
374
375   _videoFrameSize = size;
376   [self updateDrawingSize:self.frame.size];
377
378    _internalAspect = _videoFrameSize.width / _videoFrameSize.height;
379}
380
381- (void)renderFrame:(nullable RTCVideoFrame *)frame {
382    assert([NSThread isMainThread]);
383
384    if (!self.isEnabled) {
385        return;
386    }
387
388    if (frame == nil) {
389        RTCLogInfo(@"Incoming frame is nil. Exiting render callback.");
390        return;
391    }
392
393    RTCVideoFrame *videoFrame = frame;
394    // Skip rendering if we've already rendered this frame.
395    if (!videoFrame) {
396        return;
397    }
398
399    if (CGRectIsEmpty(self.bounds)) {
400        return;
401    }
402    if (CGRectIsEmpty(self.visibleRect)) {
403        return;
404    }
405    if (self.window == nil || self.superview == nil) {
406        return;
407    }
408    if ((self.window.occlusionState & NSWindowOcclusionStateVisible) == 0) {
409        return;
410    }
411
412
413    if (_frames.count >= 5) {
414        [_frames removeAllObjects];
415        [_frames addObject:videoFrame];
416        [self enqueue];
417        return;
418    }
419
420    [_frames addObject:videoFrame];
421
422    [self enqueue];
423
424}
425
426-(void)enqueue {
427    if(_frames.count > 0 && !_drawing) {
428        RTCVideoFrame *videoFrame = [_frames firstObject];
429
430        NSValue * rotationOverride = _rotationOverride;
431
432
433        int64_t timeStampNs = videoFrame.timeStampNs;
434
435        __weak VideoMetalView *weakSelf = self;
436        dispatch_block_t completion = ^{
437            __strong VideoMetalView *strongSelf = weakSelf;
438            if (strongSelf && strongSelf->_frames.count > 0) {
439                [strongSelf->_frames removeObjectAtIndex:0];
440                strongSelf->_drawing = false;
441                strongSelf->_lastFrameTimeNs = timeStampNs;
442                if (strongSelf->_deleteForegroundOnNextDrawing) {
443                    [strongSelf->_foregroundView removeFromSuperview];
444                    strongSelf->_foregroundView = nil;
445                    strongSelf->_deleteForegroundOnNextDrawing = false;
446                }
447                [strongSelf enqueue];
448            }
449        };
450
451        _videoFrame = videoFrame;
452
453        self->_drawing = true;
454
455        [_rendererI420 with:^(TGRTCMTLI420Renderer * object) {
456            object.rotationOverride = rotationOverride;
457            [object drawFrame:videoFrame];
458            dispatch_async(dispatch_get_main_queue(), completion);
459        }];
460
461    }
462}
463
464- (std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)getSink {
465    assert([NSThread isMainThread]);
466
467    return _sink;
468}
469
470- (void)setOnFirstFrameReceived:(void (^ _Nullable)(float))onFirstFrameReceived {
471    _onFirstFrameReceived = [onFirstFrameReceived copy];
472    _firstFrameReceivedReported = false;
473}
474
475- (void)setInternalOrientationAndSize:(int)internalOrientation size:(CGSize)size {
476    CGFloat aspect = 1.0f;
477    if (size.width > 1.0f && size.height > 1.0f) {
478        aspect = size.width / size.height;
479    }
480    if (_internalOrientation != internalOrientation || ABS(_internalAspect - aspect) > 0.001) {
481        RTCLogInfo(@"VideoMetalView@%lx orientation: %d, aspect: %f", (intptr_t)self, internalOrientation, (float)aspect);
482
483        _internalOrientation = internalOrientation;
484        _internalAspect = aspect;
485        if (_onOrientationUpdated) {
486            _onOrientationUpdated(internalOrientation, aspect);
487        }
488    }
489}
490
491- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated {
492    _onOrientationUpdated = [onOrientationUpdated copy];
493}
494
495- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated {
496    _onIsMirroredUpdated = [onIsMirroredUpdated copy];
497}
498
499- (void)setForceMirrored:(BOOL)forceMirrored {
500    _forceMirrored = forceMirrored;
501    [self setNeedsLayout:YES];
502}
503
504
505@end
506