1#import "VideoMetalViewMac.h" 2#import <Metal/Metal.h> 3#import <MetalKit/MetalKit.h> 4#import "TGRTCCVPixelBuffer.h" 5#import "base/RTCLogging.h" 6#import "base/RTCVideoFrame.h" 7#import "base/RTCVideoFrameBuffer.h" 8#import "components/video_frame_buffer/RTCCVPixelBuffer.h" 9#include "sdk/objc/native/api/video_frame.h" 10#include "sdk/objc/native/src/objc_frame_buffer.h" 11 12#import "api/video/video_sink_interface.h" 13#import "api/media_stream_interface.h" 14#import "rtc_base/time_utils.h" 15 16#import <SSignalKit/SSignalKit.h> 17 18 19#import "api/video/video_sink_interface.h" 20#import "api/media_stream_interface.h" 21 22#import "TGRTCMTLI420Renderer.h" 23 24#define MTKViewClass NSClassFromString(@"MTKView") 25#define TGRTCMTLI420RendererClass NSClassFromString(@"TGRTCMTLI420Renderer") 26 27SQueue *renderQueue = [[SQueue alloc] init]; 28 29namespace { 30 31static RTCVideoFrame *customToObjCVideoFrame(const webrtc::VideoFrame &frame, RTCVideoRotation &rotation) { 32 rotation = RTCVideoRotation(frame.rotation()); 33 RTCVideoFrame *videoFrame = 34 [[RTCVideoFrame alloc] initWithBuffer:webrtc::ToObjCVideoFrameBuffer(frame.video_frame_buffer()) 35 rotation:rotation 36 timeStampNs:frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec]; 37 videoFrame.timeStamp = frame.timestamp(); 38 39 return videoFrame; 40} 41 42class VideoRendererAdapterImpl : public rtc::VideoSinkInterface<webrtc::VideoFrame> { 43public: 44 VideoRendererAdapterImpl(void (^frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation)) { 45 _frameReceived = [frameReceived copy]; 46 } 47 48 void OnFrame(const webrtc::VideoFrame& nativeVideoFrame) override { 49 RTCVideoRotation rotation = RTCVideoRotation_0; 50 RTCVideoFrame* videoFrame = customToObjCVideoFrame(nativeVideoFrame, rotation); 51 52 CGSize currentSize = (videoFrame.rotation % 180 == 0) ? CGSizeMake(videoFrame.width, videoFrame.height) : CGSizeMake(videoFrame.height, videoFrame.width); 53 54 if (_frameReceived) { 55 _frameReceived(currentSize, videoFrame, rotation); 56 } 57 } 58 59private: 60 void (^_frameReceived)(CGSize, RTCVideoFrame *, RTCVideoRotation); 61}; 62 63} 64 65@interface TGCAMetalLayer : CAMetalLayer 66 67@end 68 69@implementation TGCAMetalLayer 70 71 72-(void)dealloc { 73 int bp = 0; 74 bp += 1; 75} 76@end 77 78@interface VideoMetalView () { 79 SQueueLocalObject *_rendererI420; 80 81 CAMetalLayer *_metalView; 82 NSView *_foregroundView; 83 84 CGSize _videoFrameSize; 85 RTCVideoRotation _rotation; 86 int64_t _lastFrameTimeNs; 87 88 CGSize _currentSize; 89 std::shared_ptr<VideoRendererAdapterImpl> _sink; 90 91 void (^_onFirstFrameReceived)(float); 92 bool _firstFrameReceivedReported; 93 void (^_onOrientationUpdated)(int, CGFloat); 94 void (^_onIsMirroredUpdated)(bool); 95 96 bool _didSetShouldBeMirrored; 97 bool _shouldBeMirrored; 98 bool _forceMirrored; 99 100 bool _isPaused; 101 102 NSMutableArray<RTCVideoFrame *> *_frames; 103 RTCVideoFrame *_videoFrame; 104 BOOL _drawing; 105 106 BOOL _deleteForegroundOnNextDrawing; 107} 108 109@end 110 111@implementation VideoMetalView 112 113+ (bool)isSupported { 114 return [VideoMetalView isMetalAvailable]; 115} 116 117- (instancetype)initWithFrame:(CGRect)frameRect { 118 self = [super initWithFrame:frameRect]; 119 if (self) { 120 [self configure]; 121 _lastFrameTimeNs = INT32_MAX; 122 _currentSize = CGSizeZero; 123 _frames = [[NSMutableArray alloc] init]; 124 _drawing = false; 125 _isPaused = false; 126 _deleteForegroundOnNextDrawing = false; 127 __weak VideoMetalView *weakSelf = self; 128 _sink.reset(new VideoRendererAdapterImpl(^(CGSize size, RTCVideoFrame *videoFrame, RTCVideoRotation rotation) { 129 dispatch_async(dispatch_get_main_queue(), ^{ 130 __strong VideoMetalView *strongSelf = weakSelf; 131 if (strongSelf == nil) { 132 return; 133 } 134 strongSelf->_rotation = videoFrame.rotation; 135 if (!CGSizeEqualToSize(size, strongSelf->_currentSize)) { 136 strongSelf->_currentSize = size; 137 [strongSelf setSize:size]; 138 } 139 140 int mappedValue = 0; 141 switch (rotation) { 142 case RTCVideoRotation_90: 143 mappedValue = 0; 144 break; 145 case RTCVideoRotation_180: 146 mappedValue = 1; 147 break; 148 case RTCVideoRotation_270: 149 mappedValue = 2; 150 break; 151 default: 152 mappedValue = 0; 153 break; 154 } 155 [strongSelf setInternalOrientation:mappedValue]; 156 157 [strongSelf renderFrame:videoFrame]; 158 159 if ([videoFrame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) { 160 RTCCVPixelBuffer *buffer = (RTCCVPixelBuffer*)videoFrame.buffer; 161 162 if ([buffer isKindOfClass:[TGRTCCVPixelBuffer class]]) { 163 bool shouldBeMirrored = ((TGRTCCVPixelBuffer *)buffer).shouldBeMirrored; 164 if (shouldBeMirrored != strongSelf->_shouldBeMirrored) { 165 strongSelf->_shouldBeMirrored = shouldBeMirrored; 166 if (strongSelf->_onIsMirroredUpdated) { 167 strongSelf->_onIsMirroredUpdated(strongSelf->_shouldBeMirrored); 168 } 169 } 170 } 171 } 172 173 if (!strongSelf->_firstFrameReceivedReported && strongSelf->_onFirstFrameReceived) { 174 strongSelf->_firstFrameReceivedReported = true; 175 strongSelf->_onFirstFrameReceived((float)videoFrame.width / (float)videoFrame.height); 176 } 177 }); 178 })); 179 180 } 181 return self; 182} 183 184- (BOOL)isEnabled { 185 return !_isPaused; 186} 187 188- (void)setEnabled:(BOOL)enabled { 189 _isPaused = enabled; 190} 191-(void)setIsPaused:(bool)paused { 192 _isPaused = paused; 193 [self updateDrawingSize:self.frame.size]; 194} 195-(void)renderToSize:(NSSize)size animated: (bool)animated { 196 [CATransaction begin]; 197 [CATransaction setDisableActions:YES]; 198 if (animated) { 199 if (!_foregroundView) { 200 _foregroundView = [[NSView alloc] initWithFrame:self.bounds]; 201 _foregroundView.wantsLayer = YES; 202 _foregroundView.autoresizingMask = 0; 203 _foregroundView.layer = [VideoMetalView createMetalView:self.bounds]; 204 _foregroundView.layer.contentsGravity = kCAGravityResizeAspect; 205 [self addSubview:_foregroundView]; 206 } 207 208 CAMetalLayer *layer = _metalView; 209 CAMetalLayer *foreground = (CAMetalLayer *)_foregroundView.layer; 210 211 [_rendererI420 with:^(TGRTCMTLI420Renderer * renderer) { 212 [renderer setDoubleRendering:layer foreground:foreground]; 213 }]; 214 _deleteForegroundOnNextDrawing = false; 215 } else { 216 _deleteForegroundOnNextDrawing = true; 217 CAMetalLayer *layer = _metalView; 218 219 [_rendererI420 with:^(TGRTCMTLI420Renderer * renderer) { 220 [renderer setSingleRendering:layer]; 221 }]; 222 } 223 [self updateDrawingSize:size]; 224 [CATransaction commit]; 225} 226 227- (CALayerContentsGravity)videoContentMode { 228 return _metalView.contentsGravity; 229} 230 231- (void)setVideoContentMode:(CALayerContentsGravity)mode { 232 // _metalView.contentsGravity = mode; 233} 234 235#pragma mark - Private 236 237+ (BOOL)isMetalAvailable { 238 return initMetal(); 239} 240 241+ (CAMetalLayer *)createMetalView:(CGRect)frame { 242 CAMetalLayer *layer = [[TGCAMetalLayer alloc] init]; 243 [CATransaction begin]; 244 [CATransaction setDisableActions:true]; 245 layer.framebufferOnly = true; 246 layer.opaque = false; 247// layer.cornerRadius = 4; 248 if (@available(macOS 10.13, *)) { 249 layer.displaySyncEnabled = NO; 250 } 251// layer.presentsWithTransaction = YES; 252 layer.backgroundColor = [NSColor clearColor].CGColor; 253 layer.contentsGravity = kCAGravityResizeAspectFill; 254 layer.frame = frame; 255 256 [CATransaction commit]; 257 return layer; 258} 259 260+ (TGRTCMTLI420Renderer *)createI420Renderer { 261 return [[TGRTCMTLI420RendererClass alloc] init]; 262} 263 264 265- (void)configure { 266 NSAssert([VideoMetalView isMetalAvailable], @"Metal not availiable on this device"); 267 self.wantsLayer = YES; 268 self.layerContentsRedrawPolicy = NSViewLayerContentsRedrawOnSetNeedsDisplay; 269 _metalView = [VideoMetalView createMetalView:self.bounds]; 270 271 272 self.layer = _metalView; 273 _videoFrameSize = CGSizeZero; 274 275 CAMetalLayer *layer = _metalView; 276 277 _rendererI420 = [[SQueueLocalObject alloc] initWithQueue:renderQueue generate: ^{ 278 TGRTCMTLI420Renderer *renderer = [VideoMetalView createI420Renderer]; 279 [renderer setSingleRendering:layer]; 280 return renderer; 281 }]; 282} 283 284 285-(void)setFrameSize:(NSSize)newSize { 286 [super setFrameSize:newSize]; 287 288 [self updateDrawingSize: newSize]; 289 290} 291- (void)layout { 292 [super layout]; 293} 294 295-(void)updateDrawingSize:(NSSize)size { 296 if (_isPaused) { 297 return; 298 } 299 _metalView.frame = CGRectMake(0, 0, size.width, size.height); 300 _foregroundView.frame = self.bounds; 301 if (!CGSizeEqualToSize(_videoFrameSize, CGSizeZero)) { 302 _metalView.drawableSize = [self drawableSize:size]; 303 ((CAMetalLayer *)_foregroundView.layer).drawableSize = _videoFrameSize; 304 } else { 305 _metalView.drawableSize = size; 306 ((CAMetalLayer *)_foregroundView.layer).drawableSize = size; 307 } 308 309 if(!_isPaused) { 310 RTCVideoFrame *frame = [_frames lastObject]; 311 if (frame == nil) { 312 frame = _videoFrame; 313 } 314 if (frame) { 315 [self renderFrame:frame]; 316 } 317 } 318} 319 320 321-(void)dealloc { 322 int bp = 0; 323 bp += 1; 324} 325 326#pragma mark - 327 328- (void)setRotationOverride:(NSValue *)rotationOverride { 329 _rotationOverride = rotationOverride; 330 331 [self setNeedsLayout:YES]; 332} 333 334- (RTCVideoRotation)rtcFrameRotation { 335 if (_rotationOverride) { 336 RTCVideoRotation rotation; 337 if (@available(macOS 10.13, *)) { 338 [_rotationOverride getValue:&rotation size:sizeof(rotation)]; 339 } else { 340 [_rotationOverride getValue:&rotation]; 341 } 342 return rotation; 343 } 344 345 return _rotation; 346} 347 348- (CGSize)drawableSize:(NSSize)forSize { 349 350 MTLFrameSize from; 351 MTLFrameSize to; 352 353 from.width = _videoFrameSize.width; 354 from.height = _videoFrameSize.height; 355 356 if (CGSizeEqualToSize(CGSizeZero, forSize)) { 357 to.width = _videoFrameSize.width; 358 to.height = _videoFrameSize.height; 359 } else { 360 to.width = forSize.width; 361 to.height = forSize.height; 362 } 363 364 365 MTLFrameSize size = MTLAspectFilled(to, from); 366 367 return CGSizeMake(size.width, size.height); 368} 369 370#pragma mark - RTCVideoRenderer 371 372- (void)setSize:(CGSize)size { 373 assert([NSThread isMainThread]); 374 375 _videoFrameSize = size; 376 [self updateDrawingSize:self.frame.size]; 377 378 _internalAspect = _videoFrameSize.width / _videoFrameSize.height; 379} 380 381- (void)renderFrame:(nullable RTCVideoFrame *)frame { 382 assert([NSThread isMainThread]); 383 384 if (!self.isEnabled) { 385 return; 386 } 387 388 if (frame == nil) { 389 RTCLogInfo(@"Incoming frame is nil. Exiting render callback."); 390 return; 391 } 392 393 RTCVideoFrame *videoFrame = frame; 394 // Skip rendering if we've already rendered this frame. 395 if (!videoFrame) { 396 return; 397 } 398 399 if (CGRectIsEmpty(self.bounds)) { 400 return; 401 } 402 if (CGRectIsEmpty(self.visibleRect)) { 403 return; 404 } 405 if (self.window == nil || self.superview == nil) { 406 return; 407 } 408 if ((self.window.occlusionState & NSWindowOcclusionStateVisible) == 0) { 409 return; 410 } 411 412 413 if (_frames.count >= 5) { 414 [_frames removeAllObjects]; 415 [_frames addObject:videoFrame]; 416 [self enqueue]; 417 return; 418 } 419 420 [_frames addObject:videoFrame]; 421 422 [self enqueue]; 423 424} 425 426-(void)enqueue { 427 if(_frames.count > 0 && !_drawing) { 428 RTCVideoFrame *videoFrame = [_frames firstObject]; 429 430 NSValue * rotationOverride = _rotationOverride; 431 432 433 int64_t timeStampNs = videoFrame.timeStampNs; 434 435 __weak VideoMetalView *weakSelf = self; 436 dispatch_block_t completion = ^{ 437 __strong VideoMetalView *strongSelf = weakSelf; 438 if (strongSelf && strongSelf->_frames.count > 0) { 439 [strongSelf->_frames removeObjectAtIndex:0]; 440 strongSelf->_drawing = false; 441 strongSelf->_lastFrameTimeNs = timeStampNs; 442 if (strongSelf->_deleteForegroundOnNextDrawing) { 443 [strongSelf->_foregroundView removeFromSuperview]; 444 strongSelf->_foregroundView = nil; 445 strongSelf->_deleteForegroundOnNextDrawing = false; 446 } 447 [strongSelf enqueue]; 448 } 449 }; 450 451 _videoFrame = videoFrame; 452 453 self->_drawing = true; 454 455 [_rendererI420 with:^(TGRTCMTLI420Renderer * object) { 456 object.rotationOverride = rotationOverride; 457 [object drawFrame:videoFrame]; 458 dispatch_async(dispatch_get_main_queue(), completion); 459 }]; 460 461 } 462} 463 464- (std::shared_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>)getSink { 465 assert([NSThread isMainThread]); 466 467 return _sink; 468} 469 470- (void)setOnFirstFrameReceived:(void (^ _Nullable)(float))onFirstFrameReceived { 471 _onFirstFrameReceived = [onFirstFrameReceived copy]; 472 _firstFrameReceivedReported = false; 473} 474 475- (void)setInternalOrientationAndSize:(int)internalOrientation size:(CGSize)size { 476 CGFloat aspect = 1.0f; 477 if (size.width > 1.0f && size.height > 1.0f) { 478 aspect = size.width / size.height; 479 } 480 if (_internalOrientation != internalOrientation || ABS(_internalAspect - aspect) > 0.001) { 481 RTCLogInfo(@"VideoMetalView@%lx orientation: %d, aspect: %f", (intptr_t)self, internalOrientation, (float)aspect); 482 483 _internalOrientation = internalOrientation; 484 _internalAspect = aspect; 485 if (_onOrientationUpdated) { 486 _onOrientationUpdated(internalOrientation, aspect); 487 } 488 } 489} 490 491- (void)internalSetOnOrientationUpdated:(void (^ _Nullable)(int, CGFloat))onOrientationUpdated { 492 _onOrientationUpdated = [onOrientationUpdated copy]; 493} 494 495- (void)internalSetOnIsMirroredUpdated:(void (^ _Nullable)(bool))onIsMirroredUpdated { 496 _onIsMirroredUpdated = [onIsMirroredUpdated copy]; 497} 498 499- (void)setForceMirrored:(BOOL)forceMirrored { 500 _forceMirrored = forceMirrored; 501 [self setNeedsLayout:YES]; 502} 503 504 505@end 506