1/*
2 *  Copyright 2017 The WebRTC Project Authors. All rights reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#import "RTCMTLNV12Renderer.h"
12
13#import <Metal/Metal.h>
14#import <MetalKit/MetalKit.h>
15
16#import "RTCMTLRenderer+Private.h"
17#import "base/RTCLogging.h"
18#import "base/RTCVideoFrame.h"
19#import "base/RTCVideoFrameBuffer.h"
20#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
21
22#include "rtc_base/checks.h"
23
24static NSString *const shaderSource = MTL_STRINGIFY(
25    using namespace metal;
26
27    typedef struct {
28      packed_float2 position;
29      packed_float2 texcoord;
30    } Vertex;
31
32    typedef struct {
33      float4 position[[position]];
34      float2 texcoord;
35    } Varyings;
36
37    vertex Varyings vertexPassthrough(constant Vertex *verticies[[buffer(0)]],
38                                      unsigned int vid[[vertex_id]]) {
39      Varyings out;
40      constant Vertex &v = verticies[vid];
41      out.position = float4(float2(v.position), 0.0, 1.0);
42      out.texcoord = v.texcoord;
43      return out;
44    }
45
46    // Receiving YCrCb textures.
47    fragment half4 fragmentColorConversion(
48        Varyings in[[stage_in]],
49        texture2d<float, access::sample> textureY[[texture(0)]],
50        texture2d<float, access::sample> textureCbCr[[texture(1)]]) {
51      constexpr sampler s(address::clamp_to_edge, filter::linear);
52      float y;
53      float2 uv;
54      y = textureY.sample(s, in.texcoord).r;
55      uv = textureCbCr.sample(s, in.texcoord).rg - float2(0.5, 0.5);
56
57      // Conversion for YUV to rgb from http://www.fourcc.org/fccyvrgb.php
58      float4 out = float4(y + 1.403 * uv.y, y - 0.344 * uv.x - 0.714 * uv.y, y + 1.770 * uv.x, 1.0);
59
60      return half4(out);
61    });
62
63@implementation RTCMTLNV12Renderer {
64  // Textures.
65  CVMetalTextureCacheRef _textureCache;
66  id<MTLTexture> _yTexture;
67  id<MTLTexture> _CrCbTexture;
68}
69
70- (BOOL)addRenderingDestination:(__kindof MTKView *)view {
71  if ([super addRenderingDestination:view]) {
72    return [self initializeTextureCache];
73  }
74  return NO;
75}
76
77- (BOOL)initializeTextureCache {
78  CVReturn status = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, [self currentMetalDevice],
79                                              nil, &_textureCache);
80  if (status != kCVReturnSuccess) {
81    RTCLogError(@"Metal: Failed to initialize metal texture cache. Return status is %d", status);
82    return NO;
83  }
84
85  return YES;
86}
87
88- (NSString *)shaderSource {
89  return shaderSource;
90}
91
92- (void)getWidth:(nonnull int *)width
93          height:(nonnull int *)height
94       cropWidth:(nonnull int *)cropWidth
95      cropHeight:(nonnull int *)cropHeight
96           cropX:(nonnull int *)cropX
97           cropY:(nonnull int *)cropY
98         ofFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame {
99  RTC_OBJC_TYPE(RTCCVPixelBuffer) *pixelBuffer = (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer;
100  *width = CVPixelBufferGetWidth(pixelBuffer.pixelBuffer);
101  *height = CVPixelBufferGetHeight(pixelBuffer.pixelBuffer);
102  *cropWidth = pixelBuffer.cropWidth;
103  *cropHeight = pixelBuffer.cropHeight;
104  *cropX = pixelBuffer.cropX;
105  *cropY = pixelBuffer.cropY;
106}
107
108- (BOOL)setupTexturesForFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame {
109  RTC_DCHECK([frame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]);
110  if (![super setupTexturesForFrame:frame]) {
111    return NO;
112  }
113  CVPixelBufferRef pixelBuffer = ((RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer).pixelBuffer;
114
115  id<MTLTexture> lumaTexture = nil;
116  id<MTLTexture> chromaTexture = nil;
117  CVMetalTextureRef outTexture = nullptr;
118
119  // Luma (y) texture.
120  int lumaWidth = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0);
121  int lumaHeight = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0);
122
123  int indexPlane = 0;
124  CVReturn result = CVMetalTextureCacheCreateTextureFromImage(
125      kCFAllocatorDefault, _textureCache, pixelBuffer, nil, MTLPixelFormatR8Unorm, lumaWidth,
126      lumaHeight, indexPlane, &outTexture);
127
128  if (result == kCVReturnSuccess) {
129    lumaTexture = CVMetalTextureGetTexture(outTexture);
130  }
131
132  // Same as CFRelease except it can be passed NULL without crashing.
133  CVBufferRelease(outTexture);
134  outTexture = nullptr;
135
136  // Chroma (CrCb) texture.
137  indexPlane = 1;
138  result = CVMetalTextureCacheCreateTextureFromImage(
139      kCFAllocatorDefault, _textureCache, pixelBuffer, nil, MTLPixelFormatRG8Unorm, lumaWidth / 2,
140      lumaHeight / 2, indexPlane, &outTexture);
141  if (result == kCVReturnSuccess) {
142    chromaTexture = CVMetalTextureGetTexture(outTexture);
143  }
144  CVBufferRelease(outTexture);
145
146  if (lumaTexture != nil && chromaTexture != nil) {
147    _yTexture = lumaTexture;
148    _CrCbTexture = chromaTexture;
149    return YES;
150  }
151  return NO;
152}
153
154- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder {
155  [renderEncoder setFragmentTexture:_yTexture atIndex:0];
156  [renderEncoder setFragmentTexture:_CrCbTexture atIndex:1];
157}
158
159- (void)dealloc {
160  if (_textureCache) {
161    CFRelease(_textureCache);
162  }
163}
164@end
165