1/*
2 *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#import "RTCCVPixelBuffer.h"
12
13#import "api/video_frame_buffer/RTCNativeMutableI420Buffer.h"
14
15#include "common_video/libyuv/include/webrtc_libyuv.h"
16#include "rtc_base/checks.h"
17#include "rtc_base/logging.h"
18#include "third_party/libyuv/include/libyuv.h"
19
20#if !defined(NDEBUG) && defined(WEBRTC_IOS)
21#import <UIKit/UIKit.h>
22#import <VideoToolbox/VideoToolbox.h>
23#endif
24
25@implementation RTCCVPixelBuffer {
26  int _width;
27  int _height;
28  int _bufferWidth;
29  int _bufferHeight;
30  int _cropWidth;
31  int _cropHeight;
32}
33
34@synthesize pixelBuffer = _pixelBuffer;
35@synthesize cropX = _cropX;
36@synthesize cropY = _cropY;
37@synthesize cropWidth = _cropWidth;
38@synthesize cropHeight = _cropHeight;
39
40+ (NSSet<NSNumber*>*)supportedPixelFormats {
41  return [NSSet setWithObjects:@(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange),
42                               @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange),
43                               @(kCVPixelFormatType_32BGRA),
44                               @(kCVPixelFormatType_32ARGB),
45                               nil];
46}
47
48- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer {
49  return [self initWithPixelBuffer:pixelBuffer
50                      adaptedWidth:CVPixelBufferGetWidth(pixelBuffer)
51                     adaptedHeight:CVPixelBufferGetHeight(pixelBuffer)
52                         cropWidth:CVPixelBufferGetWidth(pixelBuffer)
53                        cropHeight:CVPixelBufferGetHeight(pixelBuffer)
54                             cropX:0
55                             cropY:0];
56}
57
58- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer
59                       adaptedWidth:(int)adaptedWidth
60                      adaptedHeight:(int)adaptedHeight
61                          cropWidth:(int)cropWidth
62                         cropHeight:(int)cropHeight
63                              cropX:(int)cropX
64                              cropY:(int)cropY {
65  if (self = [super init]) {
66    _width = adaptedWidth;
67    _height = adaptedHeight;
68    _pixelBuffer = pixelBuffer;
69    _bufferWidth = CVPixelBufferGetWidth(_pixelBuffer);
70    _bufferHeight = CVPixelBufferGetHeight(_pixelBuffer);
71    _cropWidth = cropWidth;
72    _cropHeight = cropHeight;
73    // Can only crop at even pixels.
74    _cropX = cropX & ~1;
75    _cropY = cropY & ~1;
76    CVBufferRetain(_pixelBuffer);
77  }
78
79  return self;
80}
81
82- (void)dealloc {
83  CVBufferRelease(_pixelBuffer);
84}
85
86- (int)width {
87  return _width;
88}
89
90- (int)height {
91  return _height;
92}
93
94- (BOOL)requiresCropping {
95  return _cropWidth != _bufferWidth || _cropHeight != _bufferHeight;
96}
97
98- (BOOL)requiresScalingToWidth:(int)width height:(int)height {
99  return _cropWidth != width || _cropHeight != height;
100}
101
102- (int)bufferSizeForCroppingAndScalingToWidth:(int)width height:(int)height {
103  const OSType srcPixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer);
104  switch (srcPixelFormat) {
105    case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
106    case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: {
107      int srcChromaWidth = (_cropWidth + 1) / 2;
108      int srcChromaHeight = (_cropHeight + 1) / 2;
109      int dstChromaWidth = (width + 1) / 2;
110      int dstChromaHeight = (height + 1) / 2;
111
112      return srcChromaWidth * srcChromaHeight * 2 + dstChromaWidth * dstChromaHeight * 2;
113    }
114    case kCVPixelFormatType_32BGRA:
115    case kCVPixelFormatType_32ARGB: {
116      return 0;  // Scaling RGBA frames does not require a temporary buffer.
117    }
118  }
119  RTC_NOTREACHED() << "Unsupported pixel format.";
120  return 0;
121}
122
123- (BOOL)cropAndScaleTo:(CVPixelBufferRef)outputPixelBuffer
124        withTempBuffer:(nullable uint8_t*)tmpBuffer {
125  const OSType srcPixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer);
126  const OSType dstPixelFormat = CVPixelBufferGetPixelFormatType(outputPixelBuffer);
127
128  switch (srcPixelFormat) {
129    case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
130    case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: {
131      size_t dstWidth = CVPixelBufferGetWidth(outputPixelBuffer);
132      size_t dstHeight = CVPixelBufferGetHeight(outputPixelBuffer);
133      if (dstWidth > 0 && dstHeight > 0) {
134        RTC_DCHECK(dstPixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ||
135                   dstPixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange);
136        if ([self requiresScalingToWidth:dstWidth height:dstHeight]) {
137          RTC_DCHECK(tmpBuffer);
138        }
139        [self cropAndScaleNV12To:outputPixelBuffer withTempBuffer:tmpBuffer];
140      }
141      break;
142    }
143    case kCVPixelFormatType_32BGRA:
144    case kCVPixelFormatType_32ARGB: {
145      RTC_DCHECK(srcPixelFormat == dstPixelFormat);
146      [self cropAndScaleARGBTo:outputPixelBuffer];
147      break;
148    }
149    default: { RTC_NOTREACHED() << "Unsupported pixel format."; }
150  }
151
152  return YES;
153}
154
155- (id<RTCI420Buffer>)toI420 {
156  const OSType pixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer);
157
158  CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly);
159
160  RTCMutableI420Buffer* i420Buffer =
161      [[RTCMutableI420Buffer alloc] initWithWidth:[self width] height:[self height]];
162
163  switch (pixelFormat) {
164    case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
165    case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: {
166      const uint8_t* srcY =
167          static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 0));
168      const int srcYStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 0);
169      const uint8_t* srcUV =
170          static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 1));
171      const int srcUVStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 1);
172
173      // Crop just by modifying pointers.
174      srcY += srcYStride * _cropY + _cropX;
175      srcUV += srcUVStride * (_cropY / 2) + _cropX;
176
177      // TODO(magjed): Use a frame buffer pool.
178      webrtc::NV12ToI420Scaler nv12ToI420Scaler;
179      nv12ToI420Scaler.NV12ToI420Scale(srcY,
180                                       srcYStride,
181                                       srcUV,
182                                       srcUVStride,
183                                       _cropWidth,
184                                       _cropHeight,
185                                       i420Buffer.mutableDataY,
186                                       i420Buffer.strideY,
187                                       i420Buffer.mutableDataU,
188                                       i420Buffer.strideU,
189                                       i420Buffer.mutableDataV,
190                                       i420Buffer.strideV,
191                                       i420Buffer.width,
192                                       i420Buffer.height);
193      break;
194    }
195    case kCVPixelFormatType_32BGRA:
196    case kCVPixelFormatType_32ARGB: {
197      CVPixelBufferRef scaledPixelBuffer = NULL;
198      CVPixelBufferRef sourcePixelBuffer = NULL;
199      if ([self requiresCropping] ||
200          [self requiresScalingToWidth:i420Buffer.width height:i420Buffer.height]) {
201        CVPixelBufferCreate(
202            NULL, i420Buffer.width, i420Buffer.height, pixelFormat, NULL, &scaledPixelBuffer);
203        [self cropAndScaleTo:scaledPixelBuffer withTempBuffer:NULL];
204
205        CVPixelBufferLockBaseAddress(scaledPixelBuffer, kCVPixelBufferLock_ReadOnly);
206        sourcePixelBuffer = scaledPixelBuffer;
207      } else {
208        sourcePixelBuffer = _pixelBuffer;
209      }
210      const uint8_t* src = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(sourcePixelBuffer));
211      const size_t bytesPerRow = CVPixelBufferGetBytesPerRow(sourcePixelBuffer);
212
213      if (pixelFormat == kCVPixelFormatType_32BGRA) {
214        // Corresponds to libyuv::FOURCC_ARGB
215        libyuv::ARGBToI420(src,
216                           bytesPerRow,
217                           i420Buffer.mutableDataY,
218                           i420Buffer.strideY,
219                           i420Buffer.mutableDataU,
220                           i420Buffer.strideU,
221                           i420Buffer.mutableDataV,
222                           i420Buffer.strideV,
223                           i420Buffer.width,
224                           i420Buffer.height);
225      } else if (pixelFormat == kCVPixelFormatType_32ARGB) {
226        // Corresponds to libyuv::FOURCC_BGRA
227        libyuv::BGRAToI420(src,
228                           bytesPerRow,
229                           i420Buffer.mutableDataY,
230                           i420Buffer.strideY,
231                           i420Buffer.mutableDataU,
232                           i420Buffer.strideU,
233                           i420Buffer.mutableDataV,
234                           i420Buffer.strideV,
235                           i420Buffer.width,
236                           i420Buffer.height);
237      }
238
239      if (scaledPixelBuffer) {
240        CVPixelBufferUnlockBaseAddress(scaledPixelBuffer, kCVPixelBufferLock_ReadOnly);
241        CVBufferRelease(scaledPixelBuffer);
242      }
243      break;
244    }
245    default: { RTC_NOTREACHED() << "Unsupported pixel format."; }
246  }
247
248  CVPixelBufferUnlockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly);
249
250  return i420Buffer;
251}
252
253#pragma mark - Debugging
254
255#if !defined(NDEBUG) && defined(WEBRTC_IOS)
256- (id)debugQuickLookObject {
257  CGImageRef cgImage;
258  VTCreateCGImageFromCVPixelBuffer(_pixelBuffer, NULL, &cgImage);
259  UIImage *image = [UIImage imageWithCGImage:cgImage scale:1.0 orientation:UIImageOrientationUp];
260  CGImageRelease(cgImage);
261  return image;
262}
263#endif
264
265#pragma mark - Private
266
267- (void)cropAndScaleNV12To:(CVPixelBufferRef)outputPixelBuffer withTempBuffer:(uint8_t*)tmpBuffer {
268  // Prepare output pointers.
269  CVReturn cvRet = CVPixelBufferLockBaseAddress(outputPixelBuffer, 0);
270  if (cvRet != kCVReturnSuccess) {
271    RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
272  }
273  const int dstWidth = CVPixelBufferGetWidth(outputPixelBuffer);
274  const int dstHeight = CVPixelBufferGetHeight(outputPixelBuffer);
275  uint8_t* dstY =
276      reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 0));
277  const int dstYStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 0);
278  uint8_t* dstUV =
279      reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 1));
280  const int dstUVStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 1);
281
282  // Prepare source pointers.
283  CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly);
284  const uint8_t* srcY = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 0));
285  const int srcYStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 0);
286  const uint8_t* srcUV = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 1));
287  const int srcUVStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 1);
288
289  // Crop just by modifying pointers.
290  srcY += srcYStride * _cropY + _cropX;
291  srcUV += srcUVStride * (_cropY / 2) + _cropX;
292
293  webrtc::NV12Scale(tmpBuffer,
294                    srcY,
295                    srcYStride,
296                    srcUV,
297                    srcUVStride,
298                    _cropWidth,
299                    _cropHeight,
300                    dstY,
301                    dstYStride,
302                    dstUV,
303                    dstUVStride,
304                    dstWidth,
305                    dstHeight);
306
307  CVPixelBufferUnlockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly);
308  CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0);
309}
310
311- (void)cropAndScaleARGBTo:(CVPixelBufferRef)outputPixelBuffer {
312  // Prepare output pointers.
313  CVReturn cvRet = CVPixelBufferLockBaseAddress(outputPixelBuffer, 0);
314  if (cvRet != kCVReturnSuccess) {
315    RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
316  }
317  const int dstWidth = CVPixelBufferGetWidth(outputPixelBuffer);
318  const int dstHeight = CVPixelBufferGetHeight(outputPixelBuffer);
319
320  uint8_t* dst = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddress(outputPixelBuffer));
321  const int dstStride = CVPixelBufferGetBytesPerRow(outputPixelBuffer);
322
323  // Prepare source pointers.
324  CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly);
325  const uint8_t* src = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(_pixelBuffer));
326  const int srcStride = CVPixelBufferGetBytesPerRow(_pixelBuffer);
327
328  // Crop just by modifying pointers. Need to ensure that src pointer points to a byte corresponding
329  // to the start of a new pixel (byte with B for BGRA) so that libyuv scales correctly.
330  const int bytesPerPixel = 4;
331  src += srcStride * _cropY + (_cropX * bytesPerPixel);
332
333  // kCVPixelFormatType_32BGRA corresponds to libyuv::FOURCC_ARGB
334  libyuv::ARGBScale(src,
335                    srcStride,
336                    _cropWidth,
337                    _cropHeight,
338                    dst,
339                    dstStride,
340                    dstWidth,
341                    dstHeight,
342                    libyuv::kFilterBox);
343
344  CVPixelBufferUnlockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly);
345  CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0);
346}
347
348@end
349