1 /* 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef API_VIDEO_VIDEO_FRAME_BUFFER_H_ 12 #define API_VIDEO_VIDEO_FRAME_BUFFER_H_ 13 14 #include <stdint.h> 15 16 #include "api/array_view.h" 17 #include "api/scoped_refptr.h" 18 #include "rtc_base/ref_count.h" 19 #include "rtc_base/system/rtc_export.h" 20 21 namespace webrtc { 22 23 class I420BufferInterface; 24 class I420ABufferInterface; 25 class I444BufferInterface; 26 class I010BufferInterface; 27 class NV12BufferInterface; 28 29 // Base class for frame buffers of different types of pixel format and storage. 30 // The tag in type() indicates how the data is represented, and each type is 31 // implemented as a subclass. To access the pixel data, call the appropriate 32 // GetXXX() function, where XXX represents the type. There is also a function 33 // ToI420() that returns a frame buffer in I420 format, converting from the 34 // underlying representation if necessary. I420 is the most widely accepted 35 // format and serves as a fallback for video sinks that can only handle I420, 36 // e.g. the internal WebRTC software encoders. A special enum value 'kNative' is 37 // provided for external clients to implement their own frame buffer 38 // representations, e.g. as textures. The external client can produce such 39 // native frame buffers from custom video sources, and then cast it back to the 40 // correct subclass in custom video sinks. The purpose of this is to improve 41 // performance by providing an optimized path without intermediate conversions. 42 // Frame metadata such as rotation and timestamp are stored in 43 // webrtc::VideoFrame, and not here. 44 class RTC_EXPORT VideoFrameBuffer : public rtc::RefCountInterface { 45 public: 46 // New frame buffer types will be added conservatively when there is an 47 // opportunity to optimize the path between some pair of video source and 48 // video sink. 49 enum class Type { 50 kNative, 51 kI420, 52 kI420A, 53 kI444, 54 kI010, 55 kNV12, 56 }; 57 58 // This function specifies in what pixel format the data is stored in. 59 virtual Type type() const = 0; 60 61 // The resolution of the frame in pixels. For formats where some planes are 62 // subsampled, this is the highest-resolution plane. 63 virtual int width() const = 0; 64 virtual int height() const = 0; 65 66 // Returns a memory-backed frame buffer in I420 format. If the pixel data is 67 // in another format, a conversion will take place. All implementations must 68 // provide a fallback to I420 for compatibility with e.g. the internal WebRTC 69 // software encoders. 70 virtual rtc::scoped_refptr<I420BufferInterface> ToI420() = 0; 71 72 // GetI420() methods should return I420 buffer if conversion is trivial, i.e 73 // no change for binary data is needed. Otherwise these methods should return 74 // nullptr. One example of buffer with that property is 75 // WebrtcVideoFrameAdapter in Chrome - it's I420 buffer backed by a shared 76 // memory buffer. Therefore it must have type kNative. Yet, ToI420() 77 // doesn't affect binary data at all. Another example is any I420A buffer. 78 // TODO(https://crbug.com/webrtc/12021): Make this method non-virtual and 79 // behave as the other GetXXX methods below. 80 virtual const I420BufferInterface* GetI420() const; 81 82 // A format specific scale function. Default implementation works by 83 // converting to I420. But more efficient implementations may override it, 84 // especially for kNative. 85 // First, the image is cropped to |crop_width| and |crop_height| and then 86 // scaled to |scaled_width| and |scaled_height|. 87 virtual rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(int offset_x, 88 int offset_y, 89 int crop_width, 90 int crop_height, 91 int scaled_width, 92 int scaled_height); 93 94 // Alias for common use case. Scale(int scaled_width,int scaled_height)95 rtc::scoped_refptr<VideoFrameBuffer> Scale(int scaled_width, 96 int scaled_height) { 97 return CropAndScale(0, 0, width(), height(), scaled_width, scaled_height); 98 } 99 100 // These functions should only be called if type() is of the correct type. 101 // Calling with a different type will result in a crash. 102 const I420ABufferInterface* GetI420A() const; 103 const I444BufferInterface* GetI444() const; 104 const I010BufferInterface* GetI010() const; 105 const NV12BufferInterface* GetNV12() const; 106 107 // From a kNative frame, returns a VideoFrameBuffer with a pixel format in 108 // the list of types that is in the main memory with a pixel perfect 109 // conversion for encoding with a software encoder. Returns nullptr if the 110 // frame type is not supported, mapping is not possible, or if the kNative 111 // frame has not implemented this method. Only callable if type() is kNative. 112 virtual rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer( 113 rtc::ArrayView<Type> types); 114 115 protected: ~VideoFrameBuffer()116 ~VideoFrameBuffer() override {} 117 }; 118 119 // Update when VideoFrameBuffer::Type is updated. 120 const char* VideoFrameBufferTypeToString(VideoFrameBuffer::Type type); 121 122 // This interface represents planar formats. 123 class PlanarYuvBuffer : public VideoFrameBuffer { 124 public: 125 virtual int ChromaWidth() const = 0; 126 virtual int ChromaHeight() const = 0; 127 128 // Returns the number of steps(in terms of Data*() return type) between 129 // successive rows for a given plane. 130 virtual int StrideY() const = 0; 131 virtual int StrideU() const = 0; 132 virtual int StrideV() const = 0; 133 134 protected: ~PlanarYuvBuffer()135 ~PlanarYuvBuffer() override {} 136 }; 137 138 // This interface represents 8-bit color depth formats: Type::kI420, 139 // Type::kI420A and Type::kI444. 140 class PlanarYuv8Buffer : public PlanarYuvBuffer { 141 public: 142 // Returns pointer to the pixel data for a given plane. The memory is owned by 143 // the VideoFrameBuffer object and must not be freed by the caller. 144 virtual const uint8_t* DataY() const = 0; 145 virtual const uint8_t* DataU() const = 0; 146 virtual const uint8_t* DataV() const = 0; 147 148 protected: ~PlanarYuv8Buffer()149 ~PlanarYuv8Buffer() override {} 150 }; 151 152 class RTC_EXPORT I420BufferInterface : public PlanarYuv8Buffer { 153 public: 154 Type type() const override; 155 156 int ChromaWidth() const final; 157 int ChromaHeight() const final; 158 159 rtc::scoped_refptr<I420BufferInterface> ToI420() final; 160 const I420BufferInterface* GetI420() const final; 161 162 protected: ~I420BufferInterface()163 ~I420BufferInterface() override {} 164 }; 165 166 class RTC_EXPORT I420ABufferInterface : public I420BufferInterface { 167 public: 168 Type type() const final; 169 virtual const uint8_t* DataA() const = 0; 170 virtual int StrideA() const = 0; 171 172 protected: ~I420ABufferInterface()173 ~I420ABufferInterface() override {} 174 }; 175 176 class I444BufferInterface : public PlanarYuv8Buffer { 177 public: 178 Type type() const final; 179 180 int ChromaWidth() const final; 181 int ChromaHeight() const final; 182 183 protected: ~I444BufferInterface()184 ~I444BufferInterface() override {} 185 }; 186 187 // This interface represents 8-bit to 16-bit color depth formats: Type::kI010. 188 class PlanarYuv16BBuffer : public PlanarYuvBuffer { 189 public: 190 // Returns pointer to the pixel data for a given plane. The memory is owned by 191 // the VideoFrameBuffer object and must not be freed by the caller. 192 virtual const uint16_t* DataY() const = 0; 193 virtual const uint16_t* DataU() const = 0; 194 virtual const uint16_t* DataV() const = 0; 195 196 protected: ~PlanarYuv16BBuffer()197 ~PlanarYuv16BBuffer() override {} 198 }; 199 200 // Represents Type::kI010, allocates 16 bits per pixel and fills 10 least 201 // significant bits with color information. 202 class I010BufferInterface : public PlanarYuv16BBuffer { 203 public: 204 Type type() const override; 205 206 int ChromaWidth() const final; 207 int ChromaHeight() const final; 208 209 protected: ~I010BufferInterface()210 ~I010BufferInterface() override {} 211 }; 212 213 class BiplanarYuvBuffer : public VideoFrameBuffer { 214 public: 215 virtual int ChromaWidth() const = 0; 216 virtual int ChromaHeight() const = 0; 217 218 // Returns the number of steps(in terms of Data*() return type) between 219 // successive rows for a given plane. 220 virtual int StrideY() const = 0; 221 virtual int StrideUV() const = 0; 222 223 protected: ~BiplanarYuvBuffer()224 ~BiplanarYuvBuffer() override {} 225 }; 226 227 class BiplanarYuv8Buffer : public BiplanarYuvBuffer { 228 public: 229 virtual const uint8_t* DataY() const = 0; 230 virtual const uint8_t* DataUV() const = 0; 231 232 protected: ~BiplanarYuv8Buffer()233 ~BiplanarYuv8Buffer() override {} 234 }; 235 236 // Represents Type::kNV12. NV12 is full resolution Y and half-resolution 237 // interleved UV. 238 class RTC_EXPORT NV12BufferInterface : public BiplanarYuv8Buffer { 239 public: 240 Type type() const override; 241 242 int ChromaWidth() const final; 243 int ChromaHeight() const final; 244 245 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(int offset_x, 246 int offset_y, 247 int crop_width, 248 int crop_height, 249 int scaled_width, 250 int scaled_height) override; 251 252 protected: ~NV12BufferInterface()253 ~NV12BufferInterface() override {} 254 }; 255 256 } // namespace webrtc 257 258 #endif // API_VIDEO_VIDEO_FRAME_BUFFER_H_ 259