1 /* 2 * Copyright 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 package org.webrtc; 12 13 import android.graphics.Matrix; 14 import android.opengl.GLES20; 15 import java.nio.ByteBuffer; 16 import org.webrtc.VideoFrame.I420Buffer; 17 import org.webrtc.VideoFrame.TextureBuffer; 18 19 /** 20 * Class for converting OES textures to a YUV ByteBuffer. It can be constructed on any thread, but 21 * should only be operated from a single thread with an active EGL context. 22 */ 23 public class YuvConverter { 24 private static final String FRAGMENT_SHADER = 25 // Difference in texture coordinate corresponding to one 26 // sub-pixel in the x direction. 27 "uniform vec2 xUnit;\n" 28 // Color conversion coefficients, including constant term 29 + "uniform vec4 coeffs;\n" 30 + "\n" 31 + "void main() {\n" 32 // Since the alpha read from the texture is always 1, this could 33 // be written as a mat4 x vec4 multiply. However, that seems to 34 // give a worse framerate, possibly because the additional 35 // multiplies by 1.0 consume resources. TODO(nisse): Could also 36 // try to do it as a vec3 x mat3x4, followed by an add in of a 37 // constant vector. 38 + " gl_FragColor.r = coeffs.a + dot(coeffs.rgb,\n" 39 + " sample(tc - 1.5 * xUnit).rgb);\n" 40 + " gl_FragColor.g = coeffs.a + dot(coeffs.rgb,\n" 41 + " sample(tc - 0.5 * xUnit).rgb);\n" 42 + " gl_FragColor.b = coeffs.a + dot(coeffs.rgb,\n" 43 + " sample(tc + 0.5 * xUnit).rgb);\n" 44 + " gl_FragColor.a = coeffs.a + dot(coeffs.rgb,\n" 45 + " sample(tc + 1.5 * xUnit).rgb);\n" 46 + "}\n"; 47 48 private static class ShaderCallbacks implements GlGenericDrawer.ShaderCallbacks { 49 // Y'UV444 to RGB888, see https://en.wikipedia.org/wiki/YUV#Y%E2%80%B2UV444_to_RGB888_conversion 50 // We use the ITU-R BT.601 coefficients for Y, U and V. 51 // The values in Wikipedia are inaccurate, the accurate values derived from the spec are: 52 // Y = 0.299 * R + 0.587 * G + 0.114 * B 53 // U = -0.168736 * R - 0.331264 * G + 0.5 * B + 0.5 54 // V = 0.5 * R - 0.418688 * G - 0.0813124 * B + 0.5 55 // To map the Y-values to range [16-235] and U- and V-values to range [16-240], the matrix has 56 // been multiplied with matrix: 57 // {{219 / 255, 0, 0, 16 / 255}, 58 // {0, 224 / 255, 0, 16 / 255}, 59 // {0, 0, 224 / 255, 16 / 255}, 60 // {0, 0, 0, 1}} 61 private static final float[] yCoeffs = 62 new float[] {0.256788f, 0.504129f, 0.0979059f, 0.0627451f}; 63 private static final float[] uCoeffs = 64 new float[] {-0.148223f, -0.290993f, 0.439216f, 0.501961f}; 65 private static final float[] vCoeffs = 66 new float[] {0.439216f, -0.367788f, -0.0714274f, 0.501961f}; 67 68 private int xUnitLoc; 69 private int coeffsLoc; 70 71 private float[] coeffs; 72 private float stepSize; 73 setPlaneY()74 public void setPlaneY() { 75 coeffs = yCoeffs; 76 stepSize = 1.0f; 77 } 78 setPlaneU()79 public void setPlaneU() { 80 coeffs = uCoeffs; 81 stepSize = 2.0f; 82 } 83 setPlaneV()84 public void setPlaneV() { 85 coeffs = vCoeffs; 86 stepSize = 2.0f; 87 } 88 89 @Override onNewShader(GlShader shader)90 public void onNewShader(GlShader shader) { 91 xUnitLoc = shader.getUniformLocation("xUnit"); 92 coeffsLoc = shader.getUniformLocation("coeffs"); 93 } 94 95 @Override onPrepareShader(GlShader shader, float[] texMatrix, int frameWidth, int frameHeight, int viewportWidth, int viewportHeight)96 public void onPrepareShader(GlShader shader, float[] texMatrix, int frameWidth, int frameHeight, 97 int viewportWidth, int viewportHeight) { 98 GLES20.glUniform4fv(coeffsLoc, /* count= */ 1, coeffs, /* offset= */ 0); 99 // Matrix * (1;0;0;0) / (width / stepSize). Note that OpenGL uses column major order. 100 GLES20.glUniform2f( 101 xUnitLoc, stepSize * texMatrix[0] / frameWidth, stepSize * texMatrix[1] / frameWidth); 102 } 103 } 104 105 private final ThreadUtils.ThreadChecker threadChecker = new ThreadUtils.ThreadChecker(); 106 private final GlTextureFrameBuffer i420TextureFrameBuffer = 107 new GlTextureFrameBuffer(GLES20.GL_RGBA); 108 private final ShaderCallbacks shaderCallbacks = new ShaderCallbacks(); 109 private final GlGenericDrawer drawer = new GlGenericDrawer(FRAGMENT_SHADER, shaderCallbacks); 110 private final VideoFrameDrawer videoFrameDrawer; 111 112 /** 113 * This class should be constructed on a thread that has an active EGL context. 114 */ YuvConverter()115 public YuvConverter() { 116 this(new VideoFrameDrawer()); 117 } 118 YuvConverter(VideoFrameDrawer videoFrameDrawer)119 public YuvConverter(VideoFrameDrawer videoFrameDrawer) { 120 this.videoFrameDrawer = videoFrameDrawer; 121 threadChecker.detachThread(); 122 } 123 124 /** Converts the texture buffer to I420. */ convert(TextureBuffer inputTextureBuffer)125 public I420Buffer convert(TextureBuffer inputTextureBuffer) { 126 threadChecker.checkIsOnValidThread(); 127 128 TextureBuffer preparedBuffer = (TextureBuffer) videoFrameDrawer.prepareBufferForViewportSize( 129 inputTextureBuffer, inputTextureBuffer.getWidth(), inputTextureBuffer.getHeight()); 130 131 // We draw into a buffer laid out like 132 // 133 // +---------+ 134 // | | 135 // | Y | 136 // | | 137 // | | 138 // +----+----+ 139 // | U | V | 140 // | | | 141 // +----+----+ 142 // 143 // In memory, we use the same stride for all of Y, U and V. The 144 // U data starts at offset |height| * |stride| from the Y data, 145 // and the V data starts at at offset |stride/2| from the U 146 // data, with rows of U and V data alternating. 147 // 148 // Now, it would have made sense to allocate a pixel buffer with 149 // a single byte per pixel (EGL10.EGL_COLOR_BUFFER_TYPE, 150 // EGL10.EGL_LUMINANCE_BUFFER,), but that seems to be 151 // unsupported by devices. So do the following hack: Allocate an 152 // RGBA buffer, of width |stride|/4. To render each of these 153 // large pixels, sample the texture at 4 different x coordinates 154 // and store the results in the four components. 155 // 156 // Since the V data needs to start on a boundary of such a 157 // larger pixel, it is not sufficient that |stride| is even, it 158 // has to be a multiple of 8 pixels. 159 final int frameWidth = preparedBuffer.getWidth(); 160 final int frameHeight = preparedBuffer.getHeight(); 161 final int stride = ((frameWidth + 7) / 8) * 8; 162 final int uvHeight = (frameHeight + 1) / 2; 163 // Total height of the combined memory layout. 164 final int totalHeight = frameHeight + uvHeight; 165 final ByteBuffer i420ByteBuffer = JniCommon.nativeAllocateByteBuffer(stride * totalHeight); 166 // Viewport width is divided by four since we are squeezing in four color bytes in each RGBA 167 // pixel. 168 final int viewportWidth = stride / 4; 169 170 // Produce a frame buffer starting at top-left corner, not bottom-left. 171 final Matrix renderMatrix = new Matrix(); 172 renderMatrix.preTranslate(0.5f, 0.5f); 173 renderMatrix.preScale(1f, -1f); 174 renderMatrix.preTranslate(-0.5f, -0.5f); 175 176 i420TextureFrameBuffer.setSize(viewportWidth, totalHeight); 177 178 // Bind our framebuffer. 179 GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, i420TextureFrameBuffer.getFrameBufferId()); 180 GlUtil.checkNoGLES2Error("glBindFramebuffer"); 181 182 // Draw Y. 183 shaderCallbacks.setPlaneY(); 184 VideoFrameDrawer.drawTexture(drawer, preparedBuffer, renderMatrix, frameWidth, frameHeight, 185 /* viewportX= */ 0, /* viewportY= */ 0, viewportWidth, 186 /* viewportHeight= */ frameHeight); 187 188 // Draw U. 189 shaderCallbacks.setPlaneU(); 190 VideoFrameDrawer.drawTexture(drawer, preparedBuffer, renderMatrix, frameWidth, frameHeight, 191 /* viewportX= */ 0, /* viewportY= */ frameHeight, viewportWidth / 2, 192 /* viewportHeight= */ uvHeight); 193 194 // Draw V. 195 shaderCallbacks.setPlaneV(); 196 VideoFrameDrawer.drawTexture(drawer, preparedBuffer, renderMatrix, frameWidth, frameHeight, 197 /* viewportX= */ viewportWidth / 2, /* viewportY= */ frameHeight, viewportWidth / 2, 198 /* viewportHeight= */ uvHeight); 199 200 GLES20.glReadPixels(0, 0, i420TextureFrameBuffer.getWidth(), i420TextureFrameBuffer.getHeight(), 201 GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, i420ByteBuffer); 202 203 GlUtil.checkNoGLES2Error("YuvConverter.convert"); 204 205 // Restore normal framebuffer. 206 GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); 207 208 // Prepare Y, U, and V ByteBuffer slices. 209 final int yPos = 0; 210 final int uPos = yPos + stride * frameHeight; 211 // Rows of U and V alternate in the buffer, so V data starts after the first row of U. 212 final int vPos = uPos + stride / 2; 213 214 i420ByteBuffer.position(yPos); 215 i420ByteBuffer.limit(yPos + stride * frameHeight); 216 final ByteBuffer dataY = i420ByteBuffer.slice(); 217 218 i420ByteBuffer.position(uPos); 219 // The last row does not have padding. 220 final int uvSize = stride * (uvHeight - 1) + stride / 2; 221 i420ByteBuffer.limit(uPos + uvSize); 222 final ByteBuffer dataU = i420ByteBuffer.slice(); 223 224 i420ByteBuffer.position(vPos); 225 i420ByteBuffer.limit(vPos + uvSize); 226 final ByteBuffer dataV = i420ByteBuffer.slice(); 227 228 preparedBuffer.release(); 229 230 return JavaI420Buffer.wrap(frameWidth, frameHeight, dataY, stride, dataU, stride, dataV, stride, 231 () -> { JniCommon.nativeFreeByteBuffer(i420ByteBuffer); }); 232 } 233 release()234 public void release() { 235 threadChecker.checkIsOnValidThread(); 236 drawer.release(); 237 i420TextureFrameBuffer.release(); 238 videoFrameDrawer.release(); 239 // Allow this class to be reused. 240 threadChecker.detachThread(); 241 } 242 } 243