1 /* 2 * Copyright 2017 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 package org.webrtc; 12 13 import static org.junit.Assert.assertEquals; 14 import static org.junit.Assert.assertNotNull; 15 import static org.junit.Assert.assertTrue; 16 import static org.junit.Assert.fail; 17 18 import android.annotation.TargetApi; 19 import android.graphics.Matrix; 20 import android.opengl.GLES11Ext; 21 import androidx.annotation.Nullable; 22 import android.support.test.filters.SmallTest; 23 import android.util.Log; 24 import java.nio.ByteBuffer; 25 import java.util.ArrayList; 26 import java.util.List; 27 import java.util.concurrent.BlockingQueue; 28 import java.util.concurrent.LinkedBlockingQueue; 29 import java.util.concurrent.TimeUnit; 30 import org.chromium.base.test.params.BaseJUnit4RunnerDelegate; 31 import org.chromium.base.test.params.ParameterAnnotations.ClassParameter; 32 import org.chromium.base.test.params.ParameterAnnotations.UseRunnerDelegate; 33 import org.chromium.base.test.params.ParameterSet; 34 import org.chromium.base.test.params.ParameterizedRunner; 35 import org.junit.After; 36 import org.junit.Before; 37 import org.junit.Test; 38 import org.junit.runner.RunWith; 39 40 @TargetApi(16) 41 @RunWith(ParameterizedRunner.class) 42 @UseRunnerDelegate(BaseJUnit4RunnerDelegate.class) 43 public class HardwareVideoEncoderTest { 44 @ClassParameter private static List<ParameterSet> CLASS_PARAMS = new ArrayList<>(); 45 46 static { CLASS_PARAMS.add(new ParameterSet() .value(false , false ) .name(R))47 CLASS_PARAMS.add(new ParameterSet() 48 .value(false /* useTextures */, false /* useEglContext */) 49 .name("I420WithoutEglContext")); CLASS_PARAMS.add(new ParameterSet() .value(true , false ) .name(R))50 CLASS_PARAMS.add(new ParameterSet() 51 .value(true /* useTextures */, false /* useEglContext */) 52 .name("TextureWithoutEglContext")); CLASS_PARAMS.add(new ParameterSet() .value(true , true ) .name(R))53 CLASS_PARAMS.add(new ParameterSet() 54 .value(true /* useTextures */, true /* useEglContext */) 55 .name("TextureWithEglContext")); 56 } 57 58 private final boolean useTextures; 59 private final boolean useEglContext; 60 HardwareVideoEncoderTest(boolean useTextures, boolean useEglContext)61 public HardwareVideoEncoderTest(boolean useTextures, boolean useEglContext) { 62 this.useTextures = useTextures; 63 this.useEglContext = useEglContext; 64 } 65 66 final static String TAG = "HwVideoEncoderTest"; 67 68 private static final boolean ENABLE_INTEL_VP8_ENCODER = true; 69 private static final boolean ENABLE_H264_HIGH_PROFILE = true; 70 private static final VideoEncoder.Settings SETTINGS = 71 new VideoEncoder.Settings(1 /* core */, 640 /* width */, 480 /* height */, 300 /* kbps */, 72 30 /* fps */, 1 /* numberOfSimulcastStreams */, true /* automaticResizeOn */, 73 /* capabilities= */ new VideoEncoder.Capabilities(false /* lossNotification */)); 74 private static final int ENCODE_TIMEOUT_MS = 1000; 75 private static final int NUM_TEST_FRAMES = 10; 76 private static final int NUM_ENCODE_TRIES = 100; 77 private static final int ENCODE_RETRY_SLEEP_MS = 1; 78 79 // # Mock classes 80 /** 81 * Mock encoder callback that allows easy verification of the general properties of the encoded 82 * frame such as width and height. Also used from AndroidVideoDecoderInstrumentationTest. 83 */ 84 static class MockEncoderCallback implements VideoEncoder.Callback { 85 private BlockingQueue<EncodedImage> frameQueue = new LinkedBlockingQueue<>(); 86 87 @Override onEncodedFrame(EncodedImage frame, VideoEncoder.CodecSpecificInfo info)88 public void onEncodedFrame(EncodedImage frame, VideoEncoder.CodecSpecificInfo info) { 89 assertNotNull(frame); 90 assertNotNull(info); 91 92 // Make a copy because keeping a reference to the buffer is not allowed. 93 final ByteBuffer bufferCopy = ByteBuffer.allocateDirect(frame.buffer.remaining()); 94 bufferCopy.put(frame.buffer); 95 bufferCopy.rewind(); 96 97 frameQueue.offer(EncodedImage.builder() 98 .setBuffer(bufferCopy, null) 99 .setEncodedWidth(frame.encodedWidth) 100 .setEncodedHeight(frame.encodedHeight) 101 .setCaptureTimeNs(frame.captureTimeNs) 102 .setFrameType(frame.frameType) 103 .setRotation(frame.rotation) 104 .setCompleteFrame(frame.completeFrame) 105 .setQp(frame.qp) 106 .createEncodedImage()); 107 } 108 poll()109 public EncodedImage poll() { 110 try { 111 EncodedImage image = frameQueue.poll(ENCODE_TIMEOUT_MS, TimeUnit.MILLISECONDS); 112 assertNotNull("Timed out waiting for the frame to be encoded.", image); 113 return image; 114 } catch (InterruptedException e) { 115 throw new RuntimeException(e); 116 } 117 } 118 assertFrameEncoded(VideoFrame frame)119 public void assertFrameEncoded(VideoFrame frame) { 120 final VideoFrame.Buffer buffer = frame.getBuffer(); 121 final EncodedImage image = poll(); 122 assertTrue(image.buffer.capacity() > 0); 123 assertEquals(image.encodedWidth, buffer.getWidth()); 124 assertEquals(image.encodedHeight, buffer.getHeight()); 125 assertEquals(image.captureTimeNs, frame.getTimestampNs()); 126 assertEquals(image.rotation, frame.getRotation()); 127 } 128 } 129 130 /** A common base class for the texture and I420 buffer that implements reference counting. */ 131 private static abstract class MockBufferBase implements VideoFrame.Buffer { 132 protected final int width; 133 protected final int height; 134 private final Runnable releaseCallback; 135 private final Object refCountLock = new Object(); 136 private int refCount = 1; 137 MockBufferBase(int width, int height, Runnable releaseCallback)138 public MockBufferBase(int width, int height, Runnable releaseCallback) { 139 this.width = width; 140 this.height = height; 141 this.releaseCallback = releaseCallback; 142 } 143 144 @Override getWidth()145 public int getWidth() { 146 return width; 147 } 148 149 @Override getHeight()150 public int getHeight() { 151 return height; 152 } 153 154 @Override retain()155 public void retain() { 156 synchronized (refCountLock) { 157 assertTrue("Buffer retained after being destroyed.", refCount > 0); 158 ++refCount; 159 } 160 } 161 162 @Override release()163 public void release() { 164 synchronized (refCountLock) { 165 assertTrue("Buffer released too many times.", --refCount >= 0); 166 if (refCount == 0) { 167 releaseCallback.run(); 168 } 169 } 170 } 171 } 172 173 private static class MockTextureBuffer 174 extends MockBufferBase implements VideoFrame.TextureBuffer { 175 private final int textureId; 176 MockTextureBuffer(int textureId, int width, int height, Runnable releaseCallback)177 public MockTextureBuffer(int textureId, int width, int height, Runnable releaseCallback) { 178 super(width, height, releaseCallback); 179 this.textureId = textureId; 180 } 181 182 @Override getType()183 public VideoFrame.TextureBuffer.Type getType() { 184 return VideoFrame.TextureBuffer.Type.OES; 185 } 186 187 @Override getTextureId()188 public int getTextureId() { 189 return textureId; 190 } 191 192 @Override getTransformMatrix()193 public Matrix getTransformMatrix() { 194 return new Matrix(); 195 } 196 197 @Override toI420()198 public VideoFrame.I420Buffer toI420() { 199 return JavaI420Buffer.allocate(width, height); 200 } 201 202 @Override cropAndScale( int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight)203 public VideoFrame.Buffer cropAndScale( 204 int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) { 205 retain(); 206 return new MockTextureBuffer(textureId, scaleWidth, scaleHeight, this ::release); 207 } 208 } 209 210 private static class MockI420Buffer extends MockBufferBase implements VideoFrame.I420Buffer { 211 private final JavaI420Buffer realBuffer; 212 MockI420Buffer(int width, int height, Runnable releaseCallback)213 public MockI420Buffer(int width, int height, Runnable releaseCallback) { 214 super(width, height, releaseCallback); 215 realBuffer = JavaI420Buffer.allocate(width, height); 216 } 217 218 @Override getDataY()219 public ByteBuffer getDataY() { 220 return realBuffer.getDataY(); 221 } 222 223 @Override getDataU()224 public ByteBuffer getDataU() { 225 return realBuffer.getDataU(); 226 } 227 228 @Override getDataV()229 public ByteBuffer getDataV() { 230 return realBuffer.getDataV(); 231 } 232 233 @Override getStrideY()234 public int getStrideY() { 235 return realBuffer.getStrideY(); 236 } 237 238 @Override getStrideU()239 public int getStrideU() { 240 return realBuffer.getStrideU(); 241 } 242 243 @Override getStrideV()244 public int getStrideV() { 245 return realBuffer.getStrideV(); 246 } 247 248 @Override toI420()249 public VideoFrame.I420Buffer toI420() { 250 retain(); 251 return this; 252 } 253 254 @Override retain()255 public void retain() { 256 super.retain(); 257 realBuffer.retain(); 258 } 259 260 @Override release()261 public void release() { 262 super.release(); 263 realBuffer.release(); 264 } 265 266 @Override cropAndScale( int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight)267 public VideoFrame.Buffer cropAndScale( 268 int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) { 269 return realBuffer.cropAndScale(cropX, cropY, cropWidth, cropHeight, scaleWidth, scaleHeight); 270 } 271 } 272 273 // # Test fields 274 private final Object referencedFramesLock = new Object(); 275 private int referencedFrames; 276 277 private Runnable releaseFrameCallback = new Runnable() { 278 @Override 279 public void run() { 280 synchronized (referencedFramesLock) { 281 --referencedFrames; 282 } 283 } 284 }; 285 286 private EglBase14 eglBase; 287 private long lastTimestampNs; 288 289 // # Helper methods createEncoderFactory(EglBase.Context eglContext)290 private VideoEncoderFactory createEncoderFactory(EglBase.Context eglContext) { 291 return new HardwareVideoEncoderFactory( 292 eglContext, ENABLE_INTEL_VP8_ENCODER, ENABLE_H264_HIGH_PROFILE); 293 } 294 createEncoder()295 private @Nullable VideoEncoder createEncoder() { 296 VideoEncoderFactory factory = 297 createEncoderFactory(useEglContext ? eglBase.getEglBaseContext() : null); 298 VideoCodecInfo[] supportedCodecs = factory.getSupportedCodecs(); 299 return factory.createEncoder(supportedCodecs[0]); 300 } 301 generateI420Frame(int width, int height)302 private VideoFrame generateI420Frame(int width, int height) { 303 synchronized (referencedFramesLock) { 304 ++referencedFrames; 305 } 306 lastTimestampNs += TimeUnit.SECONDS.toNanos(1) / SETTINGS.maxFramerate; 307 VideoFrame.Buffer buffer = new MockI420Buffer(width, height, releaseFrameCallback); 308 return new VideoFrame(buffer, 0 /* rotation */, lastTimestampNs); 309 } 310 generateTextureFrame(int width, int height)311 private VideoFrame generateTextureFrame(int width, int height) { 312 synchronized (referencedFramesLock) { 313 ++referencedFrames; 314 } 315 final int textureId = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES); 316 lastTimestampNs += TimeUnit.SECONDS.toNanos(1) / SETTINGS.maxFramerate; 317 VideoFrame.Buffer buffer = 318 new MockTextureBuffer(textureId, width, height, releaseFrameCallback); 319 return new VideoFrame(buffer, 0 /* rotation */, lastTimestampNs); 320 } 321 generateFrame(int width, int height)322 private VideoFrame generateFrame(int width, int height) { 323 return useTextures ? generateTextureFrame(width, height) : generateI420Frame(width, height); 324 } 325 testEncodeFrame( VideoEncoder encoder, VideoFrame frame, VideoEncoder.EncodeInfo info)326 static void testEncodeFrame( 327 VideoEncoder encoder, VideoFrame frame, VideoEncoder.EncodeInfo info) { 328 int numTries = 0; 329 330 // It takes a while for the encoder to become ready so try until it accepts the frame. 331 while (true) { 332 ++numTries; 333 334 final VideoCodecStatus returnValue = encoder.encode(frame, info); 335 switch (returnValue) { 336 case OK: 337 return; // Success 338 case NO_OUTPUT: 339 if (numTries >= NUM_ENCODE_TRIES) { 340 fail("encoder.encode keeps returning NO_OUTPUT"); 341 } 342 try { 343 Thread.sleep(ENCODE_RETRY_SLEEP_MS); // Try again. 344 } catch (InterruptedException e) { 345 throw new RuntimeException(e); 346 } 347 break; 348 default: 349 fail("encoder.encode returned: " + returnValue); // Error 350 } 351 } 352 } 353 354 // # Tests 355 @Before setUp()356 public void setUp() { 357 NativeLibrary.initialize(new NativeLibrary.DefaultLoader(), TestConstants.NATIVE_LIBRARY); 358 359 eglBase = EglBase.createEgl14(EglBase.CONFIG_PLAIN); 360 eglBase.createDummyPbufferSurface(); 361 eglBase.makeCurrent(); 362 lastTimestampNs = System.nanoTime(); 363 } 364 365 @After tearDown()366 public void tearDown() { 367 eglBase.release(); 368 synchronized (referencedFramesLock) { 369 assertEquals("All frames were not released", 0, referencedFrames); 370 } 371 } 372 373 @Test 374 @SmallTest testInitialize()375 public void testInitialize() { 376 VideoEncoder encoder = createEncoder(); 377 assertEquals(VideoCodecStatus.OK, encoder.initEncode(SETTINGS, null)); 378 assertEquals(VideoCodecStatus.OK, encoder.release()); 379 } 380 381 @Test 382 @SmallTest testEncode()383 public void testEncode() { 384 VideoEncoder encoder = createEncoder(); 385 MockEncoderCallback callback = new MockEncoderCallback(); 386 assertEquals(VideoCodecStatus.OK, encoder.initEncode(SETTINGS, callback)); 387 388 for (int i = 0; i < NUM_TEST_FRAMES; i++) { 389 Log.d(TAG, "Test frame: " + i); 390 VideoFrame frame = generateFrame(SETTINGS.width, SETTINGS.height); 391 VideoEncoder.EncodeInfo info = new VideoEncoder.EncodeInfo( 392 new EncodedImage.FrameType[] {EncodedImage.FrameType.VideoFrameDelta}); 393 testEncodeFrame(encoder, frame, info); 394 395 callback.assertFrameEncoded(frame); 396 frame.release(); 397 } 398 399 assertEquals(VideoCodecStatus.OK, encoder.release()); 400 } 401 402 @Test 403 @SmallTest testEncodeAltenatingBuffers()404 public void testEncodeAltenatingBuffers() { 405 VideoEncoder encoder = createEncoder(); 406 MockEncoderCallback callback = new MockEncoderCallback(); 407 assertEquals(VideoCodecStatus.OK, encoder.initEncode(SETTINGS, callback)); 408 409 for (int i = 0; i < NUM_TEST_FRAMES; i++) { 410 Log.d(TAG, "Test frame: " + i); 411 VideoFrame frame; 412 VideoEncoder.EncodeInfo info = new VideoEncoder.EncodeInfo( 413 new EncodedImage.FrameType[] {EncodedImage.FrameType.VideoFrameDelta}); 414 415 frame = generateTextureFrame(SETTINGS.width, SETTINGS.height); 416 testEncodeFrame(encoder, frame, info); 417 callback.assertFrameEncoded(frame); 418 frame.release(); 419 420 frame = generateI420Frame(SETTINGS.width, SETTINGS.height); 421 testEncodeFrame(encoder, frame, info); 422 callback.assertFrameEncoded(frame); 423 frame.release(); 424 } 425 426 assertEquals(VideoCodecStatus.OK, encoder.release()); 427 } 428 429 @Test 430 @SmallTest testEncodeDifferentSizes()431 public void testEncodeDifferentSizes() { 432 VideoEncoder encoder = createEncoder(); 433 MockEncoderCallback callback = new MockEncoderCallback(); 434 assertEquals(VideoCodecStatus.OK, encoder.initEncode(SETTINGS, callback)); 435 436 VideoFrame frame; 437 VideoEncoder.EncodeInfo info = new VideoEncoder.EncodeInfo( 438 new EncodedImage.FrameType[] {EncodedImage.FrameType.VideoFrameDelta}); 439 440 frame = generateFrame(SETTINGS.width / 2, SETTINGS.height / 2); 441 testEncodeFrame(encoder, frame, info); 442 callback.assertFrameEncoded(frame); 443 frame.release(); 444 445 frame = generateFrame(SETTINGS.width, SETTINGS.height); 446 testEncodeFrame(encoder, frame, info); 447 callback.assertFrameEncoded(frame); 448 frame.release(); 449 450 frame = generateFrame(SETTINGS.width / 4, SETTINGS.height / 4); 451 testEncodeFrame(encoder, frame, info); 452 callback.assertFrameEncoded(frame); 453 frame.release(); 454 455 assertEquals(VideoCodecStatus.OK, encoder.release()); 456 } 457 } 458