1 /* 2 * Copyright 2016 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 package org.webrtc; 12 13 import android.graphics.Bitmap; 14 import android.graphics.Matrix; 15 import android.graphics.SurfaceTexture; 16 import android.opengl.GLES20; 17 import android.os.Handler; 18 import android.os.HandlerThread; 19 import android.os.Looper; 20 import android.os.Message; 21 import android.support.annotation.Nullable; 22 import android.view.Surface; 23 import java.nio.ByteBuffer; 24 import java.text.DecimalFormat; 25 import java.util.ArrayList; 26 import java.util.Iterator; 27 import java.util.concurrent.CountDownLatch; 28 import java.util.concurrent.TimeUnit; 29 30 /** 31 * Implements VideoSink by displaying the video stream on an EGL Surface. This class is intended to 32 * be used as a helper class for rendering on SurfaceViews and TextureViews. 33 */ 34 public class EglRenderer implements VideoSink { 35 private static final String TAG = "EglRenderer"; 36 private static final long LOG_INTERVAL_SEC = 4; 37 onFrame(Bitmap frame)38 public interface FrameListener { void onFrame(Bitmap frame); } 39 40 /** Callback for clients to be notified about errors encountered during rendering. */ 41 public static interface ErrorCallback { 42 /** Called if GLES20.GL_OUT_OF_MEMORY is encountered during rendering. */ onGlOutOfMemory()43 void onGlOutOfMemory(); 44 } 45 46 private static class FrameListenerAndParams { 47 public final FrameListener listener; 48 public final float scale; 49 public final RendererCommon.GlDrawer drawer; 50 public final boolean applyFpsReduction; 51 FrameListenerAndParams(FrameListener listener, float scale, RendererCommon.GlDrawer drawer, boolean applyFpsReduction)52 public FrameListenerAndParams(FrameListener listener, float scale, 53 RendererCommon.GlDrawer drawer, boolean applyFpsReduction) { 54 this.listener = listener; 55 this.scale = scale; 56 this.drawer = drawer; 57 this.applyFpsReduction = applyFpsReduction; 58 } 59 } 60 61 private class EglSurfaceCreation implements Runnable { 62 private Object surface; 63 64 // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. 65 @SuppressWarnings("NoSynchronizedMethodCheck") setSurface(Object surface)66 public synchronized void setSurface(Object surface) { 67 this.surface = surface; 68 } 69 70 @Override 71 // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. 72 @SuppressWarnings("NoSynchronizedMethodCheck") run()73 public synchronized void run() { 74 if (surface != null && eglBase != null && !eglBase.hasSurface()) { 75 if (surface instanceof Surface) { 76 eglBase.createSurface((Surface) surface); 77 } else if (surface instanceof SurfaceTexture) { 78 eglBase.createSurface((SurfaceTexture) surface); 79 } else { 80 throw new IllegalStateException("Invalid surface: " + surface); 81 } 82 eglBase.makeCurrent(); 83 // Necessary for YUV frames with odd width. 84 GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); 85 } 86 } 87 } 88 89 /** 90 * Handler that triggers a callback when an uncaught exception happens when handling a message. 91 */ 92 private static class HandlerWithExceptionCallback extends Handler { 93 private final Runnable exceptionCallback; 94 HandlerWithExceptionCallback(Looper looper, Runnable exceptionCallback)95 public HandlerWithExceptionCallback(Looper looper, Runnable exceptionCallback) { 96 super(looper); 97 this.exceptionCallback = exceptionCallback; 98 } 99 100 @Override dispatchMessage(Message msg)101 public void dispatchMessage(Message msg) { 102 try { 103 super.dispatchMessage(msg); 104 } catch (Exception e) { 105 Logging.e(TAG, "Exception on EglRenderer thread", e); 106 exceptionCallback.run(); 107 throw e; 108 } 109 } 110 } 111 112 protected final String name; 113 114 // |renderThreadHandler| is a handler for communicating with |renderThread|, and is synchronized 115 // on |handlerLock|. 116 private final Object handlerLock = new Object(); 117 @Nullable private Handler renderThreadHandler; 118 119 private final ArrayList<FrameListenerAndParams> frameListeners = new ArrayList<>(); 120 121 private volatile ErrorCallback errorCallback; 122 123 // Variables for fps reduction. 124 private final Object fpsReductionLock = new Object(); 125 // Time for when next frame should be rendered. 126 private long nextFrameTimeNs; 127 // Minimum duration between frames when fps reduction is active, or -1 if video is completely 128 // paused. 129 private long minRenderPeriodNs; 130 131 // EGL and GL resources for drawing YUV/OES textures. After initialization, these are only 132 // accessed from the render thread. 133 @Nullable private EglBase eglBase; 134 private final VideoFrameDrawer frameDrawer; 135 @Nullable private RendererCommon.GlDrawer drawer; 136 private boolean usePresentationTimeStamp; 137 private final Matrix drawMatrix = new Matrix(); 138 139 // Pending frame to render. Serves as a queue with size 1. Synchronized on |frameLock|. 140 private final Object frameLock = new Object(); 141 @Nullable private VideoFrame pendingFrame; 142 143 // These variables are synchronized on |layoutLock|. 144 private final Object layoutLock = new Object(); 145 private float layoutAspectRatio; 146 // If true, mirrors the video stream horizontally. 147 private boolean mirrorHorizontally; 148 // If true, mirrors the video stream vertically. 149 private boolean mirrorVertically; 150 151 // These variables are synchronized on |statisticsLock|. 152 private final Object statisticsLock = new Object(); 153 // Total number of video frames received in renderFrame() call. 154 private int framesReceived; 155 // Number of video frames dropped by renderFrame() because previous frame has not been rendered 156 // yet. 157 private int framesDropped; 158 // Number of rendered video frames. 159 private int framesRendered; 160 // Start time for counting these statistics, or 0 if we haven't started measuring yet. 161 private long statisticsStartTimeNs; 162 // Time in ns spent in renderFrameOnRenderThread() function. 163 private long renderTimeNs; 164 // Time in ns spent by the render thread in the swapBuffers() function. 165 private long renderSwapBufferTimeNs; 166 167 // Used for bitmap capturing. 168 private final GlTextureFrameBuffer bitmapTextureFramebuffer = 169 new GlTextureFrameBuffer(GLES20.GL_RGBA); 170 171 private final Runnable logStatisticsRunnable = new Runnable() { 172 @Override 173 public void run() { 174 logStatistics(); 175 synchronized (handlerLock) { 176 if (renderThreadHandler != null) { 177 renderThreadHandler.removeCallbacks(logStatisticsRunnable); 178 renderThreadHandler.postDelayed( 179 logStatisticsRunnable, TimeUnit.SECONDS.toMillis(LOG_INTERVAL_SEC)); 180 } 181 } 182 } 183 }; 184 185 private final EglSurfaceCreation eglSurfaceCreationRunnable = new EglSurfaceCreation(); 186 187 /** 188 * Standard constructor. The name will be used for the render thread name and included when 189 * logging. In order to render something, you must first call init() and createEglSurface. 190 */ EglRenderer(String name)191 public EglRenderer(String name) { 192 this(name, new VideoFrameDrawer()); 193 } 194 EglRenderer(String name, VideoFrameDrawer videoFrameDrawer)195 public EglRenderer(String name, VideoFrameDrawer videoFrameDrawer) { 196 this.name = name; 197 this.frameDrawer = videoFrameDrawer; 198 } 199 200 /** 201 * Initialize this class, sharing resources with |sharedContext|. The custom |drawer| will be used 202 * for drawing frames on the EGLSurface. This class is responsible for calling release() on 203 * |drawer|. It is allowed to call init() to reinitialize the renderer after a previous 204 * init()/release() cycle. If usePresentationTimeStamp is true, eglPresentationTimeANDROID will be 205 * set with the frame timestamps, which specifies desired presentation time and might be useful 206 * for e.g. syncing audio and video. 207 */ init(@ullable final EglBase.Context sharedContext, final int[] configAttributes, RendererCommon.GlDrawer drawer, boolean usePresentationTimeStamp)208 public void init(@Nullable final EglBase.Context sharedContext, final int[] configAttributes, 209 RendererCommon.GlDrawer drawer, boolean usePresentationTimeStamp) { 210 synchronized (handlerLock) { 211 if (renderThreadHandler != null) { 212 throw new IllegalStateException(name + "Already initialized"); 213 } 214 logD("Initializing EglRenderer"); 215 this.drawer = drawer; 216 this.usePresentationTimeStamp = usePresentationTimeStamp; 217 218 final HandlerThread renderThread = new HandlerThread(name + "EglRenderer"); 219 renderThread.start(); 220 renderThreadHandler = 221 new HandlerWithExceptionCallback(renderThread.getLooper(), new Runnable() { 222 @Override 223 public void run() { 224 synchronized (handlerLock) { 225 renderThreadHandler = null; 226 } 227 } 228 }); 229 // Create EGL context on the newly created render thread. It should be possibly to create the 230 // context on this thread and make it current on the render thread, but this causes failure on 231 // some Marvel based JB devices. https://bugs.chromium.org/p/webrtc/issues/detail?id=6350. 232 ThreadUtils.invokeAtFrontUninterruptibly(renderThreadHandler, () -> { 233 // If sharedContext is null, then texture frames are disabled. This is typically for old 234 // devices that might not be fully spec compliant, so force EGL 1.0 since EGL 1.4 has 235 // caused trouble on some weird devices. 236 if (sharedContext == null) { 237 logD("EglBase10.create context"); 238 eglBase = EglBase.createEgl10(configAttributes); 239 } else { 240 logD("EglBase.create shared context"); 241 eglBase = EglBase.create(sharedContext, configAttributes); 242 } 243 }); 244 renderThreadHandler.post(eglSurfaceCreationRunnable); 245 final long currentTimeNs = System.nanoTime(); 246 resetStatistics(currentTimeNs); 247 renderThreadHandler.postDelayed( 248 logStatisticsRunnable, TimeUnit.SECONDS.toMillis(LOG_INTERVAL_SEC)); 249 } 250 } 251 252 /** 253 * Same as above with usePresentationTimeStamp set to false. 254 * 255 * @see #init(EglBase.Context, int[], RendererCommon.GlDrawer, boolean) 256 */ init(@ullable final EglBase.Context sharedContext, final int[] configAttributes, RendererCommon.GlDrawer drawer)257 public void init(@Nullable final EglBase.Context sharedContext, final int[] configAttributes, 258 RendererCommon.GlDrawer drawer) { 259 init(sharedContext, configAttributes, drawer, /* usePresentationTimeStamp= */ false); 260 } 261 createEglSurface(Surface surface)262 public void createEglSurface(Surface surface) { 263 createEglSurfaceInternal(surface); 264 } 265 createEglSurface(SurfaceTexture surfaceTexture)266 public void createEglSurface(SurfaceTexture surfaceTexture) { 267 createEglSurfaceInternal(surfaceTexture); 268 } 269 createEglSurfaceInternal(Object surface)270 private void createEglSurfaceInternal(Object surface) { 271 eglSurfaceCreationRunnable.setSurface(surface); 272 postToRenderThread(eglSurfaceCreationRunnable); 273 } 274 275 /** 276 * Block until any pending frame is returned and all GL resources released, even if an interrupt 277 * occurs. If an interrupt occurs during release(), the interrupt flag will be set. This function 278 * should be called before the Activity is destroyed and the EGLContext is still valid. If you 279 * don't call this function, the GL resources might leak. 280 */ release()281 public void release() { 282 logD("Releasing."); 283 final CountDownLatch eglCleanupBarrier = new CountDownLatch(1); 284 synchronized (handlerLock) { 285 if (renderThreadHandler == null) { 286 logD("Already released"); 287 return; 288 } 289 renderThreadHandler.removeCallbacks(logStatisticsRunnable); 290 // Release EGL and GL resources on render thread. 291 renderThreadHandler.postAtFrontOfQueue(() -> { 292 // Detach current shader program. 293 GLES20.glUseProgram(/* program= */ 0); 294 if (drawer != null) { 295 drawer.release(); 296 drawer = null; 297 } 298 frameDrawer.release(); 299 bitmapTextureFramebuffer.release(); 300 if (eglBase != null) { 301 logD("eglBase detach and release."); 302 eglBase.detachCurrent(); 303 eglBase.release(); 304 eglBase = null; 305 } 306 frameListeners.clear(); 307 eglCleanupBarrier.countDown(); 308 }); 309 final Looper renderLooper = renderThreadHandler.getLooper(); 310 // TODO(magjed): Replace this post() with renderLooper.quitSafely() when API support >= 18. 311 renderThreadHandler.post(() -> { 312 logD("Quitting render thread."); 313 renderLooper.quit(); 314 }); 315 // Don't accept any more frames or messages to the render thread. 316 renderThreadHandler = null; 317 } 318 // Make sure the EGL/GL cleanup posted above is executed. 319 ThreadUtils.awaitUninterruptibly(eglCleanupBarrier); 320 synchronized (frameLock) { 321 if (pendingFrame != null) { 322 pendingFrame.release(); 323 pendingFrame = null; 324 } 325 } 326 logD("Releasing done."); 327 } 328 329 /** 330 * Reset the statistics logged in logStatistics(). 331 */ resetStatistics(long currentTimeNs)332 private void resetStatistics(long currentTimeNs) { 333 synchronized (statisticsLock) { 334 statisticsStartTimeNs = currentTimeNs; 335 framesReceived = 0; 336 framesDropped = 0; 337 framesRendered = 0; 338 renderTimeNs = 0; 339 renderSwapBufferTimeNs = 0; 340 } 341 } 342 printStackTrace()343 public void printStackTrace() { 344 synchronized (handlerLock) { 345 final Thread renderThread = 346 (renderThreadHandler == null) ? null : renderThreadHandler.getLooper().getThread(); 347 if (renderThread != null) { 348 final StackTraceElement[] renderStackTrace = renderThread.getStackTrace(); 349 if (renderStackTrace.length > 0) { 350 logW("EglRenderer stack trace:"); 351 for (StackTraceElement traceElem : renderStackTrace) { 352 logW(traceElem.toString()); 353 } 354 } 355 } 356 } 357 } 358 359 /** 360 * Set if the video stream should be mirrored horizontally or not. 361 */ setMirror(final boolean mirror)362 public void setMirror(final boolean mirror) { 363 logD("setMirrorHorizontally: " + mirror); 364 synchronized (layoutLock) { 365 this.mirrorHorizontally = mirror; 366 } 367 } 368 369 /** 370 * Set if the video stream should be mirrored vertically or not. 371 */ setMirrorVertically(final boolean mirrorVertically)372 public void setMirrorVertically(final boolean mirrorVertically) { 373 logD("setMirrorVertically: " + mirrorVertically); 374 synchronized (layoutLock) { 375 this.mirrorVertically = mirrorVertically; 376 } 377 } 378 379 /** 380 * Set layout aspect ratio. This is used to crop frames when rendering to avoid stretched video. 381 * Set this to 0 to disable cropping. 382 */ setLayoutAspectRatio(float layoutAspectRatio)383 public void setLayoutAspectRatio(float layoutAspectRatio) { 384 logD("setLayoutAspectRatio: " + layoutAspectRatio); 385 synchronized (layoutLock) { 386 this.layoutAspectRatio = layoutAspectRatio; 387 } 388 } 389 390 /** 391 * Limit render framerate. 392 * 393 * @param fps Limit render framerate to this value, or use Float.POSITIVE_INFINITY to disable fps 394 * reduction. 395 */ setFpsReduction(float fps)396 public void setFpsReduction(float fps) { 397 logD("setFpsReduction: " + fps); 398 synchronized (fpsReductionLock) { 399 final long previousRenderPeriodNs = minRenderPeriodNs; 400 if (fps <= 0) { 401 minRenderPeriodNs = Long.MAX_VALUE; 402 } else { 403 minRenderPeriodNs = (long) (TimeUnit.SECONDS.toNanos(1) / fps); 404 } 405 if (minRenderPeriodNs != previousRenderPeriodNs) { 406 // Fps reduction changed - reset frame time. 407 nextFrameTimeNs = System.nanoTime(); 408 } 409 } 410 } 411 disableFpsReduction()412 public void disableFpsReduction() { 413 setFpsReduction(Float.POSITIVE_INFINITY /* fps */); 414 } 415 pauseVideo()416 public void pauseVideo() { 417 setFpsReduction(0 /* fps */); 418 } 419 420 /** 421 * Register a callback to be invoked when a new video frame has been received. This version uses 422 * the drawer of the EglRenderer that was passed in init. 423 * 424 * @param listener The callback to be invoked. The callback will be invoked on the render thread. 425 * It should be lightweight and must not call removeFrameListener. 426 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is 427 * required. 428 */ addFrameListener(final FrameListener listener, final float scale)429 public void addFrameListener(final FrameListener listener, final float scale) { 430 addFrameListener(listener, scale, null, false /* applyFpsReduction */); 431 } 432 433 /** 434 * Register a callback to be invoked when a new video frame has been received. 435 * 436 * @param listener The callback to be invoked. The callback will be invoked on the render thread. 437 * It should be lightweight and must not call removeFrameListener. 438 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is 439 * required. 440 * @param drawer Custom drawer to use for this frame listener or null to use the default one. 441 */ addFrameListener( final FrameListener listener, final float scale, final RendererCommon.GlDrawer drawerParam)442 public void addFrameListener( 443 final FrameListener listener, final float scale, final RendererCommon.GlDrawer drawerParam) { 444 addFrameListener(listener, scale, drawerParam, false /* applyFpsReduction */); 445 } 446 447 /** 448 * Register a callback to be invoked when a new video frame has been received. 449 * 450 * @param listener The callback to be invoked. The callback will be invoked on the render thread. 451 * It should be lightweight and must not call removeFrameListener. 452 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is 453 * required. 454 * @param drawer Custom drawer to use for this frame listener or null to use the default one. 455 * @param applyFpsReduction This callback will not be called for frames that have been dropped by 456 * FPS reduction. 457 */ addFrameListener(final FrameListener listener, final float scale, @Nullable final RendererCommon.GlDrawer drawerParam, final boolean applyFpsReduction)458 public void addFrameListener(final FrameListener listener, final float scale, 459 @Nullable final RendererCommon.GlDrawer drawerParam, final boolean applyFpsReduction) { 460 postToRenderThread(() -> { 461 final RendererCommon.GlDrawer listenerDrawer = drawerParam == null ? drawer : drawerParam; 462 frameListeners.add( 463 new FrameListenerAndParams(listener, scale, listenerDrawer, applyFpsReduction)); 464 }); 465 } 466 467 /** 468 * Remove any pending callback that was added with addFrameListener. If the callback is not in 469 * the queue, nothing happens. It is ensured that callback won't be called after this method 470 * returns. 471 * 472 * @param runnable The callback to remove. 473 */ removeFrameListener(final FrameListener listener)474 public void removeFrameListener(final FrameListener listener) { 475 final CountDownLatch latch = new CountDownLatch(1); 476 synchronized (handlerLock) { 477 if (renderThreadHandler == null) { 478 return; 479 } 480 if (Thread.currentThread() == renderThreadHandler.getLooper().getThread()) { 481 throw new RuntimeException("removeFrameListener must not be called on the render thread."); 482 } 483 postToRenderThread(() -> { 484 latch.countDown(); 485 final Iterator<FrameListenerAndParams> iter = frameListeners.iterator(); 486 while (iter.hasNext()) { 487 if (iter.next().listener == listener) { 488 iter.remove(); 489 } 490 } 491 }); 492 } 493 ThreadUtils.awaitUninterruptibly(latch); 494 } 495 496 /** Can be set in order to be notified about errors encountered during rendering. */ setErrorCallback(ErrorCallback errorCallback)497 public void setErrorCallback(ErrorCallback errorCallback) { 498 this.errorCallback = errorCallback; 499 } 500 501 // VideoSink interface. 502 @Override onFrame(VideoFrame frame)503 public void onFrame(VideoFrame frame) { 504 synchronized (statisticsLock) { 505 ++framesReceived; 506 } 507 final boolean dropOldFrame; 508 synchronized (handlerLock) { 509 if (renderThreadHandler == null) { 510 logD("Dropping frame - Not initialized or already released."); 511 return; 512 } 513 synchronized (frameLock) { 514 dropOldFrame = (pendingFrame != null); 515 if (dropOldFrame) { 516 pendingFrame.release(); 517 } 518 pendingFrame = frame; 519 pendingFrame.retain(); 520 renderThreadHandler.post(this ::renderFrameOnRenderThread); 521 } 522 } 523 if (dropOldFrame) { 524 synchronized (statisticsLock) { 525 ++framesDropped; 526 } 527 } 528 } 529 530 /** 531 * Release EGL surface. This function will block until the EGL surface is released. 532 */ releaseEglSurface(final Runnable completionCallback)533 public void releaseEglSurface(final Runnable completionCallback) { 534 // Ensure that the render thread is no longer touching the Surface before returning from this 535 // function. 536 eglSurfaceCreationRunnable.setSurface(null /* surface */); 537 synchronized (handlerLock) { 538 if (renderThreadHandler != null) { 539 renderThreadHandler.removeCallbacks(eglSurfaceCreationRunnable); 540 renderThreadHandler.postAtFrontOfQueue(() -> { 541 if (eglBase != null) { 542 eglBase.detachCurrent(); 543 eglBase.releaseSurface(); 544 } 545 completionCallback.run(); 546 }); 547 return; 548 } 549 } 550 completionCallback.run(); 551 } 552 553 /** 554 * Private helper function to post tasks safely. 555 */ postToRenderThread(Runnable runnable)556 private void postToRenderThread(Runnable runnable) { 557 synchronized (handlerLock) { 558 if (renderThreadHandler != null) { 559 renderThreadHandler.post(runnable); 560 } 561 } 562 } 563 clearSurfaceOnRenderThread(float r, float g, float b, float a)564 private void clearSurfaceOnRenderThread(float r, float g, float b, float a) { 565 if (eglBase != null && eglBase.hasSurface()) { 566 logD("clearSurface"); 567 GLES20.glClearColor(r, g, b, a); 568 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); 569 eglBase.swapBuffers(); 570 } 571 } 572 573 /** 574 * Post a task to clear the surface to a transparent uniform color. 575 */ clearImage()576 public void clearImage() { 577 clearImage(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */); 578 } 579 580 /** 581 * Post a task to clear the surface to a specific color. 582 */ clearImage(final float r, final float g, final float b, final float a)583 public void clearImage(final float r, final float g, final float b, final float a) { 584 synchronized (handlerLock) { 585 if (renderThreadHandler == null) { 586 return; 587 } 588 renderThreadHandler.postAtFrontOfQueue(() -> clearSurfaceOnRenderThread(r, g, b, a)); 589 } 590 } 591 592 /** 593 * Renders and releases |pendingFrame|. 594 */ renderFrameOnRenderThread()595 private void renderFrameOnRenderThread() { 596 // Fetch and render |pendingFrame|. 597 final VideoFrame frame; 598 synchronized (frameLock) { 599 if (pendingFrame == null) { 600 return; 601 } 602 frame = pendingFrame; 603 pendingFrame = null; 604 } 605 if (eglBase == null || !eglBase.hasSurface()) { 606 logD("Dropping frame - No surface"); 607 frame.release(); 608 return; 609 } 610 // Check if fps reduction is active. 611 final boolean shouldRenderFrame; 612 synchronized (fpsReductionLock) { 613 if (minRenderPeriodNs == Long.MAX_VALUE) { 614 // Rendering is paused. 615 shouldRenderFrame = false; 616 } else if (minRenderPeriodNs <= 0) { 617 // FPS reduction is disabled. 618 shouldRenderFrame = true; 619 } else { 620 final long currentTimeNs = System.nanoTime(); 621 if (currentTimeNs < nextFrameTimeNs) { 622 logD("Skipping frame rendering - fps reduction is active."); 623 shouldRenderFrame = false; 624 } else { 625 nextFrameTimeNs += minRenderPeriodNs; 626 // The time for the next frame should always be in the future. 627 nextFrameTimeNs = Math.max(nextFrameTimeNs, currentTimeNs); 628 shouldRenderFrame = true; 629 } 630 } 631 } 632 633 final long startTimeNs = System.nanoTime(); 634 635 final float frameAspectRatio = frame.getRotatedWidth() / (float) frame.getRotatedHeight(); 636 final float drawnAspectRatio; 637 synchronized (layoutLock) { 638 drawnAspectRatio = layoutAspectRatio != 0f ? layoutAspectRatio : frameAspectRatio; 639 } 640 641 final float scaleX; 642 final float scaleY; 643 644 if (frameAspectRatio > drawnAspectRatio) { 645 scaleX = drawnAspectRatio / frameAspectRatio; 646 scaleY = 1f; 647 } else { 648 scaleX = 1f; 649 scaleY = frameAspectRatio / drawnAspectRatio; 650 } 651 652 drawMatrix.reset(); 653 drawMatrix.preTranslate(0.5f, 0.5f); 654 drawMatrix.preScale(mirrorHorizontally ? -1f : 1f, mirrorVertically ? -1f : 1f); 655 drawMatrix.preScale(scaleX, scaleY); 656 drawMatrix.preTranslate(-0.5f, -0.5f); 657 658 try { 659 if (shouldRenderFrame) { 660 GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */); 661 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); 662 frameDrawer.drawFrame(frame, drawer, drawMatrix, 0 /* viewportX */, 0 /* viewportY */, 663 eglBase.surfaceWidth(), eglBase.surfaceHeight()); 664 665 final long swapBuffersStartTimeNs = System.nanoTime(); 666 if (usePresentationTimeStamp) { 667 eglBase.swapBuffers(frame.getTimestampNs()); 668 } else { 669 eglBase.swapBuffers(); 670 } 671 672 final long currentTimeNs = System.nanoTime(); 673 synchronized (statisticsLock) { 674 ++framesRendered; 675 renderTimeNs += (currentTimeNs - startTimeNs); 676 renderSwapBufferTimeNs += (currentTimeNs - swapBuffersStartTimeNs); 677 } 678 } 679 680 notifyCallbacks(frame, shouldRenderFrame); 681 } catch (GlUtil.GlOutOfMemoryException e) { 682 logE("Error while drawing frame", e); 683 final ErrorCallback errorCallback = this.errorCallback; 684 if (errorCallback != null) { 685 errorCallback.onGlOutOfMemory(); 686 } 687 // Attempt to free up some resources. 688 drawer.release(); 689 frameDrawer.release(); 690 bitmapTextureFramebuffer.release(); 691 // Continue here on purpose and retry again for next frame. In worst case, this is a continous 692 // problem and no more frames will be drawn. 693 } finally { 694 frame.release(); 695 } 696 } 697 notifyCallbacks(VideoFrame frame, boolean wasRendered)698 private void notifyCallbacks(VideoFrame frame, boolean wasRendered) { 699 if (frameListeners.isEmpty()) 700 return; 701 702 drawMatrix.reset(); 703 drawMatrix.preTranslate(0.5f, 0.5f); 704 drawMatrix.preScale(mirrorHorizontally ? -1f : 1f, mirrorVertically ? -1f : 1f); 705 drawMatrix.preScale(1f, -1f); // We want the output to be upside down for Bitmap. 706 drawMatrix.preTranslate(-0.5f, -0.5f); 707 708 Iterator<FrameListenerAndParams> it = frameListeners.iterator(); 709 while (it.hasNext()) { 710 FrameListenerAndParams listenerAndParams = it.next(); 711 if (!wasRendered && listenerAndParams.applyFpsReduction) { 712 continue; 713 } 714 it.remove(); 715 716 final int scaledWidth = (int) (listenerAndParams.scale * frame.getRotatedWidth()); 717 final int scaledHeight = (int) (listenerAndParams.scale * frame.getRotatedHeight()); 718 719 if (scaledWidth == 0 || scaledHeight == 0) { 720 listenerAndParams.listener.onFrame(null); 721 continue; 722 } 723 724 bitmapTextureFramebuffer.setSize(scaledWidth, scaledHeight); 725 726 GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, bitmapTextureFramebuffer.getFrameBufferId()); 727 GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, 728 GLES20.GL_TEXTURE_2D, bitmapTextureFramebuffer.getTextureId(), 0); 729 730 GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */); 731 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); 732 frameDrawer.drawFrame(frame, listenerAndParams.drawer, drawMatrix, 0 /* viewportX */, 733 0 /* viewportY */, scaledWidth, scaledHeight); 734 735 final ByteBuffer bitmapBuffer = ByteBuffer.allocateDirect(scaledWidth * scaledHeight * 4); 736 GLES20.glViewport(0, 0, scaledWidth, scaledHeight); 737 GLES20.glReadPixels( 738 0, 0, scaledWidth, scaledHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, bitmapBuffer); 739 740 GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); 741 GlUtil.checkNoGLES2Error("EglRenderer.notifyCallbacks"); 742 743 final Bitmap bitmap = Bitmap.createBitmap(scaledWidth, scaledHeight, Bitmap.Config.ARGB_8888); 744 bitmap.copyPixelsFromBuffer(bitmapBuffer); 745 listenerAndParams.listener.onFrame(bitmap); 746 } 747 } 748 averageTimeAsString(long sumTimeNs, int count)749 private String averageTimeAsString(long sumTimeNs, int count) { 750 return (count <= 0) ? "NA" : TimeUnit.NANOSECONDS.toMicros(sumTimeNs / count) + " us"; 751 } 752 logStatistics()753 private void logStatistics() { 754 final DecimalFormat fpsFormat = new DecimalFormat("#.0"); 755 final long currentTimeNs = System.nanoTime(); 756 synchronized (statisticsLock) { 757 final long elapsedTimeNs = currentTimeNs - statisticsStartTimeNs; 758 if (elapsedTimeNs <= 0 || (minRenderPeriodNs == Long.MAX_VALUE && framesReceived == 0)) { 759 return; 760 } 761 final float renderFps = framesRendered * TimeUnit.SECONDS.toNanos(1) / (float) elapsedTimeNs; 762 logD("Duration: " + TimeUnit.NANOSECONDS.toMillis(elapsedTimeNs) + " ms." 763 + " Frames received: " + framesReceived + "." 764 + " Dropped: " + framesDropped + "." 765 + " Rendered: " + framesRendered + "." 766 + " Render fps: " + fpsFormat.format(renderFps) + "." 767 + " Average render time: " + averageTimeAsString(renderTimeNs, framesRendered) + "." 768 + " Average swapBuffer time: " 769 + averageTimeAsString(renderSwapBufferTimeNs, framesRendered) + "."); 770 resetStatistics(currentTimeNs); 771 } 772 } 773 logE(String string, Throwable e)774 private void logE(String string, Throwable e) { 775 Logging.e(TAG, name + string, e); 776 } 777 logD(String string)778 private void logD(String string) { 779 Logging.d(TAG, name + string); 780 } 781 logW(String string)782 private void logW(String string) { 783 Logging.w(TAG, name + string); 784 } 785 } 786