modules/mediastream/webmediaplayer_ms_compositor.h

// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_WEBMEDIAPLAYER_MS_COMPOSITOR_H_
#define THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_WEBMEDIAPLAYER_MS_COMPOSITOR_H_

#include <stddef.h>

#include <map>
#include <memory>
#include <vector>

#include "base/memory/weak_ptr.h"
#include "base/optional.h"
#include "base/synchronization/lock.h"
#include "base/threading/thread_checker.h"
#include "cc/layers/surface_layer.h"
#include "cc/layers/video_frame_provider.h"
#include "media/base/media_util.h"
#include "third_party/blink/public/platform/web_media_player.h"
#include "third_party/blink/public/platform/web_video_frame_submitter.h"
#include "third_party/blink/renderer/modules/mediastream/video_renderer_algorithm_wrapper.h"
#include "third_party/blink/renderer/modules/modules_export.h"
#include "third_party/blink/renderer/platform/wtf/cross_thread_functional.h"
#include "third_party/blink/renderer/platform/wtf/thread_safe_ref_counted.h"

namespace base {
class SingleThreadTaskRunner;
class WaitableEvent;
}

namespace gfx {
class Size;
}

namespace media {
class VideoRendererAlgorithm;
}

namespace viz {
class SurfaceId;
}

namespace blink {
class MediaStreamDescriptor;
class WebMediaPlayerMS;
struct WebMediaPlayerMSCompositorTraits;

// This class is designed to handle the work load on compositor thread for
// WebMediaPlayerMS. It will be instantiated on the main thread, but destroyed
// on the thread holding the last reference.
//
// WebMediaPlayerMSCompositor utilizes VideoRendererAlgorithm to store the
// incoming frames and select the best frame for rendering to maximize the
// smoothness, if REFERENCE_TIMEs are populated for incoming VideoFrames.
// Otherwise, WebMediaPlayerMSCompositor will simply store the most recent
// frame, and submit it whenever asked by the compositor.
class MODULES_EXPORT WebMediaPlayerMSCompositor
    : public cc::VideoFrameProvider,
      public WTF::ThreadSafeRefCounted<WebMediaPlayerMSCompositor,
                                       WebMediaPlayerMSCompositorTraits> {
 public:
  using OnNewFramePresentedCB = base::OnceClosure;

  WebMediaPlayerMSCompositor(
      scoped_refptr<base::SingleThreadTaskRunner> task_runner,
      scoped_refptr<base::SingleThreadTaskRunner> io_task_runner,
      MediaStreamDescriptor* media_stream_descriptor,
      std::unique_ptr<WebVideoFrameSubmitter> submitter,
      WebMediaPlayer::SurfaceLayerMode surface_layer_mode,
      const base::WeakPtr<WebMediaPlayerMS>& player);

  // Can be called from any thread.
  cc::UpdateSubmissionStateCB GetUpdateSubmissionStateCallback() {
    return update_submission_state_callback_;
  }

  void EnqueueFrame(scoped_refptr<media::VideoFrame> frame, bool is_copy);

  // Statistical data
  gfx::Size GetCurrentSize();
  base::TimeDelta GetCurrentTime();
  size_t total_frame_count();
  size_t dropped_frame_count();

  // Signals the VideoFrameSubmitter to prepare to receive BeginFrames and
  // submit video frames given by WebMediaPlayerMSCompositor.
  virtual void EnableSubmission(
      const viz::SurfaceId& id,
      media::VideoTransformation transformation,
      bool force_submit);

  // Notifies the |submitter_| that the frames must be submitted.
  void SetForceSubmit(bool force_submit);

  // Notifies the |submitter_| that the page is no longer visible.
  void SetIsPageVisible(bool is_visible);

  // VideoFrameProvider implementation.
  void SetVideoFrameProviderClient(
      cc::VideoFrameProvider::Client* client) override;
  bool UpdateCurrentFrame(base::TimeTicks deadline_min,
                          base::TimeTicks deadline_max) override;
  bool HasCurrentFrame() override;
  scoped_refptr<media::VideoFrame> GetCurrentFrame() override;
  void PutCurrentFrame() override;
  base::TimeDelta GetPreferredRenderInterval() override;

  void StartRendering();
  void StopRendering();
  void ReplaceCurrentFrameWithACopy();

  // Tell |video_frame_provider_client_| to stop using this instance in
  // preparation for dtor.
  void StopUsingProvider();

  // Sets a hook to be notified when a new frame is presented, to fulfill a
  // prending video.requestAnimationFrame() request.
  // Can be called from any thread.
  void SetOnFramePresentedCallback(OnNewFramePresentedCB presented_cb);

  // Gets the metadata for the last frame that was presented to the compositor.
  // Used to populate the VideoFrameMetadata of video.requestVideoFrameCallback
  // callbacks. See https://wicg.github.io/video-rvfc/.
  // Can be called on any thread.
  std::unique_ptr<WebMediaPlayer::VideoFramePresentationMetadata>
  GetLastPresentedFrameMetadata();

  // Sets the ForceBeginFrames flag on |submitter_|. Can be called from any
  // thread.
  //
  // The flag is used to keep receiving BeginFrame()/UpdateCurrentFrame() calls
  // even if the video element is not visible, so websites can still use the
  // requestVideoFrameCallback() API when the video is offscreen.
  void SetForceBeginFrames(bool enable);

 private:
  friend class WTF::ThreadSafeRefCounted<WebMediaPlayerMSCompositor,
                                         WebMediaPlayerMSCompositorTraits>;
  friend class WebMediaPlayerMSTest;
  friend struct WebMediaPlayerMSCompositorTraits;

  // Struct used to keep information about frames pending in
  // |rendering_frame_buffer_|.
  struct PendingFrameInfo {
    int unique_id;
    base::TimeDelta timestamp;
    base::TimeTicks reference_time;
    bool is_copy;
  };

  ~WebMediaPlayerMSCompositor() override;

  // Ran on the |video_frame_compositor_task_runner_| to initialize
  // |submitter_|
  void InitializeSubmitter();

  // Signals the VideoFrameSubmitter to stop submitting frames.
  void SetIsSurfaceVisible(bool, base::WaitableEvent*);

  // The use of std::vector here is OK because this method is bound into a
  // base::OnceCallback instance, and passed to media::VideoRendererAlgorithm
  // ctor.
  bool MapTimestampsToRenderTimeTicks(
      const std::vector<base::TimeDelta>& timestamps,
      std::vector<base::TimeTicks>* wall_clock_times);

  // For algorithm enabled case only: given the render interval, call
  // SetCurrentFrame() if a new frame is available.
  // |video_frame_provider_client_| gets notified about the new frame when it
  // calls UpdateCurrentFrame().
  void RenderUsingAlgorithm(base::TimeTicks deadline_min,
                            base::TimeTicks deadline_max);

  // For algorithm disabled case only: call SetCurrentFrame() with the current
  // frame immediately. |video_frame_provider_client_| gets notified about the
  // new frame with a DidReceiveFrame() call.
  void RenderWithoutAlgorithm(scoped_refptr<media::VideoFrame> frame,
                              bool is_copy);
  void RenderWithoutAlgorithmOnCompositor(
      scoped_refptr<media::VideoFrame> frame,
      bool is_copy);

  // Update |current_frame_| and |dropped_frame_count_|
  void SetCurrentFrame(
      scoped_refptr<media::VideoFrame> frame,
      bool is_copy,
      base::Optional<base::TimeTicks> expected_presentation_time);
  // Following the update to |current_frame_|, this will check for changes that
  // require updating video layer.
  void CheckForFrameChanges(
      bool is_first_frame,
      bool has_frame_size_changed,
      base::Optional<media::VideoRotation> new_frame_rotation,
      base::Optional<bool> new_frame_opacity);

  void StartRenderingInternal();
  void StopRenderingInternal();
  void StopUsingProviderInternal();
  void ReplaceCurrentFrameWithACopyInternal();

  void SetAlgorithmEnabledForTesting(bool algorithm_enabled);

  // Used for DCHECKs to ensure method calls executed in the correct thread,
  // which is renderer main thread in this class.
  THREAD_CHECKER(thread_checker_);

  const scoped_refptr<base::SingleThreadTaskRunner>
      video_frame_compositor_task_runner_;
  const scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_;
  const scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;

  base::WeakPtr<WebMediaPlayerMS> player_;

  // TODO(qiangchen, emircan): It might be nice to use a real MediaLog here from
  // the WebMediaPlayerMS instance, but it owns the MediaLog and this class has
  // non-deterministic destruction paths (either compositor or IO).
  media::NullMediaLog media_log_;

  size_t serial_;

  // A pointer back to the compositor to inform it about state changes. This
  // is not |nullptr| while the compositor is actively using this
  // VideoFrameProvider. This will be set to |nullptr| when the compositor stops
  // serving this VideoFrameProvider.
  cc::VideoFrameProvider::Client* video_frame_provider_client_;

  // |current_frame_| is updated only on compositor thread. The object it
  // holds can be freed on the compositor thread if it is the last to hold a
  // reference but media::VideoFrame is a thread-safe ref-pointer. It is
  // however read on the compositor and main thread so locking is required
  // around all modifications and all reads on the any thread.
  scoped_refptr<media::VideoFrame> current_frame_;

  // |rendering_frame_buffer_| stores the incoming frames, and provides a frame
  // selection method which returns the best frame for the render interval.
  std::unique_ptr<VideoRendererAlgorithmWrapper> rendering_frame_buffer_;

  // |current_frame_rendered_| is updated on compositor thread only.
  // It's used to track whether |current_frame_| was painted for detecting
  // when to increase |dropped_frame_count_|. It is also used when checking if
  // new frame for display is available in UpdateCurrentFrame().
  bool current_frame_rendered_;

  // Historical data about last rendering. These are for detecting whether
  // rendering is paused (one reason is that the tab is not in the front), in
  // which case we need to do background rendering.
  base::TimeTicks last_deadline_max_;
  base::TimeDelta last_render_length_;

  size_t total_frame_count_;
  size_t dropped_frame_count_;

  bool current_frame_is_copy_ = false;

  // Used to complete video.requestAnimationFrame() calls. Reported up via
  // GetLastPresentedFrameMetadata().
  // TODO(https://crbug.com/1050755): Improve the accuracy of these fields for
  // cases where we only use RenderWithoutAlgorithm().
  base::TimeTicks last_presentation_time_ GUARDED_BY(current_frame_lock_);
  base::TimeTicks last_expected_display_time_ GUARDED_BY(current_frame_lock_);
  size_t presented_frames_ GUARDED_BY(current_frame_lock_) = 0u;

  // The value of GetPreferredRenderInterval() the last time |current_frame_|
  // was updated. Used by GetLastPresentedFrameMetadata(), to prevent calling
  // GetPreferredRenderInterval() from the main thread.
  base::TimeDelta last_preferred_render_interval_
      GUARDED_BY(current_frame_lock_);

  bool stopped_;
  bool render_started_;

  // Called when a new frame is enqueued, either in RenderWithoutAlgorithm() or
  // in RenderUsingAlgorithm(). Used to fulfill video.requestAnimationFrame()
  // requests.
  base::Lock new_frame_presented_cb_lock_;
  OnNewFramePresentedCB new_frame_presented_cb_;

  std::unique_ptr<WebVideoFrameSubmitter> submitter_;

  // Extra information about the frames pending in |rendering_frame_buffer_|.
  WTF::Vector<PendingFrameInfo> pending_frames_info_;

  cc::UpdateSubmissionStateCB update_submission_state_callback_;

  // |current_frame_lock_| protects |current_frame_|, |rendering_frame_buffer_|,
  // |dropped_frame_count_|, and |render_started_|.
  base::Lock current_frame_lock_;

  base::WeakPtrFactory<WebMediaPlayerMSCompositor> weak_ptr_factory_{this};

  DISALLOW_COPY_AND_ASSIGN(WebMediaPlayerMSCompositor);
};

struct WebMediaPlayerMSCompositorTraits {
  // Ensure destruction occurs on main thread so that "Web" and other resources
  // are destroyed on the correct thread.
  static void Destruct(const WebMediaPlayerMSCompositor* player);
};

}  // namespace blink

#endif  // THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_WEBMEDIAPLAYER_MS_COMPOSITOR_H_