1 // Copyright 2015 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIARECORDER_VIDEO_TRACK_RECORDER_H_ 6 #define THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIARECORDER_VIDEO_TRACK_RECORDER_H_ 7 8 #include <atomic> 9 #include <memory> 10 11 #include "base/macros.h" 12 #include "base/memory/weak_ptr.h" 13 #include "base/single_thread_task_runner.h" 14 #include "base/threading/thread_checker.h" 15 #include "base/time/time.h" 16 #include "build/build_config.h" 17 #include "media/muxers/webm_muxer.h" 18 #include "media/video/video_encode_accelerator.h" 19 #include "third_party/blink/public/common/media/video_capture.h" 20 #include "third_party/blink/public/platform/web_media_stream_track.h" 21 #include "third_party/blink/public/web/modules/mediastream/encoded_video_frame.h" 22 #include "third_party/blink/public/web/modules/mediastream/media_stream_video_sink.h" 23 #include "third_party/blink/renderer/modules/mediarecorder/buildflags.h" 24 #include "third_party/blink/renderer/modules/mediarecorder/track_recorder.h" 25 #include "third_party/blink/renderer/modules/modules_export.h" 26 #include "third_party/blink/renderer/platform/heap/thread_state.h" 27 #include "third_party/blink/renderer/platform/wtf/cross_thread_copier.h" 28 #include "third_party/blink/renderer/platform/wtf/functional.h" 29 #include "third_party/blink/renderer/platform/wtf/hash_map.h" 30 #include "third_party/blink/renderer/platform/wtf/thread_safe_ref_counted.h" 31 #include "third_party/skia/include/core/SkBitmap.h" 32 33 namespace base { 34 class Thread; 35 } // namespace base 36 37 namespace cc { 38 class PaintCanvas; 39 } // namespace cc 40 41 namespace media { 42 class PaintCanvasVideoRenderer; 43 class VideoFrame; 44 } // namespace media 45 46 namespace video_track_recorder { 47 #if defined(OS_ANDROID) 48 const int kVEAEncoderMinResolutionWidth = 176; 49 const int kVEAEncoderMinResolutionHeight = 144; 50 #else 51 const int kVEAEncoderMinResolutionWidth = 640; 52 const int kVEAEncoderMinResolutionHeight = 480; 53 #endif 54 } // namespace video_track_recorder 55 56 namespace WTF { 57 58 template <> 59 struct CrossThreadCopier<media::WebmMuxer::VideoParameters> { 60 STATIC_ONLY(CrossThreadCopier); 61 using Type = media::WebmMuxer::VideoParameters; 62 static Type Copy(Type pointer) { return pointer; } 63 }; 64 65 } // namespace WTF 66 67 namespace blink { 68 69 class MediaStreamVideoTrack; 70 class Thread; 71 72 // Base class serving as interface for eventually saving encoded frames stemming 73 // from media from a source. 74 class VideoTrackRecorder : public TrackRecorder<MediaStreamVideoSink> { 75 public: 76 // Do not change the order of codecs; add new ones right before LAST. 77 enum class CodecId { 78 VP8, 79 VP9, 80 #if BUILDFLAG(RTC_USE_H264) 81 H264, 82 #endif 83 LAST 84 }; 85 86 using OnEncodedVideoCB = base::RepeatingCallback<void( 87 const media::WebmMuxer::VideoParameters& params, 88 std::string encoded_data, 89 std::string encoded_alpha, 90 base::TimeTicks capture_timestamp, 91 bool is_key_frame)>; 92 using OnErrorCB = base::RepeatingClosure; 93 94 // Wraps a counter in a class in order to enable use of base::WeakPtr<>. 95 // See https://crbug.com/859610 for why this was added. 96 class Counter { 97 public: 98 Counter(); 99 ~Counter(); 100 uint32_t count() const { return count_; } 101 void IncreaseCount(); 102 void DecreaseCount(); 103 base::WeakPtr<Counter> GetWeakPtr(); 104 105 private: 106 uint32_t count_; 107 base::WeakPtrFactory<Counter> weak_factory_{this}; 108 }; 109 110 // Base class to describe a generic Encoder, encapsulating all actual encoder 111 // (re)configurations, encoding and delivery of received frames. This class is 112 // ref-counted to allow the MediaStreamVideoTrack to hold a reference to it 113 // (via the callback that MediaStreamVideoSink passes along) and to jump back 114 // and forth to an internal encoder thread. Moreover, this class: 115 // - is created on its parent's thread (usually the main Render thread), that 116 // is, |main_task_runner_|. 117 // - receives VideoFrames on |origin_task_runner_| and runs OnEncodedVideoCB 118 // on that thread as well. This task runner is cached on first frame arrival, 119 // and is supposed to be the render IO thread (but this is not enforced); 120 // - uses an internal |encoding_task_runner_| for actual encoder interactions, 121 // namely configuration, encoding (which might take some time) and 122 // destruction. This task runner can be passed on the creation. If nothing is 123 // passed, a new encoding thread is created and used. 124 class Encoder : public WTF::ThreadSafeRefCounted<Encoder> { 125 public: 126 Encoder(const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_cb, 127 int32_t bits_per_second, 128 scoped_refptr<base::SingleThreadTaskRunner> main_task_runner, 129 scoped_refptr<base::SingleThreadTaskRunner> encoding_task_runner = 130 nullptr); 131 132 // Start encoding |frame|, returning via |on_encoded_video_cb_|. This 133 // call will also trigger an encode configuration upon first frame arrival 134 // or parameter change, and an EncodeOnEncodingTaskRunner() to actually 135 // encode the frame. If the |frame|'s data is not directly available (e.g. 136 // it's a texture) then RetrieveFrameOnMainThread() is called, and if even 137 // that fails, black frames are sent instead. 138 void StartFrameEncode(scoped_refptr<media::VideoFrame> frame, 139 base::TimeTicks capture_timestamp); 140 void RetrieveFrameOnMainThread(scoped_refptr<media::VideoFrame> video_frame, 141 base::TimeTicks capture_timestamp); 142 143 using OnEncodedVideoInternalCB = WTF::CrossThreadFunction<void( 144 const media::WebmMuxer::VideoParameters& params, 145 std::string encoded_data, 146 std::string encoded_alpha, 147 base::TimeTicks capture_timestamp, 148 bool is_key_frame)>; 149 150 static void OnFrameEncodeCompleted( 151 const OnEncodedVideoInternalCB& on_encoded_video_cb, 152 const media::WebmMuxer::VideoParameters& params, 153 std::string data, 154 std::string alpha_data, 155 base::TimeTicks capture_timestamp, 156 bool keyframe); 157 158 void SetPaused(bool paused); 159 virtual bool CanEncodeAlphaChannel(); 160 161 protected: 162 friend class WTF::ThreadSafeRefCounted<Encoder>; 163 friend class VideoTrackRecorderTest; 164 165 // This destructor may run on either |main_task_runner|, 166 // |encoding_task_runner|, or |origin_task_runner_|. Main ownership lies 167 // with VideoTrackRecorder. Shared ownership is handed out to 168 // asynchronous tasks running on |encoding_task_runner| for encoding. Shared 169 // ownership is also handed out to a MediaStreamVideoTrack which pushes 170 // frames on |origin_task_runner_|. Each of these may end up being the last 171 // reference. 172 virtual ~Encoder(); 173 174 virtual void EncodeOnEncodingTaskRunner( 175 scoped_refptr<media::VideoFrame> frame, 176 base::TimeTicks capture_timestamp) = 0; 177 178 // Called when the frame reference is released after encode. 179 void FrameReleased(scoped_refptr<media::VideoFrame> frame); 180 181 // A helper function to convert the given |frame| to an I420 video frame. 182 // Used mainly by the software encoders since I420 is the only supported 183 // pixel format. The function is best-effort. If for any reason the 184 // conversion fails, the original |frame| will be returned. 185 static scoped_refptr<media::VideoFrame> ConvertToI420ForSoftwareEncoder( 186 scoped_refptr<media::VideoFrame> frame); 187 188 // Used to shutdown properly on the same thread we were created. 189 const scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_; 190 191 // Task runner where frames to encode and reply callbacks must happen. 192 scoped_refptr<base::SingleThreadTaskRunner> origin_task_runner_; 193 194 // Task runner where encoding interactions happen. 195 scoped_refptr<base::SingleThreadTaskRunner> encoding_task_runner_; 196 197 // Optional thread for encoding. Active for the lifetime of VpxEncoder. 198 std::unique_ptr<Thread> encoding_thread_; 199 200 // While |paused_|, frames are not encoded. Used only from 201 // |encoding_thread_|. 202 // Use an atomic variable since it can be set on the main thread and read 203 // on the io thread at the same time. 204 std::atomic_bool paused_; 205 206 // This callback should be exercised on IO thread. 207 const OnEncodedVideoCB on_encoded_video_cb_; 208 209 // Target bitrate for video encoding. If 0, a standard bitrate is used. 210 const int32_t bits_per_second_; 211 212 // Number of frames that we keep the reference alive for encode. 213 // Operated and released exclusively on |origin_task_runner_|. 214 std::unique_ptr<Counter> num_frames_in_encode_; 215 216 // Used to retrieve incoming opaque VideoFrames (i.e. VideoFrames backed by 217 // textures). Created on-demand on |main_task_runner_|. 218 std::unique_ptr<media::PaintCanvasVideoRenderer> video_renderer_; 219 SkBitmap bitmap_; 220 std::unique_ptr<cc::PaintCanvas> canvas_; 221 222 DISALLOW_COPY_AND_ASSIGN(Encoder); 223 }; 224 225 // Class to encapsulate the enumeration of CodecIds/VideoCodecProfiles 226 // supported by the VEA underlying platform. Provides methods to query the 227 // preferred CodecId and to check if a given CodecId is supported. 228 class MODULES_EXPORT CodecEnumerator { 229 public: 230 CodecEnumerator(const media::VideoEncodeAccelerator::SupportedProfiles& 231 vea_supported_profiles); 232 ~CodecEnumerator(); 233 234 // Returns the first CodecId that has an associated VEA VideoCodecProfile, 235 // or VP8 if none available. 236 CodecId GetPreferredCodecId() const; 237 238 // Returns VEA's first supported VideoCodedProfile for a given CodecId, or 239 // VIDEO_CODEC_PROFILE_UNKNOWN otherwise. 240 media::VideoCodecProfile GetFirstSupportedVideoCodecProfile( 241 CodecId codec) const; 242 243 // Returns a list of supported media::VEA::SupportedProfile for a given 244 // CodecId, or empty vector if CodecId is unsupported. 245 media::VideoEncodeAccelerator::SupportedProfiles GetSupportedProfiles( 246 CodecId codec) const; 247 248 private: 249 // VEA-supported profiles grouped by CodecId. 250 HashMap<CodecId, media::VideoEncodeAccelerator::SupportedProfiles> 251 supported_profiles_; 252 CodecId preferred_codec_id_ = CodecId::LAST; 253 254 DISALLOW_COPY_AND_ASSIGN(CodecEnumerator); 255 }; 256 257 explicit VideoTrackRecorder(base::OnceClosure on_track_source_ended_cb); 258 259 virtual void Pause() = 0; 260 virtual void Resume() = 0; 261 virtual void OnVideoFrameForTesting(scoped_refptr<media::VideoFrame> frame, 262 base::TimeTicks capture_time) {} 263 virtual void OnEncodedVideoFrameForTesting( 264 scoped_refptr<EncodedVideoFrame> frame, 265 base::TimeTicks capture_time) {} 266 }; 267 268 // VideoTrackRecorderImpl uses the inherited WebMediaStreamSink and encodes the 269 // video frames received from a Stream Video Track. This class is constructed 270 // and used on a single thread, namely the main Render thread. This mirrors the 271 // other MediaStreamVideo* classes that are constructed/configured on Main 272 // Render thread but that pass frames on Render IO thread. It has an internal 273 // Encoder with its own threading subtleties, see the implementation file. 274 class MODULES_EXPORT VideoTrackRecorderImpl : public VideoTrackRecorder { 275 public: 276 static CodecId GetPreferredCodecId(); 277 278 // Returns true if the device has a hardware accelerated encoder which can 279 // encode video of the given |width|x|height| and |framerate| to specific 280 // |codec|. 281 // Note: default framerate value means no restriction. 282 static bool CanUseAcceleratedEncoder(CodecId codec, 283 size_t width, 284 size_t height, 285 double framerate = 0.0); 286 287 VideoTrackRecorderImpl( 288 CodecId codec, 289 MediaStreamComponent* track, 290 OnEncodedVideoCB on_encoded_video_cb, 291 base::OnceClosure on_track_source_ended_cb, 292 int32_t bits_per_second, 293 scoped_refptr<base::SingleThreadTaskRunner> main_task_runner); 294 ~VideoTrackRecorderImpl() override; 295 296 void Pause() override; 297 void Resume() override; 298 void OnVideoFrameForTesting(scoped_refptr<media::VideoFrame> frame, 299 base::TimeTicks capture_time) override; 300 301 private: 302 friend class VideoTrackRecorderTest; 303 void InitializeEncoder(CodecId codec, 304 const OnEncodedVideoCB& on_encoded_video_cb, 305 int32_t bits_per_second, 306 bool allow_vea_encoder, 307 scoped_refptr<media::VideoFrame> frame, 308 base::TimeTicks capture_time); 309 void OnError(); 310 311 void ConnectToTrack(const VideoCaptureDeliverFrameCB& callback); 312 void DisconnectFromTrack(); 313 314 // Used to check that we are destroyed on the same thread we were created. 315 THREAD_CHECKER(main_thread_checker_); 316 317 // We need to hold on to the Blink track to remove ourselves on dtor. 318 Persistent<MediaStreamComponent> track_; 319 320 // Inner class to encode using whichever codec is configured. 321 scoped_refptr<Encoder> encoder_; 322 323 base::RepeatingCallback<void(bool allow_vea_encoder, 324 scoped_refptr<media::VideoFrame> frame, 325 base::TimeTicks capture_time)> 326 initialize_encoder_cb_; 327 328 bool should_pause_encoder_on_initialization_; 329 330 scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_; 331 base::WeakPtrFactory<VideoTrackRecorderImpl> weak_factory_{this}; 332 333 DISALLOW_COPY_AND_ASSIGN(VideoTrackRecorderImpl); 334 }; 335 336 // VideoTrackRecorderPassthrough uses the inherited WebMediaStreamSink to 337 // dispatch EncodedVideoFrame content received from a MediaStreamVideoTrack. 338 class MODULES_EXPORT VideoTrackRecorderPassthrough : public VideoTrackRecorder { 339 public: 340 VideoTrackRecorderPassthrough( 341 MediaStreamComponent* track, 342 OnEncodedVideoCB on_encoded_video_cb, 343 base::OnceClosure on_track_source_ended_cb, 344 scoped_refptr<base::SingleThreadTaskRunner> main_task_runner); 345 ~VideoTrackRecorderPassthrough() override; 346 347 // VideoTrackRecorderBase 348 void Pause() override; 349 void Resume() override; 350 void OnEncodedVideoFrameForTesting(scoped_refptr<EncodedVideoFrame> frame, 351 base::TimeTicks capture_time) override; 352 353 private: 354 void RequestRefreshFrame(); 355 void DisconnectFromTrack(); 356 void HandleEncodedVideoFrame(scoped_refptr<EncodedVideoFrame> encoded_frame, 357 base::TimeTicks estimated_capture_time); 358 359 // Used to check that we are destroyed on the same thread we were created. 360 THREAD_CHECKER(main_thread_checker_); 361 362 // This enum class tracks encoded frame waiting and dispatching state. This 363 // is needed to guarantee we're dispatching decodable content to 364 // |on_encoded_video_cb|. Examples of times where this is needed is 365 // startup and Pause/Resume. 366 enum class KeyFrameState { 367 kWaitingForKeyFrame, 368 kKeyFrameReceivedOK, 369 kPaused 370 }; 371 372 // We need to hold on to the Blink track to remove ourselves on dtor. 373 const Persistent<MediaStreamComponent> track_; 374 KeyFrameState state_; 375 const scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_; 376 const OnEncodedVideoCB callback_; 377 base::WeakPtrFactory<VideoTrackRecorderPassthrough> weak_factory_{this}; 378 379 DISALLOW_COPY_AND_ASSIGN(VideoTrackRecorderPassthrough); 380 }; 381 } // namespace blink 382 383 #endif // THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIARECORDER_VIDEO_TRACK_RECORDER_H_ 384