1 /* 2 * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 // Declaration of abstract class VideoCapturer 12 13 #ifndef WEBRTC_MEDIA_BASE_VIDEOCAPTURER_H_ 14 #define WEBRTC_MEDIA_BASE_VIDEOCAPTURER_H_ 15 16 #include <stdint.h> 17 18 #include <algorithm> 19 #include <memory> 20 #include <string> 21 #include <vector> 22 23 // TODO(nisse): Transition hack, some downstream applications expect that 24 // including this file declares I420Buffer and NativeHandleBuffer. Delete after 25 // users of these classes are fixed to include the right headers. 26 #include "webrtc/api/video/i420_buffer.h" 27 #include "webrtc/common_video/include/video_frame_buffer.h" 28 29 #include "webrtc/base/constructormagic.h" 30 #include "webrtc/base/criticalsection.h" 31 #include "webrtc/media/base/videosourceinterface.h" 32 #include "webrtc/base/sigslot.h" 33 #include "webrtc/base/thread_checker.h" 34 #include "webrtc/base/timestampaligner.h" 35 #include "webrtc/media/base/videoadapter.h" 36 #include "webrtc/media/base/videobroadcaster.h" 37 #include "webrtc/media/base/videocommon.h" 38 39 namespace webrtc { 40 class VideoFrame; 41 } 42 43 namespace cricket { 44 45 // Current state of the capturer. 46 enum CaptureState { 47 CS_STOPPED, // The capturer has been stopped or hasn't started yet. 48 CS_STARTING, // The capturer is in the process of starting. Note, it may 49 // still fail to start. 50 CS_RUNNING, // The capturer has been started successfully and is now 51 // capturing. 52 CS_FAILED, // The capturer failed to start. 53 }; 54 55 // VideoCapturer is an abstract class that defines the interfaces for video 56 // capturing. The subclasses implement the video capturer for various types of 57 // capturers and various platforms. 58 // 59 // The captured frames may need to be adapted (for example, cropping). 60 // Video adaptation is built into and enabled by default. After a frame has 61 // been captured from the device, it is sent to the video adapter, then out to 62 // the sinks. 63 // 64 // Programming model: 65 // Create an object of a subclass of VideoCapturer 66 // Initialize 67 // SignalStateChange.connect() 68 // AddOrUpdateSink() 69 // Find the capture format for Start() by either calling GetSupportedFormats() 70 // and selecting one of the supported or calling GetBestCaptureFormat(). 71 // video_adapter()->OnOutputFormatRequest(desired_encoding_format) 72 // Start() 73 // GetCaptureFormat() optionally 74 // Stop() 75 // 76 // Assumption: 77 // The Start() and Stop() methods are called by a single thread (E.g., the 78 // media engine thread). Hence, the VideoCapture subclasses dont need to be 79 // thread safe. 80 // 81 class VideoCapturer : public sigslot::has_slots<>, 82 public rtc::VideoSourceInterface<webrtc::VideoFrame> { 83 public: 84 VideoCapturer(); 85 ~VideoCapturer()86 virtual ~VideoCapturer() {} 87 88 // Gets the id of the underlying device, which is available after the capturer 89 // is initialized. Can be used to determine if two capturers reference the 90 // same device. GetId()91 const std::string& GetId() const { return id_; } 92 93 // Get the capture formats supported by the video capturer. The supported 94 // formats are non empty after the device has been opened successfully. 95 const std::vector<VideoFormat>* GetSupportedFormats() const; 96 97 // Get the best capture format for the desired format. The best format is the 98 // same as one of the supported formats except that the frame interval may be 99 // different. If the application asks for 16x9 and the camera does not support 100 // 16x9 HD or the application asks for 16x10, we find the closest 4x3 and then 101 // crop; Otherwise, we find what the application asks for. Note that we assume 102 // that for HD, the desired format is always 16x9. The subclasses can override 103 // the default implementation. 104 // Parameters 105 // desired: the input desired format. If desired.fourcc is not kAnyFourcc, 106 // the best capture format has the exactly same fourcc. Otherwise, 107 // the best capture format uses a fourcc in GetPreferredFourccs(). 108 // best_format: the output of the best capture format. 109 // Return false if there is no such a best format, that is, the desired format 110 // is not supported. 111 virtual bool GetBestCaptureFormat(const VideoFormat& desired, 112 VideoFormat* best_format); 113 114 // TODO(hellner): deprecate (make private) the Start API in favor of this one. 115 // Also remove CS_STARTING as it is implied by the return 116 // value of StartCapturing(). 117 bool StartCapturing(const VideoFormat& capture_format); 118 // Start the video capturer with the specified capture format. 119 // Parameter 120 // capture_format: The caller got this parameter by either calling 121 // GetSupportedFormats() and selecting one of the supported 122 // or calling GetBestCaptureFormat(). 123 // Return 124 // CS_STARTING: The capturer is trying to start. Success or failure will 125 // be notified via the |SignalStateChange| callback. 126 // CS_RUNNING: if the capturer is started and capturing. 127 // CS_FAILED: if the capturer failes to start.. 128 // CS_NO_DEVICE: if the capturer has no device and fails to start. 129 virtual CaptureState Start(const VideoFormat& capture_format) = 0; 130 131 // Get the current capture format, which is set by the Start() call. 132 // Note that the width and height of the captured frames may differ from the 133 // capture format. For example, the capture format is HD but the captured 134 // frames may be smaller than HD. GetCaptureFormat()135 const VideoFormat* GetCaptureFormat() const { 136 return capture_format_.get(); 137 } 138 139 // Stop the video capturer. 140 virtual void Stop() = 0; 141 // Check if the video capturer is running. 142 virtual bool IsRunning() = 0; capture_state()143 CaptureState capture_state() const { 144 return capture_state_; 145 } 146 apply_rotation()147 virtual bool apply_rotation() { return apply_rotation_; } 148 149 // Returns true if the capturer is screencasting. This can be used to 150 // implement screencast specific behavior. 151 virtual bool IsScreencast() const = 0; 152 153 // Caps the VideoCapturer's format according to max_format. It can e.g. be 154 // used to prevent cameras from capturing at a resolution or framerate that 155 // the capturer is capable of but not performing satisfactorily at. 156 // The capping is an upper bound for each component of the capturing format. 157 // The fourcc component is ignored. 158 void ConstrainSupportedFormats(const VideoFormat& max_format); 159 set_enable_camera_list(bool enable_camera_list)160 void set_enable_camera_list(bool enable_camera_list) { 161 enable_camera_list_ = enable_camera_list; 162 } enable_camera_list()163 bool enable_camera_list() { 164 return enable_camera_list_; 165 } 166 167 // Signal all capture state changes that are not a direct result of calling 168 // Start(). 169 sigslot::signal2<VideoCapturer*, CaptureState> SignalStateChange; 170 171 // If true, run video adaptation. By default, video adaptation is enabled 172 // and users must call video_adapter()->OnOutputFormatRequest() 173 // to receive frames. enable_video_adapter()174 bool enable_video_adapter() const { return enable_video_adapter_; } set_enable_video_adapter(bool enable_video_adapter)175 void set_enable_video_adapter(bool enable_video_adapter) { 176 enable_video_adapter_ = enable_video_adapter; 177 } 178 179 bool GetInputSize(int* width, int* height); 180 181 // Implements VideoSourceInterface 182 void AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink, 183 const rtc::VideoSinkWants& wants) override; 184 void RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override; 185 186 protected: 187 // OnSinkWantsChanged can be overridden to change the default behavior 188 // when a sink changes its VideoSinkWants by calling AddOrUpdateSink. 189 virtual void OnSinkWantsChanged(const rtc::VideoSinkWants& wants); 190 191 // Reports the appropriate frame size after adaptation. Returns true 192 // if a frame is wanted. Returns false if there are no interested 193 // sinks, or if the VideoAdapter decides to drop the frame. 194 195 // This function also implements timestamp translation/filtering. 196 // |camera_time_ns| is the camera's timestamp for the captured 197 // frame; it is expected to have good accuracy, but it may use an 198 // arbitrary epoch and a small possibly free-running with a frequency 199 // slightly different from the system clock. |system_time_us| is the 200 // monotonic system time (in the same scale as rtc::TimeMicros) when 201 // the frame was captured; the application is expected to read the 202 // system time as soon as possible after frame capture, but it may 203 // suffer scheduling jitter or poor system clock resolution. The 204 // output |translated_camera_time_us| is a combined timestamp, 205 // taking advantage of the supposedly higher accuracy in the camera 206 // timestamp, but using the same epoch and frequency as system time. 207 bool AdaptFrame(int width, 208 int height, 209 int64_t camera_time_us, 210 int64_t system_time_us, 211 int* out_width, 212 int* out_height, 213 int* crop_width, 214 int* crop_height, 215 int* crop_x, 216 int* crop_y, 217 int64_t* translated_camera_time_us); 218 219 // Called when a frame has been captured and converted to a 220 // VideoFrame. OnFrame can be called directly by an implementation 221 // that does not use SignalFrameCaptured or OnFrameCaptured. The 222 // orig_width and orig_height are used only to produce stats. 223 void OnFrame(const webrtc::VideoFrame& frame, 224 int orig_width, 225 int orig_height); 226 video_adapter()227 VideoAdapter* video_adapter() { return &video_adapter_; } 228 229 void SetCaptureState(CaptureState state); 230 231 // subclasses override this virtual method to provide a vector of fourccs, in 232 // order of preference, that are expected by the media engine. 233 virtual bool GetPreferredFourccs(std::vector<uint32_t>* fourccs) = 0; 234 235 // mutators to set private attributes SetId(const std::string & id)236 void SetId(const std::string& id) { 237 id_ = id; 238 } 239 SetCaptureFormat(const VideoFormat * format)240 void SetCaptureFormat(const VideoFormat* format) { 241 capture_format_.reset(format ? new VideoFormat(*format) : NULL); 242 } 243 244 void SetSupportedFormats(const std::vector<VideoFormat>& formats); 245 246 private: 247 void Construct(); 248 // Get the distance between the desired format and the supported format. 249 // Return the max distance if they mismatch. See the implementation for 250 // details. 251 int64_t GetFormatDistance(const VideoFormat& desired, 252 const VideoFormat& supported); 253 254 // Updates filtered_supported_formats_ so that it contains the formats in 255 // supported_formats_ that fulfill all applied restrictions. 256 void UpdateFilteredSupportedFormats(); 257 // Returns true if format doesn't fulfill all applied restrictions. 258 bool ShouldFilterFormat(const VideoFormat& format) const; 259 260 void UpdateInputSize(int width, int height); 261 262 rtc::ThreadChecker thread_checker_; 263 std::string id_; 264 CaptureState capture_state_; 265 std::unique_ptr<VideoFormat> capture_format_; 266 std::vector<VideoFormat> supported_formats_; 267 std::unique_ptr<VideoFormat> max_format_; 268 std::vector<VideoFormat> filtered_supported_formats_; 269 270 bool enable_camera_list_; 271 int scaled_width_; // Current output size from ComputeScale. 272 int scaled_height_; 273 274 rtc::VideoBroadcaster broadcaster_; 275 bool enable_video_adapter_; 276 VideoAdapter video_adapter_; 277 278 rtc::CriticalSection frame_stats_crit_; 279 // The captured frame size before potential adapation. 280 bool input_size_valid_ GUARDED_BY(frame_stats_crit_) = false; 281 int input_width_ GUARDED_BY(frame_stats_crit_); 282 int input_height_ GUARDED_BY(frame_stats_crit_); 283 284 // Whether capturer should apply rotation to the frame before 285 // passing it on to the registered sinks. 286 bool apply_rotation_; 287 288 // State for the timestamp translation. 289 rtc::TimestampAligner timestamp_aligner_; 290 RTC_DISALLOW_COPY_AND_ASSIGN(VideoCapturer); 291 }; 292 293 } // namespace cricket 294 295 #endif // WEBRTC_MEDIA_BASE_VIDEOCAPTURER_H_ 296