1 // Copyright 2015 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef REMOTING_PROTOCOL_VIDEO_FRAME_PUMP_H_ 6 #define REMOTING_PROTOCOL_VIDEO_FRAME_PUMP_H_ 7 8 #include <stddef.h> 9 #include <stdint.h> 10 11 #include <memory> 12 #include <vector> 13 14 #include "base/macros.h" 15 #include "base/threading/thread_checker.h" 16 #include "base/time/time.h" 17 #include "base/timer/timer.h" 18 #include "remoting/codec/video_encoder.h" 19 #include "remoting/proto/video.pb.h" 20 #include "remoting/protocol/capture_scheduler.h" 21 #include "remoting/protocol/video_stream.h" 22 #include "third_party/webrtc/modules/desktop_capture/desktop_capturer.h" 23 24 namespace base { 25 class SingleThreadTaskRunner; 26 } // namespace base 27 28 namespace remoting { 29 namespace protocol { 30 31 class VideoFeedbackStub; 32 class VideoStub; 33 34 // Class responsible for scheduling frame captures from a screen capturer., 35 // delivering them to a VideoEncoder to encode, and 36 // finally passing the encoded video packets to the specified VideoStub to send 37 // on the network. 38 // 39 // THREADING 40 // 41 // This class is supplied TaskRunners to use for capture, encode and network 42 // operations. Capture, encode and network transmission tasks are interleaved 43 // as illustrated below: 44 // 45 // | CAPTURE ENCODE NETWORK 46 // | ............. 47 // | . Capture . 48 // | ............. 49 // | ............ 50 // | . . 51 // | ............. . . 52 // | . Capture . . Encode . 53 // | ............. . . 54 // | . . 55 // | ............ 56 // | ............. ............ .......... 57 // | . Capture . . . . Send . 58 // | ............. . . .......... 59 // | . Encode . 60 // | . . 61 // | . . 62 // | ............ 63 // | Time 64 // v 65 // 66 // VideoFramePump would ideally schedule captures so as to saturate the slowest 67 // of the capture, encode and network processes. However, it also needs to 68 // rate-limit captures to avoid overloading the host system, either by consuming 69 // too much CPU, or hogging the host's graphics subsystem. 70 class VideoFramePump : public VideoStream, 71 public webrtc::DesktopCapturer::Callback { 72 public: 73 // Creates a VideoFramePump running capture, encode and network tasks on the 74 // supplied TaskRunners. Video will be pumped to |video_stub|, which must 75 // outlive the pump.. 76 VideoFramePump(scoped_refptr<base::SingleThreadTaskRunner> encode_task_runner, 77 std::unique_ptr<webrtc::DesktopCapturer> capturer, 78 std::unique_ptr<VideoEncoder> encoder, 79 protocol::VideoStub* video_stub); 80 ~VideoFramePump() override; 81 82 // VideoStream interface. 83 void SetEventTimestampsSource(scoped_refptr<InputEventTimestampsSource> 84 event_timestamps_source) override; 85 void Pause(bool pause) override; 86 void SetLosslessEncode(bool want_lossless) override; 87 void SetLosslessColor(bool want_lossless) override; 88 void SetObserver(Observer* observer) override; 89 void SelectSource(int id) override; 90 video_feedback_stub()91 protocol::VideoFeedbackStub* video_feedback_stub() { 92 return &capture_scheduler_; 93 } 94 95 private: 96 struct FrameTimestamps { 97 FrameTimestamps(); 98 ~FrameTimestamps(); 99 100 // The following field is not-null for a single frame after each incoming 101 // input event. 102 InputEventTimestamps input_event_timestamps; 103 104 base::TimeTicks capture_started_time; 105 base::TimeTicks capture_ended_time; 106 base::TimeTicks encode_started_time; 107 base::TimeTicks encode_ended_time; 108 base::TimeTicks can_send_time; 109 }; 110 111 struct PacketWithTimestamps { 112 PacketWithTimestamps(std::unique_ptr<VideoPacket> packet, 113 std::unique_ptr<FrameTimestamps> timestamps); 114 ~PacketWithTimestamps(); 115 116 std::unique_ptr<VideoPacket> packet; 117 std::unique_ptr<FrameTimestamps> timestamps; 118 }; 119 120 // webrtc::DesktopCapturer::Callback interface. 121 void OnCaptureResult(webrtc::DesktopCapturer::Result result, 122 std::unique_ptr<webrtc::DesktopFrame> frame) override; 123 124 // Callback for CaptureScheduler. 125 void CaptureNextFrame(); 126 127 // Task running on the encoder thread to encode the |frame|. 128 static std::unique_ptr<PacketWithTimestamps> EncodeFrame( 129 VideoEncoder* encoder, 130 std::unique_ptr<webrtc::DesktopFrame> frame, 131 std::unique_ptr<FrameTimestamps> timestamps); 132 133 // Task called when a frame has finished encoding. 134 void OnFrameEncoded(std::unique_ptr<PacketWithTimestamps> packet); 135 136 // Sends |packet| to the client. 137 void SendPacket(std::unique_ptr<PacketWithTimestamps> packet); 138 139 // Helper called from SendPacket() to calculate timing fields in the |packet| 140 // before sending it. 141 void UpdateFrameTimers(VideoPacket* packet, FrameTimestamps* timestamps); 142 143 // Callback passed to |video_stub_|. 144 void OnVideoPacketSent(); 145 146 // Called by |keep_alive_timer_|. 147 void SendKeepAlivePacket(); 148 149 // Callback for |video_stub_| called after a keep-alive packet is sent. 150 void OnKeepAlivePacketSent(); 151 152 // Task runner used to run |encoder_|. 153 scoped_refptr<base::SingleThreadTaskRunner> encode_task_runner_; 154 155 // Capturer used to capture the screen. 156 std::unique_ptr<webrtc::DesktopCapturer> capturer_; 157 158 // Used to encode captured frames. Always accessed on the encode thread. 159 std::unique_ptr<VideoEncoder> encoder_; 160 161 scoped_refptr<InputEventTimestampsSource> event_timestamps_source_; 162 163 // Interface through which video frames are passed to the client. 164 protocol::VideoStub* video_stub_; 165 166 Observer* observer_ = nullptr; 167 webrtc::DesktopSize frame_size_; 168 webrtc::DesktopVector frame_dpi_; 169 170 // Timer used to ensure that we send empty keep-alive frames to the client 171 // even when the video stream is paused or encoder is busy. 172 base::RetainingOneShotTimer keep_alive_timer_; 173 174 // CaptureScheduler calls CaptureNextFrame() whenever a new frame needs to be 175 // captured. 176 CaptureScheduler capture_scheduler_; 177 178 // Timestamps for the frame that's being captured. 179 std::unique_ptr<FrameTimestamps> captured_frame_timestamps_; 180 181 bool send_pending_ = false; 182 183 std::vector<std::unique_ptr<PacketWithTimestamps>> pending_packets_; 184 185 base::ThreadChecker thread_checker_; 186 187 base::WeakPtrFactory<VideoFramePump> weak_factory_{this}; 188 189 DISALLOW_COPY_AND_ASSIGN(VideoFramePump); 190 }; 191 192 } // namespace protocol 193 } // namespace remoting 194 195 #endif // REMOTING_PROTOCOL_VIDEO_FRAME_PUMP_H_ 196