1 // Copyright 2015 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #ifndef REMOTING_PROTOCOL_VIDEO_FRAME_PUMP_H_
6 #define REMOTING_PROTOCOL_VIDEO_FRAME_PUMP_H_
7 
8 #include <stddef.h>
9 #include <stdint.h>
10 
11 #include <memory>
12 #include <vector>
13 
14 #include "base/macros.h"
15 #include "base/threading/thread_checker.h"
16 #include "base/time/time.h"
17 #include "base/timer/timer.h"
18 #include "remoting/codec/video_encoder.h"
19 #include "remoting/proto/video.pb.h"
20 #include "remoting/protocol/capture_scheduler.h"
21 #include "remoting/protocol/video_stream.h"
22 #include "third_party/webrtc/modules/desktop_capture/desktop_capturer.h"
23 
24 namespace base {
25 class SingleThreadTaskRunner;
26 }  // namespace base
27 
28 namespace remoting {
29 namespace protocol {
30 
31 class VideoFeedbackStub;
32 class VideoStub;
33 
34 // Class responsible for scheduling frame captures from a screen capturer.,
35 // delivering them to a VideoEncoder to encode, and
36 // finally passing the encoded video packets to the specified VideoStub to send
37 // on the network.
38 //
39 // THREADING
40 //
41 // This class is supplied TaskRunners to use for capture, encode and network
42 // operations.  Capture, encode and network transmission tasks are interleaved
43 // as illustrated below:
44 //
45 // |       CAPTURE       ENCODE     NETWORK
46 // |    .............
47 // |    .  Capture  .
48 // |    .............
49 // |                  ............
50 // |                  .          .
51 // |    ............. .          .
52 // |    .  Capture  . .  Encode  .
53 // |    ............. .          .
54 // |                  .          .
55 // |                  ............
56 // |    ............. ............ ..........
57 // |    .  Capture  . .          . .  Send  .
58 // |    ............. .          . ..........
59 // |                  .  Encode  .
60 // |                  .          .
61 // |                  .          .
62 // |                  ............
63 // | Time
64 // v
65 //
66 // VideoFramePump would ideally schedule captures so as to saturate the slowest
67 // of the capture, encode and network processes.  However, it also needs to
68 // rate-limit captures to avoid overloading the host system, either by consuming
69 // too much CPU, or hogging the host's graphics subsystem.
70 class VideoFramePump : public VideoStream,
71                        public webrtc::DesktopCapturer::Callback {
72  public:
73   // Creates a VideoFramePump running capture, encode and network tasks on the
74   // supplied TaskRunners. Video will be pumped to |video_stub|, which must
75   // outlive the pump..
76   VideoFramePump(scoped_refptr<base::SingleThreadTaskRunner> encode_task_runner,
77                  std::unique_ptr<webrtc::DesktopCapturer> capturer,
78                  std::unique_ptr<VideoEncoder> encoder,
79                  protocol::VideoStub* video_stub);
80   ~VideoFramePump() override;
81 
82   // VideoStream interface.
83   void SetEventTimestampsSource(scoped_refptr<InputEventTimestampsSource>
84                                     event_timestamps_source) override;
85   void Pause(bool pause) override;
86   void SetLosslessEncode(bool want_lossless) override;
87   void SetLosslessColor(bool want_lossless) override;
88   void SetObserver(Observer* observer) override;
89   void SelectSource(int id) override;
90 
video_feedback_stub()91   protocol::VideoFeedbackStub* video_feedback_stub() {
92     return &capture_scheduler_;
93   }
94 
95  private:
96   struct FrameTimestamps {
97     FrameTimestamps();
98     ~FrameTimestamps();
99 
100     // The following field is not-null for a single frame after each incoming
101     // input event.
102     InputEventTimestamps input_event_timestamps;
103 
104     base::TimeTicks capture_started_time;
105     base::TimeTicks capture_ended_time;
106     base::TimeTicks encode_started_time;
107     base::TimeTicks encode_ended_time;
108     base::TimeTicks can_send_time;
109   };
110 
111   struct PacketWithTimestamps {
112     PacketWithTimestamps(std::unique_ptr<VideoPacket> packet,
113                          std::unique_ptr<FrameTimestamps> timestamps);
114     ~PacketWithTimestamps();
115 
116     std::unique_ptr<VideoPacket> packet;
117     std::unique_ptr<FrameTimestamps> timestamps;
118   };
119 
120   // webrtc::DesktopCapturer::Callback interface.
121   void OnCaptureResult(webrtc::DesktopCapturer::Result result,
122                        std::unique_ptr<webrtc::DesktopFrame> frame) override;
123 
124   // Callback for CaptureScheduler.
125   void CaptureNextFrame();
126 
127   // Task running on the encoder thread to encode the |frame|.
128   static std::unique_ptr<PacketWithTimestamps> EncodeFrame(
129       VideoEncoder* encoder,
130       std::unique_ptr<webrtc::DesktopFrame> frame,
131       std::unique_ptr<FrameTimestamps> timestamps);
132 
133   // Task called when a frame has finished encoding.
134   void OnFrameEncoded(std::unique_ptr<PacketWithTimestamps> packet);
135 
136   // Sends |packet| to the client.
137   void SendPacket(std::unique_ptr<PacketWithTimestamps> packet);
138 
139   // Helper called from SendPacket() to calculate timing fields in the |packet|
140   // before sending it.
141   void UpdateFrameTimers(VideoPacket* packet, FrameTimestamps* timestamps);
142 
143   // Callback passed to |video_stub_|.
144   void OnVideoPacketSent();
145 
146   // Called by |keep_alive_timer_|.
147   void SendKeepAlivePacket();
148 
149   // Callback for |video_stub_| called after a keep-alive packet is sent.
150   void OnKeepAlivePacketSent();
151 
152   // Task runner used to run |encoder_|.
153   scoped_refptr<base::SingleThreadTaskRunner> encode_task_runner_;
154 
155   // Capturer used to capture the screen.
156   std::unique_ptr<webrtc::DesktopCapturer> capturer_;
157 
158   // Used to encode captured frames. Always accessed on the encode thread.
159   std::unique_ptr<VideoEncoder> encoder_;
160 
161   scoped_refptr<InputEventTimestampsSource> event_timestamps_source_;
162 
163   // Interface through which video frames are passed to the client.
164   protocol::VideoStub* video_stub_;
165 
166   Observer* observer_ = nullptr;
167   webrtc::DesktopSize frame_size_;
168   webrtc::DesktopVector frame_dpi_;
169 
170   // Timer used to ensure that we send empty keep-alive frames to the client
171   // even when the video stream is paused or encoder is busy.
172   base::RetainingOneShotTimer keep_alive_timer_;
173 
174   // CaptureScheduler calls CaptureNextFrame() whenever a new frame needs to be
175   // captured.
176   CaptureScheduler capture_scheduler_;
177 
178   // Timestamps for the frame that's being captured.
179   std::unique_ptr<FrameTimestamps> captured_frame_timestamps_;
180 
181   bool send_pending_ = false;
182 
183   std::vector<std::unique_ptr<PacketWithTimestamps>> pending_packets_;
184 
185   base::ThreadChecker thread_checker_;
186 
187   base::WeakPtrFactory<VideoFramePump> weak_factory_{this};
188 
189   DISALLOW_COPY_AND_ASSIGN(VideoFramePump);
190 };
191 
192 }  // namespace protocol
193 }  // namespace remoting
194 
195 #endif  // REMOTING_PROTOCOL_VIDEO_FRAME_PUMP_H_
196