1 // Copyright 2015 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "remoting/protocol/video_frame_pump.h"
6
7 #include <algorithm>
8 #include <memory>
9 #include <utility>
10
11 #include "base/bind.h"
12 #include "base/callback.h"
13 #include "base/check.h"
14 #include "base/single_thread_task_runner.h"
15 #include "base/task_runner_util.h"
16 #include "base/time/time.h"
17 #include "remoting/base/constants.h"
18 #include "remoting/proto/control.pb.h"
19 #include "remoting/proto/video.pb.h"
20 #include "remoting/protocol/video_stub.h"
21 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
22
23 namespace remoting {
24 namespace protocol {
25
26 // Interval between empty keep-alive frames. These frames are sent only when the
27 // stream is paused or inactive for some other reason (e.g. when blocked on
28 // capturer). To prevent PseudoTCP from resetting congestion window this value
29 // must be smaller than the minimum RTO used in PseudoTCP, which is 250ms.
30 static const int kKeepAlivePacketIntervalMs = 200;
31
32 VideoFramePump::FrameTimestamps::FrameTimestamps() = default;
33 VideoFramePump::FrameTimestamps::~FrameTimestamps() = default;
34
PacketWithTimestamps(std::unique_ptr<VideoPacket> packet,std::unique_ptr<FrameTimestamps> timestamps)35 VideoFramePump::PacketWithTimestamps::PacketWithTimestamps(
36 std::unique_ptr<VideoPacket> packet,
37 std::unique_ptr<FrameTimestamps> timestamps)
38 : packet(std::move(packet)), timestamps(std::move(timestamps)) {}
39
40 VideoFramePump::PacketWithTimestamps::~PacketWithTimestamps() = default;
41
VideoFramePump(scoped_refptr<base::SingleThreadTaskRunner> encode_task_runner,std::unique_ptr<webrtc::DesktopCapturer> capturer,std::unique_ptr<VideoEncoder> encoder,protocol::VideoStub * video_stub)42 VideoFramePump::VideoFramePump(
43 scoped_refptr<base::SingleThreadTaskRunner> encode_task_runner,
44 std::unique_ptr<webrtc::DesktopCapturer> capturer,
45 std::unique_ptr<VideoEncoder> encoder,
46 protocol::VideoStub* video_stub)
47 : encode_task_runner_(encode_task_runner),
48 capturer_(std::move(capturer)),
49 encoder_(std::move(encoder)),
50 video_stub_(video_stub),
51 keep_alive_timer_(
52 FROM_HERE,
53 base::TimeDelta::FromMilliseconds(kKeepAlivePacketIntervalMs),
54 base::BindRepeating(&VideoFramePump::SendKeepAlivePacket,
55 base::Unretained(this))),
56 capture_scheduler_(base::BindRepeating(&VideoFramePump::CaptureNextFrame,
57 base::Unretained(this))) {
58 DCHECK(encoder_);
59 DCHECK(video_stub_);
60
61 capturer_->Start(this);
62 capture_scheduler_.Start();
63 }
64
~VideoFramePump()65 VideoFramePump::~VideoFramePump() {
66 DCHECK(thread_checker_.CalledOnValidThread());
67 encode_task_runner_->DeleteSoon(FROM_HERE, encoder_.release());
68 }
69
SetEventTimestampsSource(scoped_refptr<InputEventTimestampsSource> event_timestamps_source)70 void VideoFramePump::SetEventTimestampsSource(
71 scoped_refptr<InputEventTimestampsSource> event_timestamps_source) {
72 DCHECK(thread_checker_.CalledOnValidThread());
73
74 event_timestamps_source_ = event_timestamps_source;
75 }
76
Pause(bool pause)77 void VideoFramePump::Pause(bool pause) {
78 DCHECK(thread_checker_.CalledOnValidThread());
79
80 capture_scheduler_.Pause(pause);
81 }
82
SetLosslessEncode(bool want_lossless)83 void VideoFramePump::SetLosslessEncode(bool want_lossless) {
84 DCHECK(thread_checker_.CalledOnValidThread());
85
86 encode_task_runner_->PostTask(
87 FROM_HERE,
88 base::BindOnce(&VideoEncoder::SetLosslessEncode,
89 base::Unretained(encoder_.get()), want_lossless));
90 }
91
SetLosslessColor(bool want_lossless)92 void VideoFramePump::SetLosslessColor(bool want_lossless) {
93 DCHECK(thread_checker_.CalledOnValidThread());
94
95 encode_task_runner_->PostTask(
96 FROM_HERE,
97 base::BindOnce(&VideoEncoder::SetLosslessColor,
98 base::Unretained(encoder_.get()), want_lossless));
99 }
100
SetObserver(Observer * observer)101 void VideoFramePump::SetObserver(Observer* observer) {
102 DCHECK(thread_checker_.CalledOnValidThread());
103 observer_ = observer;
104 }
105
SelectSource(int id)106 void VideoFramePump::SelectSource(int id) {}
107
OnCaptureResult(webrtc::DesktopCapturer::Result result,std::unique_ptr<webrtc::DesktopFrame> frame)108 void VideoFramePump::OnCaptureResult(
109 webrtc::DesktopCapturer::Result result,
110 std::unique_ptr<webrtc::DesktopFrame> frame) {
111 DCHECK(thread_checker_.CalledOnValidThread());
112
113 capture_scheduler_.OnCaptureCompleted();
114
115 captured_frame_timestamps_->capture_ended_time = base::TimeTicks::Now();
116
117 if (frame) {
118 webrtc::DesktopVector dpi =
119 frame->dpi().is_zero() ? webrtc::DesktopVector(kDefaultDpi, kDefaultDpi)
120 : frame->dpi();
121 if (!frame_size_.equals(frame->size()) || !frame_dpi_.equals(dpi)) {
122 frame_size_ = frame->size();
123 frame_dpi_ = dpi;
124 if (observer_)
125 observer_->OnVideoSizeChanged(this, frame_size_, frame_dpi_);
126 }
127 }
128
129 // Even when |frame| is nullptr we still need to post it to the encode thread
130 // to make sure frames are freed in the same order they are received and
131 // that we don't start capturing frame n+2 before frame n is freed.
132 base::PostTaskAndReplyWithResult(
133 encode_task_runner_.get(), FROM_HERE,
134 base::BindOnce(&VideoFramePump::EncodeFrame, encoder_.get(),
135 std::move(frame), std::move(captured_frame_timestamps_)),
136 base::BindOnce(&VideoFramePump::OnFrameEncoded,
137 weak_factory_.GetWeakPtr()));
138 }
139
CaptureNextFrame()140 void VideoFramePump::CaptureNextFrame() {
141 DCHECK(thread_checker_.CalledOnValidThread());
142
143 captured_frame_timestamps_.reset(new FrameTimestamps());
144 captured_frame_timestamps_->capture_started_time = base::TimeTicks::Now();
145
146 if (event_timestamps_source_) {
147 captured_frame_timestamps_->input_event_timestamps =
148 event_timestamps_source_->TakeLastEventTimestamps();
149 }
150
151 capturer_->CaptureFrame();
152 }
153
154 // static
155 std::unique_ptr<VideoFramePump::PacketWithTimestamps>
EncodeFrame(VideoEncoder * encoder,std::unique_ptr<webrtc::DesktopFrame> frame,std::unique_ptr<FrameTimestamps> timestamps)156 VideoFramePump::EncodeFrame(VideoEncoder* encoder,
157 std::unique_ptr<webrtc::DesktopFrame> frame,
158 std::unique_ptr<FrameTimestamps> timestamps) {
159 timestamps->encode_started_time = base::TimeTicks::Now();
160
161 std::unique_ptr<VideoPacket> packet;
162 // If |frame| is non-NULL then let the encoder process it.
163 if (frame)
164 packet = encoder->Encode(*frame);
165
166 // If |frame| is NULL, or the encoder returned nothing, return an empty
167 // packet.
168 if (!packet)
169 packet.reset(new VideoPacket());
170
171 if (frame)
172 packet->set_capture_time_ms(frame->capture_time_ms());
173
174 timestamps->encode_ended_time = base::TimeTicks::Now();
175 packet->set_encode_time_ms(
176 (timestamps->encode_ended_time - timestamps->encode_started_time)
177 .InMilliseconds());
178
179 return std::make_unique<PacketWithTimestamps>(std::move(packet),
180 std::move(timestamps));
181 }
182
OnFrameEncoded(std::unique_ptr<PacketWithTimestamps> packet)183 void VideoFramePump::OnFrameEncoded(
184 std::unique_ptr<PacketWithTimestamps> packet) {
185 DCHECK(thread_checker_.CalledOnValidThread());
186
187 capture_scheduler_.OnFrameEncoded(packet->packet.get());
188
189 if (send_pending_) {
190 pending_packets_.push_back(std::move(packet));
191 } else {
192 SendPacket(std::move(packet));
193 }
194 }
195
SendPacket(std::unique_ptr<PacketWithTimestamps> packet)196 void VideoFramePump::SendPacket(std::unique_ptr<PacketWithTimestamps> packet) {
197 DCHECK(thread_checker_.CalledOnValidThread());
198 DCHECK(!send_pending_);
199
200 packet->timestamps->can_send_time = base::TimeTicks::Now();
201 UpdateFrameTimers(packet->packet.get(), packet->timestamps.get());
202
203 send_pending_ = true;
204 video_stub_->ProcessVideoPacket(
205 std::move(packet->packet),
206 base::BindOnce(&VideoFramePump::OnVideoPacketSent,
207 weak_factory_.GetWeakPtr()));
208 }
209
UpdateFrameTimers(VideoPacket * packet,FrameTimestamps * timestamps)210 void VideoFramePump::UpdateFrameTimers(VideoPacket* packet,
211 FrameTimestamps* timestamps) {
212 if (!timestamps->input_event_timestamps.is_null()) {
213 packet->set_capture_pending_time_ms(
214 (timestamps->capture_started_time -
215 timestamps->input_event_timestamps.host_timestamp)
216 .InMilliseconds());
217 packet->set_latest_event_timestamp(
218 timestamps->input_event_timestamps.client_timestamp.ToInternalValue());
219 }
220
221 packet->set_capture_overhead_time_ms(
222 (timestamps->capture_ended_time - timestamps->capture_started_time)
223 .InMilliseconds() -
224 packet->capture_time_ms());
225
226 packet->set_encode_pending_time_ms(
227 (timestamps->encode_started_time - timestamps->capture_ended_time)
228 .InMilliseconds());
229
230 packet->set_send_pending_time_ms(
231 (timestamps->can_send_time - timestamps->encode_ended_time)
232 .InMilliseconds());
233 }
234
OnVideoPacketSent()235 void VideoFramePump::OnVideoPacketSent() {
236 DCHECK(thread_checker_.CalledOnValidThread());
237
238 send_pending_ = false;
239 capture_scheduler_.OnFrameSent();
240 keep_alive_timer_.Reset();
241
242 // Send next packet if any.
243 if (!pending_packets_.empty()) {
244 std::unique_ptr<PacketWithTimestamps> next =
245 std::move(pending_packets_.front());
246 pending_packets_.erase(pending_packets_.begin());
247 SendPacket(std::move(next));
248 }
249 }
250
SendKeepAlivePacket()251 void VideoFramePump::SendKeepAlivePacket() {
252 DCHECK(thread_checker_.CalledOnValidThread());
253
254 video_stub_->ProcessVideoPacket(
255 std::make_unique<VideoPacket>(),
256 base::BindOnce(&VideoFramePump::OnKeepAlivePacketSent,
257 weak_factory_.GetWeakPtr()));
258 }
259
OnKeepAlivePacketSent()260 void VideoFramePump::OnKeepAlivePacketSent() {
261 DCHECK(thread_checker_.CalledOnValidThread());
262
263 keep_alive_timer_.Reset();
264 }
265
266 } // namespace protocol
267 } // namespace remoting
268