1 /*
2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "video/rtp_streams_synchronizer2.h"
12
13 #include "absl/types/optional.h"
14 #include "call/syncable.h"
15 #include "rtc_base/checks.h"
16 #include "rtc_base/logging.h"
17 #include "rtc_base/time_utils.h"
18 #include "rtc_base/trace_event.h"
19 #include "system_wrappers/include/rtp_to_ntp_estimator.h"
20
21 namespace webrtc {
22 namespace internal {
23 namespace {
24 // Time interval for logging stats.
25 constexpr int64_t kStatsLogIntervalMs = 10000;
26 constexpr TimeDelta kSyncInterval = TimeDelta::Millis(1000);
27
UpdateMeasurements(StreamSynchronization::Measurements * stream,const Syncable::Info & info)28 bool UpdateMeasurements(StreamSynchronization::Measurements* stream,
29 const Syncable::Info& info) {
30 stream->latest_timestamp = info.latest_received_capture_timestamp;
31 stream->latest_receive_time_ms = info.latest_receive_time_ms;
32 bool new_rtcp_sr = false;
33 return stream->rtp_to_ntp.UpdateMeasurements(
34 info.capture_time_ntp_secs, info.capture_time_ntp_frac,
35 info.capture_time_source_clock, &new_rtcp_sr);
36 }
37
38 } // namespace
39
RtpStreamsSynchronizer(TaskQueueBase * main_queue,Syncable * syncable_video)40 RtpStreamsSynchronizer::RtpStreamsSynchronizer(TaskQueueBase* main_queue,
41 Syncable* syncable_video)
42 : task_queue_(main_queue),
43 syncable_video_(syncable_video),
44 last_stats_log_ms_(rtc::TimeMillis()) {
45 RTC_DCHECK(syncable_video);
46 }
47
~RtpStreamsSynchronizer()48 RtpStreamsSynchronizer::~RtpStreamsSynchronizer() {
49 RTC_DCHECK_RUN_ON(&main_checker_);
50 repeating_task_.Stop();
51 }
52
ConfigureSync(Syncable * syncable_audio)53 void RtpStreamsSynchronizer::ConfigureSync(Syncable* syncable_audio) {
54 RTC_DCHECK_RUN_ON(&main_checker_);
55
56 // Prevent expensive no-ops.
57 if (syncable_audio == syncable_audio_)
58 return;
59
60 syncable_audio_ = syncable_audio;
61 sync_.reset(nullptr);
62 if (!syncable_audio_) {
63 repeating_task_.Stop();
64 return;
65 }
66
67 sync_.reset(
68 new StreamSynchronization(syncable_video_->id(), syncable_audio_->id()));
69
70 if (repeating_task_.Running())
71 return;
72
73 repeating_task_ =
74 RepeatingTaskHandle::DelayedStart(task_queue_, kSyncInterval, [this]() {
75 UpdateDelay();
76 return kSyncInterval;
77 });
78 }
79
UpdateDelay()80 void RtpStreamsSynchronizer::UpdateDelay() {
81 RTC_DCHECK_RUN_ON(&main_checker_);
82
83 if (!syncable_audio_)
84 return;
85
86 RTC_DCHECK(sync_.get());
87
88 bool log_stats = false;
89 const int64_t now_ms = rtc::TimeMillis();
90 if (now_ms - last_stats_log_ms_ > kStatsLogIntervalMs) {
91 last_stats_log_ms_ = now_ms;
92 log_stats = true;
93 }
94
95 int64_t last_audio_receive_time_ms =
96 audio_measurement_.latest_receive_time_ms;
97 absl::optional<Syncable::Info> audio_info = syncable_audio_->GetInfo();
98 if (!audio_info || !UpdateMeasurements(&audio_measurement_, *audio_info)) {
99 return;
100 }
101
102 if (last_audio_receive_time_ms == audio_measurement_.latest_receive_time_ms) {
103 // No new audio packet has been received since last update.
104 return;
105 }
106
107 int64_t last_video_receive_ms = video_measurement_.latest_receive_time_ms;
108 absl::optional<Syncable::Info> video_info = syncable_video_->GetInfo();
109 if (!video_info || !UpdateMeasurements(&video_measurement_, *video_info)) {
110 return;
111 }
112
113 if (last_video_receive_ms == video_measurement_.latest_receive_time_ms) {
114 // No new video packet has been received since last update.
115 return;
116 }
117
118 int relative_delay_ms;
119 // Calculate how much later or earlier the audio stream is compared to video.
120 if (!sync_->ComputeRelativeDelay(audio_measurement_, video_measurement_,
121 &relative_delay_ms)) {
122 return;
123 }
124
125 if (log_stats) {
126 RTC_LOG(LS_INFO) << "Sync info stats: " << now_ms
127 << ", {ssrc: " << sync_->audio_stream_id() << ", "
128 << "cur_delay_ms: " << audio_info->current_delay_ms
129 << "} {ssrc: " << sync_->video_stream_id() << ", "
130 << "cur_delay_ms: " << video_info->current_delay_ms
131 << "} {relative_delay_ms: " << relative_delay_ms << "} ";
132 }
133
134 TRACE_COUNTER1("webrtc", "SyncCurrentVideoDelay",
135 video_info->current_delay_ms);
136 TRACE_COUNTER1("webrtc", "SyncCurrentAudioDelay",
137 audio_info->current_delay_ms);
138 TRACE_COUNTER1("webrtc", "SyncRelativeDelay", relative_delay_ms);
139
140 int target_audio_delay_ms = 0;
141 int target_video_delay_ms = video_info->current_delay_ms;
142 // Calculate the necessary extra audio delay and desired total video
143 // delay to get the streams in sync.
144 if (!sync_->ComputeDelays(relative_delay_ms, audio_info->current_delay_ms,
145 &target_audio_delay_ms, &target_video_delay_ms)) {
146 return;
147 }
148
149 if (log_stats) {
150 RTC_LOG(LS_INFO) << "Sync delay stats: " << now_ms
151 << ", {ssrc: " << sync_->audio_stream_id() << ", "
152 << "target_delay_ms: " << target_audio_delay_ms
153 << "} {ssrc: " << sync_->video_stream_id() << ", "
154 << "target_delay_ms: " << target_video_delay_ms << "} ";
155 }
156
157 if (!syncable_audio_->SetMinimumPlayoutDelay(target_audio_delay_ms)) {
158 sync_->ReduceAudioDelay();
159 }
160 if (!syncable_video_->SetMinimumPlayoutDelay(target_video_delay_ms)) {
161 sync_->ReduceVideoDelay();
162 }
163 }
164
165 // TODO(https://bugs.webrtc.org/7065): Move RtpToNtpEstimator out of
166 // RtpStreamsSynchronizer and into respective receive stream to always populate
167 // the estimated playout timestamp.
GetStreamSyncOffsetInMs(uint32_t rtp_timestamp,int64_t render_time_ms,int64_t * video_playout_ntp_ms,int64_t * stream_offset_ms,double * estimated_freq_khz) const168 bool RtpStreamsSynchronizer::GetStreamSyncOffsetInMs(
169 uint32_t rtp_timestamp,
170 int64_t render_time_ms,
171 int64_t* video_playout_ntp_ms,
172 int64_t* stream_offset_ms,
173 double* estimated_freq_khz) const {
174 RTC_DCHECK_RUN_ON(&main_checker_);
175
176 if (!syncable_audio_)
177 return false;
178
179 uint32_t audio_rtp_timestamp;
180 int64_t time_ms;
181 if (!syncable_audio_->GetPlayoutRtpTimestamp(&audio_rtp_timestamp,
182 &time_ms)) {
183 return false;
184 }
185
186 int64_t latest_audio_ntp;
187 if (!audio_measurement_.rtp_to_ntp.Estimate(audio_rtp_timestamp,
188 &latest_audio_ntp)) {
189 return false;
190 }
191
192 syncable_audio_->SetEstimatedPlayoutNtpTimestampMs(latest_audio_ntp, time_ms);
193
194 int64_t latest_video_ntp;
195 if (!video_measurement_.rtp_to_ntp.Estimate(rtp_timestamp,
196 &latest_video_ntp)) {
197 return false;
198 }
199
200 // Current audio ntp.
201 int64_t now_ms = rtc::TimeMillis();
202 latest_audio_ntp += (now_ms - time_ms);
203
204 // Remove video playout delay.
205 int64_t time_to_render_ms = render_time_ms - now_ms;
206 if (time_to_render_ms > 0)
207 latest_video_ntp -= time_to_render_ms;
208
209 *video_playout_ntp_ms = latest_video_ntp;
210 *stream_offset_ms = latest_audio_ntp - latest_video_ntp;
211 *estimated_freq_khz = video_measurement_.rtp_to_ntp.params()->frequency_khz;
212 return true;
213 }
214
215 } // namespace internal
216 } // namespace webrtc
217