1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "video/rtp_streams_synchronizer.h"
12 
13 #include "absl/types/optional.h"
14 #include "call/syncable.h"
15 #include "rtc_base/checks.h"
16 #include "rtc_base/logging.h"
17 #include "rtc_base/time_utils.h"
18 #include "rtc_base/trace_event.h"
19 #include "system_wrappers/include/rtp_to_ntp_estimator.h"
20 
21 namespace webrtc {
22 namespace {
23 // Time interval for logging stats.
24 constexpr int64_t kStatsLogIntervalMs = 10000;
25 
UpdateMeasurements(StreamSynchronization::Measurements * stream,const Syncable::Info & info)26 bool UpdateMeasurements(StreamSynchronization::Measurements* stream,
27                         const Syncable::Info& info) {
28   RTC_DCHECK(stream);
29   stream->latest_timestamp = info.latest_received_capture_timestamp;
30   stream->latest_receive_time_ms = info.latest_receive_time_ms;
31   bool new_rtcp_sr = false;
32   if (!stream->rtp_to_ntp.UpdateMeasurements(
33           info.capture_time_ntp_secs, info.capture_time_ntp_frac,
34           info.capture_time_source_clock, &new_rtcp_sr)) {
35     return false;
36   }
37   return true;
38 }
39 }  // namespace
40 
RtpStreamsSynchronizer(Syncable * syncable_video)41 RtpStreamsSynchronizer::RtpStreamsSynchronizer(Syncable* syncable_video)
42     : syncable_video_(syncable_video),
43       syncable_audio_(nullptr),
44       sync_(),
45       last_sync_time_(rtc::TimeNanos()),
46       last_stats_log_ms_(rtc::TimeMillis()) {
47   RTC_DCHECK(syncable_video);
48   process_thread_checker_.Detach();
49 }
50 
51 RtpStreamsSynchronizer::~RtpStreamsSynchronizer() = default;
52 
ConfigureSync(Syncable * syncable_audio)53 void RtpStreamsSynchronizer::ConfigureSync(Syncable* syncable_audio) {
54   MutexLock lock(&mutex_);
55   if (syncable_audio == syncable_audio_) {
56     // This prevents expensive no-ops.
57     return;
58   }
59 
60   syncable_audio_ = syncable_audio;
61   sync_.reset(nullptr);
62   if (syncable_audio_) {
63     sync_.reset(new StreamSynchronization(syncable_video_->id(),
64                                           syncable_audio_->id()));
65   }
66 }
67 
TimeUntilNextProcess()68 int64_t RtpStreamsSynchronizer::TimeUntilNextProcess() {
69   RTC_DCHECK_RUN_ON(&process_thread_checker_);
70   const int64_t kSyncIntervalMs = 1000;
71   return kSyncIntervalMs -
72          (rtc::TimeNanos() - last_sync_time_) / rtc::kNumNanosecsPerMillisec;
73 }
74 
Process()75 void RtpStreamsSynchronizer::Process() {
76   RTC_DCHECK_RUN_ON(&process_thread_checker_);
77   last_sync_time_ = rtc::TimeNanos();
78 
79   MutexLock lock(&mutex_);
80   if (!syncable_audio_) {
81     return;
82   }
83   RTC_DCHECK(sync_.get());
84 
85   bool log_stats = false;
86   const int64_t now_ms = rtc::TimeMillis();
87   if (now_ms - last_stats_log_ms_ > kStatsLogIntervalMs) {
88     last_stats_log_ms_ = now_ms;
89     log_stats = true;
90   }
91 
92   int64_t last_audio_receive_time_ms =
93       audio_measurement_.latest_receive_time_ms;
94   absl::optional<Syncable::Info> audio_info = syncable_audio_->GetInfo();
95   if (!audio_info || !UpdateMeasurements(&audio_measurement_, *audio_info)) {
96     return;
97   }
98 
99   if (last_audio_receive_time_ms == audio_measurement_.latest_receive_time_ms) {
100     // No new audio packet has been received since last update.
101     return;
102   }
103 
104   int64_t last_video_receive_ms = video_measurement_.latest_receive_time_ms;
105   absl::optional<Syncable::Info> video_info = syncable_video_->GetInfo();
106   if (!video_info || !UpdateMeasurements(&video_measurement_, *video_info)) {
107     return;
108   }
109 
110   if (last_video_receive_ms == video_measurement_.latest_receive_time_ms) {
111     // No new video packet has been received since last update.
112     return;
113   }
114 
115   int relative_delay_ms;
116   // Calculate how much later or earlier the audio stream is compared to video.
117   if (!sync_->ComputeRelativeDelay(audio_measurement_, video_measurement_,
118                                    &relative_delay_ms)) {
119     return;
120   }
121 
122   if (log_stats) {
123     RTC_LOG(LS_INFO) << "Sync info stats: " << now_ms
124                      << ", {ssrc: " << sync_->audio_stream_id() << ", "
125                      << "cur_delay_ms: " << audio_info->current_delay_ms
126                      << "} {ssrc: " << sync_->video_stream_id() << ", "
127                      << "cur_delay_ms: " << video_info->current_delay_ms
128                      << "} {relative_delay_ms: " << relative_delay_ms << "} ";
129   }
130 
131   TRACE_COUNTER1("webrtc", "SyncCurrentVideoDelay",
132                  video_info->current_delay_ms);
133   TRACE_COUNTER1("webrtc", "SyncCurrentAudioDelay",
134                  audio_info->current_delay_ms);
135   TRACE_COUNTER1("webrtc", "SyncRelativeDelay", relative_delay_ms);
136 
137   int target_audio_delay_ms = 0;
138   int target_video_delay_ms = video_info->current_delay_ms;
139   // Calculate the necessary extra audio delay and desired total video
140   // delay to get the streams in sync.
141   if (!sync_->ComputeDelays(relative_delay_ms, audio_info->current_delay_ms,
142                             &target_audio_delay_ms, &target_video_delay_ms)) {
143     return;
144   }
145 
146   if (log_stats) {
147     RTC_LOG(LS_INFO) << "Sync delay stats: " << now_ms
148                      << ", {ssrc: " << sync_->audio_stream_id() << ", "
149                      << "target_delay_ms: " << target_audio_delay_ms
150                      << "} {ssrc: " << sync_->video_stream_id() << ", "
151                      << "target_delay_ms: " << target_video_delay_ms << "} ";
152   }
153 
154   syncable_audio_->SetMinimumPlayoutDelay(target_audio_delay_ms);
155   syncable_video_->SetMinimumPlayoutDelay(target_video_delay_ms);
156 }
157 
158 // TODO(https://bugs.webrtc.org/7065): Move RtpToNtpEstimator out of
159 // RtpStreamsSynchronizer and into respective receive stream to always populate
160 // the estimated playout timestamp.
GetStreamSyncOffsetInMs(uint32_t rtp_timestamp,int64_t render_time_ms,int64_t * video_playout_ntp_ms,int64_t * stream_offset_ms,double * estimated_freq_khz) const161 bool RtpStreamsSynchronizer::GetStreamSyncOffsetInMs(
162     uint32_t rtp_timestamp,
163     int64_t render_time_ms,
164     int64_t* video_playout_ntp_ms,
165     int64_t* stream_offset_ms,
166     double* estimated_freq_khz) const {
167   MutexLock lock(&mutex_);
168   if (!syncable_audio_) {
169     return false;
170   }
171 
172   uint32_t audio_rtp_timestamp;
173   int64_t time_ms;
174   if (!syncable_audio_->GetPlayoutRtpTimestamp(&audio_rtp_timestamp,
175                                                &time_ms)) {
176     return false;
177   }
178 
179   int64_t latest_audio_ntp;
180   if (!audio_measurement_.rtp_to_ntp.Estimate(audio_rtp_timestamp,
181                                               &latest_audio_ntp)) {
182     return false;
183   }
184 
185   syncable_audio_->SetEstimatedPlayoutNtpTimestampMs(latest_audio_ntp, time_ms);
186 
187   int64_t latest_video_ntp;
188   if (!video_measurement_.rtp_to_ntp.Estimate(rtp_timestamp,
189                                               &latest_video_ntp)) {
190     return false;
191   }
192 
193   // Current audio ntp.
194   int64_t now_ms = rtc::TimeMillis();
195   latest_audio_ntp += (now_ms - time_ms);
196 
197   // Remove video playout delay.
198   int64_t time_to_render_ms = render_time_ms - now_ms;
199   if (time_to_render_ms > 0)
200     latest_video_ntp -= time_to_render_ms;
201 
202   *video_playout_ntp_ms = latest_video_ntp;
203   *stream_offset_ms = latest_audio_ntp - latest_video_ntp;
204   *estimated_freq_khz = video_measurement_.rtp_to_ntp.params()->frequency_khz;
205   return true;
206 }
207 
208 }  // namespace webrtc
209