1 /*
2 * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "video/frame_encode_metadata_writer.h"
12
13 #include <algorithm>
14 #include <utility>
15
16 #include "common_video/h264/sps_vui_rewriter.h"
17 #include "modules/include/module_common_types_public.h"
18 #include "modules/video_coding/include/video_coding_defines.h"
19 #include "rtc_base/logging.h"
20 #include "rtc_base/ref_counted_object.h"
21 #include "rtc_base/time_utils.h"
22
23 namespace webrtc {
24 namespace {
25 const int kMessagesThrottlingThreshold = 2;
26 const int kThrottleRatio = 100000;
27
28 class EncodedImageBufferWrapper : public EncodedImageBufferInterface {
29 public:
EncodedImageBufferWrapper(rtc::Buffer && buffer)30 explicit EncodedImageBufferWrapper(rtc::Buffer&& buffer)
31 : buffer_(std::move(buffer)) {}
32
data() const33 const uint8_t* data() const override { return buffer_.data(); }
data()34 uint8_t* data() override { return buffer_.data(); }
size() const35 size_t size() const override { return buffer_.size(); }
36
37 private:
38 rtc::Buffer buffer_;
39 };
40
41 } // namespace
42
43 FrameEncodeMetadataWriter::TimingFramesLayerInfo::TimingFramesLayerInfo() =
44 default;
45 FrameEncodeMetadataWriter::TimingFramesLayerInfo::~TimingFramesLayerInfo() =
46 default;
47
FrameEncodeMetadataWriter(EncodedImageCallback * frame_drop_callback)48 FrameEncodeMetadataWriter::FrameEncodeMetadataWriter(
49 EncodedImageCallback* frame_drop_callback)
50 : frame_drop_callback_(frame_drop_callback),
51 internal_source_(false),
52 framerate_fps_(0),
53 last_timing_frame_time_ms_(-1),
54 reordered_frames_logged_messages_(0),
55 stalled_encoder_logged_messages_(0) {
56 codec_settings_.timing_frame_thresholds = {-1, 0};
57 }
~FrameEncodeMetadataWriter()58 FrameEncodeMetadataWriter::~FrameEncodeMetadataWriter() {}
59
OnEncoderInit(const VideoCodec & codec,bool internal_source)60 void FrameEncodeMetadataWriter::OnEncoderInit(const VideoCodec& codec,
61 bool internal_source) {
62 MutexLock lock(&lock_);
63 codec_settings_ = codec;
64 internal_source_ = internal_source;
65 }
66
OnSetRates(const VideoBitrateAllocation & bitrate_allocation,uint32_t framerate_fps)67 void FrameEncodeMetadataWriter::OnSetRates(
68 const VideoBitrateAllocation& bitrate_allocation,
69 uint32_t framerate_fps) {
70 MutexLock lock(&lock_);
71 framerate_fps_ = framerate_fps;
72 const size_t num_spatial_layers = NumSpatialLayers();
73 if (timing_frames_info_.size() < num_spatial_layers) {
74 timing_frames_info_.resize(num_spatial_layers);
75 }
76 for (size_t i = 0; i < num_spatial_layers; ++i) {
77 timing_frames_info_[i].target_bitrate_bytes_per_sec =
78 bitrate_allocation.GetSpatialLayerSum(i) / 8;
79 }
80 }
81
OnEncodeStarted(const VideoFrame & frame)82 void FrameEncodeMetadataWriter::OnEncodeStarted(const VideoFrame& frame) {
83 MutexLock lock(&lock_);
84 if (internal_source_) {
85 return;
86 }
87
88 const size_t num_spatial_layers = NumSpatialLayers();
89 timing_frames_info_.resize(num_spatial_layers);
90 FrameMetadata metadata;
91 metadata.rtp_timestamp = frame.timestamp();
92 metadata.encode_start_time_ms = rtc::TimeMillis();
93 metadata.ntp_time_ms = frame.ntp_time_ms();
94 metadata.timestamp_us = frame.timestamp_us();
95 metadata.rotation = frame.rotation();
96 metadata.color_space = frame.color_space();
97 metadata.packet_infos = frame.packet_infos();
98 for (size_t si = 0; si < num_spatial_layers; ++si) {
99 RTC_DCHECK(timing_frames_info_[si].frames.empty() ||
100 rtc::TimeDiff(
101 frame.render_time_ms(),
102 timing_frames_info_[si].frames.back().timestamp_us / 1000) >=
103 0);
104 // If stream is disabled due to low bandwidth OnEncodeStarted still will be
105 // called and have to be ignored.
106 if (timing_frames_info_[si].target_bitrate_bytes_per_sec == 0)
107 continue;
108 if (timing_frames_info_[si].frames.size() == kMaxEncodeStartTimeListSize) {
109 ++stalled_encoder_logged_messages_;
110 if (stalled_encoder_logged_messages_ <= kMessagesThrottlingThreshold ||
111 stalled_encoder_logged_messages_ % kThrottleRatio == 0) {
112 RTC_LOG(LS_WARNING) << "Too many frames in the encode_start_list."
113 " Did encoder stall?";
114 if (stalled_encoder_logged_messages_ == kMessagesThrottlingThreshold) {
115 RTC_LOG(LS_WARNING)
116 << "Too many log messages. Further stalled encoder"
117 "warnings will be throttled.";
118 }
119 }
120 frame_drop_callback_->OnDroppedFrame(
121 EncodedImageCallback::DropReason::kDroppedByEncoder);
122 timing_frames_info_[si].frames.pop_front();
123 }
124 timing_frames_info_[si].frames.emplace_back(metadata);
125 }
126 }
127
FillTimingInfo(size_t simulcast_svc_idx,EncodedImage * encoded_image)128 void FrameEncodeMetadataWriter::FillTimingInfo(size_t simulcast_svc_idx,
129 EncodedImage* encoded_image) {
130 MutexLock lock(&lock_);
131 absl::optional<size_t> outlier_frame_size;
132 absl::optional<int64_t> encode_start_ms;
133 uint8_t timing_flags = VideoSendTiming::kNotTriggered;
134
135 int64_t encode_done_ms = rtc::TimeMillis();
136
137 // Encoders with internal sources do not call OnEncodeStarted
138 // |timing_frames_info_| may be not filled here.
139 if (!internal_source_) {
140 encode_start_ms =
141 ExtractEncodeStartTimeAndFillMetadata(simulcast_svc_idx, encoded_image);
142 }
143
144 if (timing_frames_info_.size() > simulcast_svc_idx) {
145 size_t target_bitrate =
146 timing_frames_info_[simulcast_svc_idx].target_bitrate_bytes_per_sec;
147 if (framerate_fps_ > 0 && target_bitrate > 0) {
148 // framerate and target bitrate were reported by encoder.
149 size_t average_frame_size = target_bitrate / framerate_fps_;
150 outlier_frame_size.emplace(
151 average_frame_size *
152 codec_settings_.timing_frame_thresholds.outlier_ratio_percent / 100);
153 }
154 }
155
156 // Outliers trigger timing frames, but do not affect scheduled timing
157 // frames.
158 if (outlier_frame_size && encoded_image->size() >= *outlier_frame_size) {
159 timing_flags |= VideoSendTiming::kTriggeredBySize;
160 }
161
162 // Check if it's time to send a timing frame.
163 int64_t timing_frame_delay_ms =
164 encoded_image->capture_time_ms_ - last_timing_frame_time_ms_;
165 // Trigger threshold if it's a first frame, too long passed since the last
166 // timing frame, or we already sent timing frame on a different simulcast
167 // stream with the same capture time.
168 if (last_timing_frame_time_ms_ == -1 ||
169 timing_frame_delay_ms >=
170 codec_settings_.timing_frame_thresholds.delay_ms ||
171 timing_frame_delay_ms == 0) {
172 timing_flags |= VideoSendTiming::kTriggeredByTimer;
173 last_timing_frame_time_ms_ = encoded_image->capture_time_ms_;
174 }
175
176 // Workaround for chromoting encoder: it passes encode start and finished
177 // timestamps in |timing_| field, but they (together with capture timestamp)
178 // are not in the WebRTC clock.
179 if (internal_source_ && encoded_image->timing_.encode_finish_ms > 0 &&
180 encoded_image->timing_.encode_start_ms > 0) {
181 int64_t clock_offset_ms =
182 encode_done_ms - encoded_image->timing_.encode_finish_ms;
183 // Translate capture timestamp to local WebRTC clock.
184 encoded_image->capture_time_ms_ += clock_offset_ms;
185 encoded_image->SetTimestamp(
186 static_cast<uint32_t>(encoded_image->capture_time_ms_ * 90));
187 encode_start_ms.emplace(encoded_image->timing_.encode_start_ms +
188 clock_offset_ms);
189 }
190
191 // If encode start is not available that means that encoder uses internal
192 // source. In that case capture timestamp may be from a different clock with a
193 // drift relative to rtc::TimeMillis(). We can't use it for Timing frames,
194 // because to being sent in the network capture time required to be less than
195 // all the other timestamps.
196 if (encode_start_ms) {
197 encoded_image->SetEncodeTime(*encode_start_ms, encode_done_ms);
198 encoded_image->timing_.flags = timing_flags;
199 } else {
200 encoded_image->timing_.flags = VideoSendTiming::kInvalid;
201 }
202 }
203
UpdateBitstream(const CodecSpecificInfo * codec_specific_info,EncodedImage * encoded_image)204 void FrameEncodeMetadataWriter::UpdateBitstream(
205 const CodecSpecificInfo* codec_specific_info,
206 EncodedImage* encoded_image) {
207 if (!codec_specific_info ||
208 codec_specific_info->codecType != kVideoCodecH264 ||
209 encoded_image->_frameType != VideoFrameType::kVideoFrameKey) {
210 return;
211 }
212
213 // Make sure that the data is not copied if owned by EncodedImage.
214 const EncodedImage& buffer = *encoded_image;
215 rtc::Buffer modified_buffer =
216 SpsVuiRewriter::ParseOutgoingBitstreamAndRewrite(
217 buffer, encoded_image->ColorSpace());
218
219 encoded_image->SetEncodedData(
220 new rtc::RefCountedObject<EncodedImageBufferWrapper>(
221 std::move(modified_buffer)));
222 }
223
Reset()224 void FrameEncodeMetadataWriter::Reset() {
225 MutexLock lock(&lock_);
226 for (auto& info : timing_frames_info_) {
227 info.frames.clear();
228 }
229 last_timing_frame_time_ms_ = -1;
230 reordered_frames_logged_messages_ = 0;
231 stalled_encoder_logged_messages_ = 0;
232 }
233
234 absl::optional<int64_t>
ExtractEncodeStartTimeAndFillMetadata(size_t simulcast_svc_idx,EncodedImage * encoded_image)235 FrameEncodeMetadataWriter::ExtractEncodeStartTimeAndFillMetadata(
236 size_t simulcast_svc_idx,
237 EncodedImage* encoded_image) {
238 absl::optional<int64_t> result;
239 size_t num_simulcast_svc_streams = timing_frames_info_.size();
240 if (simulcast_svc_idx < num_simulcast_svc_streams) {
241 auto metadata_list = &timing_frames_info_[simulcast_svc_idx].frames;
242 // Skip frames for which there was OnEncodeStarted but no OnEncodedImage
243 // call. These are dropped by encoder internally.
244 // Because some hardware encoders don't preserve capture timestamp we
245 // use RTP timestamps here.
246 while (!metadata_list->empty() &&
247 IsNewerTimestamp(encoded_image->Timestamp(),
248 metadata_list->front().rtp_timestamp)) {
249 frame_drop_callback_->OnDroppedFrame(
250 EncodedImageCallback::DropReason::kDroppedByEncoder);
251 metadata_list->pop_front();
252 }
253
254 encoded_image->content_type_ =
255 (codec_settings_.mode == VideoCodecMode::kScreensharing)
256 ? VideoContentType::SCREENSHARE
257 : VideoContentType::UNSPECIFIED;
258
259 if (!metadata_list->empty() &&
260 metadata_list->front().rtp_timestamp == encoded_image->Timestamp()) {
261 result.emplace(metadata_list->front().encode_start_time_ms);
262 encoded_image->capture_time_ms_ =
263 metadata_list->front().timestamp_us / 1000;
264 encoded_image->ntp_time_ms_ = metadata_list->front().ntp_time_ms;
265 encoded_image->rotation_ = metadata_list->front().rotation;
266 encoded_image->SetColorSpace(metadata_list->front().color_space);
267 encoded_image->SetPacketInfos(metadata_list->front().packet_infos);
268 metadata_list->pop_front();
269 } else {
270 ++reordered_frames_logged_messages_;
271 if (reordered_frames_logged_messages_ <= kMessagesThrottlingThreshold ||
272 reordered_frames_logged_messages_ % kThrottleRatio == 0) {
273 RTC_LOG(LS_WARNING) << "Frame with no encode started time recordings. "
274 "Encoder may be reordering frames "
275 "or not preserving RTP timestamps.";
276 if (reordered_frames_logged_messages_ == kMessagesThrottlingThreshold) {
277 RTC_LOG(LS_WARNING) << "Too many log messages. Further frames "
278 "reordering warnings will be throttled.";
279 }
280 }
281 }
282 }
283 return result;
284 }
285
NumSpatialLayers() const286 size_t FrameEncodeMetadataWriter::NumSpatialLayers() const {
287 size_t num_spatial_layers = codec_settings_.numberOfSimulcastStreams;
288 if (codec_settings_.codecType == kVideoCodecVP9) {
289 num_spatial_layers = std::max(
290 num_spatial_layers,
291 static_cast<size_t>(codec_settings_.VP9().numberOfSpatialLayers));
292 }
293 return std::max(num_spatial_layers, size_t{1});
294 }
295
296 } // namespace webrtc
297