1 /*
2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <vector>
12
13 #include "modules/video_coding/encoded_frame.h"
14 #include "modules/video_coding/generic_encoder.h"
15 #include "modules/video_coding/include/video_coding_defines.h"
16 #include "test/gtest.h"
17
18 namespace webrtc {
19 namespace test {
20 namespace {
FrameSize(const size_t & min_frame_size,const size_t & max_frame_size,const int & s,const int & i)21 inline size_t FrameSize(const size_t& min_frame_size,
22 const size_t& max_frame_size,
23 const int& s,
24 const int& i) {
25 return min_frame_size + (s + 1) * i % (max_frame_size - min_frame_size);
26 }
27
28 class FakeEncodedImageCallback : public EncodedImageCallback {
29 public:
FakeEncodedImageCallback()30 FakeEncodedImageCallback()
31 : last_frame_was_timing_(false),
32 num_frames_dropped_(0),
33 last_capture_timestamp_(-1) {}
OnEncodedImage(const EncodedImage & encoded_image,const CodecSpecificInfo * codec_specific_info,const RTPFragmentationHeader * fragmentation)34 Result OnEncodedImage(const EncodedImage& encoded_image,
35 const CodecSpecificInfo* codec_specific_info,
36 const RTPFragmentationHeader* fragmentation) override {
37 last_frame_was_timing_ =
38 encoded_image.timing_.flags != TimingFrameFlags::kInvalid &&
39 encoded_image.timing_.flags != TimingFrameFlags::kNotTriggered;
40 last_capture_timestamp_ = encoded_image.capture_time_ms_;
41 return Result(Result::OK);
42 };
43
OnDroppedFrame(DropReason reason)44 void OnDroppedFrame(DropReason reason) override { ++num_frames_dropped_; }
45
WasTimingFrame()46 bool WasTimingFrame() { return last_frame_was_timing_; }
47
GetNumFramesDropped()48 size_t GetNumFramesDropped() { return num_frames_dropped_; }
49
GetLastCaptureTimestamp()50 int64_t GetLastCaptureTimestamp() { return last_capture_timestamp_; }
51
52 private:
53 bool last_frame_was_timing_;
54 size_t num_frames_dropped_;
55 int64_t last_capture_timestamp_;
56 };
57
58 enum class FrameType {
59 kNormal,
60 kTiming,
61 kDropped,
62 };
63
64 // Emulates |num_frames| on |num_streams| frames with capture timestamps
65 // increased by 1 from 0. Size of each frame is between
66 // |min_frame_size| and |max_frame_size|, outliers are counted relatevely to
67 // |average_frame_sizes[]| for each stream.
GetTimingFrames(const int64_t delay_ms,const size_t min_frame_size,const size_t max_frame_size,std::vector<size_t> average_frame_sizes,const int num_streams,const int num_frames)68 std::vector<std::vector<FrameType>> GetTimingFrames(
69 const int64_t delay_ms,
70 const size_t min_frame_size,
71 const size_t max_frame_size,
72 std::vector<size_t> average_frame_sizes,
73 const int num_streams,
74 const int num_frames) {
75 FakeEncodedImageCallback sink;
76 VCMEncodedFrameCallback callback(&sink, nullptr);
77 const size_t kFramerate = 30;
78 callback.SetTimingFramesThresholds(
79 {delay_ms, kDefaultOutlierFrameSizePercent});
80 callback.OnFrameRateChanged(kFramerate);
81 int s, i;
82 std::vector<std::vector<FrameType>> result(num_streams);
83 for (s = 0; s < num_streams; ++s)
84 callback.OnTargetBitrateChanged(average_frame_sizes[s] * kFramerate, s);
85 int64_t current_timestamp = 0;
86 for (i = 0; i < num_frames; ++i) {
87 current_timestamp += 1;
88 for (s = 0; s < num_streams; ++s) {
89 // every (5+s)-th frame is dropped on s-th stream by design.
90 bool dropped = i % (5 + s) == 0;
91
92 EncodedImage image;
93 CodecSpecificInfo codec_specific;
94 image._length = FrameSize(min_frame_size, max_frame_size, s, i);
95 image.capture_time_ms_ = current_timestamp;
96 image._timeStamp = static_cast<uint32_t>(current_timestamp * 90);
97 codec_specific.codecType = kVideoCodecGeneric;
98 codec_specific.codecSpecific.generic.simulcast_idx = s;
99 callback.OnEncodeStarted(static_cast<uint32_t>(current_timestamp * 90),
100 current_timestamp, s);
101 if (dropped) {
102 result[s].push_back(FrameType::kDropped);
103 continue;
104 }
105 callback.OnEncodedImage(image, &codec_specific, nullptr);
106 if (sink.WasTimingFrame()) {
107 result[s].push_back(FrameType::kTiming);
108 } else {
109 result[s].push_back(FrameType::kNormal);
110 }
111 }
112 }
113 return result;
114 }
115 } // namespace
116
TEST(TestVCMEncodedFrameCallback,MarksTimingFramesPeriodicallyTogether)117 TEST(TestVCMEncodedFrameCallback, MarksTimingFramesPeriodicallyTogether) {
118 const int64_t kDelayMs = 29;
119 const size_t kMinFrameSize = 10;
120 const size_t kMaxFrameSize = 20;
121 const int kNumFrames = 1000;
122 const int kNumStreams = 3;
123 // No outliers as 1000 is larger than anything from range [10,20].
124 const std::vector<size_t> kAverageSize = {1000, 1000, 1000};
125 auto frames = GetTimingFrames(kDelayMs, kMinFrameSize, kMaxFrameSize,
126 kAverageSize, kNumStreams, kNumFrames);
127 // Timing frames should be tirggered every delayMs.
128 // As no outliers are expected, frames on all streams have to be
129 // marked together.
130 int last_timing_frame = -1;
131 for (int i = 0; i < kNumFrames; ++i) {
132 int num_normal = 0;
133 int num_timing = 0;
134 int num_dropped = 0;
135 for (int s = 0; s < kNumStreams; ++s) {
136 if (frames[s][i] == FrameType::kTiming) {
137 ++num_timing;
138 } else if (frames[s][i] == FrameType::kNormal) {
139 ++num_normal;
140 } else {
141 ++num_dropped;
142 }
143 }
144 // Can't have both normal and timing frames at the same timstamp.
145 EXPECT_TRUE(num_timing == 0 || num_normal == 0);
146 if (num_dropped < kNumStreams) {
147 if (last_timing_frame == -1 || i >= last_timing_frame + kDelayMs) {
148 // If didn't have timing frames for a period, current sent frame has to
149 // be one. No normal frames should be sent.
150 EXPECT_EQ(num_normal, 0);
151 } else {
152 // No unneeded timing frames should be sent.
153 EXPECT_EQ(num_timing, 0);
154 }
155 }
156 if (num_timing > 0)
157 last_timing_frame = i;
158 }
159 }
160
TEST(TestVCMEncodedFrameCallback,MarksOutliers)161 TEST(TestVCMEncodedFrameCallback, MarksOutliers) {
162 const int64_t kDelayMs = 29;
163 const size_t kMinFrameSize = 2495;
164 const size_t kMaxFrameSize = 2505;
165 const int kNumFrames = 1000;
166 const int kNumStreams = 3;
167 // Possible outliers as 1000 lies in range [995, 1005].
168 const std::vector<size_t> kAverageSize = {998, 1000, 1004};
169 auto frames = GetTimingFrames(kDelayMs, kMinFrameSize, kMaxFrameSize,
170 kAverageSize, kNumStreams, kNumFrames);
171 // All outliers should be marked.
172 for (int i = 0; i < kNumFrames; ++i) {
173 for (int s = 0; s < kNumStreams; ++s) {
174 if (FrameSize(kMinFrameSize, kMaxFrameSize, s, i) >=
175 kAverageSize[s] * kDefaultOutlierFrameSizePercent / 100) {
176 // Too big frame. May be dropped or timing, but not normal.
177 EXPECT_NE(frames[s][i], FrameType::kNormal);
178 }
179 }
180 }
181 }
182
TEST(TestVCMEncodedFrameCallback,NoTimingFrameIfNoEncodeStartTime)183 TEST(TestVCMEncodedFrameCallback, NoTimingFrameIfNoEncodeStartTime) {
184 EncodedImage image;
185 CodecSpecificInfo codec_specific;
186 int64_t timestamp = 1;
187 image._length = 500;
188 image.capture_time_ms_ = timestamp;
189 image._timeStamp = static_cast<uint32_t>(timestamp * 90);
190 codec_specific.codecType = kVideoCodecGeneric;
191 codec_specific.codecSpecific.generic.simulcast_idx = 0;
192 FakeEncodedImageCallback sink;
193 VCMEncodedFrameCallback callback(&sink, nullptr);
194 VideoCodec::TimingFrameTriggerThresholds thresholds;
195 thresholds.delay_ms = 1; // Make all frames timing frames.
196 callback.SetTimingFramesThresholds(thresholds);
197 callback.OnTargetBitrateChanged(500, 0);
198
199 // Verify a single frame works with encode start time set.
200 callback.OnEncodeStarted(static_cast<uint32_t>(timestamp * 90), timestamp, 0);
201 callback.OnEncodedImage(image, &codec_specific, nullptr);
202 EXPECT_TRUE(sink.WasTimingFrame());
203
204 // New frame, now skip OnEncodeStarted. Should not result in timing frame.
205 image.capture_time_ms_ = ++timestamp;
206 image._timeStamp = static_cast<uint32_t>(timestamp * 90);
207 callback.OnEncodedImage(image, &codec_specific, nullptr);
208 EXPECT_FALSE(sink.WasTimingFrame());
209 }
210
TEST(TestVCMEncodedFrameCallback,NotifiesAboutDroppedFrames)211 TEST(TestVCMEncodedFrameCallback, NotifiesAboutDroppedFrames) {
212 EncodedImage image;
213 CodecSpecificInfo codec_specific;
214 const int64_t kTimestampMs1 = 47721840;
215 const int64_t kTimestampMs2 = 47721850;
216 const int64_t kTimestampMs3 = 47721860;
217 const int64_t kTimestampMs4 = 47721870;
218 codec_specific.codecType = kVideoCodecGeneric;
219 codec_specific.codecSpecific.generic.simulcast_idx = 0;
220 FakeEncodedImageCallback sink;
221 VCMEncodedFrameCallback callback(&sink, nullptr);
222 // Any non-zero bitrate needed to be set before the first frame.
223 callback.OnTargetBitrateChanged(500, 0);
224 image.capture_time_ms_ = kTimestampMs1;
225 image._timeStamp = static_cast<uint32_t>(image.capture_time_ms_ * 90);
226 callback.OnEncodeStarted(image._timeStamp, image.capture_time_ms_, 0);
227 EXPECT_EQ(0u, sink.GetNumFramesDropped());
228 callback.OnEncodedImage(image, &codec_specific, nullptr);
229
230 image.capture_time_ms_ = kTimestampMs2;
231 image._timeStamp = static_cast<uint32_t>(image.capture_time_ms_ * 90);
232 callback.OnEncodeStarted(image._timeStamp, image.capture_time_ms_, 0);
233 // No OnEncodedImageCall for timestamp2. Yet, at this moment it's not known
234 // that frame with timestamp2 was dropped.
235 EXPECT_EQ(0u, sink.GetNumFramesDropped());
236
237 image.capture_time_ms_ = kTimestampMs3;
238 image._timeStamp = static_cast<uint32_t>(image.capture_time_ms_ * 90);
239 callback.OnEncodeStarted(image._timeStamp, image.capture_time_ms_, 0);
240 callback.OnEncodedImage(image, &codec_specific, nullptr);
241 EXPECT_EQ(1u, sink.GetNumFramesDropped());
242
243 image.capture_time_ms_ = kTimestampMs4;
244 image._timeStamp = static_cast<uint32_t>(image.capture_time_ms_ * 90);
245 callback.OnEncodeStarted(image._timeStamp, image.capture_time_ms_, 0);
246 callback.OnEncodedImage(image, &codec_specific, nullptr);
247 EXPECT_EQ(1u, sink.GetNumFramesDropped());
248 }
249
TEST(TestVCMEncodedFrameCallback,RestoresCaptureTimestamps)250 TEST(TestVCMEncodedFrameCallback, RestoresCaptureTimestamps) {
251 EncodedImage image;
252 CodecSpecificInfo codec_specific;
253 const int64_t kTimestampMs = 123456;
254 codec_specific.codecType = kVideoCodecGeneric;
255 codec_specific.codecSpecific.generic.simulcast_idx = 0;
256 FakeEncodedImageCallback sink;
257 VCMEncodedFrameCallback callback(&sink, nullptr);
258 // Any non-zero bitrate needed to be set before the first frame.
259 callback.OnTargetBitrateChanged(500, 0);
260 image.capture_time_ms_ = kTimestampMs; // Incorrect timesetamp.
261 image._timeStamp = static_cast<uint32_t>(image.capture_time_ms_ * 90);
262 callback.OnEncodeStarted(image._timeStamp, image.capture_time_ms_, 0);
263 image.capture_time_ms_ = 0; // Incorrect timesetamp.
264 callback.OnEncodedImage(image, &codec_specific, nullptr);
265 EXPECT_EQ(kTimestampMs, sink.GetLastCaptureTimestamp());
266 }
267
268 } // namespace test
269 } // namespace webrtc
270