1 /*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <algorithm>
12 #include <limits>
13 #include <utility>
14
15 #include "api/video/i420_buffer.h"
16 #include "media/base/videoadapter.h"
17 #include "modules/video_coding/codecs/vp8/temporal_layers.h"
18 #include "modules/video_coding/utility/default_video_bitrate_allocator.h"
19 #include "rtc_base/fakeclock.h"
20 #include "rtc_base/logging.h"
21 #include "system_wrappers/include/metrics_default.h"
22 #include "system_wrappers/include/sleep.h"
23 #include "test/encoder_settings.h"
24 #include "test/fake_encoder.h"
25 #include "test/frame_generator.h"
26 #include "test/gmock.h"
27 #include "test/gtest.h"
28 #include "video/send_statistics_proxy.h"
29 #include "video/video_stream_encoder.h"
30
31 namespace {
32 const int kMinPixelsPerFrame = 320 * 180;
33 const int kMinFramerateFps = 2;
34 const int64_t kFrameTimeoutMs = 100;
35 const unsigned char kNumSlDummy = 0;
36 } // namespace
37
38 namespace webrtc {
39
40 using DegredationPreference = VideoSendStream::DegradationPreference;
41 using ScaleReason = AdaptationObserverInterface::AdaptReason;
42 using ::testing::_;
43 using ::testing::Return;
44
45 namespace {
46 const size_t kMaxPayloadLength = 1440;
47 const int kTargetBitrateBps = 1000000;
48 const int kLowTargetBitrateBps = kTargetBitrateBps / 10;
49 const int kMaxInitialFramedrop = 4;
50 const int kDefaultFramerate = 30;
51
52 class TestBuffer : public webrtc::I420Buffer {
53 public:
TestBuffer(rtc::Event * event,int width,int height)54 TestBuffer(rtc::Event* event, int width, int height)
55 : I420Buffer(width, height), event_(event) {}
56
57 private:
58 friend class rtc::RefCountedObject<TestBuffer>;
~TestBuffer()59 ~TestBuffer() override {
60 if (event_)
61 event_->Set();
62 }
63 rtc::Event* const event_;
64 };
65
66 class CpuOveruseDetectorProxy : public OveruseFrameDetector {
67 public:
CpuOveruseDetectorProxy(const CpuOveruseOptions & options,AdaptationObserverInterface * overuse_observer,EncodedFrameObserver * encoder_timing_,CpuOveruseMetricsObserver * metrics_observer)68 CpuOveruseDetectorProxy(const CpuOveruseOptions& options,
69 AdaptationObserverInterface* overuse_observer,
70 EncodedFrameObserver* encoder_timing_,
71 CpuOveruseMetricsObserver* metrics_observer)
72 : OveruseFrameDetector(options,
73 overuse_observer,
74 encoder_timing_,
75 metrics_observer),
76 last_target_framerate_fps_(-1) {}
~CpuOveruseDetectorProxy()77 virtual ~CpuOveruseDetectorProxy() {}
78
OnTargetFramerateUpdated(int framerate_fps)79 void OnTargetFramerateUpdated(int framerate_fps) override {
80 rtc::CritScope cs(&lock_);
81 last_target_framerate_fps_ = framerate_fps;
82 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
83 }
84
GetLastTargetFramerate()85 int GetLastTargetFramerate() {
86 rtc::CritScope cs(&lock_);
87 return last_target_framerate_fps_;
88 }
89
90 private:
91 rtc::CriticalSection lock_;
92 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
93 };
94
95 class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
96 public:
VideoStreamEncoderUnderTest(SendStatisticsProxy * stats_proxy,const VideoSendStream::Config::EncoderSettings & settings)97 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
98 const VideoSendStream::Config::EncoderSettings& settings)
99 : VideoStreamEncoder(
100 1 /* number_of_cores */,
101 stats_proxy,
102 settings,
103 nullptr /* pre_encode_callback */,
104 nullptr /* encoder_timing */,
105 std::unique_ptr<OveruseFrameDetector>(
106 overuse_detector_proxy_ = new CpuOveruseDetectorProxy(
107 GetCpuOveruseOptions(settings.full_overuse_time),
108 this,
109 nullptr,
110 stats_proxy))) {}
111
PostTaskAndWait(bool down,AdaptReason reason)112 void PostTaskAndWait(bool down, AdaptReason reason) {
113 rtc::Event event(false, false);
114 encoder_queue()->PostTask([this, &event, reason, down] {
115 down ? AdaptDown(reason) : AdaptUp(reason);
116 event.Set();
117 });
118 ASSERT_TRUE(event.Wait(5000));
119 }
120
121 // This is used as a synchronisation mechanism, to make sure that the
122 // encoder queue is not blocked before we start sending it frames.
WaitUntilTaskQueueIsIdle()123 void WaitUntilTaskQueueIsIdle() {
124 rtc::Event event(false, false);
125 encoder_queue()->PostTask([&event] {
126 event.Set();
127 });
128 ASSERT_TRUE(event.Wait(5000));
129 }
130
TriggerCpuOveruse()131 void TriggerCpuOveruse() { PostTaskAndWait(true, AdaptReason::kCpu); }
132
TriggerCpuNormalUsage()133 void TriggerCpuNormalUsage() { PostTaskAndWait(false, AdaptReason::kCpu); }
134
TriggerQualityLow()135 void TriggerQualityLow() { PostTaskAndWait(true, AdaptReason::kQuality); }
136
TriggerQualityHigh()137 void TriggerQualityHigh() { PostTaskAndWait(false, AdaptReason::kQuality); }
138
139 CpuOveruseDetectorProxy* overuse_detector_proxy_;
140 };
141
142 class VideoStreamFactory
143 : public VideoEncoderConfig::VideoStreamFactoryInterface {
144 public:
VideoStreamFactory(size_t num_temporal_layers,int framerate)145 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
146 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
147 EXPECT_GT(num_temporal_layers, 0u);
148 EXPECT_GT(framerate, 0);
149 }
150
151 private:
CreateEncoderStreams(int width,int height,const VideoEncoderConfig & encoder_config)152 std::vector<VideoStream> CreateEncoderStreams(
153 int width,
154 int height,
155 const VideoEncoderConfig& encoder_config) override {
156 std::vector<VideoStream> streams =
157 test::CreateVideoStreams(width, height, encoder_config);
158 for (VideoStream& stream : streams) {
159 stream.temporal_layer_thresholds_bps.resize(num_temporal_layers_ - 1);
160 stream.max_framerate = framerate_;
161 }
162 return streams;
163 }
164
165 const size_t num_temporal_layers_;
166 const int framerate_;
167 };
168
169
170 class AdaptingFrameForwarder : public test::FrameForwarder {
171 public:
AdaptingFrameForwarder()172 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
~AdaptingFrameForwarder()173 ~AdaptingFrameForwarder() override {}
174
set_adaptation_enabled(bool enabled)175 void set_adaptation_enabled(bool enabled) {
176 rtc::CritScope cs(&crit_);
177 adaptation_enabled_ = enabled;
178 }
179
adaption_enabled() const180 bool adaption_enabled() const {
181 rtc::CritScope cs(&crit_);
182 return adaptation_enabled_;
183 }
184
last_wants() const185 rtc::VideoSinkWants last_wants() const {
186 rtc::CritScope cs(&crit_);
187 return last_wants_;
188 }
189
IncomingCapturedFrame(const VideoFrame & video_frame)190 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
191 int cropped_width = 0;
192 int cropped_height = 0;
193 int out_width = 0;
194 int out_height = 0;
195 if (adaption_enabled()) {
196 if (adapter_.AdaptFrameResolution(
197 video_frame.width(), video_frame.height(),
198 video_frame.timestamp_us() * 1000, &cropped_width,
199 &cropped_height, &out_width, &out_height)) {
200 VideoFrame adapted_frame(new rtc::RefCountedObject<TestBuffer>(
201 nullptr, out_width, out_height),
202 99, 99, kVideoRotation_0);
203 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
204 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
205 }
206 } else {
207 test::FrameForwarder::IncomingCapturedFrame(video_frame);
208 }
209 }
210
AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame> * sink,const rtc::VideoSinkWants & wants)211 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
212 const rtc::VideoSinkWants& wants) override {
213 rtc::CritScope cs(&crit_);
214 last_wants_ = sink_wants();
215 adapter_.OnResolutionFramerateRequest(wants.target_pixel_count,
216 wants.max_pixel_count,
217 wants.max_framerate_fps);
218 test::FrameForwarder::AddOrUpdateSink(sink, wants);
219 }
220 cricket::VideoAdapter adapter_;
221 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
222 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
223 };
224
225 class MockableSendStatisticsProxy : public SendStatisticsProxy {
226 public:
MockableSendStatisticsProxy(Clock * clock,const VideoSendStream::Config & config,VideoEncoderConfig::ContentType content_type)227 MockableSendStatisticsProxy(Clock* clock,
228 const VideoSendStream::Config& config,
229 VideoEncoderConfig::ContentType content_type)
230 : SendStatisticsProxy(clock, config, content_type) {}
231
GetStats()232 VideoSendStream::Stats GetStats() override {
233 rtc::CritScope cs(&lock_);
234 if (mock_stats_)
235 return *mock_stats_;
236 return SendStatisticsProxy::GetStats();
237 }
238
SetMockStats(const VideoSendStream::Stats & stats)239 void SetMockStats(const VideoSendStream::Stats& stats) {
240 rtc::CritScope cs(&lock_);
241 mock_stats_.emplace(stats);
242 }
243
ResetMockStats()244 void ResetMockStats() {
245 rtc::CritScope cs(&lock_);
246 mock_stats_.reset();
247 }
248
249 private:
250 rtc::CriticalSection lock_;
251 rtc::Optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
252 };
253
254 class MockBitrateObserver : public VideoBitrateAllocationObserver {
255 public:
256 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const BitrateAllocation&));
257 };
258
259 } // namespace
260
261 class VideoStreamEncoderTest : public ::testing::Test {
262 public:
263 static const int kDefaultTimeoutMs = 30 * 1000;
264
VideoStreamEncoderTest()265 VideoStreamEncoderTest()
266 : video_send_config_(VideoSendStream::Config(nullptr)),
267 codec_width_(320),
268 codec_height_(240),
269 max_framerate_(30),
270 fake_encoder_(),
271 stats_proxy_(new MockableSendStatisticsProxy(
272 Clock::GetRealTimeClock(),
273 video_send_config_,
274 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
275 sink_(&fake_encoder_) {}
276
SetUp()277 void SetUp() override {
278 metrics::Reset();
279 video_send_config_ = VideoSendStream::Config(nullptr);
280 video_send_config_.encoder_settings.encoder = &fake_encoder_;
281 video_send_config_.encoder_settings.payload_name = "FAKE";
282 video_send_config_.encoder_settings.payload_type = 125;
283
284 VideoEncoderConfig video_encoder_config;
285 test::FillEncoderConfiguration(1, &video_encoder_config);
286 video_encoder_config.video_stream_factory =
287 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
288 video_encoder_config_ = video_encoder_config.Copy();
289
290 // Framerate limit is specified by the VideoStreamFactory.
291 std::vector<VideoStream> streams =
292 video_encoder_config.video_stream_factory->CreateEncoderStreams(
293 codec_width_, codec_height_, video_encoder_config);
294 max_framerate_ = streams[0].max_framerate;
295 fake_clock_.SetTimeMicros(1234);
296
297 ConfigureEncoder(std::move(video_encoder_config), true /* nack_enabled */);
298 }
299
ConfigureEncoder(VideoEncoderConfig video_encoder_config,bool nack_enabled)300 void ConfigureEncoder(VideoEncoderConfig video_encoder_config,
301 bool nack_enabled) {
302 if (video_stream_encoder_)
303 video_stream_encoder_->Stop();
304 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
305 stats_proxy_.get(), video_send_config_.encoder_settings));
306 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
307 video_stream_encoder_->SetSource(
308 &video_source_,
309 VideoSendStream::DegradationPreference::kMaintainFramerate);
310 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
311 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
312 kMaxPayloadLength, nack_enabled);
313 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
314 }
315
ResetEncoder(const std::string & payload_name,size_t num_streams,size_t num_temporal_layers,unsigned char num_spatial_layers,bool nack_enabled,bool screenshare)316 void ResetEncoder(const std::string& payload_name,
317 size_t num_streams,
318 size_t num_temporal_layers,
319 unsigned char num_spatial_layers,
320 bool nack_enabled,
321 bool screenshare) {
322 video_send_config_.encoder_settings.payload_name = payload_name;
323
324 VideoEncoderConfig video_encoder_config;
325 video_encoder_config.number_of_streams = num_streams;
326 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
327 video_encoder_config.video_stream_factory =
328 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
329 kDefaultFramerate);
330 video_encoder_config.content_type =
331 screenshare ? VideoEncoderConfig::ContentType::kScreen
332 : VideoEncoderConfig::ContentType::kRealtimeVideo;
333 if (payload_name == "VP9") {
334 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
335 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
336 video_encoder_config.encoder_specific_settings =
337 new rtc::RefCountedObject<
338 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
339 }
340 ConfigureEncoder(std::move(video_encoder_config), nack_enabled);
341 }
342
CreateFrame(int64_t ntp_time_ms,rtc::Event * destruction_event) const343 VideoFrame CreateFrame(int64_t ntp_time_ms,
344 rtc::Event* destruction_event) const {
345 VideoFrame frame(new rtc::RefCountedObject<TestBuffer>(
346 destruction_event, codec_width_, codec_height_),
347 99, 99, kVideoRotation_0);
348 frame.set_ntp_time_ms(ntp_time_ms);
349 return frame;
350 }
351
CreateFrame(int64_t ntp_time_ms,int width,int height) const352 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
353 VideoFrame frame(
354 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height), 99, 99,
355 kVideoRotation_0);
356 frame.set_ntp_time_ms(ntp_time_ms);
357 frame.set_timestamp_us(ntp_time_ms * 1000);
358 return frame;
359 }
360
VerifyNoLimitation(const rtc::VideoSinkWants & wants)361 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
362 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
363 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
364 EXPECT_FALSE(wants.target_pixel_count);
365 }
366
VerifyFpsEqResolutionEq(const rtc::VideoSinkWants & wants1,const rtc::VideoSinkWants & wants2)367 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
368 const rtc::VideoSinkWants& wants2) {
369 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
370 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
371 }
372
VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants & wants1,const rtc::VideoSinkWants & wants2)373 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
374 const rtc::VideoSinkWants& wants2) {
375 EXPECT_EQ(std::numeric_limits<int>::max(), wants1.max_framerate_fps);
376 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
377 EXPECT_GT(wants1.max_pixel_count, 0);
378 }
379
VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants & wants1,const rtc::VideoSinkWants & wants2)380 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
381 const rtc::VideoSinkWants& wants2) {
382 EXPECT_EQ(std::numeric_limits<int>::max(), wants1.max_framerate_fps);
383 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
384 }
385
VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants & wants1,const rtc::VideoSinkWants & wants2)386 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
387 const rtc::VideoSinkWants& wants2) {
388 EXPECT_EQ(std::numeric_limits<int>::max(), wants1.max_framerate_fps);
389 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
390 }
391
VerifyFpsLtResolutionEq(const rtc::VideoSinkWants & wants1,const rtc::VideoSinkWants & wants2)392 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
393 const rtc::VideoSinkWants& wants2) {
394 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
395 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
396 }
397
VerifyFpsGtResolutionEq(const rtc::VideoSinkWants & wants1,const rtc::VideoSinkWants & wants2)398 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
399 const rtc::VideoSinkWants& wants2) {
400 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
401 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
402 }
403
VerifyFpsEqResolutionLt(const rtc::VideoSinkWants & wants1,const rtc::VideoSinkWants & wants2)404 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
405 const rtc::VideoSinkWants& wants2) {
406 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
407 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
408 EXPECT_GT(wants1.max_pixel_count, 0);
409 }
410
VerifyFpsEqResolutionGt(const rtc::VideoSinkWants & wants1,const rtc::VideoSinkWants & wants2)411 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
412 const rtc::VideoSinkWants& wants2) {
413 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
414 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
415 }
416
VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants & wants,int pixel_count)417 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
418 int pixel_count) {
419 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
420 EXPECT_LT(wants.max_pixel_count, pixel_count);
421 EXPECT_GT(wants.max_pixel_count, 0);
422 }
423
VerifyFpsLtResolutionMax(const rtc::VideoSinkWants & wants,int fps)424 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
425 EXPECT_LT(wants.max_framerate_fps, fps);
426 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
427 EXPECT_FALSE(wants.target_pixel_count);
428 }
429
VerifyFpsEqResolutionMax(const rtc::VideoSinkWants & wants,int expected_fps)430 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
431 int expected_fps) {
432 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
433 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
434 EXPECT_FALSE(wants.target_pixel_count);
435 }
436
WaitForEncodedFrame(int64_t expected_ntp_time)437 void WaitForEncodedFrame(int64_t expected_ntp_time) {
438 sink_.WaitForEncodedFrame(expected_ntp_time);
439 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
440 }
441
TimedWaitForEncodedFrame(int64_t expected_ntp_time,int64_t timeout_ms)442 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
443 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
444 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
445 return ok;
446 }
447
WaitForEncodedFrame(uint32_t expected_width,uint32_t expected_height)448 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
449 sink_.WaitForEncodedFrame(expected_width, expected_height);
450 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
451 }
452
ExpectDroppedFrame()453 void ExpectDroppedFrame() {
454 sink_.ExpectDroppedFrame();
455 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
456 }
457
WaitForFrame(int64_t timeout_ms)458 bool WaitForFrame(int64_t timeout_ms) {
459 bool ok = sink_.WaitForFrame(timeout_ms);
460 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
461 return ok;
462 }
463
464 class TestEncoder : public test::FakeEncoder {
465 public:
TestEncoder()466 TestEncoder()
467 : FakeEncoder(Clock::GetRealTimeClock()),
468 continue_encode_event_(false, false) {}
469
codec_config() const470 VideoCodec codec_config() const {
471 rtc::CritScope lock(&crit_sect_);
472 return config_;
473 }
474
BlockNextEncode()475 void BlockNextEncode() {
476 rtc::CritScope lock(&local_crit_sect_);
477 block_next_encode_ = true;
478 }
479
GetScalingSettings() const480 VideoEncoder::ScalingSettings GetScalingSettings() const override {
481 rtc::CritScope lock(&local_crit_sect_);
482 if (quality_scaling_)
483 return VideoEncoder::ScalingSettings(true, 1, 2, kMinPixelsPerFrame);
484 return VideoEncoder::ScalingSettings(false);
485 }
486
ContinueEncode()487 void ContinueEncode() { continue_encode_event_.Set(); }
488
CheckLastTimeStampsMatch(int64_t ntp_time_ms,uint32_t timestamp) const489 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
490 uint32_t timestamp) const {
491 rtc::CritScope lock(&local_crit_sect_);
492 EXPECT_EQ(timestamp_, timestamp);
493 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
494 }
495
SetQualityScaling(bool b)496 void SetQualityScaling(bool b) {
497 rtc::CritScope lock(&local_crit_sect_);
498 quality_scaling_ = b;
499 }
500
ForceInitEncodeFailure(bool force_failure)501 void ForceInitEncodeFailure(bool force_failure) {
502 rtc::CritScope lock(&local_crit_sect_);
503 force_init_encode_failed_ = force_failure;
504 }
505
506 private:
Encode(const VideoFrame & input_image,const CodecSpecificInfo * codec_specific_info,const std::vector<FrameType> * frame_types)507 int32_t Encode(const VideoFrame& input_image,
508 const CodecSpecificInfo* codec_specific_info,
509 const std::vector<FrameType>* frame_types) override {
510 bool block_encode;
511 {
512 rtc::CritScope lock(&local_crit_sect_);
513 EXPECT_GT(input_image.timestamp(), timestamp_);
514 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
515 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
516
517 timestamp_ = input_image.timestamp();
518 ntp_time_ms_ = input_image.ntp_time_ms();
519 last_input_width_ = input_image.width();
520 last_input_height_ = input_image.height();
521 block_encode = block_next_encode_;
522 block_next_encode_ = false;
523 }
524 int32_t result =
525 FakeEncoder::Encode(input_image, codec_specific_info, frame_types);
526 if (block_encode)
527 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
528 return result;
529 }
530
InitEncode(const VideoCodec * config,int32_t number_of_cores,size_t max_payload_size)531 int32_t InitEncode(const VideoCodec* config,
532 int32_t number_of_cores,
533 size_t max_payload_size) override {
534 int res =
535 FakeEncoder::InitEncode(config, number_of_cores, max_payload_size);
536 rtc::CritScope lock(&local_crit_sect_);
537 if (config->codecType == kVideoCodecVP8 && config->VP8().tl_factory) {
538 // Simulate setting up temporal layers, in order to validate the life
539 // cycle of these objects.
540 int num_streams = std::max<int>(1, config->numberOfSimulcastStreams);
541 int num_temporal_layers =
542 std::max<int>(1, config->VP8().numberOfTemporalLayers);
543 for (int i = 0; i < num_streams; ++i) {
544 allocated_temporal_layers_.emplace_back(
545 config->VP8().tl_factory->Create(i, num_temporal_layers, 42));
546 }
547 }
548 if (force_init_encode_failed_)
549 return -1;
550 return res;
551 }
552
553 rtc::CriticalSection local_crit_sect_;
554 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
555 rtc::Event continue_encode_event_;
556 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
557 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
558 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
559 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
560 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
561 std::vector<std::unique_ptr<TemporalLayers>> allocated_temporal_layers_
562 RTC_GUARDED_BY(local_crit_sect_);
563 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
564 };
565
566 class TestSink : public VideoStreamEncoder::EncoderSink {
567 public:
TestSink(TestEncoder * test_encoder)568 explicit TestSink(TestEncoder* test_encoder)
569 : test_encoder_(test_encoder), encoded_frame_event_(false, false) {}
570
WaitForEncodedFrame(int64_t expected_ntp_time)571 void WaitForEncodedFrame(int64_t expected_ntp_time) {
572 EXPECT_TRUE(
573 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
574 }
575
TimedWaitForEncodedFrame(int64_t expected_ntp_time,int64_t timeout_ms)576 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
577 int64_t timeout_ms) {
578 uint32_t timestamp = 0;
579 if (!encoded_frame_event_.Wait(timeout_ms))
580 return false;
581 {
582 rtc::CritScope lock(&crit_);
583 timestamp = last_timestamp_;
584 }
585 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
586 return true;
587 }
588
WaitForEncodedFrame(uint32_t expected_width,uint32_t expected_height)589 void WaitForEncodedFrame(uint32_t expected_width,
590 uint32_t expected_height) {
591 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
592 CheckLastFrameSizeMathces(expected_width, expected_height);
593 }
594
CheckLastFrameSizeMathces(uint32_t expected_width,uint32_t expected_height)595 void CheckLastFrameSizeMathces(uint32_t expected_width,
596 uint32_t expected_height) {
597 uint32_t width = 0;
598 uint32_t height = 0;
599 {
600 rtc::CritScope lock(&crit_);
601 width = last_width_;
602 height = last_height_;
603 }
604 EXPECT_EQ(expected_height, height);
605 EXPECT_EQ(expected_width, width);
606 }
607
ExpectDroppedFrame()608 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
609
WaitForFrame(int64_t timeout_ms)610 bool WaitForFrame(int64_t timeout_ms) {
611 return encoded_frame_event_.Wait(timeout_ms);
612 }
613
SetExpectNoFrames()614 void SetExpectNoFrames() {
615 rtc::CritScope lock(&crit_);
616 expect_frames_ = false;
617 }
618
number_of_reconfigurations() const619 int number_of_reconfigurations() const {
620 rtc::CritScope lock(&crit_);
621 return number_of_reconfigurations_;
622 }
623
last_min_transmit_bitrate() const624 int last_min_transmit_bitrate() const {
625 rtc::CritScope lock(&crit_);
626 return min_transmit_bitrate_bps_;
627 }
628
629 private:
OnEncodedImage(const EncodedImage & encoded_image,const CodecSpecificInfo * codec_specific_info,const RTPFragmentationHeader * fragmentation)630 Result OnEncodedImage(
631 const EncodedImage& encoded_image,
632 const CodecSpecificInfo* codec_specific_info,
633 const RTPFragmentationHeader* fragmentation) override {
634 rtc::CritScope lock(&crit_);
635 EXPECT_TRUE(expect_frames_);
636 last_timestamp_ = encoded_image._timeStamp;
637 last_width_ = encoded_image._encodedWidth;
638 last_height_ = encoded_image._encodedHeight;
639 encoded_frame_event_.Set();
640 return Result(Result::OK, last_timestamp_);
641 }
642
OnEncoderConfigurationChanged(std::vector<VideoStream> streams,int min_transmit_bitrate_bps)643 void OnEncoderConfigurationChanged(std::vector<VideoStream> streams,
644 int min_transmit_bitrate_bps) override {
645 rtc::CriticalSection crit_;
646 ++number_of_reconfigurations_;
647 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
648 }
649
650 rtc::CriticalSection crit_;
651 TestEncoder* test_encoder_;
652 rtc::Event encoded_frame_event_;
653 uint32_t last_timestamp_ = 0;
654 uint32_t last_height_ = 0;
655 uint32_t last_width_ = 0;
656 bool expect_frames_ = true;
657 int number_of_reconfigurations_ = 0;
658 int min_transmit_bitrate_bps_ = 0;
659 };
660
661 VideoSendStream::Config video_send_config_;
662 VideoEncoderConfig video_encoder_config_;
663 int codec_width_;
664 int codec_height_;
665 int max_framerate_;
666 TestEncoder fake_encoder_;
667 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
668 TestSink sink_;
669 AdaptingFrameForwarder video_source_;
670 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
671 rtc::ScopedFakeClock fake_clock_;
672 };
673
TEST_F(VideoStreamEncoderTest,EncodeOneFrame)674 TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
675 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
676 rtc::Event frame_destroyed_event(false, false);
677 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
678 WaitForEncodedFrame(1);
679 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
680 video_stream_encoder_->Stop();
681 }
682
TEST_F(VideoStreamEncoderTest,DropsFramesBeforeFirstOnBitrateUpdated)683 TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
684 // Dropped since no target bitrate has been set.
685 rtc::Event frame_destroyed_event(false, false);
686 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
687 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
688
689 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
690
691 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
692 WaitForEncodedFrame(2);
693 video_stream_encoder_->Stop();
694 }
695
TEST_F(VideoStreamEncoderTest,DropsFramesWhenRateSetToZero)696 TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
697 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
698 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
699 WaitForEncodedFrame(1);
700
701 video_stream_encoder_->OnBitrateUpdated(0, 0, 0);
702 // Dropped since bitrate is zero.
703 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
704
705 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
706 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
707 WaitForEncodedFrame(3);
708 video_stream_encoder_->Stop();
709 }
710
TEST_F(VideoStreamEncoderTest,DropsFramesWithSameOrOldNtpTimestamp)711 TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
712 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
713 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
714 WaitForEncodedFrame(1);
715
716 // This frame will be dropped since it has the same ntp timestamp.
717 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
718
719 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
720 WaitForEncodedFrame(2);
721 video_stream_encoder_->Stop();
722 }
723
TEST_F(VideoStreamEncoderTest,DropsFrameAfterStop)724 TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
725 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
726
727 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
728 WaitForEncodedFrame(1);
729
730 video_stream_encoder_->Stop();
731 sink_.SetExpectNoFrames();
732 rtc::Event frame_destroyed_event(false, false);
733 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
734 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
735 }
736
TEST_F(VideoStreamEncoderTest,DropsPendingFramesOnSlowEncode)737 TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
738 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
739
740 fake_encoder_.BlockNextEncode();
741 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
742 WaitForEncodedFrame(1);
743 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
744 // call to ContinueEncode.
745 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
746 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
747 fake_encoder_.ContinueEncode();
748 WaitForEncodedFrame(3);
749
750 video_stream_encoder_->Stop();
751 }
752
TEST_F(VideoStreamEncoderTest,ConfigureEncoderTriggersOnEncoderConfigurationChanged)753 TEST_F(VideoStreamEncoderTest,
754 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
755 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
756 EXPECT_EQ(0, sink_.number_of_reconfigurations());
757
758 // Capture a frame and wait for it to synchronize with the encoder thread.
759 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
760 WaitForEncodedFrame(1);
761 // The encoder will have been configured once when the first frame is
762 // received.
763 EXPECT_EQ(1, sink_.number_of_reconfigurations());
764
765 VideoEncoderConfig video_encoder_config;
766 test::FillEncoderConfiguration(1, &video_encoder_config);
767 video_encoder_config.min_transmit_bitrate_bps = 9999;
768 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
769 kMaxPayloadLength,
770 true /* nack_enabled */);
771
772 // Capture a frame and wait for it to synchronize with the encoder thread.
773 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
774 WaitForEncodedFrame(2);
775 EXPECT_EQ(2, sink_.number_of_reconfigurations());
776 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
777
778 video_stream_encoder_->Stop();
779 }
780
TEST_F(VideoStreamEncoderTest,FrameResolutionChangeReconfigureEncoder)781 TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
782 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
783
784 // Capture a frame and wait for it to synchronize with the encoder thread.
785 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
786 WaitForEncodedFrame(1);
787 // The encoder will have been configured once.
788 EXPECT_EQ(1, sink_.number_of_reconfigurations());
789 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
790 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
791
792 codec_width_ *= 2;
793 codec_height_ *= 2;
794 // Capture a frame with a higher resolution and wait for it to synchronize
795 // with the encoder thread.
796 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
797 WaitForEncodedFrame(2);
798 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
799 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
800 EXPECT_EQ(2, sink_.number_of_reconfigurations());
801
802 video_stream_encoder_->Stop();
803 }
804
TEST_F(VideoStreamEncoderTest,Vp8ResilienceIsOffFor1S1TLWithNackEnabled)805 TEST_F(VideoStreamEncoderTest, Vp8ResilienceIsOffFor1S1TLWithNackEnabled) {
806 const bool kNackEnabled = true;
807 const size_t kNumStreams = 1;
808 const size_t kNumTl = 1;
809 ResetEncoder("VP8", kNumStreams, kNumTl, kNumSlDummy, kNackEnabled, false);
810 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
811
812 // Capture a frame and wait for it to synchronize with the encoder thread.
813 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
814 WaitForEncodedFrame(1);
815 // The encoder have been configured once when the first frame is received.
816 EXPECT_EQ(1, sink_.number_of_reconfigurations());
817 EXPECT_EQ(kVideoCodecVP8, fake_encoder_.codec_config().codecType);
818 EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
819 EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP8()->numberOfTemporalLayers);
820 // Resilience is off for no temporal layers with nack on.
821 EXPECT_EQ(kResilienceOff, fake_encoder_.codec_config().VP8()->resilience);
822 video_stream_encoder_->Stop();
823 }
824
TEST_F(VideoStreamEncoderTest,Vp8ResilienceIsOffFor2S1TlWithNackEnabled)825 TEST_F(VideoStreamEncoderTest, Vp8ResilienceIsOffFor2S1TlWithNackEnabled) {
826 const bool kNackEnabled = true;
827 const size_t kNumStreams = 2;
828 const size_t kNumTl = 1;
829 ResetEncoder("VP8", kNumStreams, kNumTl, kNumSlDummy, kNackEnabled, false);
830 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
831
832 // Capture a frame and wait for it to synchronize with the encoder thread.
833 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
834 WaitForEncodedFrame(1);
835 // The encoder have been configured once when the first frame is received.
836 EXPECT_EQ(1, sink_.number_of_reconfigurations());
837 EXPECT_EQ(kVideoCodecVP8, fake_encoder_.codec_config().codecType);
838 EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
839 EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP8()->numberOfTemporalLayers);
840 // Resilience is off for no temporal layers and >1 streams with nack on.
841 EXPECT_EQ(kResilienceOff, fake_encoder_.codec_config().VP8()->resilience);
842 video_stream_encoder_->Stop();
843 }
844
TEST_F(VideoStreamEncoderTest,Vp8ResilienceIsOnFor1S1TLWithNackDisabled)845 TEST_F(VideoStreamEncoderTest, Vp8ResilienceIsOnFor1S1TLWithNackDisabled) {
846 const bool kNackEnabled = false;
847 const size_t kNumStreams = 1;
848 const size_t kNumTl = 1;
849 ResetEncoder("VP8", kNumStreams, kNumTl, kNumSlDummy, kNackEnabled, false);
850 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
851
852 // Capture a frame and wait for it to synchronize with the encoder thread.
853 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
854 WaitForEncodedFrame(1);
855 // The encoder have been configured once when the first frame is received.
856 EXPECT_EQ(1, sink_.number_of_reconfigurations());
857 EXPECT_EQ(kVideoCodecVP8, fake_encoder_.codec_config().codecType);
858 EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
859 EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP8()->numberOfTemporalLayers);
860 // Resilience is on for no temporal layers with nack off.
861 EXPECT_EQ(kResilientStream, fake_encoder_.codec_config().VP8()->resilience);
862 video_stream_encoder_->Stop();
863 }
864
TEST_F(VideoStreamEncoderTest,Vp8ResilienceIsOnFor1S2TlWithNackEnabled)865 TEST_F(VideoStreamEncoderTest, Vp8ResilienceIsOnFor1S2TlWithNackEnabled) {
866 const bool kNackEnabled = true;
867 const size_t kNumStreams = 1;
868 const size_t kNumTl = 2;
869 ResetEncoder("VP8", kNumStreams, kNumTl, kNumSlDummy, kNackEnabled, false);
870 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
871
872 // Capture a frame and wait for it to synchronize with the encoder thread.
873 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
874 WaitForEncodedFrame(1);
875 // The encoder have been configured once when the first frame is received.
876 EXPECT_EQ(1, sink_.number_of_reconfigurations());
877 EXPECT_EQ(kVideoCodecVP8, fake_encoder_.codec_config().codecType);
878 EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
879 EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP8()->numberOfTemporalLayers);
880 // Resilience is on for temporal layers.
881 EXPECT_EQ(kResilientStream, fake_encoder_.codec_config().VP8()->resilience);
882 video_stream_encoder_->Stop();
883 }
884
TEST_F(VideoStreamEncoderTest,Vp9ResilienceIsOffFor1SL1TLWithNackEnabled)885 TEST_F(VideoStreamEncoderTest, Vp9ResilienceIsOffFor1SL1TLWithNackEnabled) {
886 const bool kNackEnabled = true;
887 const size_t kNumStreams = 1;
888 const size_t kNumTl = 1;
889 const unsigned char kNumSl = 1;
890 ResetEncoder("VP9", kNumStreams, kNumTl, kNumSl, kNackEnabled, false);
891 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
892
893 // Capture a frame and wait for it to synchronize with the encoder thread.
894 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
895 sink_.WaitForEncodedFrame(1);
896 // The encoder have been configured once when the first frame is received.
897 EXPECT_EQ(1, sink_.number_of_reconfigurations());
898 EXPECT_EQ(kVideoCodecVP9, fake_encoder_.codec_config().codecType);
899 EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
900 EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP9()->numberOfTemporalLayers);
901 EXPECT_EQ(kNumSl, fake_encoder_.codec_config().VP9()->numberOfSpatialLayers);
902 // Resilience is off for no spatial and temporal layers with nack on.
903 EXPECT_FALSE(fake_encoder_.codec_config().VP9()->resilienceOn);
904 video_stream_encoder_->Stop();
905 }
906
TEST_F(VideoStreamEncoderTest,Vp9ResilienceIsOnFor1SL1TLWithNackDisabled)907 TEST_F(VideoStreamEncoderTest, Vp9ResilienceIsOnFor1SL1TLWithNackDisabled) {
908 const bool kNackEnabled = false;
909 const size_t kNumStreams = 1;
910 const size_t kNumTl = 1;
911 const unsigned char kNumSl = 1;
912 ResetEncoder("VP9", kNumStreams, kNumTl, kNumSl, kNackEnabled, false);
913 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
914
915 // Capture a frame and wait for it to synchronize with the encoder thread.
916 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
917 sink_.WaitForEncodedFrame(1);
918 // The encoder have been configured once when the first frame is received.
919 EXPECT_EQ(1, sink_.number_of_reconfigurations());
920 EXPECT_EQ(kVideoCodecVP9, fake_encoder_.codec_config().codecType);
921 EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
922 EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP9()->numberOfTemporalLayers);
923 EXPECT_EQ(kNumSl, fake_encoder_.codec_config().VP9()->numberOfSpatialLayers);
924 // Resilience is on if nack is off.
925 EXPECT_TRUE(fake_encoder_.codec_config().VP9()->resilienceOn);
926 video_stream_encoder_->Stop();
927 }
928
TEST_F(VideoStreamEncoderTest,Vp9ResilienceIsOnFor2SL1TLWithNackEnabled)929 TEST_F(VideoStreamEncoderTest, Vp9ResilienceIsOnFor2SL1TLWithNackEnabled) {
930 const bool kNackEnabled = true;
931 const size_t kNumStreams = 1;
932 const size_t kNumTl = 1;
933 const unsigned char kNumSl = 2;
934 ResetEncoder("VP9", kNumStreams, kNumTl, kNumSl, kNackEnabled, false);
935 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
936
937 // Capture a frame and wait for it to synchronize with the encoder thread.
938 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
939 sink_.WaitForEncodedFrame(1);
940 // The encoder have been configured once when the first frame is received.
941 EXPECT_EQ(1, sink_.number_of_reconfigurations());
942 EXPECT_EQ(kVideoCodecVP9, fake_encoder_.codec_config().codecType);
943 EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
944 EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP9()->numberOfTemporalLayers);
945 EXPECT_EQ(kNumSl, fake_encoder_.codec_config().VP9()->numberOfSpatialLayers);
946 // Resilience is on for spatial layers.
947 EXPECT_TRUE(fake_encoder_.codec_config().VP9()->resilienceOn);
948 video_stream_encoder_->Stop();
949 }
950
TEST_F(VideoStreamEncoderTest,Vp9ResilienceIsOnFor1SL2TLWithNackEnabled)951 TEST_F(VideoStreamEncoderTest, Vp9ResilienceIsOnFor1SL2TLWithNackEnabled) {
952 const bool kNackEnabled = true;
953 const size_t kNumStreams = 1;
954 const size_t kNumTl = 2;
955 const unsigned char kNumSl = 1;
956 ResetEncoder("VP9", kNumStreams, kNumTl, kNumSl, kNackEnabled, false);
957 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
958
959 // Capture a frame and wait for it to synchronize with the encoder thread.
960 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
961 sink_.WaitForEncodedFrame(1);
962 // The encoder have been configured once when the first frame is received.
963 EXPECT_EQ(1, sink_.number_of_reconfigurations());
964 EXPECT_EQ(kVideoCodecVP9, fake_encoder_.codec_config().codecType);
965 EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
966 EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP9()->numberOfTemporalLayers);
967 EXPECT_EQ(kNumSl, fake_encoder_.codec_config().VP9()->numberOfSpatialLayers);
968 // Resilience is on for temporal layers.
969 EXPECT_TRUE(fake_encoder_.codec_config().VP9()->resilienceOn);
970 video_stream_encoder_->Stop();
971 }
972
TEST_F(VideoStreamEncoderTest,SwitchSourceDeregisterEncoderAsSink)973 TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
974 EXPECT_TRUE(video_source_.has_sinks());
975 test::FrameForwarder new_video_source;
976 video_stream_encoder_->SetSource(
977 &new_video_source,
978 VideoSendStream::DegradationPreference::kMaintainFramerate);
979 EXPECT_FALSE(video_source_.has_sinks());
980 EXPECT_TRUE(new_video_source.has_sinks());
981
982 video_stream_encoder_->Stop();
983 }
984
TEST_F(VideoStreamEncoderTest,SinkWantsRotationApplied)985 TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
986 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
987 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
988 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
989 video_stream_encoder_->Stop();
990 }
991
TEST_F(VideoStreamEncoderTest,SinkWantsFromOveruseDetector)992 TEST_F(VideoStreamEncoderTest, SinkWantsFromOveruseDetector) {
993 const int kMaxDowngrades = VideoStreamEncoder::kMaxCpuResolutionDowngrades;
994 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
995
996 VerifyNoLimitation(video_source_.sink_wants());
997
998 int frame_width = 1280;
999 int frame_height = 720;
1000
1001 // Trigger CPU overuse kMaxCpuDowngrades times. Every time, VideoStreamEncoder
1002 // should request lower resolution.
1003 for (int i = 1; i <= kMaxDowngrades; ++i) {
1004 video_source_.IncomingCapturedFrame(
1005 CreateFrame(i, frame_width, frame_height));
1006 WaitForEncodedFrame(i);
1007
1008 video_stream_encoder_->TriggerCpuOveruse();
1009
1010 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1011 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1012 frame_width * frame_height);
1013 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1014 EXPECT_EQ(i, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1015
1016 frame_width /= 2;
1017 frame_height /= 2;
1018 }
1019
1020 // Trigger CPU overuse one more time. This should not trigger a request for
1021 // lower resolution.
1022 rtc::VideoSinkWants current_wants = video_source_.sink_wants();
1023 video_source_.IncomingCapturedFrame(
1024 CreateFrame(kMaxDowngrades + 1, frame_width, frame_height));
1025 WaitForEncodedFrame(kMaxDowngrades + 1);
1026 video_stream_encoder_->TriggerCpuOveruse();
1027 EXPECT_EQ(video_source_.sink_wants().target_pixel_count,
1028 current_wants.target_pixel_count);
1029 EXPECT_EQ(video_source_.sink_wants().max_pixel_count,
1030 current_wants.max_pixel_count);
1031 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1032 EXPECT_EQ(kMaxDowngrades,
1033 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1034
1035 // Trigger CPU normal use.
1036 video_stream_encoder_->TriggerCpuNormalUsage();
1037 EXPECT_EQ(frame_width * frame_height * 5 / 3,
1038 video_source_.sink_wants().target_pixel_count.value_or(0));
1039 EXPECT_EQ(frame_width * frame_height * 4,
1040 video_source_.sink_wants().max_pixel_count);
1041 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1042 EXPECT_EQ(kMaxDowngrades + 1,
1043 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1044
1045 video_stream_encoder_->Stop();
1046 }
1047
TEST_F(VideoStreamEncoderTest,TestMaxCpuResolutionDowngrades_BalancedMode_NoFpsLimit)1048 TEST_F(VideoStreamEncoderTest,
1049 TestMaxCpuResolutionDowngrades_BalancedMode_NoFpsLimit) {
1050 const int kMaxDowngrades = VideoStreamEncoder::kMaxCpuResolutionDowngrades;
1051 const int kWidth = 1280;
1052 const int kHeight = 720;
1053 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1054
1055 // Enable kBalanced preference, no initial limitation.
1056 AdaptingFrameForwarder source;
1057 source.set_adaptation_enabled(true);
1058 video_stream_encoder_->SetSource(
1059 &source,
1060 VideoSendStream::DegradationPreference::kBalanced);
1061 VerifyNoLimitation(source.sink_wants());
1062 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1063 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1064
1065 // Trigger adapt down kMaxCpuDowngrades times.
1066 int t = 1;
1067 for (int i = 1; i <= kMaxDowngrades; ++i) {
1068 source.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1069 sink_.WaitForEncodedFrame(t++);
1070 video_stream_encoder_->TriggerCpuOveruse();
1071 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
1072 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1073 EXPECT_EQ(i, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1074 }
1075
1076 // Trigger adapt down, max cpu downgrades reach, expect no change.
1077 rtc::VideoSinkWants last_wants = source.sink_wants();
1078 source.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1079 sink_.WaitForEncodedFrame(t++);
1080 video_stream_encoder_->TriggerCpuOveruse();
1081 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
1082 EXPECT_EQ(last_wants.max_pixel_count, source.sink_wants().max_pixel_count);
1083 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1084 EXPECT_EQ(kMaxDowngrades,
1085 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1086
1087 // Trigger adapt up kMaxCpuDowngrades times.
1088 for (int i = 1; i <= kMaxDowngrades; ++i) {
1089 source.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1090 sink_.WaitForEncodedFrame(t++);
1091 video_stream_encoder_->TriggerCpuNormalUsage();
1092 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
1093 EXPECT_GT(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
1094 EXPECT_EQ(kMaxDowngrades + i,
1095 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1096 }
1097
1098 VerifyNoLimitation(source.sink_wants());
1099 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1100
1101 video_stream_encoder_->Stop();
1102 }
TEST_F(VideoStreamEncoderTest,SinkWantsStoredByDegradationPreference)1103 TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
1104 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1105 VerifyNoLimitation(video_source_.sink_wants());
1106
1107 const int kFrameWidth = 1280;
1108 const int kFrameHeight = 720;
1109 const int kFrameIntervalMs = 1000 / 30;
1110
1111 int frame_timestamp = 1;
1112
1113 video_source_.IncomingCapturedFrame(
1114 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
1115 WaitForEncodedFrame(frame_timestamp);
1116 frame_timestamp += kFrameIntervalMs;
1117
1118 // Trigger CPU overuse.
1119 video_stream_encoder_->TriggerCpuOveruse();
1120 video_source_.IncomingCapturedFrame(
1121 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
1122 WaitForEncodedFrame(frame_timestamp);
1123 frame_timestamp += kFrameIntervalMs;
1124
1125 // Default degradation preference is maintain-framerate, so will lower max
1126 // wanted resolution.
1127 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1128 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1129 kFrameWidth * kFrameHeight);
1130 EXPECT_EQ(std::numeric_limits<int>::max(),
1131 video_source_.sink_wants().max_framerate_fps);
1132
1133 // Set new source, switch to maintain-resolution.
1134 test::FrameForwarder new_video_source;
1135 video_stream_encoder_->SetSource(
1136 &new_video_source,
1137 VideoSendStream::DegradationPreference::kMaintainResolution);
1138
1139 // Initially no degradation registered.
1140 VerifyNoLimitation(new_video_source.sink_wants());
1141
1142 // Force an input frame rate to be available, or the adaptation call won't
1143 // know what framerate to adapt form.
1144 const int kInputFps = 30;
1145 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1146 stats.input_frame_rate = kInputFps;
1147 stats_proxy_->SetMockStats(stats);
1148
1149 video_stream_encoder_->TriggerCpuOveruse();
1150 new_video_source.IncomingCapturedFrame(
1151 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
1152 WaitForEncodedFrame(frame_timestamp);
1153 frame_timestamp += kFrameIntervalMs;
1154
1155 // Some framerate constraint should be set.
1156 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1157 EXPECT_EQ(std::numeric_limits<int>::max(),
1158 new_video_source.sink_wants().max_pixel_count);
1159 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
1160
1161 // Turn off degradation completely.
1162 video_stream_encoder_->SetSource(
1163 &new_video_source,
1164 VideoSendStream::DegradationPreference::kDegradationDisabled);
1165 VerifyNoLimitation(new_video_source.sink_wants());
1166
1167 video_stream_encoder_->TriggerCpuOveruse();
1168 new_video_source.IncomingCapturedFrame(
1169 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
1170 WaitForEncodedFrame(frame_timestamp);
1171 frame_timestamp += kFrameIntervalMs;
1172
1173 // Still no degradation.
1174 VerifyNoLimitation(new_video_source.sink_wants());
1175
1176 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
1177 video_stream_encoder_->SetSource(
1178 &new_video_source,
1179 VideoSendStream::DegradationPreference::kMaintainFramerate);
1180 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
1181 kFrameWidth * kFrameHeight);
1182 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1183 EXPECT_EQ(std::numeric_limits<int>::max(),
1184 new_video_source.sink_wants().max_framerate_fps);
1185
1186 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
1187 video_stream_encoder_->SetSource(
1188 &new_video_source,
1189 VideoSendStream::DegradationPreference::kMaintainResolution);
1190 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1191 EXPECT_EQ(std::numeric_limits<int>::max(),
1192 new_video_source.sink_wants().max_pixel_count);
1193 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
1194
1195 video_stream_encoder_->Stop();
1196 }
1197
TEST_F(VideoStreamEncoderTest,StatsTracksQualityAdaptationStats)1198 TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
1199 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1200
1201 const int kWidth = 1280;
1202 const int kHeight = 720;
1203 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1204 WaitForEncodedFrame(1);
1205 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1206 EXPECT_FALSE(stats.bw_limited_resolution);
1207 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1208
1209 // Trigger adapt down.
1210 video_stream_encoder_->TriggerQualityLow();
1211 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1212 WaitForEncodedFrame(2);
1213
1214 stats = stats_proxy_->GetStats();
1215 EXPECT_TRUE(stats.bw_limited_resolution);
1216 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1217
1218 // Trigger adapt up.
1219 video_stream_encoder_->TriggerQualityHigh();
1220 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
1221 WaitForEncodedFrame(3);
1222
1223 stats = stats_proxy_->GetStats();
1224 EXPECT_FALSE(stats.bw_limited_resolution);
1225 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
1226 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1227
1228 video_stream_encoder_->Stop();
1229 }
1230
TEST_F(VideoStreamEncoderTest,StatsTracksCpuAdaptationStats)1231 TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
1232 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1233
1234 const int kWidth = 1280;
1235 const int kHeight = 720;
1236 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1237 WaitForEncodedFrame(1);
1238 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1239 EXPECT_FALSE(stats.cpu_limited_resolution);
1240 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1241
1242 // Trigger CPU overuse.
1243 video_stream_encoder_->TriggerCpuOveruse();
1244 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1245 WaitForEncodedFrame(2);
1246
1247 stats = stats_proxy_->GetStats();
1248 EXPECT_TRUE(stats.cpu_limited_resolution);
1249 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1250
1251 // Trigger CPU normal use.
1252 video_stream_encoder_->TriggerCpuNormalUsage();
1253 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
1254 WaitForEncodedFrame(3);
1255
1256 stats = stats_proxy_->GetStats();
1257 EXPECT_FALSE(stats.cpu_limited_resolution);
1258 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1259 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1260
1261 video_stream_encoder_->Stop();
1262 }
1263
TEST_F(VideoStreamEncoderTest,SwitchingSourceKeepsCpuAdaptation)1264 TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
1265 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1266
1267 const int kWidth = 1280;
1268 const int kHeight = 720;
1269 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1270 WaitForEncodedFrame(1);
1271 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1272 EXPECT_FALSE(stats.bw_limited_resolution);
1273 EXPECT_FALSE(stats.cpu_limited_resolution);
1274 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1275
1276 // Trigger CPU overuse.
1277 video_stream_encoder_->TriggerCpuOveruse();
1278 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1279 WaitForEncodedFrame(2);
1280 stats = stats_proxy_->GetStats();
1281 EXPECT_FALSE(stats.bw_limited_resolution);
1282 EXPECT_TRUE(stats.cpu_limited_resolution);
1283 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1284
1285 // Set new source with adaptation still enabled.
1286 test::FrameForwarder new_video_source;
1287 video_stream_encoder_->SetSource(
1288 &new_video_source,
1289 VideoSendStream::DegradationPreference::kMaintainFramerate);
1290
1291 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
1292 WaitForEncodedFrame(3);
1293 stats = stats_proxy_->GetStats();
1294 EXPECT_FALSE(stats.bw_limited_resolution);
1295 EXPECT_TRUE(stats.cpu_limited_resolution);
1296 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1297
1298 // Set adaptation disabled.
1299 video_stream_encoder_->SetSource(
1300 &new_video_source,
1301 VideoSendStream::DegradationPreference::kDegradationDisabled);
1302
1303 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
1304 WaitForEncodedFrame(4);
1305 stats = stats_proxy_->GetStats();
1306 EXPECT_FALSE(stats.bw_limited_resolution);
1307 EXPECT_FALSE(stats.cpu_limited_resolution);
1308 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1309
1310 // Set adaptation back to enabled.
1311 video_stream_encoder_->SetSource(
1312 &new_video_source,
1313 VideoSendStream::DegradationPreference::kMaintainFramerate);
1314
1315 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
1316 WaitForEncodedFrame(5);
1317 stats = stats_proxy_->GetStats();
1318 EXPECT_FALSE(stats.bw_limited_resolution);
1319 EXPECT_TRUE(stats.cpu_limited_resolution);
1320 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1321
1322 // Trigger CPU normal use.
1323 video_stream_encoder_->TriggerCpuNormalUsage();
1324 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
1325 WaitForEncodedFrame(6);
1326 stats = stats_proxy_->GetStats();
1327 EXPECT_FALSE(stats.bw_limited_resolution);
1328 EXPECT_FALSE(stats.cpu_limited_resolution);
1329 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1330 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1331
1332 video_stream_encoder_->Stop();
1333 }
1334
TEST_F(VideoStreamEncoderTest,SwitchingSourceKeepsQualityAdaptation)1335 TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
1336 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1337
1338 const int kWidth = 1280;
1339 const int kHeight = 720;
1340 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1341 WaitForEncodedFrame(1);
1342 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1343 EXPECT_FALSE(stats.bw_limited_resolution);
1344 EXPECT_FALSE(stats.bw_limited_framerate);
1345 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1346
1347 // Set new source with adaptation still enabled.
1348 test::FrameForwarder new_video_source;
1349 video_stream_encoder_->SetSource(
1350 &new_video_source,
1351 VideoSendStream::DegradationPreference::kBalanced);
1352
1353 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1354 WaitForEncodedFrame(2);
1355 stats = stats_proxy_->GetStats();
1356 EXPECT_FALSE(stats.bw_limited_resolution);
1357 EXPECT_FALSE(stats.bw_limited_framerate);
1358 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1359
1360 // Trigger adapt down.
1361 video_stream_encoder_->TriggerQualityLow();
1362 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
1363 WaitForEncodedFrame(3);
1364 stats = stats_proxy_->GetStats();
1365 EXPECT_TRUE(stats.bw_limited_resolution);
1366 EXPECT_FALSE(stats.bw_limited_framerate);
1367 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1368
1369 // Set new source with adaptation still enabled.
1370 video_stream_encoder_->SetSource(
1371 &new_video_source,
1372 VideoSendStream::DegradationPreference::kBalanced);
1373
1374 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
1375 WaitForEncodedFrame(4);
1376 stats = stats_proxy_->GetStats();
1377 EXPECT_TRUE(stats.bw_limited_resolution);
1378 EXPECT_FALSE(stats.bw_limited_framerate);
1379 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1380
1381 // Disable resolution scaling.
1382 video_stream_encoder_->SetSource(
1383 &new_video_source,
1384 VideoSendStream::DegradationPreference::kMaintainResolution);
1385
1386 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
1387 WaitForEncodedFrame(5);
1388 stats = stats_proxy_->GetStats();
1389 EXPECT_FALSE(stats.bw_limited_resolution);
1390 EXPECT_FALSE(stats.bw_limited_framerate);
1391 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1392 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1393
1394 video_stream_encoder_->Stop();
1395 }
1396
TEST_F(VideoStreamEncoderTest,QualityAdaptationStatsAreResetWhenScalerIsDisabled)1397 TEST_F(VideoStreamEncoderTest,
1398 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
1399 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1400
1401 const int kWidth = 1280;
1402 const int kHeight = 720;
1403 video_source_.set_adaptation_enabled(true);
1404 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1405 WaitForEncodedFrame(1);
1406 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1407 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1408 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1409
1410 // Trigger adapt down.
1411 video_stream_encoder_->TriggerQualityLow();
1412 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1413 WaitForEncodedFrame(2);
1414 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1415 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1416 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1417
1418 // Trigger overuse.
1419 video_stream_encoder_->TriggerCpuOveruse();
1420 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
1421 WaitForEncodedFrame(3);
1422 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1423 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1424 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1425
1426 // Set source with adaptation still enabled but quality scaler is off.
1427 fake_encoder_.SetQualityScaling(false);
1428 video_stream_encoder_->SetSource(
1429 &video_source_,
1430 VideoSendStream::DegradationPreference::kMaintainFramerate);
1431
1432 video_source_.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
1433 WaitForEncodedFrame(4);
1434 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1435 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1436 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1437
1438 video_stream_encoder_->Stop();
1439 }
1440
TEST_F(VideoStreamEncoderTest,StatsTracksCpuAdaptationStatsWhenSwitchingSource)1441 TEST_F(VideoStreamEncoderTest,
1442 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
1443 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1444
1445 const int kWidth = 1280;
1446 const int kHeight = 720;
1447 int sequence = 1;
1448
1449 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
1450 WaitForEncodedFrame(sequence++);
1451 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1452 EXPECT_FALSE(stats.cpu_limited_resolution);
1453 EXPECT_FALSE(stats.cpu_limited_framerate);
1454 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1455
1456 // Trigger CPU overuse, should now adapt down.
1457 video_stream_encoder_->TriggerCpuOveruse();
1458 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
1459 WaitForEncodedFrame(sequence++);
1460 stats = stats_proxy_->GetStats();
1461 EXPECT_TRUE(stats.cpu_limited_resolution);
1462 EXPECT_FALSE(stats.cpu_limited_framerate);
1463 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1464
1465 // Set new source with adaptation still enabled.
1466 test::FrameForwarder new_video_source;
1467 video_stream_encoder_->SetSource(
1468 &new_video_source,
1469 VideoSendStream::DegradationPreference::kMaintainFramerate);
1470
1471 new_video_source.IncomingCapturedFrame(
1472 CreateFrame(sequence, kWidth, kHeight));
1473 WaitForEncodedFrame(sequence++);
1474 stats = stats_proxy_->GetStats();
1475 EXPECT_TRUE(stats.cpu_limited_resolution);
1476 EXPECT_FALSE(stats.cpu_limited_framerate);
1477 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1478
1479 // Set cpu adaptation by frame dropping.
1480 video_stream_encoder_->SetSource(
1481 &new_video_source,
1482 VideoSendStream::DegradationPreference::kMaintainResolution);
1483 new_video_source.IncomingCapturedFrame(
1484 CreateFrame(sequence, kWidth, kHeight));
1485 WaitForEncodedFrame(sequence++);
1486 stats = stats_proxy_->GetStats();
1487 // Not adapted at first.
1488 EXPECT_FALSE(stats.cpu_limited_resolution);
1489 EXPECT_FALSE(stats.cpu_limited_framerate);
1490 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1491
1492 // Force an input frame rate to be available, or the adaptation call won't
1493 // know what framerate to adapt from.
1494 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1495 mock_stats.input_frame_rate = 30;
1496 stats_proxy_->SetMockStats(mock_stats);
1497 video_stream_encoder_->TriggerCpuOveruse();
1498 stats_proxy_->ResetMockStats();
1499
1500 new_video_source.IncomingCapturedFrame(
1501 CreateFrame(sequence, kWidth, kHeight));
1502 WaitForEncodedFrame(sequence++);
1503
1504 // Framerate now adapted.
1505 stats = stats_proxy_->GetStats();
1506 EXPECT_FALSE(stats.cpu_limited_resolution);
1507 EXPECT_TRUE(stats.cpu_limited_framerate);
1508 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1509
1510 // Disable CPU adaptation.
1511 video_stream_encoder_->SetSource(
1512 &new_video_source,
1513 VideoSendStream::DegradationPreference::kDegradationDisabled);
1514 new_video_source.IncomingCapturedFrame(
1515 CreateFrame(sequence, kWidth, kHeight));
1516 WaitForEncodedFrame(sequence++);
1517
1518 stats = stats_proxy_->GetStats();
1519 EXPECT_FALSE(stats.cpu_limited_resolution);
1520 EXPECT_FALSE(stats.cpu_limited_framerate);
1521 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1522
1523 // Try to trigger overuse. Should not succeed.
1524 stats_proxy_->SetMockStats(mock_stats);
1525 video_stream_encoder_->TriggerCpuOveruse();
1526 stats_proxy_->ResetMockStats();
1527
1528 stats = stats_proxy_->GetStats();
1529 EXPECT_FALSE(stats.cpu_limited_resolution);
1530 EXPECT_FALSE(stats.cpu_limited_framerate);
1531 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1532
1533 // Switch back the source with resolution adaptation enabled.
1534 video_stream_encoder_->SetSource(
1535 &video_source_,
1536 VideoSendStream::DegradationPreference::kMaintainFramerate);
1537 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
1538 WaitForEncodedFrame(sequence++);
1539 stats = stats_proxy_->GetStats();
1540 EXPECT_TRUE(stats.cpu_limited_resolution);
1541 EXPECT_FALSE(stats.cpu_limited_framerate);
1542 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1543
1544 // Trigger CPU normal usage.
1545 video_stream_encoder_->TriggerCpuNormalUsage();
1546 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
1547 WaitForEncodedFrame(sequence++);
1548 stats = stats_proxy_->GetStats();
1549 EXPECT_FALSE(stats.cpu_limited_resolution);
1550 EXPECT_FALSE(stats.cpu_limited_framerate);
1551 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
1552
1553 // Back to the source with adaptation off, set it back to maintain-resolution.
1554 video_stream_encoder_->SetSource(
1555 &new_video_source,
1556 VideoSendStream::DegradationPreference::kMaintainResolution);
1557 new_video_source.IncomingCapturedFrame(
1558 CreateFrame(sequence, kWidth, kHeight));
1559 WaitForEncodedFrame(sequence++);
1560 stats = stats_proxy_->GetStats();
1561 // Disabled, since we previously switched the source to disabled.
1562 EXPECT_FALSE(stats.cpu_limited_resolution);
1563 EXPECT_TRUE(stats.cpu_limited_framerate);
1564 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
1565
1566 // Trigger CPU normal usage.
1567 video_stream_encoder_->TriggerCpuNormalUsage();
1568 new_video_source.IncomingCapturedFrame(
1569 CreateFrame(sequence, kWidth, kHeight));
1570 WaitForEncodedFrame(sequence++);
1571 stats = stats_proxy_->GetStats();
1572 EXPECT_FALSE(stats.cpu_limited_resolution);
1573 EXPECT_FALSE(stats.cpu_limited_framerate);
1574 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
1575 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1576
1577 video_stream_encoder_->Stop();
1578 }
1579
TEST_F(VideoStreamEncoderTest,StatsTracksPreferredBitrate)1580 TEST_F(VideoStreamEncoderTest, StatsTracksPreferredBitrate) {
1581 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1582
1583 const int kWidth = 1280;
1584 const int kHeight = 720;
1585 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1586 WaitForEncodedFrame(1);
1587
1588 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1589 EXPECT_EQ(video_encoder_config_.max_bitrate_bps,
1590 stats.preferred_media_bitrate_bps);
1591
1592 video_stream_encoder_->Stop();
1593 }
1594
TEST_F(VideoStreamEncoderTest,ScalingUpAndDownDoesNothingWithMaintainResolution)1595 TEST_F(VideoStreamEncoderTest,
1596 ScalingUpAndDownDoesNothingWithMaintainResolution) {
1597 const int kWidth = 1280;
1598 const int kHeight = 720;
1599 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1600
1601 // Expect no scaling to begin with.
1602 VerifyNoLimitation(video_source_.sink_wants());
1603
1604 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1605 WaitForEncodedFrame(1);
1606
1607 // Trigger scale down.
1608 video_stream_encoder_->TriggerQualityLow();
1609
1610 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1611 WaitForEncodedFrame(2);
1612
1613 // Expect a scale down.
1614 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
1615 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
1616
1617 // Set resolution scaling disabled.
1618 test::FrameForwarder new_video_source;
1619 video_stream_encoder_->SetSource(
1620 &new_video_source,
1621 VideoSendStream::DegradationPreference::kMaintainResolution);
1622
1623 // Trigger scale down.
1624 video_stream_encoder_->TriggerQualityLow();
1625 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
1626 WaitForEncodedFrame(3);
1627
1628 // Expect no scaling.
1629 EXPECT_EQ(std::numeric_limits<int>::max(),
1630 new_video_source.sink_wants().max_pixel_count);
1631
1632 // Trigger scale up.
1633 video_stream_encoder_->TriggerQualityHigh();
1634 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
1635 WaitForEncodedFrame(4);
1636
1637 // Expect nothing to change, still no scaling.
1638 EXPECT_EQ(std::numeric_limits<int>::max(),
1639 new_video_source.sink_wants().max_pixel_count);
1640
1641 video_stream_encoder_->Stop();
1642 }
1643
TEST_F(VideoStreamEncoderTest,SkipsSameAdaptDownRequest_MaintainFramerateMode)1644 TEST_F(VideoStreamEncoderTest,
1645 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
1646 const int kWidth = 1280;
1647 const int kHeight = 720;
1648 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1649
1650 // Enable kMaintainFramerate preference, no initial limitation.
1651 test::FrameForwarder source;
1652 video_stream_encoder_->SetSource(
1653 &source, VideoSendStream::DegradationPreference::kMaintainFramerate);
1654
1655 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1656 WaitForEncodedFrame(1);
1657 VerifyNoLimitation(source.sink_wants());
1658 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1659 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1660
1661 // Trigger adapt down, expect scaled down resolution.
1662 video_stream_encoder_->TriggerCpuOveruse();
1663 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
1664 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
1665 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1666 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1667
1668 // Trigger adapt down for same input resolution, expect no change.
1669 video_stream_encoder_->TriggerCpuOveruse();
1670 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1671 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1672 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1673
1674 video_stream_encoder_->Stop();
1675 }
1676
TEST_F(VideoStreamEncoderTest,SkipsSameOrLargerAdaptDownRequest_BalancedMode)1677 TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
1678 const int kWidth = 1280;
1679 const int kHeight = 720;
1680 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1681
1682 // Enable kBalanced preference, no initial limitation.
1683 test::FrameForwarder source;
1684 video_stream_encoder_->SetSource(
1685 &source,
1686 VideoSendStream::DegradationPreference::kBalanced);
1687 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1688 sink_.WaitForEncodedFrame(1);
1689 VerifyNoLimitation(source.sink_wants());
1690
1691 // Trigger adapt down, expect scaled down resolution.
1692 video_stream_encoder_->TriggerQualityLow();
1693 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
1694 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1695 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1696 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
1697
1698 // Trigger adapt down for same input resolution, expect no change.
1699 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1700 sink_.WaitForEncodedFrame(2);
1701 video_stream_encoder_->TriggerQualityLow();
1702 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1703 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1704 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1705
1706 // Trigger adapt down for larger input resolution, expect no change.
1707 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
1708 sink_.WaitForEncodedFrame(3);
1709 video_stream_encoder_->TriggerQualityLow();
1710 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1711 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1712 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1713
1714 video_stream_encoder_->Stop();
1715 }
1716
TEST_F(VideoStreamEncoderTest,NoChangeForInitialNormalUsage_MaintainFramerateMode)1717 TEST_F(VideoStreamEncoderTest,
1718 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
1719 const int kWidth = 1280;
1720 const int kHeight = 720;
1721 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1722
1723 // Enable kMaintainFramerate preference, no initial limitation.
1724 test::FrameForwarder source;
1725 video_stream_encoder_->SetSource(
1726 &source, VideoSendStream::DegradationPreference::kMaintainFramerate);
1727
1728 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1729 WaitForEncodedFrame(kWidth, kHeight);
1730 VerifyNoLimitation(source.sink_wants());
1731 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1732 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1733
1734 // Trigger adapt up, expect no change.
1735 video_stream_encoder_->TriggerCpuNormalUsage();
1736 VerifyNoLimitation(source.sink_wants());
1737 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1738 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1739
1740 video_stream_encoder_->Stop();
1741 }
1742
TEST_F(VideoStreamEncoderTest,NoChangeForInitialNormalUsage_MaintainResolutionMode)1743 TEST_F(VideoStreamEncoderTest,
1744 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
1745 const int kWidth = 1280;
1746 const int kHeight = 720;
1747 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1748
1749 // Enable kMaintainResolution preference, no initial limitation.
1750 test::FrameForwarder source;
1751 video_stream_encoder_->SetSource(
1752 &source, VideoSendStream::DegradationPreference::kMaintainResolution);
1753
1754 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1755 WaitForEncodedFrame(kWidth, kHeight);
1756 VerifyNoLimitation(source.sink_wants());
1757 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1758 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1759
1760 // Trigger adapt up, expect no change.
1761 video_stream_encoder_->TriggerCpuNormalUsage();
1762 VerifyNoLimitation(source.sink_wants());
1763 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1764 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1765
1766 video_stream_encoder_->Stop();
1767 }
1768
TEST_F(VideoStreamEncoderTest,NoChangeForInitialNormalUsage_BalancedMode)1769 TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
1770 const int kWidth = 1280;
1771 const int kHeight = 720;
1772 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1773
1774 // Enable kBalanced preference, no initial limitation.
1775 test::FrameForwarder source;
1776 video_stream_encoder_->SetSource(
1777 &source,
1778 VideoSendStream::DegradationPreference::kBalanced);
1779
1780 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1781 sink_.WaitForEncodedFrame(kWidth, kHeight);
1782 VerifyNoLimitation(source.sink_wants());
1783 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1784 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1785 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1786
1787 // Trigger adapt up, expect no change.
1788 video_stream_encoder_->TriggerQualityHigh();
1789 VerifyNoLimitation(source.sink_wants());
1790 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1791 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1792 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1793
1794 video_stream_encoder_->Stop();
1795 }
1796
TEST_F(VideoStreamEncoderTest,NoChangeForInitialNormalUsage_DisabledMode)1797 TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
1798 const int kWidth = 1280;
1799 const int kHeight = 720;
1800 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1801
1802 // Enable kDegradationDisabled preference, no initial limitation.
1803 test::FrameForwarder source;
1804 video_stream_encoder_->SetSource(
1805 &source, VideoSendStream::DegradationPreference::kDegradationDisabled);
1806
1807 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1808 sink_.WaitForEncodedFrame(kWidth, kHeight);
1809 VerifyNoLimitation(source.sink_wants());
1810 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1811 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1812 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1813
1814 // Trigger adapt up, expect no change.
1815 video_stream_encoder_->TriggerQualityHigh();
1816 VerifyNoLimitation(source.sink_wants());
1817 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1818 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1819 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1820
1821 video_stream_encoder_->Stop();
1822 }
1823
TEST_F(VideoStreamEncoderTest,AdaptsResolutionForLowQuality_MaintainFramerateMode)1824 TEST_F(VideoStreamEncoderTest,
1825 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
1826 const int kWidth = 1280;
1827 const int kHeight = 720;
1828 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1829
1830 // Enable kMaintainFramerate preference, no initial limitation.
1831 AdaptingFrameForwarder source;
1832 source.set_adaptation_enabled(true);
1833 video_stream_encoder_->SetSource(
1834 &source, VideoSendStream::DegradationPreference::kMaintainFramerate);
1835
1836 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1837 WaitForEncodedFrame(1);
1838 VerifyNoLimitation(source.sink_wants());
1839 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1840 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1841
1842 // Trigger adapt down, expect scaled down resolution.
1843 video_stream_encoder_->TriggerQualityLow();
1844 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1845 WaitForEncodedFrame(2);
1846 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
1847 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1848 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1849
1850 // Trigger adapt up, expect no restriction.
1851 video_stream_encoder_->TriggerQualityHigh();
1852 VerifyNoLimitation(source.sink_wants());
1853 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1854 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1855 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1856
1857 video_stream_encoder_->Stop();
1858 }
1859
TEST_F(VideoStreamEncoderTest,AdaptsFramerateForLowQuality_MaintainResolutionMode)1860 TEST_F(VideoStreamEncoderTest,
1861 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
1862 const int kWidth = 1280;
1863 const int kHeight = 720;
1864 const int kInputFps = 30;
1865 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1866
1867 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1868 stats.input_frame_rate = kInputFps;
1869 stats_proxy_->SetMockStats(stats);
1870
1871 // Expect no scaling to begin with (preference: kMaintainFramerate).
1872 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1873 sink_.WaitForEncodedFrame(1);
1874 VerifyNoLimitation(video_source_.sink_wants());
1875
1876 // Trigger adapt down, expect scaled down resolution.
1877 video_stream_encoder_->TriggerQualityLow();
1878 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1879 sink_.WaitForEncodedFrame(2);
1880 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
1881
1882 // Enable kMaintainResolution preference.
1883 test::FrameForwarder new_video_source;
1884 video_stream_encoder_->SetSource(
1885 &new_video_source,
1886 VideoSendStream::DegradationPreference::kMaintainResolution);
1887 VerifyNoLimitation(new_video_source.sink_wants());
1888
1889 // Trigger adapt down, expect reduced framerate.
1890 video_stream_encoder_->TriggerQualityLow();
1891 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
1892 sink_.WaitForEncodedFrame(3);
1893 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
1894
1895 // Trigger adapt up, expect no restriction.
1896 video_stream_encoder_->TriggerQualityHigh();
1897 VerifyNoLimitation(new_video_source.sink_wants());
1898
1899 video_stream_encoder_->Stop();
1900 }
1901
TEST_F(VideoStreamEncoderTest,DoesNotScaleBelowSetResolutionLimit)1902 TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
1903 const int kWidth = 1280;
1904 const int kHeight = 720;
1905 const size_t kNumFrames = 10;
1906
1907 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1908
1909 // Enable adapter, expected input resolutions when downscaling:
1910 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
1911 video_source_.set_adaptation_enabled(true);
1912
1913 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1914 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1915
1916 int downscales = 0;
1917 for (size_t i = 1; i <= kNumFrames; i++) {
1918 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
1919 WaitForEncodedFrame(i);
1920
1921 // Trigger scale down.
1922 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
1923 video_stream_encoder_->TriggerQualityLow();
1924 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
1925
1926 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
1927 ++downscales;
1928
1929 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1930 EXPECT_EQ(downscales,
1931 stats_proxy_->GetStats().number_of_quality_adapt_changes);
1932 EXPECT_GT(downscales, 0);
1933 }
1934 video_stream_encoder_->Stop();
1935 }
1936
TEST_F(VideoStreamEncoderTest,AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode)1937 TEST_F(VideoStreamEncoderTest,
1938 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
1939 const int kWidth = 1280;
1940 const int kHeight = 720;
1941 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1942
1943 // Enable kMaintainFramerate preference, no initial limitation.
1944 AdaptingFrameForwarder source;
1945 source.set_adaptation_enabled(true);
1946 video_stream_encoder_->SetSource(
1947 &source, VideoSendStream::DegradationPreference::kMaintainFramerate);
1948
1949 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1950 WaitForEncodedFrame(kWidth, kHeight);
1951 VerifyNoLimitation(source.sink_wants());
1952 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1953 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1954
1955 // Trigger adapt down, expect scaled down resolution.
1956 video_stream_encoder_->TriggerCpuOveruse();
1957 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1958 WaitForEncodedFrame(2);
1959 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
1960 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1961 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1962
1963 // Trigger adapt up, expect no restriction.
1964 video_stream_encoder_->TriggerCpuNormalUsage();
1965 source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
1966 WaitForEncodedFrame(kWidth, kHeight);
1967 VerifyNoLimitation(source.sink_wants());
1968 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1969 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1970
1971 // Trigger adapt down, expect scaled down resolution.
1972 video_stream_encoder_->TriggerCpuOveruse();
1973 source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
1974 WaitForEncodedFrame(4);
1975 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
1976 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1977 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1978
1979 // Trigger adapt up, expect no restriction.
1980 video_stream_encoder_->TriggerCpuNormalUsage();
1981 source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
1982 sink_.WaitForEncodedFrame(kWidth, kHeight);
1983 VerifyNoLimitation(source.sink_wants());
1984 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1985 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1986
1987 video_stream_encoder_->Stop();
1988 }
1989
TEST_F(VideoStreamEncoderTest,AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit)1990 TEST_F(VideoStreamEncoderTest,
1991 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
1992 const int kWidth = 1280;
1993 const int kHeight = 720;
1994 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
1995
1996 // Enable kBalanced preference, no initial limitation.
1997 AdaptingFrameForwarder source;
1998 source.set_adaptation_enabled(true);
1999 video_stream_encoder_->SetSource(
2000 &source,
2001 VideoSendStream::DegradationPreference::kBalanced);
2002
2003 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2004 sink_.WaitForEncodedFrame(kWidth, kHeight);
2005 VerifyNoLimitation(source.sink_wants());
2006 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2007 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2008
2009 // Trigger adapt down, expect scaled down resolution.
2010 video_stream_encoder_->TriggerQualityLow();
2011 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2012 sink_.WaitForEncodedFrame(2);
2013 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2014 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2015 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2016
2017 // Trigger adapt up, expect no restriction.
2018 video_stream_encoder_->TriggerQualityHigh();
2019 source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2020 sink_.WaitForEncodedFrame(kWidth, kHeight);
2021 VerifyNoLimitation(source.sink_wants());
2022 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2023 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2024
2025 // Trigger adapt down, expect scaled down resolution.
2026 video_stream_encoder_->TriggerQualityLow();
2027 source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
2028 sink_.WaitForEncodedFrame(4);
2029 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2030 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2031 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2032
2033 // Trigger adapt up, expect no restriction.
2034 video_stream_encoder_->TriggerQualityHigh();
2035 source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
2036 sink_.WaitForEncodedFrame(kWidth, kHeight);
2037 VerifyNoLimitation(source.sink_wants());
2038 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2039 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2040
2041 video_stream_encoder_->Stop();
2042 }
2043
TEST_F(VideoStreamEncoderTest,AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode)2044 TEST_F(VideoStreamEncoderTest,
2045 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
2046 const int kWidth = 1280;
2047 const int kHeight = 720;
2048 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2049
2050 // Enable kMaintainFramerate preference, no initial limitation.
2051 AdaptingFrameForwarder source;
2052 source.set_adaptation_enabled(true);
2053 video_stream_encoder_->SetSource(
2054 &source, VideoSendStream::DegradationPreference::kMaintainFramerate);
2055
2056 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2057 WaitForEncodedFrame(kWidth, kHeight);
2058 VerifyNoLimitation(source.sink_wants());
2059 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2060 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2061 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2062 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2063
2064 // Trigger cpu adapt down, expect scaled down resolution (960x540).
2065 video_stream_encoder_->TriggerCpuOveruse();
2066 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2067 WaitForEncodedFrame(2);
2068 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2069 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2070 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2071 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2072 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2073
2074 // Trigger cpu adapt down, expect scaled down resolution (640x360).
2075 video_stream_encoder_->TriggerCpuOveruse();
2076 source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2077 WaitForEncodedFrame(3);
2078 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
2079 rtc::VideoSinkWants last_wants = source.sink_wants();
2080 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2081 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2082 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2083 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2084
2085 // Trigger cpu adapt down, max cpu downgrades reached, expect no change.
2086 video_stream_encoder_->TriggerCpuOveruse();
2087 source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
2088 WaitForEncodedFrame(4);
2089 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
2090 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2091 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2092 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2093 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2094
2095 // Trigger quality adapt down, expect scaled down resolution (480x270).
2096 video_stream_encoder_->TriggerQualityLow();
2097 source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
2098 WaitForEncodedFrame(5);
2099 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
2100 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2101 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2102 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2103 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2104
2105 // Trigger cpu adapt up, expect upscaled resolution (640x360).
2106 video_stream_encoder_->TriggerCpuNormalUsage();
2107 source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
2108 WaitForEncodedFrame(6);
2109 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
2110 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2111 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2112 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2113 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2114
2115 // Trigger cpu adapt up, expect upscaled resolution (960x540).
2116 video_stream_encoder_->TriggerCpuNormalUsage();
2117 source.IncomingCapturedFrame(CreateFrame(7, kWidth, kHeight));
2118 WaitForEncodedFrame(7);
2119 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
2120 last_wants = source.sink_wants();
2121 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2122 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2123 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2124 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2125
2126 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
2127 video_stream_encoder_->TriggerCpuNormalUsage();
2128 source.IncomingCapturedFrame(CreateFrame(8, kWidth, kHeight));
2129 WaitForEncodedFrame(8);
2130 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
2131 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2132 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2133 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2134 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2135
2136 // Trigger quality adapt up, expect no restriction (1280x720).
2137 video_stream_encoder_->TriggerQualityHigh();
2138 source.IncomingCapturedFrame(CreateFrame(9, kWidth, kHeight));
2139 WaitForEncodedFrame(kWidth, kHeight);
2140 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
2141 VerifyNoLimitation(source.sink_wants());
2142 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2143 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2144 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2145 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2146
2147 video_stream_encoder_->Stop();
2148 }
2149
TEST_F(VideoStreamEncoderTest,CpuLimitedHistogramIsReported)2150 TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
2151 const int kWidth = 640;
2152 const int kHeight = 360;
2153
2154 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2155
2156 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
2157 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
2158 WaitForEncodedFrame(i);
2159 }
2160
2161 video_stream_encoder_->TriggerCpuOveruse();
2162 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
2163 video_source_.IncomingCapturedFrame(CreateFrame(
2164 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
2165 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
2166 }
2167
2168 video_stream_encoder_->Stop();
2169 video_stream_encoder_.reset();
2170 stats_proxy_.reset();
2171
2172 EXPECT_EQ(1,
2173 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
2174 EXPECT_EQ(
2175 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
2176 }
2177
TEST_F(VideoStreamEncoderTest,CpuLimitedHistogramIsNotReportedForDisabledDegradation)2178 TEST_F(VideoStreamEncoderTest,
2179 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
2180 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2181 const int kWidth = 640;
2182 const int kHeight = 360;
2183
2184 video_stream_encoder_->SetSource(
2185 &video_source_,
2186 VideoSendStream::DegradationPreference::kDegradationDisabled);
2187
2188 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
2189 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
2190 WaitForEncodedFrame(i);
2191 }
2192
2193 video_stream_encoder_->Stop();
2194 video_stream_encoder_.reset();
2195 stats_proxy_.reset();
2196
2197 EXPECT_EQ(0,
2198 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
2199 }
2200
TEST_F(VideoStreamEncoderTest,CallsBitrateObserver)2201 TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
2202 MockBitrateObserver bitrate_observer;
2203 video_stream_encoder_->SetBitrateObserver(&bitrate_observer);
2204
2205 const int kDefaultFps = 30;
2206 const BitrateAllocation expected_bitrate =
2207 DefaultVideoBitrateAllocator(fake_encoder_.codec_config())
2208 .GetAllocation(kLowTargetBitrateBps, kDefaultFps);
2209
2210 // First called on bitrate updated, then again on first frame.
2211 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
2212 .Times(2);
2213 video_stream_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0);
2214
2215 const int64_t kStartTimeMs = 1;
2216 video_source_.IncomingCapturedFrame(
2217 CreateFrame(kStartTimeMs, codec_width_, codec_height_));
2218 WaitForEncodedFrame(kStartTimeMs);
2219
2220 // Not called on second frame.
2221 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
2222 .Times(0);
2223 video_source_.IncomingCapturedFrame(
2224 CreateFrame(kStartTimeMs + 1, codec_width_, codec_height_));
2225 WaitForEncodedFrame(kStartTimeMs + 1);
2226
2227 // Called after a process interval.
2228 const int64_t kProcessIntervalMs =
2229 vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
2230 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec *
2231 (kProcessIntervalMs + (1000 / kDefaultFps)));
2232 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
2233 .Times(1);
2234 video_source_.IncomingCapturedFrame(CreateFrame(
2235 kStartTimeMs + kProcessIntervalMs, codec_width_, codec_height_));
2236 WaitForEncodedFrame(kStartTimeMs + kProcessIntervalMs);
2237
2238 video_stream_encoder_->Stop();
2239 }
2240
TEST_F(VideoStreamEncoderTest,OveruseDetectorUpdatedOnReconfigureAndAdaption)2241 TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
2242 const int kFrameWidth = 1280;
2243 const int kFrameHeight = 720;
2244 const int kFramerate = 24;
2245
2246 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2247 test::FrameForwarder source;
2248 video_stream_encoder_->SetSource(
2249 &source, VideoSendStream::DegradationPreference::kMaintainResolution);
2250
2251 // Insert a single frame, triggering initial configuration.
2252 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2253 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2254
2255 EXPECT_EQ(
2256 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2257 kDefaultFramerate);
2258
2259 // Trigger reconfigure encoder (without resetting the entire instance).
2260 VideoEncoderConfig video_encoder_config;
2261 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2262 video_encoder_config.number_of_streams = 1;
2263 video_encoder_config.video_stream_factory =
2264 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
2265 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
2266 kMaxPayloadLength, false);
2267 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2268
2269 // Detector should be updated with fps limit from codec config.
2270 EXPECT_EQ(
2271 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2272 kFramerate);
2273
2274 // Trigger overuse, max framerate should be reduced.
2275 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2276 stats.input_frame_rate = kFramerate;
2277 stats_proxy_->SetMockStats(stats);
2278 video_stream_encoder_->TriggerCpuOveruse();
2279 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2280 int adapted_framerate =
2281 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
2282 EXPECT_LT(adapted_framerate, kFramerate);
2283
2284 // Trigger underuse, max framerate should go back to codec configured fps.
2285 // Set extra low fps, to make sure it's actually reset, not just incremented.
2286 stats = stats_proxy_->GetStats();
2287 stats.input_frame_rate = adapted_framerate / 2;
2288 stats_proxy_->SetMockStats(stats);
2289 video_stream_encoder_->TriggerCpuNormalUsage();
2290 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2291 EXPECT_EQ(
2292 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2293 kFramerate);
2294
2295 video_stream_encoder_->Stop();
2296 }
2297
TEST_F(VideoStreamEncoderTest,OveruseDetectorUpdatedRespectsFramerateAfterUnderuse)2298 TEST_F(VideoStreamEncoderTest,
2299 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
2300 const int kFrameWidth = 1280;
2301 const int kFrameHeight = 720;
2302 const int kLowFramerate = 15;
2303 const int kHighFramerate = 25;
2304
2305 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2306 test::FrameForwarder source;
2307 video_stream_encoder_->SetSource(
2308 &source, VideoSendStream::DegradationPreference::kMaintainResolution);
2309
2310 // Trigger initial configuration.
2311 VideoEncoderConfig video_encoder_config;
2312 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2313 video_encoder_config.number_of_streams = 1;
2314 video_encoder_config.video_stream_factory =
2315 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
2316 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2317 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
2318 kMaxPayloadLength, false);
2319 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2320
2321 EXPECT_EQ(
2322 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2323 kLowFramerate);
2324
2325 // Trigger overuse, max framerate should be reduced.
2326 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2327 stats.input_frame_rate = kLowFramerate;
2328 stats_proxy_->SetMockStats(stats);
2329 video_stream_encoder_->TriggerCpuOveruse();
2330 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2331 int adapted_framerate =
2332 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
2333 EXPECT_LT(adapted_framerate, kLowFramerate);
2334
2335 // Reconfigure the encoder with a new (higher max framerate), max fps should
2336 // still respect the adaptation.
2337 video_encoder_config.video_stream_factory =
2338 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
2339 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2340 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
2341 kMaxPayloadLength, false);
2342 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2343
2344 EXPECT_EQ(
2345 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2346 adapted_framerate);
2347
2348 // Trigger underuse, max framerate should go back to codec configured fps.
2349 stats = stats_proxy_->GetStats();
2350 stats.input_frame_rate = adapted_framerate;
2351 stats_proxy_->SetMockStats(stats);
2352 video_stream_encoder_->TriggerCpuNormalUsage();
2353 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2354 EXPECT_EQ(
2355 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2356 kHighFramerate);
2357
2358 video_stream_encoder_->Stop();
2359 }
2360
TEST_F(VideoStreamEncoderTest,OveruseDetectorUpdatedOnDegradationPreferenceChange)2361 TEST_F(VideoStreamEncoderTest,
2362 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
2363 const int kFrameWidth = 1280;
2364 const int kFrameHeight = 720;
2365 const int kFramerate = 24;
2366
2367 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2368 test::FrameForwarder source;
2369 video_stream_encoder_->SetSource(
2370 &source, VideoSendStream::DegradationPreference::kMaintainResolution);
2371
2372 // Trigger initial configuration.
2373 VideoEncoderConfig video_encoder_config;
2374 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2375 video_encoder_config.number_of_streams = 1;
2376 video_encoder_config.video_stream_factory =
2377 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
2378 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2379 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
2380 kMaxPayloadLength, false);
2381 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2382
2383 EXPECT_EQ(
2384 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2385 kFramerate);
2386
2387 // Trigger overuse, max framerate should be reduced.
2388 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2389 stats.input_frame_rate = kFramerate;
2390 stats_proxy_->SetMockStats(stats);
2391 video_stream_encoder_->TriggerCpuOveruse();
2392 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2393 int adapted_framerate =
2394 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
2395 EXPECT_LT(adapted_framerate, kFramerate);
2396
2397 // Change degradation preference to not enable framerate scaling. Target
2398 // framerate should be changed to codec defined limit.
2399 video_stream_encoder_->SetSource(
2400 &source, VideoSendStream::DegradationPreference::kMaintainFramerate);
2401 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2402 EXPECT_EQ(
2403 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2404 kFramerate);
2405
2406 video_stream_encoder_->Stop();
2407 }
2408
TEST_F(VideoStreamEncoderTest,DropsFramesAndScalesWhenBitrateIsTooLow)2409 TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
2410 const int kTooLowBitrateForFrameSizeBps = 10000;
2411 video_stream_encoder_->OnBitrateUpdated(kTooLowBitrateForFrameSizeBps, 0, 0);
2412 const int kWidth = 640;
2413 const int kHeight = 360;
2414
2415 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2416
2417 // Expect to drop this frame, the wait should time out.
2418 ExpectDroppedFrame();
2419
2420 // Expect the sink_wants to specify a scaled frame.
2421 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
2422
2423 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
2424
2425 // Next frame is scaled.
2426 video_source_.IncomingCapturedFrame(
2427 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
2428
2429 // Expect to drop this frame, the wait should time out.
2430 ExpectDroppedFrame();
2431
2432 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
2433
2434 video_stream_encoder_->Stop();
2435 }
2436
TEST_F(VideoStreamEncoderTest,NumberOfDroppedFramesLimitedWhenBitrateIsTooLow)2437 TEST_F(VideoStreamEncoderTest,
2438 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
2439 const int kTooLowBitrateForFrameSizeBps = 10000;
2440 video_stream_encoder_->OnBitrateUpdated(kTooLowBitrateForFrameSizeBps, 0, 0);
2441 const int kWidth = 640;
2442 const int kHeight = 360;
2443
2444 // We expect the n initial frames to get dropped.
2445 int i;
2446 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
2447 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
2448 ExpectDroppedFrame();
2449 }
2450 // The n+1th frame should not be dropped, even though it's size is too large.
2451 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
2452 WaitForEncodedFrame(i);
2453
2454 // Expect the sink_wants to specify a scaled frame.
2455 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
2456
2457 video_stream_encoder_->Stop();
2458 }
2459
TEST_F(VideoStreamEncoderTest,InitialFrameDropOffWithMaintainResolutionPreference)2460 TEST_F(VideoStreamEncoderTest,
2461 InitialFrameDropOffWithMaintainResolutionPreference) {
2462 const int kWidth = 640;
2463 const int kHeight = 360;
2464 video_stream_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0);
2465
2466 // Set degradation preference.
2467 video_stream_encoder_->SetSource(
2468 &video_source_,
2469 VideoSendStream::DegradationPreference::kMaintainResolution);
2470
2471 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2472 // Frame should not be dropped, even if it's too large.
2473 WaitForEncodedFrame(1);
2474
2475 video_stream_encoder_->Stop();
2476 }
2477
TEST_F(VideoStreamEncoderTest,InitialFrameDropOffWhenEncoderDisabledScaling)2478 TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
2479 const int kWidth = 640;
2480 const int kHeight = 360;
2481 fake_encoder_.SetQualityScaling(false);
2482 video_stream_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0);
2483
2484 // Force quality scaler reconfiguration by resetting the source.
2485 video_stream_encoder_->SetSource(
2486 &video_source_,
2487 VideoSendStream::DegradationPreference::kBalanced);
2488
2489 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2490 // Frame should not be dropped, even if it's too large.
2491 WaitForEncodedFrame(1);
2492
2493 video_stream_encoder_->Stop();
2494 fake_encoder_.SetQualityScaling(true);
2495 }
2496
TEST_F(VideoStreamEncoderTest,ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode)2497 TEST_F(VideoStreamEncoderTest,
2498 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
2499 const int kTooSmallWidth = 10;
2500 const int kTooSmallHeight = 10;
2501 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2502
2503 // Enable kMaintainFramerate preference, no initial limitation.
2504 test::FrameForwarder source;
2505 video_stream_encoder_->SetSource(
2506 &source, VideoSendStream::DegradationPreference::kMaintainFramerate);
2507 VerifyNoLimitation(source.sink_wants());
2508 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2509
2510 // Trigger adapt down, too small frame, expect no change.
2511 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
2512 WaitForEncodedFrame(1);
2513 video_stream_encoder_->TriggerCpuOveruse();
2514 VerifyNoLimitation(source.sink_wants());
2515 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2516 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2517
2518 video_stream_encoder_->Stop();
2519 }
2520
TEST_F(VideoStreamEncoderTest,ResolutionNotAdaptedForTooSmallFrame_BalancedMode)2521 TEST_F(VideoStreamEncoderTest,
2522 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
2523 const int kTooSmallWidth = 10;
2524 const int kTooSmallHeight = 10;
2525 const int kFpsLimit = 7;
2526 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2527
2528 // Enable kBalanced preference, no initial limitation.
2529 test::FrameForwarder source;
2530 video_stream_encoder_->SetSource(
2531 &source,
2532 VideoSendStream::DegradationPreference::kBalanced);
2533 VerifyNoLimitation(source.sink_wants());
2534 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2535 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2536
2537 // Trigger adapt down, expect limited framerate.
2538 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
2539 WaitForEncodedFrame(1);
2540 video_stream_encoder_->TriggerQualityLow();
2541 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
2542 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2543 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2544 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2545
2546 // Trigger adapt down, too small frame, expect no change.
2547 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
2548 WaitForEncodedFrame(2);
2549 video_stream_encoder_->TriggerQualityLow();
2550 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
2551 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2552 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2553 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2554
2555 video_stream_encoder_->Stop();
2556 }
2557
TEST_F(VideoStreamEncoderTest,FailingInitEncodeDoesntCauseCrash)2558 TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
2559 fake_encoder_.ForceInitEncodeFailure(true);
2560 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2561 ResetEncoder("VP8", 2, 1, 1, true, false);
2562 const int kFrameWidth = 1280;
2563 const int kFrameHeight = 720;
2564 video_source_.IncomingCapturedFrame(
2565 CreateFrame(1, kFrameWidth, kFrameHeight));
2566 ExpectDroppedFrame();
2567 video_stream_encoder_->Stop();
2568 }
2569
2570 // TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
TEST_F(VideoStreamEncoderTest,AdaptsResolutionOnOveruse_MaintainFramerateMode)2571 TEST_F(VideoStreamEncoderTest,
2572 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
2573 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2574
2575 const int kFrameWidth = 1280;
2576 const int kFrameHeight = 720;
2577 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
2578 // requested by
2579 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
2580 video_source_.set_adaptation_enabled(true);
2581
2582 video_source_.IncomingCapturedFrame(
2583 CreateFrame(1, kFrameWidth, kFrameHeight));
2584 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
2585
2586 // Trigger CPU overuse, downscale by 3/4.
2587 video_stream_encoder_->TriggerCpuOveruse();
2588 video_source_.IncomingCapturedFrame(
2589 CreateFrame(2, kFrameWidth, kFrameHeight));
2590 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
2591
2592 // Trigger CPU normal use, return to original resolution.
2593 video_stream_encoder_->TriggerCpuNormalUsage();
2594 video_source_.IncomingCapturedFrame(
2595 CreateFrame(3, kFrameWidth, kFrameHeight));
2596 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
2597
2598 video_stream_encoder_->Stop();
2599 }
2600
TEST_F(VideoStreamEncoderTest,AdaptsFramerateOnOveruse_MaintainResolutionMode)2601 TEST_F(VideoStreamEncoderTest,
2602 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
2603 const int kFrameWidth = 1280;
2604 const int kFrameHeight = 720;
2605 int kFrameIntervalMs = rtc::kNumMillisecsPerSec / max_framerate_;
2606
2607 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2608 video_stream_encoder_->SetSource(
2609 &video_source_,
2610 VideoSendStream::DegradationPreference::kMaintainResolution);
2611 video_source_.set_adaptation_enabled(true);
2612
2613 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
2614
2615 video_source_.IncomingCapturedFrame(
2616 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
2617 WaitForEncodedFrame(timestamp_ms);
2618
2619 // Try to trigger overuse. No fps estimate available => no effect.
2620 video_stream_encoder_->TriggerCpuOveruse();
2621
2622 // Insert frames for one second to get a stable estimate.
2623 for (int i = 0; i < max_framerate_; ++i) {
2624 timestamp_ms += kFrameIntervalMs;
2625 video_source_.IncomingCapturedFrame(
2626 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
2627 WaitForEncodedFrame(timestamp_ms);
2628 }
2629
2630 // Trigger CPU overuse, reduce framerate by 2/3.
2631 video_stream_encoder_->TriggerCpuOveruse();
2632 int num_frames_dropped = 0;
2633 for (int i = 0; i < max_framerate_; ++i) {
2634 timestamp_ms += kFrameIntervalMs;
2635 video_source_.IncomingCapturedFrame(
2636 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
2637 if (!WaitForFrame(kFrameTimeoutMs)) {
2638 ++num_frames_dropped;
2639 } else {
2640 sink_.CheckLastFrameSizeMathces(kFrameWidth, kFrameHeight);
2641 }
2642 }
2643
2644 // Add some slack to account for frames dropped by the frame dropper.
2645 const int kErrorMargin = 1;
2646 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
2647 kErrorMargin);
2648
2649 // Trigger CPU overuse, reduce framerate by 2/3 again.
2650 video_stream_encoder_->TriggerCpuOveruse();
2651 num_frames_dropped = 0;
2652 for (int i = 0; i < max_framerate_; ++i) {
2653 timestamp_ms += kFrameIntervalMs;
2654 video_source_.IncomingCapturedFrame(
2655 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
2656 if (!WaitForFrame(kFrameTimeoutMs)) {
2657 ++num_frames_dropped;
2658 } else {
2659 sink_.CheckLastFrameSizeMathces(kFrameWidth, kFrameHeight);
2660 }
2661 }
2662 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
2663 kErrorMargin);
2664
2665 // Go back up one step.
2666 video_stream_encoder_->TriggerCpuNormalUsage();
2667 num_frames_dropped = 0;
2668 for (int i = 0; i < max_framerate_; ++i) {
2669 timestamp_ms += kFrameIntervalMs;
2670 video_source_.IncomingCapturedFrame(
2671 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
2672 if (!WaitForFrame(kFrameTimeoutMs)) {
2673 ++num_frames_dropped;
2674 } else {
2675 sink_.CheckLastFrameSizeMathces(kFrameWidth, kFrameHeight);
2676 }
2677 }
2678 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
2679 kErrorMargin);
2680
2681 // Go back up to original mode.
2682 video_stream_encoder_->TriggerCpuNormalUsage();
2683 num_frames_dropped = 0;
2684 for (int i = 0; i < max_framerate_; ++i) {
2685 timestamp_ms += kFrameIntervalMs;
2686 video_source_.IncomingCapturedFrame(
2687 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
2688 if (!WaitForFrame(kFrameTimeoutMs)) {
2689 ++num_frames_dropped;
2690 } else {
2691 sink_.CheckLastFrameSizeMathces(kFrameWidth, kFrameHeight);
2692 }
2693 }
2694 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
2695
2696 video_stream_encoder_->Stop();
2697 }
2698
TEST_F(VideoStreamEncoderTest,DoesntAdaptDownPastMinFramerate)2699 TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
2700 const int kFramerateFps = 5;
2701 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
2702 const int kMinFpsFrameInterval = rtc::kNumMillisecsPerSec / kMinFramerateFps;
2703 const int kFrameWidth = 1280;
2704 const int kFrameHeight = 720;
2705
2706 // Reconfigure encoder with two temporal layers and screensharing, which will
2707 // disable frame dropping and make testing easier.
2708 ResetEncoder("VP8", 1, 2, 1, true, true);
2709
2710 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2711 video_stream_encoder_->SetSource(
2712 &video_source_,
2713 VideoSendStream::DegradationPreference::kMaintainResolution);
2714 video_source_.set_adaptation_enabled(true);
2715
2716 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
2717
2718 // Trigger overuse as much as we can.
2719 for (int i = 0; i < VideoStreamEncoder::kMaxCpuResolutionDowngrades; ++i) {
2720 // Insert frames to get a new fps estimate...
2721 for (int j = 0; j < kFramerateFps; ++j) {
2722 video_source_.IncomingCapturedFrame(
2723 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
2724 timestamp_ms += kFrameIntervalMs;
2725 }
2726 // ...and then try to adapt again.
2727 video_stream_encoder_->TriggerCpuOveruse();
2728 }
2729
2730 // Drain any frame in the pipeline.
2731 WaitForFrame(kDefaultTimeoutMs);
2732
2733 // Insert frames at min fps, all should go through.
2734 for (int i = 0; i < 10; ++i) {
2735 timestamp_ms += kMinFpsFrameInterval;
2736 video_source_.IncomingCapturedFrame(
2737 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
2738 WaitForEncodedFrame(timestamp_ms);
2739 }
2740
2741 video_stream_encoder_->Stop();
2742 }
2743
TEST_F(VideoStreamEncoderTest,AdaptsResolutionAndFramerateForLowQuality_BalancedMode)2744 TEST_F(VideoStreamEncoderTest,
2745 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
2746 const int kWidth = 1280;
2747 const int kHeight = 720;
2748 const int64_t kFrameIntervalMs = 150;
2749 int64_t timestamp_ms = kFrameIntervalMs;
2750 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2751
2752 // Enable kBalanced preference, no initial limitation.
2753 AdaptingFrameForwarder source;
2754 source.set_adaptation_enabled(true);
2755 video_stream_encoder_->SetSource(
2756 &source,
2757 VideoSendStream::DegradationPreference::kBalanced);
2758 timestamp_ms += kFrameIntervalMs;
2759 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2760 WaitForEncodedFrame(kWidth, kHeight);
2761 VerifyNoLimitation(source.sink_wants());
2762 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2763 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2764 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2765
2766 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
2767 video_stream_encoder_->TriggerQualityLow();
2768 timestamp_ms += kFrameIntervalMs;
2769 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2770 WaitForEncodedFrame(timestamp_ms);
2771 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2772 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2773 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2774 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2775
2776 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
2777 video_stream_encoder_->TriggerQualityLow();
2778 timestamp_ms += kFrameIntervalMs;
2779 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2780 WaitForEncodedFrame(timestamp_ms);
2781 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
2782 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2783 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2784 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2785
2786 // Trigger adapt down, expect reduced fps (640x360@15fps).
2787 video_stream_encoder_->TriggerQualityLow();
2788 timestamp_ms += kFrameIntervalMs;
2789 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2790 WaitForEncodedFrame(timestamp_ms);
2791 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2792 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2793 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2794 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2795
2796 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
2797 video_stream_encoder_->TriggerQualityLow();
2798 timestamp_ms += kFrameIntervalMs;
2799 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2800 WaitForEncodedFrame(timestamp_ms);
2801 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
2802 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2803 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2804 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2805
2806 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
2807 video_stream_encoder_->TriggerQualityLow();
2808 timestamp_ms += kFrameIntervalMs;
2809 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2810 WaitForEncodedFrame(timestamp_ms);
2811 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2812 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2813 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2814 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2815
2816 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
2817 video_stream_encoder_->TriggerQualityLow();
2818 timestamp_ms += kFrameIntervalMs;
2819 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2820 WaitForEncodedFrame(timestamp_ms);
2821 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
2822 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2823 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2824 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2825
2826 // Trigger adapt down, expect reduced fps (320x180@7fps).
2827 video_stream_encoder_->TriggerQualityLow();
2828 timestamp_ms += kFrameIntervalMs;
2829 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2830 WaitForEncodedFrame(timestamp_ms);
2831 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2832 rtc::VideoSinkWants last_wants = source.sink_wants();
2833 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2834 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2835 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2836
2837 // Trigger adapt down, min resolution reached, expect no change.
2838 video_stream_encoder_->TriggerQualityLow();
2839 timestamp_ms += kFrameIntervalMs;
2840 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2841 WaitForEncodedFrame(timestamp_ms);
2842 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
2843 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2844 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2845 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2846
2847 // Trigger adapt down, expect expect increased fps (320x180@10fps).
2848 video_stream_encoder_->TriggerQualityHigh();
2849 timestamp_ms += kFrameIntervalMs;
2850 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2851 WaitForEncodedFrame(timestamp_ms);
2852 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
2853 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2854 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2855 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2856
2857 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
2858 video_stream_encoder_->TriggerQualityHigh();
2859 timestamp_ms += kFrameIntervalMs;
2860 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2861 WaitForEncodedFrame(timestamp_ms);
2862 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
2863 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2864 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2865 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2866
2867 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
2868 video_stream_encoder_->TriggerQualityHigh();
2869 timestamp_ms += kFrameIntervalMs;
2870 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2871 WaitForEncodedFrame(timestamp_ms);
2872 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
2873 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2874 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2875 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2876
2877 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
2878 video_stream_encoder_->TriggerQualityHigh();
2879 timestamp_ms += kFrameIntervalMs;
2880 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2881 WaitForEncodedFrame(timestamp_ms);
2882 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
2883 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2884 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2885 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2886
2887 // Trigger adapt up, expect increased fps (640x360@30fps).
2888 video_stream_encoder_->TriggerQualityHigh();
2889 timestamp_ms += kFrameIntervalMs;
2890 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2891 WaitForEncodedFrame(timestamp_ms);
2892 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
2893 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2894 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2895 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2896
2897 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
2898 video_stream_encoder_->TriggerQualityHigh();
2899 timestamp_ms += kFrameIntervalMs;
2900 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2901 WaitForEncodedFrame(timestamp_ms);
2902 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
2903 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2904 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2905 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2906
2907 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
2908 video_stream_encoder_->TriggerQualityHigh();
2909 timestamp_ms += kFrameIntervalMs;
2910 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2911 WaitForEncodedFrame(kWidth, kHeight);
2912 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
2913 VerifyNoLimitation(source.sink_wants());
2914 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2915 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2916 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2917
2918 // Trigger adapt up, expect no change.
2919 video_stream_encoder_->TriggerQualityHigh();
2920 VerifyNoLimitation(source.sink_wants());
2921 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2922
2923 video_stream_encoder_->Stop();
2924 }
2925
TEST_F(VideoStreamEncoderTest,AdaptWithTwoReasonsAndDifferentOrder_Framerate)2926 TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
2927 const int kWidth = 1280;
2928 const int kHeight = 720;
2929 const int64_t kFrameIntervalMs = 150;
2930 int64_t timestamp_ms = kFrameIntervalMs;
2931 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2932
2933 // Enable kBalanced preference, no initial limitation.
2934 AdaptingFrameForwarder source;
2935 source.set_adaptation_enabled(true);
2936 video_stream_encoder_->SetSource(
2937 &source,
2938 VideoSendStream::DegradationPreference::kBalanced);
2939 timestamp_ms += kFrameIntervalMs;
2940 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2941 WaitForEncodedFrame(kWidth, kHeight);
2942 VerifyNoLimitation(source.sink_wants());
2943 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2944 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2945 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2946 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2947 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2948 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2949
2950 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
2951 video_stream_encoder_->TriggerCpuOveruse();
2952 timestamp_ms += kFrameIntervalMs;
2953 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2954 WaitForEncodedFrame(timestamp_ms);
2955 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2956 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2957 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2958 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2959 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2960 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2961 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2962
2963 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
2964 video_stream_encoder_->TriggerCpuOveruse();
2965 timestamp_ms += kFrameIntervalMs;
2966 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2967 WaitForEncodedFrame(timestamp_ms);
2968 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
2969 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2970 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2971 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2972 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2973 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2974 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2975
2976 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
2977 video_stream_encoder_->TriggerQualityLow();
2978 timestamp_ms += kFrameIntervalMs;
2979 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2980 WaitForEncodedFrame(timestamp_ms);
2981 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2982 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2983 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2984 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2985 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2986 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2987 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2988
2989 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
2990 video_stream_encoder_->TriggerCpuNormalUsage();
2991 timestamp_ms += kFrameIntervalMs;
2992 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2993 WaitForEncodedFrame(timestamp_ms);
2994 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
2995 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2996 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2997 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2998 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2999 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3000 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3001
3002 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
3003 video_stream_encoder_->TriggerQualityHigh();
3004 timestamp_ms += kFrameIntervalMs;
3005 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3006 WaitForEncodedFrame(timestamp_ms);
3007 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3008 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3009 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3010 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3011 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3012 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3013 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3014
3015 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
3016 video_stream_encoder_->TriggerCpuNormalUsage();
3017 timestamp_ms += kFrameIntervalMs;
3018 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3019 WaitForEncodedFrame(kWidth, kHeight);
3020 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3021 VerifyNoLimitation(source.sink_wants());
3022 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3023 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3024 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3025 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3026 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3027 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3028
3029 // Trigger adapt up, expect no change.
3030 video_stream_encoder_->TriggerQualityHigh();
3031 VerifyNoLimitation(source.sink_wants());
3032 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3033 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3034
3035 video_stream_encoder_->Stop();
3036 }
3037
TEST_F(VideoStreamEncoderTest,AdaptWithTwoReasonsAndDifferentOrder_Resolution)3038 TEST_F(VideoStreamEncoderTest,
3039 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
3040 const int kWidth = 640;
3041 const int kHeight = 360;
3042 const int kFpsLimit = 15;
3043 const int64_t kFrameIntervalMs = 150;
3044 int64_t timestamp_ms = kFrameIntervalMs;
3045 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3046
3047 // Enable kBalanced preference, no initial limitation.
3048 AdaptingFrameForwarder source;
3049 source.set_adaptation_enabled(true);
3050 video_stream_encoder_->SetSource(
3051 &source,
3052 VideoSendStream::DegradationPreference::kBalanced);
3053 timestamp_ms += kFrameIntervalMs;
3054 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3055 WaitForEncodedFrame(kWidth, kHeight);
3056 VerifyNoLimitation(source.sink_wants());
3057 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3058 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3059 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3060 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3061 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3062 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3063
3064 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
3065 video_stream_encoder_->TriggerCpuOveruse();
3066 timestamp_ms += kFrameIntervalMs;
3067 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3068 WaitForEncodedFrame(timestamp_ms);
3069 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3070 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3071 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3072 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3073 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3074 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3075 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3076
3077 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
3078 video_stream_encoder_->TriggerQualityLow();
3079 timestamp_ms += kFrameIntervalMs;
3080 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3081 WaitForEncodedFrame(timestamp_ms);
3082 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3083 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3084 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3085 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3086 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3087 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3088 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3089
3090 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
3091 video_stream_encoder_->TriggerCpuNormalUsage();
3092 timestamp_ms += kFrameIntervalMs;
3093 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3094 WaitForEncodedFrame(timestamp_ms);
3095 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3096 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3097 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3098 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3099 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3100 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3101 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3102
3103 // Trigger quality adapt up, expect increased fps (640x360@30fps).
3104 video_stream_encoder_->TriggerQualityHigh();
3105 timestamp_ms += kFrameIntervalMs;
3106 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3107 WaitForEncodedFrame(timestamp_ms);
3108 VerifyNoLimitation(source.sink_wants());
3109 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3110 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3111 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3112 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3113 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3114 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3115
3116 // Trigger adapt up, expect no change.
3117 video_stream_encoder_->TriggerQualityHigh();
3118 VerifyNoLimitation(source.sink_wants());
3119 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3120 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3121
3122 video_stream_encoder_->Stop();
3123 }
3124
TEST_F(VideoStreamEncoderTest,AcceptsFullHdAdaptedDownSimulcastFrames)3125 TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
3126 // Simulates simulcast behavior and makes highest stream resolutions divisible
3127 // by 4.
3128 class CroppingVideoStreamFactory
3129 : public VideoEncoderConfig::VideoStreamFactoryInterface {
3130 public:
3131 explicit CroppingVideoStreamFactory(size_t num_temporal_layers,
3132 int framerate)
3133 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
3134 EXPECT_GT(num_temporal_layers, 0u);
3135 EXPECT_GT(framerate, 0);
3136 }
3137
3138 private:
3139 std::vector<VideoStream> CreateEncoderStreams(
3140 int width,
3141 int height,
3142 const VideoEncoderConfig& encoder_config) override {
3143 std::vector<VideoStream> streams =
3144 test::CreateVideoStreams(width - width % 4, height - height % 4,
3145 encoder_config);
3146 for (VideoStream& stream : streams) {
3147 stream.temporal_layer_thresholds_bps.resize(num_temporal_layers_ - 1);
3148 stream.max_framerate = framerate_;
3149 }
3150 return streams;
3151 }
3152
3153 const size_t num_temporal_layers_;
3154 const int framerate_;
3155 };
3156
3157 const int kFrameWidth = 1920;
3158 const int kFrameHeight = 1080;
3159 // 3/4 of 1920.
3160 const int kAdaptedFrameWidth = 1440;
3161 // 3/4 of 1080 rounded down to multiple of 4.
3162 const int kAdaptedFrameHeight = 808;
3163 const int kFramerate = 24;
3164
3165 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3166 // Trigger reconfigure encoder (without resetting the entire instance).
3167 VideoEncoderConfig video_encoder_config;
3168 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3169 video_encoder_config.number_of_streams = 1;
3170 video_encoder_config.video_stream_factory =
3171 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
3172 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
3173 kMaxPayloadLength, false);
3174 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3175
3176 video_source_.set_adaptation_enabled(true);
3177
3178 video_source_.IncomingCapturedFrame(
3179 CreateFrame(1, kFrameWidth, kFrameHeight));
3180 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
3181
3182 // Trigger CPU overuse, downscale by 3/4.
3183 video_stream_encoder_->TriggerCpuOveruse();
3184 video_source_.IncomingCapturedFrame(
3185 CreateFrame(2, kFrameWidth, kFrameHeight));
3186 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
3187
3188 video_stream_encoder_->Stop();
3189 }
3190
TEST_F(VideoStreamEncoderTest,PeriodicallyUpdatesChannelParameters)3191 TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
3192 const int kFrameWidth = 1280;
3193 const int kFrameHeight = 720;
3194 const int kLowFps = 2;
3195 const int kHighFps = 30;
3196
3197 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3198
3199 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3200 max_framerate_ = kLowFps;
3201
3202 // Insert 2 seconds of 2fps video.
3203 for (int i = 0; i < kLowFps * 2; ++i) {
3204 video_source_.IncomingCapturedFrame(
3205 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3206 WaitForEncodedFrame(timestamp_ms);
3207 timestamp_ms += 1000 / kLowFps;
3208 }
3209
3210 // Make sure encoder is updated with new target.
3211 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3212 video_source_.IncomingCapturedFrame(
3213 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3214 WaitForEncodedFrame(timestamp_ms);
3215 timestamp_ms += 1000 / kLowFps;
3216
3217 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
3218
3219 // Insert 30fps frames for just a little more than the forced update period.
3220 const int kVcmTimerIntervalFrames =
3221 (vcm::VCMProcessTimer::kDefaultProcessIntervalMs * kHighFps) / 1000;
3222 const int kFrameIntervalMs = 1000 / kHighFps;
3223 max_framerate_ = kHighFps;
3224 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
3225 video_source_.IncomingCapturedFrame(
3226 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3227 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
3228 // be dropped if the encoder hans't been updated with the new higher target
3229 // framerate yet, causing it to overshoot the target bitrate and then
3230 // suffering the wrath of the media optimizer.
3231 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
3232 timestamp_ms += kFrameIntervalMs;
3233 }
3234
3235 // Don expect correct measurement just yet, but it should be higher than
3236 // before.
3237 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
3238
3239 video_stream_encoder_->Stop();
3240 }
3241
TEST_F(VideoStreamEncoderTest,DoesNotUpdateBitrateAllocationWhenSuspended)3242 TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
3243 const int kFrameWidth = 1280;
3244 const int kFrameHeight = 720;
3245 const int kTargetBitrateBps = 1000000;
3246
3247 MockBitrateObserver bitrate_observer;
3248 video_stream_encoder_->SetBitrateObserver(&bitrate_observer);
3249
3250 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
3251 // Initial bitrate update.
3252 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3253 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3254
3255 // Insert a first video frame, causes another bitrate update.
3256 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3257 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
3258 video_source_.IncomingCapturedFrame(
3259 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3260 WaitForEncodedFrame(timestamp_ms);
3261
3262 // Next, simulate video suspension due to pacer queue overrun.
3263 video_stream_encoder_->OnBitrateUpdated(0, 0, 1);
3264
3265 // Skip ahead until a new periodic parameter update should have occured.
3266 timestamp_ms += vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
3267 fake_clock_.AdvanceTimeMicros(
3268 vcm::VCMProcessTimer::kDefaultProcessIntervalMs *
3269 rtc::kNumMicrosecsPerMillisec);
3270
3271 // Bitrate observer should not be called.
3272 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
3273 video_source_.IncomingCapturedFrame(
3274 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3275 ExpectDroppedFrame();
3276
3277 video_stream_encoder_->Stop();
3278 }
3279
3280 } // namespace webrtc
3281