1 /*
2  *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include <cstdint>
12 #include <memory>
13 
14 #include "api/test/create_network_emulation_manager.h"
15 #include "api/test/create_peerconnection_quality_test_fixture.h"
16 #include "api/test/network_emulation_manager.h"
17 #include "api/test/peerconnection_quality_test_fixture.h"
18 #include "call/simulated_network.h"
19 #include "system_wrappers/include/field_trial.h"
20 #include "test/field_trial.h"
21 #include "test/gtest.h"
22 #include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h"
23 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
24 #include "test/pc/e2e/network_quality_metrics_reporter.h"
25 #include "test/testsupport/file_utils.h"
26 
27 namespace webrtc {
28 namespace webrtc_pc_e2e {
29 namespace {
30 
31 class PeerConnectionE2EQualityTestSmokeTest : public ::testing::Test {
32  public:
33   using PeerConfigurer = PeerConnectionE2EQualityTestFixture::PeerConfigurer;
34   using RunParams = PeerConnectionE2EQualityTestFixture::RunParams;
35   using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig;
36   using VideoCodecConfig =
37       PeerConnectionE2EQualityTestFixture::VideoCodecConfig;
38   using AudioConfig = PeerConnectionE2EQualityTestFixture::AudioConfig;
39   using ScreenShareConfig =
40       PeerConnectionE2EQualityTestFixture::ScreenShareConfig;
41   using ScrollingParams = PeerConnectionE2EQualityTestFixture::ScrollingParams;
42   using VideoSimulcastConfig =
43       PeerConnectionE2EQualityTestFixture::VideoSimulcastConfig;
44   using EchoEmulationConfig =
45       PeerConnectionE2EQualityTestFixture::EchoEmulationConfig;
46 
RunTest(const std::string & test_case_name,const RunParams & run_params,rtc::FunctionView<void (PeerConfigurer *)> alice_configurer,rtc::FunctionView<void (PeerConfigurer *)> bob_configurer)47   void RunTest(const std::string& test_case_name,
48                const RunParams& run_params,
49                rtc::FunctionView<void(PeerConfigurer*)> alice_configurer,
50                rtc::FunctionView<void(PeerConfigurer*)> bob_configurer) {
51     // Setup emulated network
52     std::unique_ptr<NetworkEmulationManager> network_emulation_manager =
53         CreateNetworkEmulationManager();
54 
55     auto alice_network_behavior =
56         std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig());
57     SimulatedNetwork* alice_network_behavior_ptr = alice_network_behavior.get();
58     EmulatedNetworkNode* alice_node =
59         network_emulation_manager->CreateEmulatedNode(
60             std::move(alice_network_behavior));
61     EmulatedNetworkNode* bob_node =
62         network_emulation_manager->CreateEmulatedNode(
63             std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig()));
64     auto* alice_endpoint =
65         network_emulation_manager->CreateEndpoint(EmulatedEndpointConfig());
66     EmulatedEndpoint* bob_endpoint =
67         network_emulation_manager->CreateEndpoint(EmulatedEndpointConfig());
68     network_emulation_manager->CreateRoute(alice_endpoint, {alice_node},
69                                            bob_endpoint);
70     network_emulation_manager->CreateRoute(bob_endpoint, {bob_node},
71                                            alice_endpoint);
72 
73     // Create analyzers.
74     std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer =
75         std::make_unique<DefaultVideoQualityAnalyzer>();
76     // This is only done for the sake of smoke testing. In general there should
77     // be no need to explicitly pull data from analyzers after the run.
78     auto* video_analyzer_ptr =
79         static_cast<DefaultVideoQualityAnalyzer*>(video_quality_analyzer.get());
80 
81     auto fixture = CreatePeerConnectionE2EQualityTestFixture(
82         test_case_name, /*audio_quality_analyzer=*/nullptr,
83         std::move(video_quality_analyzer));
84     fixture->ExecuteAt(TimeDelta::Seconds(2),
85                        [alice_network_behavior_ptr](TimeDelta) {
86                          BuiltInNetworkBehaviorConfig config;
87                          config.loss_percent = 5;
88                          alice_network_behavior_ptr->SetConfig(config);
89                        });
90 
91     // Setup components. We need to provide rtc::NetworkManager compatible with
92     // emulated network layer.
93     EmulatedNetworkManagerInterface* alice_network =
94         network_emulation_manager->CreateEmulatedNetworkManagerInterface(
95             {alice_endpoint});
96     EmulatedNetworkManagerInterface* bob_network =
97         network_emulation_manager->CreateEmulatedNetworkManagerInterface(
98             {bob_endpoint});
99 
100     fixture->AddPeer(alice_network->network_thread(),
101                      alice_network->network_manager(), alice_configurer);
102     fixture->AddPeer(bob_network->network_thread(),
103                      bob_network->network_manager(), bob_configurer);
104     fixture->AddQualityMetricsReporter(
105         std::make_unique<NetworkQualityMetricsReporter>(alice_network,
106                                                         bob_network));
107 
108     fixture->Run(run_params);
109 
110     EXPECT_GE(fixture->GetRealTestDuration(), run_params.run_duration);
111     for (auto stream_label : video_analyzer_ptr->GetKnownVideoStreams()) {
112       FrameCounters stream_conters =
113           video_analyzer_ptr->GetPerStreamCounters().at(stream_label);
114       // On some devices the pipeline can be too slow, so we actually can't
115       // force real constraints here. Lets just check, that at least 1
116       // frame passed whole pipeline.
117       int64_t expected_min_fps = run_params.run_duration.seconds() * 30;
118       EXPECT_GE(stream_conters.captured, expected_min_fps);
119       EXPECT_GE(stream_conters.pre_encoded, 1);
120       EXPECT_GE(stream_conters.encoded, 1);
121       EXPECT_GE(stream_conters.received, 1);
122       EXPECT_GE(stream_conters.decoded, 1);
123       EXPECT_GE(stream_conters.rendered, 1);
124     }
125   }
126 };
127 
128 }  // namespace
129 
130 // IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
131 #if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
132 #define MAYBE_Smoke DISABLED_Smoke
133 #else
134 #define MAYBE_Smoke Smoke
135 #endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest,MAYBE_Smoke)136 TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Smoke) {
137   RunParams run_params(TimeDelta::Seconds(2));
138   run_params.video_codecs = {
139       VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})};
140   run_params.use_flex_fec = true;
141   run_params.use_ulp_fec = true;
142   run_params.video_encoder_bitrate_multiplier = 1.1;
143   test::ScopedFieldTrials field_trials(
144       std::string(field_trial::GetFieldTrialString()) +
145       "WebRTC-UseStandardBytesStats/Enabled/");
146   RunTest(
147       "smoke", run_params,
148       [](PeerConfigurer* alice) {
149         VideoConfig video(640, 360, 30);
150         video.stream_label = "alice-video";
151         video.sync_group = "alice-media";
152         alice->AddVideoConfig(std::move(video));
153 
154         AudioConfig audio;
155         audio.stream_label = "alice-audio";
156         audio.mode = AudioConfig::Mode::kFile;
157         audio.input_file_name =
158             test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
159         audio.sampling_frequency_in_hz = 48000;
160         audio.sync_group = "alice-media";
161         alice->SetAudioConfig(std::move(audio));
162       },
163       [](PeerConfigurer* bob) {
164         VideoConfig video(640, 360, 30);
165         video.stream_label = "bob-video";
166         video.temporal_layers_count = 2;
167         bob->AddVideoConfig(std::move(video));
168 
169         VideoConfig screenshare(640, 360, 30);
170         screenshare.stream_label = "bob-screenshare";
171         screenshare.screen_share_config =
172             ScreenShareConfig(TimeDelta::Seconds(2));
173         screenshare.screen_share_config->scrolling_params = ScrollingParams(
174             TimeDelta::Millis(1800), kDefaultSlidesWidth, kDefaultSlidesHeight);
175         bob->AddVideoConfig(screenshare);
176 
177         AudioConfig audio;
178         audio.stream_label = "bob-audio";
179         audio.mode = AudioConfig::Mode::kFile;
180         audio.input_file_name =
181             test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
182         bob->SetAudioConfig(std::move(audio));
183       });
184 }
185 
186 // IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
187 #if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
188 #define MAYBE_Echo DISABLED_Echo
189 #else
190 #define MAYBE_Echo Echo
191 #endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest,MAYBE_Echo)192 TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Echo) {
193   RunParams run_params(TimeDelta::Seconds(2));
194   run_params.echo_emulation_config = EchoEmulationConfig();
195   RunTest(
196       "smoke", run_params,
197       [](PeerConfigurer* alice) {
198         AudioConfig audio;
199         audio.stream_label = "alice-audio";
200         audio.mode = AudioConfig::Mode::kFile;
201         audio.input_file_name =
202             test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
203         audio.sampling_frequency_in_hz = 48000;
204         alice->SetAudioConfig(std::move(audio));
205       },
206       [](PeerConfigurer* bob) {
207         AudioConfig audio;
208         audio.stream_label = "bob-audio";
209         audio.mode = AudioConfig::Mode::kFile;
210         audio.input_file_name =
211             test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
212         bob->SetAudioConfig(std::move(audio));
213       });
214 }
215 
216 // IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
217 #if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
218 #define MAYBE_Simulcast DISABLED_Simulcast
219 #else
220 #define MAYBE_Simulcast Simulcast
221 #endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest,MAYBE_Simulcast)222 TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Simulcast) {
223   RunParams run_params(TimeDelta::Seconds(2));
224   run_params.video_codecs = {VideoCodecConfig(cricket::kVp8CodecName)};
225   RunTest(
226       "simulcast", run_params,
227       [](PeerConfigurer* alice) {
228         VideoConfig simulcast(1280, 720, 30);
229         simulcast.stream_label = "alice-simulcast";
230         simulcast.simulcast_config = VideoSimulcastConfig(3, 0);
231         alice->AddVideoConfig(std::move(simulcast));
232 
233         AudioConfig audio;
234         audio.stream_label = "alice-audio";
235         audio.mode = AudioConfig::Mode::kFile;
236         audio.input_file_name =
237             test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
238         alice->SetAudioConfig(std::move(audio));
239       },
240       [](PeerConfigurer* bob) {
241         VideoConfig video(640, 360, 30);
242         video.stream_label = "bob-video";
243         bob->AddVideoConfig(std::move(video));
244 
245         AudioConfig audio;
246         audio.stream_label = "bob-audio";
247         audio.mode = AudioConfig::Mode::kFile;
248         audio.input_file_name =
249             test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
250         bob->SetAudioConfig(std::move(audio));
251       });
252 }
253 
254 // IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
255 #if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
256 #define MAYBE_Svc DISABLED_Svc
257 #else
258 #define MAYBE_Svc Svc
259 #endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest,MAYBE_Svc)260 TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Svc) {
261   RunParams run_params(TimeDelta::Seconds(2));
262   run_params.video_codecs = {VideoCodecConfig(cricket::kVp9CodecName)};
263   RunTest(
264       "simulcast", run_params,
265       [](PeerConfigurer* alice) {
266         VideoConfig simulcast(1280, 720, 30);
267         simulcast.stream_label = "alice-svc";
268         // Because we have network with packets loss we can analyze only the
269         // highest spatial layer in SVC mode.
270         simulcast.simulcast_config = VideoSimulcastConfig(3, 2);
271         alice->AddVideoConfig(std::move(simulcast));
272 
273         AudioConfig audio;
274         audio.stream_label = "alice-audio";
275         audio.mode = AudioConfig::Mode::kFile;
276         audio.input_file_name =
277             test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
278         alice->SetAudioConfig(std::move(audio));
279       },
280       [](PeerConfigurer* bob) {
281         VideoConfig video(640, 360, 30);
282         video.stream_label = "bob-video";
283         bob->AddVideoConfig(std::move(video));
284 
285         AudioConfig audio;
286         audio.stream_label = "bob-audio";
287         audio.mode = AudioConfig::Mode::kFile;
288         audio.input_file_name =
289             test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
290         bob->SetAudioConfig(std::move(audio));
291       });
292 }
293 
294 // IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
295 #if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
296 #define MAYBE_HighBitrate DISABLED_HighBitrate
297 #else
298 #define MAYBE_HighBitrate HighBitrate
299 #endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest,MAYBE_HighBitrate)300 TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_HighBitrate) {
301   RunParams run_params(TimeDelta::Seconds(2));
302   run_params.video_codecs = {
303       VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})};
304 
305   RunTest(
306       "smoke", run_params,
307       [](PeerConfigurer* alice) {
308         PeerConnectionInterface::BitrateParameters bitrate_params;
309         bitrate_params.current_bitrate_bps = 3'000'000;
310         bitrate_params.max_bitrate_bps = 3'000'000;
311         alice->SetBitrateParameters(bitrate_params);
312         VideoConfig video(800, 600, 30);
313         video.stream_label = "alice-video";
314         video.min_encode_bitrate_bps = 500'000;
315         video.max_encode_bitrate_bps = 3'000'000;
316         alice->AddVideoConfig(std::move(video));
317 
318         AudioConfig audio;
319         audio.stream_label = "alice-audio";
320         audio.mode = AudioConfig::Mode::kFile;
321         audio.input_file_name =
322             test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
323         audio.sampling_frequency_in_hz = 48000;
324         alice->SetAudioConfig(std::move(audio));
325       },
326       [](PeerConfigurer* bob) {
327         PeerConnectionInterface::BitrateParameters bitrate_params;
328         bitrate_params.current_bitrate_bps = 3'000'000;
329         bitrate_params.max_bitrate_bps = 3'000'000;
330         bob->SetBitrateParameters(bitrate_params);
331         VideoConfig video(800, 600, 30);
332         video.stream_label = "bob-video";
333         video.min_encode_bitrate_bps = 500'000;
334         video.max_encode_bitrate_bps = 3'000'000;
335         bob->AddVideoConfig(std::move(video));
336 
337         AudioConfig audio;
338         audio.stream_label = "bob-audio";
339         audio.mode = AudioConfig::Mode::kFile;
340         audio.input_file_name =
341             test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
342         bob->SetAudioConfig(std::move(audio));
343       });
344 }
345 
346 }  // namespace webrtc_pc_e2e
347 }  // namespace webrtc
348