1 /*
2 * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <cstdint>
12 #include <memory>
13
14 #include "api/test/create_network_emulation_manager.h"
15 #include "api/test/create_peerconnection_quality_test_fixture.h"
16 #include "api/test/network_emulation_manager.h"
17 #include "api/test/peerconnection_quality_test_fixture.h"
18 #include "call/simulated_network.h"
19 #include "system_wrappers/include/field_trial.h"
20 #include "test/field_trial.h"
21 #include "test/gtest.h"
22 #include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h"
23 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
24 #include "test/pc/e2e/network_quality_metrics_reporter.h"
25 #include "test/testsupport/file_utils.h"
26
27 namespace webrtc {
28 namespace webrtc_pc_e2e {
29 namespace {
30
31 class PeerConnectionE2EQualityTestSmokeTest : public ::testing::Test {
32 public:
33 using PeerConfigurer = PeerConnectionE2EQualityTestFixture::PeerConfigurer;
34 using RunParams = PeerConnectionE2EQualityTestFixture::RunParams;
35 using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig;
36 using VideoCodecConfig =
37 PeerConnectionE2EQualityTestFixture::VideoCodecConfig;
38 using AudioConfig = PeerConnectionE2EQualityTestFixture::AudioConfig;
39 using ScreenShareConfig =
40 PeerConnectionE2EQualityTestFixture::ScreenShareConfig;
41 using ScrollingParams = PeerConnectionE2EQualityTestFixture::ScrollingParams;
42 using VideoSimulcastConfig =
43 PeerConnectionE2EQualityTestFixture::VideoSimulcastConfig;
44 using EchoEmulationConfig =
45 PeerConnectionE2EQualityTestFixture::EchoEmulationConfig;
46
RunTest(const std::string & test_case_name,const RunParams & run_params,rtc::FunctionView<void (PeerConfigurer *)> alice_configurer,rtc::FunctionView<void (PeerConfigurer *)> bob_configurer)47 void RunTest(const std::string& test_case_name,
48 const RunParams& run_params,
49 rtc::FunctionView<void(PeerConfigurer*)> alice_configurer,
50 rtc::FunctionView<void(PeerConfigurer*)> bob_configurer) {
51 // Setup emulated network
52 std::unique_ptr<NetworkEmulationManager> network_emulation_manager =
53 CreateNetworkEmulationManager();
54
55 auto alice_network_behavior =
56 std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig());
57 SimulatedNetwork* alice_network_behavior_ptr = alice_network_behavior.get();
58 EmulatedNetworkNode* alice_node =
59 network_emulation_manager->CreateEmulatedNode(
60 std::move(alice_network_behavior));
61 EmulatedNetworkNode* bob_node =
62 network_emulation_manager->CreateEmulatedNode(
63 std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig()));
64 auto* alice_endpoint =
65 network_emulation_manager->CreateEndpoint(EmulatedEndpointConfig());
66 EmulatedEndpoint* bob_endpoint =
67 network_emulation_manager->CreateEndpoint(EmulatedEndpointConfig());
68 network_emulation_manager->CreateRoute(alice_endpoint, {alice_node},
69 bob_endpoint);
70 network_emulation_manager->CreateRoute(bob_endpoint, {bob_node},
71 alice_endpoint);
72
73 // Create analyzers.
74 std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer =
75 std::make_unique<DefaultVideoQualityAnalyzer>();
76 // This is only done for the sake of smoke testing. In general there should
77 // be no need to explicitly pull data from analyzers after the run.
78 auto* video_analyzer_ptr =
79 static_cast<DefaultVideoQualityAnalyzer*>(video_quality_analyzer.get());
80
81 auto fixture = CreatePeerConnectionE2EQualityTestFixture(
82 test_case_name, /*audio_quality_analyzer=*/nullptr,
83 std::move(video_quality_analyzer));
84 fixture->ExecuteAt(TimeDelta::Seconds(2),
85 [alice_network_behavior_ptr](TimeDelta) {
86 BuiltInNetworkBehaviorConfig config;
87 config.loss_percent = 5;
88 alice_network_behavior_ptr->SetConfig(config);
89 });
90
91 // Setup components. We need to provide rtc::NetworkManager compatible with
92 // emulated network layer.
93 EmulatedNetworkManagerInterface* alice_network =
94 network_emulation_manager->CreateEmulatedNetworkManagerInterface(
95 {alice_endpoint});
96 EmulatedNetworkManagerInterface* bob_network =
97 network_emulation_manager->CreateEmulatedNetworkManagerInterface(
98 {bob_endpoint});
99
100 fixture->AddPeer(alice_network->network_thread(),
101 alice_network->network_manager(), alice_configurer);
102 fixture->AddPeer(bob_network->network_thread(),
103 bob_network->network_manager(), bob_configurer);
104 fixture->AddQualityMetricsReporter(
105 std::make_unique<NetworkQualityMetricsReporter>(alice_network,
106 bob_network));
107
108 fixture->Run(run_params);
109
110 EXPECT_GE(fixture->GetRealTestDuration(), run_params.run_duration);
111 for (auto stream_label : video_analyzer_ptr->GetKnownVideoStreams()) {
112 FrameCounters stream_conters =
113 video_analyzer_ptr->GetPerStreamCounters().at(stream_label);
114 // On some devices the pipeline can be too slow, so we actually can't
115 // force real constraints here. Lets just check, that at least 1
116 // frame passed whole pipeline.
117 int64_t expected_min_fps = run_params.run_duration.seconds() * 30;
118 EXPECT_GE(stream_conters.captured, expected_min_fps);
119 EXPECT_GE(stream_conters.pre_encoded, 1);
120 EXPECT_GE(stream_conters.encoded, 1);
121 EXPECT_GE(stream_conters.received, 1);
122 EXPECT_GE(stream_conters.decoded, 1);
123 EXPECT_GE(stream_conters.rendered, 1);
124 }
125 }
126 };
127
128 } // namespace
129
130 // IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
131 #if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
132 #define MAYBE_Smoke DISABLED_Smoke
133 #else
134 #define MAYBE_Smoke Smoke
135 #endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest,MAYBE_Smoke)136 TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Smoke) {
137 RunParams run_params(TimeDelta::Seconds(2));
138 run_params.video_codecs = {
139 VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})};
140 run_params.use_flex_fec = true;
141 run_params.use_ulp_fec = true;
142 run_params.video_encoder_bitrate_multiplier = 1.1;
143 test::ScopedFieldTrials field_trials(
144 std::string(field_trial::GetFieldTrialString()) +
145 "WebRTC-UseStandardBytesStats/Enabled/");
146 RunTest(
147 "smoke", run_params,
148 [](PeerConfigurer* alice) {
149 VideoConfig video(640, 360, 30);
150 video.stream_label = "alice-video";
151 video.sync_group = "alice-media";
152 alice->AddVideoConfig(std::move(video));
153
154 AudioConfig audio;
155 audio.stream_label = "alice-audio";
156 audio.mode = AudioConfig::Mode::kFile;
157 audio.input_file_name =
158 test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
159 audio.sampling_frequency_in_hz = 48000;
160 audio.sync_group = "alice-media";
161 alice->SetAudioConfig(std::move(audio));
162 },
163 [](PeerConfigurer* bob) {
164 VideoConfig video(640, 360, 30);
165 video.stream_label = "bob-video";
166 video.temporal_layers_count = 2;
167 bob->AddVideoConfig(std::move(video));
168
169 VideoConfig screenshare(640, 360, 30);
170 screenshare.stream_label = "bob-screenshare";
171 screenshare.screen_share_config =
172 ScreenShareConfig(TimeDelta::Seconds(2));
173 screenshare.screen_share_config->scrolling_params = ScrollingParams(
174 TimeDelta::Millis(1800), kDefaultSlidesWidth, kDefaultSlidesHeight);
175 bob->AddVideoConfig(screenshare);
176
177 AudioConfig audio;
178 audio.stream_label = "bob-audio";
179 audio.mode = AudioConfig::Mode::kFile;
180 audio.input_file_name =
181 test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
182 bob->SetAudioConfig(std::move(audio));
183 });
184 }
185
186 // IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
187 #if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
188 #define MAYBE_Echo DISABLED_Echo
189 #else
190 #define MAYBE_Echo Echo
191 #endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest,MAYBE_Echo)192 TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Echo) {
193 RunParams run_params(TimeDelta::Seconds(2));
194 run_params.echo_emulation_config = EchoEmulationConfig();
195 RunTest(
196 "smoke", run_params,
197 [](PeerConfigurer* alice) {
198 AudioConfig audio;
199 audio.stream_label = "alice-audio";
200 audio.mode = AudioConfig::Mode::kFile;
201 audio.input_file_name =
202 test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
203 audio.sampling_frequency_in_hz = 48000;
204 alice->SetAudioConfig(std::move(audio));
205 },
206 [](PeerConfigurer* bob) {
207 AudioConfig audio;
208 audio.stream_label = "bob-audio";
209 audio.mode = AudioConfig::Mode::kFile;
210 audio.input_file_name =
211 test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
212 bob->SetAudioConfig(std::move(audio));
213 });
214 }
215
216 // IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
217 #if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
218 #define MAYBE_Simulcast DISABLED_Simulcast
219 #else
220 #define MAYBE_Simulcast Simulcast
221 #endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest,MAYBE_Simulcast)222 TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Simulcast) {
223 RunParams run_params(TimeDelta::Seconds(2));
224 run_params.video_codecs = {VideoCodecConfig(cricket::kVp8CodecName)};
225 RunTest(
226 "simulcast", run_params,
227 [](PeerConfigurer* alice) {
228 VideoConfig simulcast(1280, 720, 30);
229 simulcast.stream_label = "alice-simulcast";
230 simulcast.simulcast_config = VideoSimulcastConfig(3, 0);
231 alice->AddVideoConfig(std::move(simulcast));
232
233 AudioConfig audio;
234 audio.stream_label = "alice-audio";
235 audio.mode = AudioConfig::Mode::kFile;
236 audio.input_file_name =
237 test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
238 alice->SetAudioConfig(std::move(audio));
239 },
240 [](PeerConfigurer* bob) {
241 VideoConfig video(640, 360, 30);
242 video.stream_label = "bob-video";
243 bob->AddVideoConfig(std::move(video));
244
245 AudioConfig audio;
246 audio.stream_label = "bob-audio";
247 audio.mode = AudioConfig::Mode::kFile;
248 audio.input_file_name =
249 test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
250 bob->SetAudioConfig(std::move(audio));
251 });
252 }
253
254 // IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
255 #if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
256 #define MAYBE_Svc DISABLED_Svc
257 #else
258 #define MAYBE_Svc Svc
259 #endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest,MAYBE_Svc)260 TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Svc) {
261 RunParams run_params(TimeDelta::Seconds(2));
262 run_params.video_codecs = {VideoCodecConfig(cricket::kVp9CodecName)};
263 RunTest(
264 "simulcast", run_params,
265 [](PeerConfigurer* alice) {
266 VideoConfig simulcast(1280, 720, 30);
267 simulcast.stream_label = "alice-svc";
268 // Because we have network with packets loss we can analyze only the
269 // highest spatial layer in SVC mode.
270 simulcast.simulcast_config = VideoSimulcastConfig(3, 2);
271 alice->AddVideoConfig(std::move(simulcast));
272
273 AudioConfig audio;
274 audio.stream_label = "alice-audio";
275 audio.mode = AudioConfig::Mode::kFile;
276 audio.input_file_name =
277 test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
278 alice->SetAudioConfig(std::move(audio));
279 },
280 [](PeerConfigurer* bob) {
281 VideoConfig video(640, 360, 30);
282 video.stream_label = "bob-video";
283 bob->AddVideoConfig(std::move(video));
284
285 AudioConfig audio;
286 audio.stream_label = "bob-audio";
287 audio.mode = AudioConfig::Mode::kFile;
288 audio.input_file_name =
289 test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
290 bob->SetAudioConfig(std::move(audio));
291 });
292 }
293
294 // IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
295 #if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
296 #define MAYBE_HighBitrate DISABLED_HighBitrate
297 #else
298 #define MAYBE_HighBitrate HighBitrate
299 #endif
TEST_F(PeerConnectionE2EQualityTestSmokeTest,MAYBE_HighBitrate)300 TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_HighBitrate) {
301 RunParams run_params(TimeDelta::Seconds(2));
302 run_params.video_codecs = {
303 VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})};
304
305 RunTest(
306 "smoke", run_params,
307 [](PeerConfigurer* alice) {
308 PeerConnectionInterface::BitrateParameters bitrate_params;
309 bitrate_params.current_bitrate_bps = 3'000'000;
310 bitrate_params.max_bitrate_bps = 3'000'000;
311 alice->SetBitrateParameters(bitrate_params);
312 VideoConfig video(800, 600, 30);
313 video.stream_label = "alice-video";
314 video.min_encode_bitrate_bps = 500'000;
315 video.max_encode_bitrate_bps = 3'000'000;
316 alice->AddVideoConfig(std::move(video));
317
318 AudioConfig audio;
319 audio.stream_label = "alice-audio";
320 audio.mode = AudioConfig::Mode::kFile;
321 audio.input_file_name =
322 test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
323 audio.sampling_frequency_in_hz = 48000;
324 alice->SetAudioConfig(std::move(audio));
325 },
326 [](PeerConfigurer* bob) {
327 PeerConnectionInterface::BitrateParameters bitrate_params;
328 bitrate_params.current_bitrate_bps = 3'000'000;
329 bitrate_params.max_bitrate_bps = 3'000'000;
330 bob->SetBitrateParameters(bitrate_params);
331 VideoConfig video(800, 600, 30);
332 video.stream_label = "bob-video";
333 video.min_encode_bitrate_bps = 500'000;
334 video.max_encode_bitrate_bps = 3'000'000;
335 bob->AddVideoConfig(std::move(video));
336
337 AudioConfig audio;
338 audio.stream_label = "bob-audio";
339 audio.mode = AudioConfig::Mode::kFile;
340 audio.input_file_name =
341 test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
342 bob->SetAudioConfig(std::move(audio));
343 });
344 }
345
346 } // namespace webrtc_pc_e2e
347 } // namespace webrtc
348