1 // Copyright 2016 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include <string>
6
7 #include "base/command_line.h"
8 #include "base/test/test_timeouts.h"
9 #include "base/threading/thread_restrictions.h"
10 #include "base/time/time.h"
11 #include "build/build_config.h"
12 #include "chrome/browser/media/webrtc/test_stats_dictionary.h"
13 #include "chrome/browser/media/webrtc/webrtc_browsertest_base.h"
14 #include "chrome/browser/media/webrtc/webrtc_browsertest_common.h"
15 #include "content/public/common/content_switches.h"
16 #include "content/public/test/browser_test.h"
17 #include "media/base/media_switches.h"
18 #include "testing/perf/perf_test.h"
19 #include "third_party/blink/public/common/features.h"
20
21 namespace content {
22
23 namespace {
24
25 const char kMainWebrtcTestHtmlPage[] = "/webrtc/webrtc_jsep01_test.html";
26
27 const char kInboundRtp[] = "inbound-rtp";
28 const char kOutboundRtp[] = "outbound-rtp";
29
30 enum class GetStatsVariation {
31 PROMISE_BASED,
32 CALLBACK_BASED
33 };
34
35 // Sums up "RTC[In/Out]boundRTPStreamStats.bytes_[received/sent]" values.
GetTotalRTPStreamBytes(TestStatsReportDictionary * report,const char * type,const char * media_type)36 double GetTotalRTPStreamBytes(
37 TestStatsReportDictionary* report, const char* type,
38 const char* media_type) {
39 DCHECK(type == kInboundRtp || type == kOutboundRtp);
40 const char* bytes_name =
41 (type == kInboundRtp) ? "bytesReceived" : "bytesSent";
42 double total_bytes = 0.0;
43 report->ForEach([&type, &bytes_name, &media_type, &total_bytes](
44 const TestStatsDictionary& stats) {
45 if (stats.GetString("type") == type &&
46 stats.GetString("mediaType") == media_type) {
47 total_bytes += stats.GetNumber(bytes_name);
48 }
49 });
50 return total_bytes;
51 }
52
GetAudioBytesSent(TestStatsReportDictionary * report)53 double GetAudioBytesSent(TestStatsReportDictionary* report) {
54 return GetTotalRTPStreamBytes(report, kOutboundRtp, "audio");
55 }
56
GetAudioBytesReceived(TestStatsReportDictionary * report)57 double GetAudioBytesReceived(TestStatsReportDictionary* report) {
58 return GetTotalRTPStreamBytes(report, kInboundRtp, "audio");
59 }
60
GetVideoBytesSent(TestStatsReportDictionary * report)61 double GetVideoBytesSent(TestStatsReportDictionary* report) {
62 return GetTotalRTPStreamBytes(report, kOutboundRtp, "video");
63 }
64
GetVideoBytesReceived(TestStatsReportDictionary * report)65 double GetVideoBytesReceived(TestStatsReportDictionary* report) {
66 return GetTotalRTPStreamBytes(report, kInboundRtp, "video");
67 }
68
69 // Performance browsertest for WebRTC. This test is manual since it takes long
70 // to execute and requires the reference files provided by the webrtc.DEPS
71 // solution (which is only available on WebRTC internal bots).
72 // Gets its metrics from the standards conformant "RTCPeerConnection.getStats".
73 class WebRtcStatsPerfBrowserTest : public WebRtcTestBase {
74 public:
SetUpInProcessBrowserTestFixture()75 void SetUpInProcessBrowserTestFixture() override {
76 DetectErrorsInJavaScript();
77 }
78
SetUpCommandLine(base::CommandLine * command_line)79 void SetUpCommandLine(base::CommandLine* command_line) override {
80 // Ensure the infobar is enabled, since we expect that in this test.
81 EXPECT_FALSE(command_line->HasSwitch(switches::kUseFakeUIForMediaStream));
82
83 // Play a suitable, somewhat realistic video file.
84 base::FilePath input_video = test::GetReferenceFilesDir()
85 .Append(test::kReferenceFileName360p)
86 .AddExtension(test::kY4mFileExtension);
87 command_line->AppendSwitchPath(switches::kUseFileForFakeVideoCapture,
88 input_video);
89 }
90
StartCall(const std::string & audio_codec,const std::string & video_codec,bool prefer_hw_video_codec,const std::string & video_codec_profile)91 void StartCall(const std::string& audio_codec,
92 const std::string& video_codec,
93 bool prefer_hw_video_codec,
94 const std::string& video_codec_profile) {
95 ASSERT_TRUE(test::HasReferenceFilesInCheckout());
96 ASSERT_TRUE(embedded_test_server()->Start());
97
98 ASSERT_GE(TestTimeouts::test_launcher_timeout().InSeconds(), 100)
99 << "This is a long-running test; you must specify "
100 "--test-launcher-timeout to have a value of at least 100000.";
101
102 ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 100)
103 << "This is a long-running test; you must specify "
104 "--ui-test-action-max-timeout to have a value of at least 100000.";
105
106 ASSERT_LT(TestTimeouts::action_max_timeout(),
107 TestTimeouts::test_launcher_timeout())
108 << "action_max_timeout needs to be strictly-less-than "
109 "test_launcher_timeout";
110
111 left_tab_ = OpenTestPageAndGetUserMediaInNewTab(kMainWebrtcTestHtmlPage);
112 right_tab_ = OpenTestPageAndGetUserMediaInNewTab(kMainWebrtcTestHtmlPage);
113
114 SetupPeerconnectionWithLocalStream(left_tab_);
115 SetupPeerconnectionWithLocalStream(right_tab_);
116 SetDefaultAudioCodec(left_tab_, audio_codec);
117 SetDefaultAudioCodec(right_tab_, audio_codec);
118 SetDefaultVideoCodec(left_tab_, video_codec, prefer_hw_video_codec,
119 video_codec_profile);
120 SetDefaultVideoCodec(right_tab_, video_codec, prefer_hw_video_codec,
121 video_codec_profile);
122 CreateDataChannel(left_tab_, "data");
123 CreateDataChannel(right_tab_, "data");
124 NegotiateCall(left_tab_, right_tab_);
125 StartDetectingVideo(left_tab_, "remote-view");
126 StartDetectingVideo(right_tab_, "remote-view");
127 WaitForVideoToPlay(left_tab_);
128 WaitForVideoToPlay(right_tab_);
129 }
130
EndCall()131 void EndCall() {
132 if (left_tab_)
133 HangUp(left_tab_);
134 if (right_tab_)
135 HangUp(right_tab_);
136 }
137
RunsAudioAndVideoCallCollectingMetricsWithAudioCodec(const std::string & audio_codec)138 void RunsAudioAndVideoCallCollectingMetricsWithAudioCodec(
139 const std::string& audio_codec) {
140 RunsAudioAndVideoCallCollectingMetrics(
141 audio_codec, kUseDefaultVideoCodec, false /* prefer_hw_video_codec */,
142 "" /* video_codec_profile */, "" /* video_codec_print_modifier */);
143 }
144
RunsAudioAndVideoCallCollectingMetricsWithVideoCodec(const std::string & video_codec,bool prefer_hw_video_codec=false,const std::string & video_codec_profile=std::string (),const std::string & video_codec_print_modifier=std::string ())145 void RunsAudioAndVideoCallCollectingMetricsWithVideoCodec(
146 const std::string& video_codec,
147 bool prefer_hw_video_codec = false,
148 const std::string& video_codec_profile = std::string(),
149 const std::string& video_codec_print_modifier = std::string()) {
150 RunsAudioAndVideoCallCollectingMetrics(
151 kUseDefaultAudioCodec, video_codec, prefer_hw_video_codec,
152 video_codec_profile, video_codec_print_modifier);
153 }
154
RunsAudioAndVideoCallCollectingMetrics(const std::string & audio_codec,const std::string & video_codec,bool prefer_hw_video_codec,const std::string & video_codec_profile,const std::string & video_codec_print_modifier)155 void RunsAudioAndVideoCallCollectingMetrics(
156 const std::string& audio_codec,
157 const std::string& video_codec,
158 bool prefer_hw_video_codec,
159 const std::string& video_codec_profile,
160 const std::string& video_codec_print_modifier) {
161 StartCall(audio_codec, video_codec, prefer_hw_video_codec,
162 video_codec_profile);
163
164 // Call for 60 seconds so that values may stabilize, bandwidth ramp up, etc.
165 test::SleepInJavascript(left_tab_, 60000);
166
167 // The ramp-up may vary greatly and impact the resulting total bytes, to get
168 // reliable measurements we do two measurements, at 60 and 70 seconds and
169 // look at the average bytes/second in that window.
170 double audio_bytes_sent_before = 0.0;
171 double audio_bytes_received_before = 0.0;
172 double video_bytes_sent_before = 0.0;
173 double video_bytes_received_before = 0.0;
174
175 scoped_refptr<TestStatsReportDictionary> report =
176 GetStatsReportDictionary(left_tab_);
177 if (audio_codec != kUseDefaultAudioCodec) {
178 audio_bytes_sent_before = GetAudioBytesSent(report.get());
179 audio_bytes_received_before = GetAudioBytesReceived(report.get());
180
181 }
182 if (video_codec != kUseDefaultVideoCodec) {
183 video_bytes_sent_before = GetVideoBytesSent(report.get());
184 video_bytes_received_before = GetVideoBytesReceived(report.get());
185 }
186
187 double measure_duration_seconds = 10.0;
188 test::SleepInJavascript(left_tab_, static_cast<int>(
189 measure_duration_seconds * base::Time::kMillisecondsPerSecond));
190
191 report = GetStatsReportDictionary(left_tab_);
192 if (audio_codec != kUseDefaultAudioCodec) {
193 double audio_bytes_sent_after = GetAudioBytesSent(report.get());
194 double audio_bytes_received_after = GetAudioBytesReceived(report.get());
195
196 double audio_send_rate =
197 (audio_bytes_sent_after - audio_bytes_sent_before) /
198 measure_duration_seconds;
199 double audio_receive_rate =
200 (audio_bytes_received_after - audio_bytes_received_before) /
201 measure_duration_seconds;
202
203 std::string audio_codec_modifier = "_" + audio_codec;
204 perf_test::PrintResult(
205 "audio", audio_codec_modifier, "send_rate", audio_send_rate,
206 "bytes/second", false);
207 perf_test::PrintResult(
208 "audio", audio_codec_modifier, "receive_rate", audio_receive_rate,
209 "bytes/second", false);
210 }
211 if (video_codec != kUseDefaultVideoCodec) {
212 double video_bytes_sent_after = GetVideoBytesSent(report.get());
213 double video_bytes_received_after = GetVideoBytesReceived(report.get());
214
215 double video_send_rate =
216 (video_bytes_sent_after - video_bytes_sent_before) /
217 measure_duration_seconds;
218 double video_receive_rate =
219 (video_bytes_received_after - video_bytes_received_before) /
220 measure_duration_seconds;
221
222 std::string video_codec_modifier =
223 "_" + (video_codec_print_modifier.empty()
224 ? video_codec
225 : video_codec_print_modifier);
226 perf_test::PrintResult("video", video_codec_modifier, "send_rate",
227 video_send_rate, "bytes/second", false);
228 perf_test::PrintResult(
229 "video", video_codec_modifier, "receive_rate", video_receive_rate,
230 "bytes/second", false);
231 }
232
233 EndCall();
234 }
235
RunsAudioAndVideoCallMeasuringGetStatsPerformance(GetStatsVariation variation)236 void RunsAudioAndVideoCallMeasuringGetStatsPerformance(
237 GetStatsVariation variation) {
238 EXPECT_TRUE(base::TimeTicks::IsHighResolution());
239
240 StartCall(kUseDefaultAudioCodec, kUseDefaultVideoCodec,
241 false /* prefer_hw_video_codec */, "");
242
243 double invocation_time = 0.0;
244 switch (variation) {
245 case GetStatsVariation::PROMISE_BASED:
246 invocation_time = (MeasureGetStatsPerformance(left_tab_) +
247 MeasureGetStatsPerformance(right_tab_)) / 2.0;
248 break;
249 case GetStatsVariation::CALLBACK_BASED:
250 invocation_time =
251 (MeasureGetStatsCallbackPerformance(left_tab_) +
252 MeasureGetStatsCallbackPerformance(right_tab_)) / 2.0;
253 break;
254 }
255 perf_test::PrintResult(
256 "getStats",
257 (variation == GetStatsVariation::PROMISE_BASED) ?
258 "_promise" : "_callback",
259 "invocation_time",
260 invocation_time,
261 "milliseconds",
262 false);
263
264 EndCall();
265 }
266
267 private:
268 content::WebContents* left_tab_ = nullptr;
269 content::WebContents* right_tab_ = nullptr;
270 };
271
IN_PROC_BROWSER_TEST_F(WebRtcStatsPerfBrowserTest,MANUAL_RunsAudioAndVideoCallCollectingMetrics_AudioCodec_opus)272 IN_PROC_BROWSER_TEST_F(
273 WebRtcStatsPerfBrowserTest,
274 MANUAL_RunsAudioAndVideoCallCollectingMetrics_AudioCodec_opus) {
275 base::ScopedAllowBlockingForTesting allow_blocking;
276 RunsAudioAndVideoCallCollectingMetricsWithAudioCodec("opus");
277 }
278
IN_PROC_BROWSER_TEST_F(WebRtcStatsPerfBrowserTest,MANUAL_RunsAudioAndVideoCallCollectingMetrics_AudioCodec_ISAC)279 IN_PROC_BROWSER_TEST_F(
280 WebRtcStatsPerfBrowserTest,
281 MANUAL_RunsAudioAndVideoCallCollectingMetrics_AudioCodec_ISAC) {
282 base::ScopedAllowBlockingForTesting allow_blocking;
283 RunsAudioAndVideoCallCollectingMetricsWithAudioCodec("ISAC");
284 }
285
IN_PROC_BROWSER_TEST_F(WebRtcStatsPerfBrowserTest,MANUAL_RunsAudioAndVideoCallCollectingMetrics_AudioCodec_G722)286 IN_PROC_BROWSER_TEST_F(
287 WebRtcStatsPerfBrowserTest,
288 MANUAL_RunsAudioAndVideoCallCollectingMetrics_AudioCodec_G722) {
289 base::ScopedAllowBlockingForTesting allow_blocking;
290 RunsAudioAndVideoCallCollectingMetricsWithAudioCodec("G722");
291 }
292
IN_PROC_BROWSER_TEST_F(WebRtcStatsPerfBrowserTest,MANUAL_RunsAudioAndVideoCallCollectingMetrics_AudioCodec_PCMU)293 IN_PROC_BROWSER_TEST_F(
294 WebRtcStatsPerfBrowserTest,
295 MANUAL_RunsAudioAndVideoCallCollectingMetrics_AudioCodec_PCMU) {
296 base::ScopedAllowBlockingForTesting allow_blocking;
297 RunsAudioAndVideoCallCollectingMetricsWithAudioCodec("PCMU");
298 }
299
IN_PROC_BROWSER_TEST_F(WebRtcStatsPerfBrowserTest,MANUAL_RunsAudioAndVideoCallCollectingMetrics_AudioCodec_PCMA)300 IN_PROC_BROWSER_TEST_F(
301 WebRtcStatsPerfBrowserTest,
302 MANUAL_RunsAudioAndVideoCallCollectingMetrics_AudioCodec_PCMA) {
303 base::ScopedAllowBlockingForTesting allow_blocking;
304 RunsAudioAndVideoCallCollectingMetricsWithAudioCodec("PCMA");
305 }
306
IN_PROC_BROWSER_TEST_F(WebRtcStatsPerfBrowserTest,MANUAL_RunsAudioAndVideoCallCollectingMetrics_VideoCodec_VP8)307 IN_PROC_BROWSER_TEST_F(
308 WebRtcStatsPerfBrowserTest,
309 MANUAL_RunsAudioAndVideoCallCollectingMetrics_VideoCodec_VP8) {
310 base::ScopedAllowBlockingForTesting allow_blocking;
311 RunsAudioAndVideoCallCollectingMetricsWithVideoCodec("VP8");
312 }
313
IN_PROC_BROWSER_TEST_F(WebRtcStatsPerfBrowserTest,MANUAL_RunsAudioAndVideoCallCollectingMetrics_VideoCodec_VP9)314 IN_PROC_BROWSER_TEST_F(
315 WebRtcStatsPerfBrowserTest,
316 MANUAL_RunsAudioAndVideoCallCollectingMetrics_VideoCodec_VP9) {
317 base::ScopedAllowBlockingForTesting allow_blocking;
318 RunsAudioAndVideoCallCollectingMetricsWithVideoCodec("VP9");
319 }
320
IN_PROC_BROWSER_TEST_F(WebRtcStatsPerfBrowserTest,MANUAL_RunsAudioAndVideoCallCollectingMetrics_VideoCodec_VP9Profile2)321 IN_PROC_BROWSER_TEST_F(
322 WebRtcStatsPerfBrowserTest,
323 MANUAL_RunsAudioAndVideoCallCollectingMetrics_VideoCodec_VP9Profile2) {
324 base::ScopedAllowBlockingForTesting allow_blocking;
325 RunsAudioAndVideoCallCollectingMetricsWithVideoCodec(
326 "VP9", true /* prefer_hw_video_codec */,
327 WebRtcTestBase::kVP9Profile2Specifier, "VP9p2");
328 }
329
330 #if BUILDFLAG(RTC_USE_H264)
331
IN_PROC_BROWSER_TEST_F(WebRtcStatsPerfBrowserTest,MANUAL_RunsAudioAndVideoCallCollectingMetrics_VideoCodec_H264)332 IN_PROC_BROWSER_TEST_F(
333 WebRtcStatsPerfBrowserTest,
334 MANUAL_RunsAudioAndVideoCallCollectingMetrics_VideoCodec_H264) {
335 base::ScopedAllowBlockingForTesting allow_blocking;
336 // Only run test if run-time feature corresponding to |rtc_use_h264| is on.
337 if (!base::FeatureList::IsEnabled(
338 blink::features::kWebRtcH264WithOpenH264FFmpeg)) {
339 LOG(WARNING) << "Run-time feature WebRTC-H264WithOpenH264FFmpeg disabled. "
340 "Skipping WebRtcPerfBrowserTest."
341 "MANUAL_RunsAudioAndVideoCallCollectingMetrics_VideoCodec_"
342 "H264 (test "
343 "\"OK\")";
344 return;
345 }
346 RunsAudioAndVideoCallCollectingMetricsWithVideoCodec(
347 "H264", true /* prefer_hw_video_codec */);
348 }
349
350 #endif // BUILDFLAG(RTC_USE_H264)
351
IN_PROC_BROWSER_TEST_F(WebRtcStatsPerfBrowserTest,MANUAL_RunsAudioAndVideoCallMeasuringGetStatsPerformance_Promise)352 IN_PROC_BROWSER_TEST_F(
353 WebRtcStatsPerfBrowserTest,
354 MANUAL_RunsAudioAndVideoCallMeasuringGetStatsPerformance_Promise) {
355 base::ScopedAllowBlockingForTesting allow_blocking;
356 RunsAudioAndVideoCallMeasuringGetStatsPerformance(
357 GetStatsVariation::PROMISE_BASED);
358 }
359
IN_PROC_BROWSER_TEST_F(WebRtcStatsPerfBrowserTest,MANUAL_RunsAudioAndVideoCallMeasuringGetStatsPerformance_Callback)360 IN_PROC_BROWSER_TEST_F(
361 WebRtcStatsPerfBrowserTest,
362 MANUAL_RunsAudioAndVideoCallMeasuringGetStatsPerformance_Callback) {
363 base::ScopedAllowBlockingForTesting allow_blocking;
364 RunsAudioAndVideoCallMeasuringGetStatsPerformance(
365 GetStatsVariation::CALLBACK_BASED);
366 }
367
368 } // namespace
369
370 } // namespace content
371