1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include <memory>
6
7 #include "base/command_line.h"
8 #include "base/files/file_util.h"
9 #include "base/json/json_reader.h"
10 #include "base/macros.h"
11 #include "base/strings/string_split.h"
12 #include "base/test/test_timeouts.h"
13 #include "base/threading/thread_restrictions.h"
14 #include "base/time/time.h"
15 #include "build/build_config.h"
16 #include "chrome/browser/browser_process.h"
17 #include "chrome/browser/media/webrtc/webrtc_browsertest_base.h"
18 #include "chrome/browser/media/webrtc/webrtc_browsertest_common.h"
19 #include "chrome/browser/media/webrtc/webrtc_browsertest_perf.h"
20 #include "chrome/browser/ui/browser.h"
21 #include "chrome/browser/ui/browser_tabstrip.h"
22 #include "chrome/browser/ui/tabs/tab_strip_model.h"
23 #include "chrome/common/chrome_switches.h"
24 #include "chrome/test/base/in_process_browser_test.h"
25 #include "chrome/test/base/ui_test_utils.h"
26 #include "content/public/common/content_switches.h"
27 #include "content/public/test/browser_test.h"
28 #include "content/public/test/browser_test_utils.h"
29 #include "media/base/media_switches.h"
30 #include "net/test/embedded_test_server/embedded_test_server.h"
31 #include "testing/perf/perf_test.h"
32 #include "third_party/blink/public/common/features.h"
33
34 static const char kMainWebrtcTestHtmlPage[] =
35 "/webrtc/webrtc_jsep01_test.html";
36
MakePerfTestLabel(std::string base,bool opus_dtx)37 std::string MakePerfTestLabel(std::string base, bool opus_dtx) {
38 if (opus_dtx) {
39 return base + "_with_opus_dtx";
40 }
41 return base;
42 }
43
44 // Performance browsertest for WebRTC. This test is manual since it takes long
45 // to execute and requires the reference files provided by the webrtc.DEPS
46 // solution (which is only available on WebRTC internal bots).
47 // Gets its metrics from "chrome://webrtc-internals".
48 class WebRtcInternalsPerfBrowserTest : public WebRtcTestBase {
49 public:
SetUpInProcessBrowserTestFixture()50 void SetUpInProcessBrowserTestFixture() override {
51 DetectErrorsInJavaScript(); // Look for errors in our rather complex js.
52 }
53
SetUpCommandLine(base::CommandLine * command_line)54 void SetUpCommandLine(base::CommandLine* command_line) override {
55 // Ensure the infobar is enabled, since we expect that in this test.
56 EXPECT_FALSE(command_line->HasSwitch(switches::kUseFakeUIForMediaStream));
57
58 // Play a suitable, somewhat realistic video file.
59 base::FilePath input_video = test::GetReferenceFilesDir()
60 .Append(test::kReferenceFileName360p)
61 .AddExtension(test::kY4mFileExtension);
62 command_line->AppendSwitchPath(switches::kUseFileForFakeVideoCapture,
63 input_video);
64 }
65
66 // Tries to extract data from peerConnectionDataStore in the webrtc-internals
67 // tab. The caller owns the parsed data. Returns NULL on failure.
GetWebrtcInternalsData(content::WebContents * webrtc_internals_tab)68 base::DictionaryValue* GetWebrtcInternalsData(
69 content::WebContents* webrtc_internals_tab) {
70 std::string all_stats_json = ExecuteJavascript(
71 "window.domAutomationController.send("
72 " JSON.stringify(peerConnectionDataStore));",
73 webrtc_internals_tab);
74
75 std::unique_ptr<base::Value> parsed_json =
76 base::JSONReader::ReadDeprecated(all_stats_json);
77 base::DictionaryValue* result;
78 if (parsed_json.get() && parsed_json->GetAsDictionary(&result)) {
79 ignore_result(parsed_json.release());
80 return result;
81 }
82
83 return NULL;
84 }
85
GetDataOnPeerConnection(const base::DictionaryValue * all_data,int peer_connection_index)86 const base::DictionaryValue* GetDataOnPeerConnection(
87 const base::DictionaryValue* all_data,
88 int peer_connection_index) {
89 base::DictionaryValue::Iterator iterator(*all_data);
90
91 for (int i = 0; i < peer_connection_index && !iterator.IsAtEnd();
92 --peer_connection_index) {
93 iterator.Advance();
94 }
95
96 const base::DictionaryValue* result;
97 if (!iterator.IsAtEnd() && iterator.value().GetAsDictionary(&result))
98 return result;
99
100 return NULL;
101 }
102
MeasureWebRtcInternalsData(int duration_msec)103 std::unique_ptr<base::DictionaryValue> MeasureWebRtcInternalsData(
104 int duration_msec) {
105 chrome::AddTabAt(browser(), GURL(url::kAboutBlankURL), -1, true);
106 ui_test_utils::NavigateToURL(browser(), GURL("chrome://webrtc-internals"));
107 content::WebContents* webrtc_internals_tab =
108 browser()->tab_strip_model()->GetActiveWebContents();
109
110 // TODO(https://crbug.com/1004239): Stop relying on the legacy getStats()
111 // API.
112 ChangeToLegacyGetStats(webrtc_internals_tab);
113 test::SleepInJavascript(webrtc_internals_tab, duration_msec);
114
115 return std::unique_ptr<base::DictionaryValue>(
116 GetWebrtcInternalsData(webrtc_internals_tab));
117 }
118
RunsAudioVideoCall60SecsAndLogsInternalMetrics(const std::string & video_codec,bool prefer_hw_video_codec=false,const std::string & video_codec_profile=std::string (),const std::string & video_codec_print_modifier=std::string ())119 void RunsAudioVideoCall60SecsAndLogsInternalMetrics(
120 const std::string& video_codec,
121 bool prefer_hw_video_codec = false,
122 const std::string& video_codec_profile = std::string(),
123 const std::string& video_codec_print_modifier = std::string()) {
124 ASSERT_TRUE(test::HasReferenceFilesInCheckout());
125 ASSERT_TRUE(embedded_test_server()->Start());
126
127 ASSERT_GE(TestTimeouts::test_launcher_timeout().InSeconds(), 100)
128 << "This is a long-running test; you must specify "
129 "--test-launcher-timeout to have a value of at least 100000.";
130 ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 100)
131 << "This is a long-running test; you must specify "
132 "--ui-test-action-max-timeout to have a value of at least 100000.";
133 ASSERT_LT(TestTimeouts::action_max_timeout(),
134 TestTimeouts::test_launcher_timeout())
135 << "action_max_timeout needs to be strictly-less-than "
136 "test_launcher_timeout";
137
138 content::WebContents* left_tab =
139 OpenTestPageAndGetUserMediaInNewTab(kMainWebrtcTestHtmlPage);
140 content::WebContents* right_tab =
141 OpenTestPageAndGetUserMediaInNewTab(kMainWebrtcTestHtmlPage);
142
143 SetupPeerconnectionWithLocalStream(left_tab);
144 SetupPeerconnectionWithLocalStream(right_tab);
145
146 if (!video_codec.empty()) {
147 SetDefaultVideoCodec(left_tab, video_codec, prefer_hw_video_codec,
148 video_codec_profile);
149 SetDefaultVideoCodec(right_tab, video_codec, prefer_hw_video_codec,
150 video_codec_profile);
151 }
152 NegotiateCall(left_tab, right_tab);
153
154 StartDetectingVideo(left_tab, "remote-view");
155 StartDetectingVideo(right_tab, "remote-view");
156
157 WaitForVideoToPlay(left_tab);
158 WaitForVideoToPlay(right_tab);
159
160 // Let values stabilize, bandwidth ramp up, etc.
161 test::SleepInJavascript(left_tab, 60000);
162
163 // Start measurements.
164 std::unique_ptr<base::DictionaryValue> all_data =
165 MeasureWebRtcInternalsData(10000);
166 ASSERT_TRUE(all_data.get() != NULL);
167
168 const base::DictionaryValue* first_pc_dict =
169 GetDataOnPeerConnection(all_data.get(), 0);
170 ASSERT_TRUE(first_pc_dict != NULL);
171 const std::string print_modifier = video_codec_print_modifier.empty()
172 ? video_codec
173 : video_codec_print_modifier;
174 test::PrintBweForVideoMetrics(*first_pc_dict, "", print_modifier);
175 test::PrintMetricsForAllStreams(*first_pc_dict, "", print_modifier);
176
177 HangUp(left_tab);
178 HangUp(right_tab);
179 }
180
RunsOneWayCall60SecsAndLogsInternalMetrics(const std::string & video_codec,bool opus_dtx)181 void RunsOneWayCall60SecsAndLogsInternalMetrics(
182 const std::string& video_codec,
183 bool opus_dtx) {
184 ASSERT_TRUE(test::HasReferenceFilesInCheckout());
185 ASSERT_TRUE(embedded_test_server()->Start());
186
187 ASSERT_GE(TestTimeouts::test_launcher_timeout().InSeconds(), 100)
188 << "This is a long-running test; you must specify "
189 "--test-launcher-timeout to have a value of at least 100000.";
190 ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 100)
191 << "This is a long-running test; you must specify "
192 "--ui-test-action-max-timeout to have a value of at least 100000.";
193 ASSERT_LT(TestTimeouts::action_max_timeout(),
194 TestTimeouts::test_launcher_timeout())
195 << "action_max_timeout needs to be strictly-less-than "
196 "test_launcher_timeout";
197
198 content::WebContents* left_tab =
199 OpenTestPageAndGetUserMediaInNewTab(kMainWebrtcTestHtmlPage);
200 content::WebContents* right_tab =
201 OpenTestPageAndGetUserMediaInNewTab(kMainWebrtcTestHtmlPage);
202
203 SetupPeerconnectionWithLocalStream(left_tab);
204 SetupPeerconnectionWithoutLocalStream(right_tab);
205
206 if (!video_codec.empty()) {
207 SetDefaultVideoCodec(left_tab, video_codec, false /* prefer_hw_codec */);
208 SetDefaultVideoCodec(right_tab, video_codec, false /* prefer_hw_codec */);
209 }
210 if (opus_dtx) {
211 EnableOpusDtx(left_tab);
212 EnableOpusDtx(right_tab);
213 }
214 NegotiateCall(left_tab, right_tab);
215
216 // Remote video will only play in one tab since the call is one-way.
217 StartDetectingVideo(right_tab, "remote-view");
218 WaitForVideoToPlay(right_tab);
219
220 // Let values stabilize, bandwidth ramp up, etc.
221 test::SleepInJavascript(left_tab, 60000);
222
223 std::unique_ptr<base::DictionaryValue> all_data =
224 MeasureWebRtcInternalsData(10000);
225 ASSERT_TRUE(all_data.get() != NULL);
226
227 // This assumes the sending peer connection is always listed first in the
228 // data store, and the receiving second.
229 const base::DictionaryValue* first_pc_dict =
230 GetDataOnPeerConnection(all_data.get(), 0);
231 ASSERT_TRUE(first_pc_dict != NULL);
232 test::PrintBweForVideoMetrics(
233 *first_pc_dict, MakePerfTestLabel("_sendonly", opus_dtx), video_codec);
234 test::PrintMetricsForSendStreams(
235 *first_pc_dict, MakePerfTestLabel("_sendonly", opus_dtx), video_codec);
236
237 const base::DictionaryValue* second_pc_dict =
238 GetDataOnPeerConnection(all_data.get(), 1);
239 ASSERT_TRUE(second_pc_dict != NULL);
240 test::PrintBweForVideoMetrics(
241 *second_pc_dict, MakePerfTestLabel("_recvonly", opus_dtx), video_codec);
242 test::PrintMetricsForRecvStreams(
243 *second_pc_dict, MakePerfTestLabel("_recvonly", opus_dtx), video_codec);
244
245 HangUp(left_tab);
246 HangUp(right_tab);
247 }
248 };
249
250 // This is manual for its long execution time.
251
IN_PROC_BROWSER_TEST_F(WebRtcInternalsPerfBrowserTest,MANUAL_RunsAudioVideoCall60SecsAndLogsInternalMetricsVp8)252 IN_PROC_BROWSER_TEST_F(
253 WebRtcInternalsPerfBrowserTest,
254 MANUAL_RunsAudioVideoCall60SecsAndLogsInternalMetricsVp8) {
255 base::ScopedAllowBlockingForTesting allow_blocking;
256 RunsAudioVideoCall60SecsAndLogsInternalMetrics("VP8");
257 }
258
IN_PROC_BROWSER_TEST_F(WebRtcInternalsPerfBrowserTest,MANUAL_RunsAudioVideoCall60SecsAndLogsInternalMetricsVp9)259 IN_PROC_BROWSER_TEST_F(
260 WebRtcInternalsPerfBrowserTest,
261 MANUAL_RunsAudioVideoCall60SecsAndLogsInternalMetricsVp9) {
262 base::ScopedAllowBlockingForTesting allow_blocking;
263 RunsAudioVideoCall60SecsAndLogsInternalMetrics("VP9");
264 }
265
IN_PROC_BROWSER_TEST_F(WebRtcInternalsPerfBrowserTest,MANUAL_RunsAudioVideoCall60SecsAndLogsInternalMetricsVp9Profile2)266 IN_PROC_BROWSER_TEST_F(
267 WebRtcInternalsPerfBrowserTest,
268 MANUAL_RunsAudioVideoCall60SecsAndLogsInternalMetricsVp9Profile2) {
269 base::ScopedAllowBlockingForTesting allow_blocking;
270 RunsAudioVideoCall60SecsAndLogsInternalMetrics(
271 "VP9", true /* prefer_hw_video_codec */,
272 WebRtcTestBase::kVP9Profile2Specifier, "VP9p2");
273 }
274
275 #if BUILDFLAG(RTC_USE_H264)
276
IN_PROC_BROWSER_TEST_F(WebRtcInternalsPerfBrowserTest,MANUAL_RunsAudioVideoCall60SecsAndLogsInternalMetricsH264)277 IN_PROC_BROWSER_TEST_F(
278 WebRtcInternalsPerfBrowserTest,
279 MANUAL_RunsAudioVideoCall60SecsAndLogsInternalMetricsH264) {
280 base::ScopedAllowBlockingForTesting allow_blocking;
281 // Only run test if run-time feature corresponding to |rtc_use_h264| is on.
282 if (!base::FeatureList::IsEnabled(
283 blink::features::kWebRtcH264WithOpenH264FFmpeg)) {
284 LOG(WARNING)
285 << "Run-time feature WebRTC-H264WithOpenH264FFmpeg disabled. "
286 "Skipping WebRtcInternalsPerfBrowserTest."
287 "MANUAL_RunsAudioVideoCall60SecsAndLogsInternalMetricsH264 (test "
288 "\"OK\")";
289 return;
290 }
291 RunsAudioVideoCall60SecsAndLogsInternalMetrics(
292 "H264", true /* prefer_hw_video_codec */);
293 }
294
295 #endif // BUILDFLAG(RTC_USE_H264)
296
IN_PROC_BROWSER_TEST_F(WebRtcInternalsPerfBrowserTest,MANUAL_RunsOneWayCall60SecsAndLogsInternalMetricsDefault)297 IN_PROC_BROWSER_TEST_F(
298 WebRtcInternalsPerfBrowserTest,
299 MANUAL_RunsOneWayCall60SecsAndLogsInternalMetricsDefault) {
300 base::ScopedAllowBlockingForTesting allow_blocking;
301 RunsOneWayCall60SecsAndLogsInternalMetrics("", false);
302 }
303
IN_PROC_BROWSER_TEST_F(WebRtcInternalsPerfBrowserTest,MANUAL_RunsOneWayCall60SecsAndLogsInternalMetricsWithOpusDtx)304 IN_PROC_BROWSER_TEST_F(
305 WebRtcInternalsPerfBrowserTest,
306 MANUAL_RunsOneWayCall60SecsAndLogsInternalMetricsWithOpusDtx) {
307 base::ScopedAllowBlockingForTesting allow_blocking;
308 RunsOneWayCall60SecsAndLogsInternalMetrics("", true);
309 }
310