1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include <stddef.h>
6
7 #include "base/base64.h"
8 #include "base/command_line.h"
9 #include "base/environment.h"
10 #include "base/files/file.h"
11 #include "base/files/file_util.h"
12 #include "base/files/scoped_temp_dir.h"
13 #include "base/path_service.h"
14 #include "base/process/launch.h"
15 #include "base/strings/string_number_conversions.h"
16 #include "base/strings/string_split.h"
17 #include "base/strings/stringprintf.h"
18 #include "base/test/test_timeouts.h"
19 #include "base/threading/thread_restrictions.h"
20 #include "base/time/time.h"
21 #include "build/build_config.h"
22 #include "chrome/browser/chrome_notification_types.h"
23 #include "chrome/browser/infobars/infobar_service.h"
24 #include "chrome/browser/media/webrtc/webrtc_browsertest_base.h"
25 #include "chrome/browser/media/webrtc/webrtc_browsertest_common.h"
26 #include "chrome/browser/media/webrtc/webrtc_browsertest_perf.h"
27 #include "chrome/browser/profiles/profile.h"
28 #include "chrome/browser/ui/browser.h"
29 #include "chrome/browser/ui/browser_tabstrip.h"
30 #include "chrome/browser/ui/tabs/tab_strip_model.h"
31 #include "chrome/common/chrome_switches.h"
32 #include "chrome/test/base/in_process_browser_test.h"
33 #include "components/infobars/core/infobar.h"
34 #include "content/public/browser/notification_service.h"
35 #include "content/public/test/browser_test.h"
36 #include "content/public/test/browser_test_utils.h"
37 #include "media/base/media_switches.h"
38 #include "net/test/embedded_test_server/embedded_test_server.h"
39 #include "net/test/python_utils.h"
40 #include "testing/perf/perf_test.h"
41 #include "third_party/blink/public/common/features.h"
42 #include "ui/gl/gl_switches.h"
43
44 namespace {
MakeLabel(const char * test_name,const std::string & video_codec)45 std::string MakeLabel(const char* test_name, const std::string& video_codec) {
46 std::string codec_label = video_codec.empty() ? "" : "_" + video_codec;
47 return base::StringPrintf("%s%s", test_name, codec_label.c_str());
48 }
49 } // namespace
50
51 static const base::FilePath::CharType kFrameAnalyzerExecutable[] =
52 #if defined(OS_WIN)
53 FILE_PATH_LITERAL("frame_analyzer.exe");
54 #else
55 FILE_PATH_LITERAL("frame_analyzer");
56 #endif
57
58 static const base::FilePath::CharType kCapturedYuvFileName[] =
59 FILE_PATH_LITERAL("captured_video.yuv");
60 static const base::FilePath::CharType kCapturedWebmFileName[] =
61 FILE_PATH_LITERAL("captured_video.webm");
62 static const char kMainWebrtcTestHtmlPage[] =
63 "/webrtc/webrtc_jsep01_test.html";
64 static const char kCapturingWebrtcHtmlPage[] =
65 "/webrtc/webrtc_video_quality_test.html";
66
67 static const struct VideoQualityTestConfig {
68 const char* test_name;
69 int width;
70 int height;
71 const base::FilePath::CharType* reference_video;
72 const char* constraints;
73 } kVideoConfigurations[] = {
74 { "360p", 640, 360,
75 test::kReferenceFileName360p,
76 WebRtcTestBase::kAudioVideoCallConstraints360p },
77 { "720p", 1280, 720,
78 test::kReferenceFileName720p,
79 WebRtcTestBase::kAudioVideoCallConstraints720p },
80 };
81
82 // Test the video quality of the WebRTC output.
83 //
84 // Prerequisites: This test case must run on a machine with a chrome playing
85 // the video from the reference files located in GetReferenceFilesDir().
86 // The file kReferenceY4mFileName.kY4mFileExtension is played using a
87 // FileVideoCaptureDevice and its sibling with kYuvFileExtension is used for
88 // comparison.
89 //
90 // You must also compile the frame_analyzer target before you run this
91 // test to get all the tools built.
92 //
93 // The test runs several custom binaries - rgba_to_i420 converter and
94 // frame_analyzer. Both tools can be found under third_party/webrtc/rtc_tools.
95 // The test also runs a stand alone Python implementation of a WebSocket server
96 // (pywebsocket) and a barcode_decoder script.
97 class WebRtcVideoQualityBrowserTest : public WebRtcTestBase,
98 public testing::WithParamInterface<VideoQualityTestConfig> {
99 public:
WebRtcVideoQualityBrowserTest()100 WebRtcVideoQualityBrowserTest()
101 : environment_(base::Environment::Create()) {
102 test_config_ = GetParam();
103 }
104
SetUpInProcessBrowserTestFixture()105 void SetUpInProcessBrowserTestFixture() override {
106 DetectErrorsInJavaScript(); // Look for errors in our rather complex js.
107
108 ASSERT_TRUE(temp_working_dir_.CreateUniqueTempDir());
109 }
110
SetUpCommandLine(base::CommandLine * command_line)111 void SetUpCommandLine(base::CommandLine* command_line) override {
112 // Set up the command line option with the expected file name. We will check
113 // its existence in HasAllRequiredResources().
114 webrtc_reference_video_y4m_ = test::GetReferenceFilesDir()
115 .Append(test_config_.reference_video)
116 .AddExtension(test::kY4mFileExtension);
117 command_line->AppendSwitchPath(switches::kUseFileForFakeVideoCapture,
118 webrtc_reference_video_y4m_);
119
120 // The video playback will not work without a GPU, so force its use here.
121 command_line->AppendSwitch(switches::kUseGpuInTests);
122 }
123
124 // Writes the captured video to a webm file.
WriteCapturedWebmVideo(content::WebContents * capturing_tab,const base::FilePath & webm_video_filename)125 void WriteCapturedWebmVideo(content::WebContents* capturing_tab,
126 const base::FilePath& webm_video_filename) {
127 std::string base64_encoded_video =
128 ExecuteJavascript("getRecordedVideoAsBase64()", capturing_tab);
129 std::string recorded_video;
130 ASSERT_TRUE(base::Base64Decode(base64_encoded_video, &recorded_video));
131 base::File video_file(webm_video_filename,
132 base::File::FLAG_CREATE | base::File::FLAG_WRITE);
133 size_t written =
134 video_file.Write(0, recorded_video.c_str(), recorded_video.length());
135 ASSERT_EQ(recorded_video.length(), written);
136 }
137
138 // Runs ffmpeg on the captured webm video and writes it to a yuv video file.
RunWebmToI420Converter(const base::FilePath & webm_video_filename,const base::FilePath & yuv_video_filename,const int width,const int height)139 bool RunWebmToI420Converter(const base::FilePath& webm_video_filename,
140 const base::FilePath& yuv_video_filename,
141 const int width,
142 const int height) {
143 base::FilePath path_to_ffmpeg = test::GetToolForPlatform("ffmpeg");
144 if (!base::PathExists(path_to_ffmpeg)) {
145 LOG(ERROR) << "Missing ffmpeg: should be in " << path_to_ffmpeg.value();
146 return false;
147 }
148
149 // Set up ffmpeg to output at a certain resolution (-s) and bitrate (-b:v).
150 // This is needed because WebRTC is free to start the call at a lower
151 // resolution before ramping up. Without these flags, ffmpeg would output a
152 // video in the inital lower resolution, causing the SSIM and PSNR results
153 // to become meaningless.
154 base::CommandLine ffmpeg_command(path_to_ffmpeg);
155 ffmpeg_command.AppendArg("-i");
156 ffmpeg_command.AppendArgPath(webm_video_filename);
157 ffmpeg_command.AppendArg("-s");
158 ffmpeg_command.AppendArg(base::StringPrintf("%dx%d", width, height));
159 ffmpeg_command.AppendArg("-b:v");
160 ffmpeg_command.AppendArg(base::StringPrintf("%d", 120 * width * height));
161 ffmpeg_command.AppendArg("-vsync");
162 ffmpeg_command.AppendArg("passthrough");
163 ffmpeg_command.AppendArgPath(yuv_video_filename);
164
165 // We produce an output file that will later be used as an input to the
166 // barcode decoder and frame analyzer tools.
167 DVLOG(0) << "Running " << ffmpeg_command.GetCommandLineString();
168 std::string result;
169 bool ok = base::GetAppOutputAndError(ffmpeg_command, &result);
170 DVLOG(0) << "Output was:\n\n" << result;
171 return ok;
172 }
173
174 // Compares the |captured_video_filename| with the |reference_video_filename|.
175 //
176 // The barcode decoder decodes the captured video containing barcodes overlaid
177 // into every frame of the video. It produces a set of PNG images.
178 // The frames should be of size |width| x |height|.
179 // All measurements calculated are printed as perf parsable numbers to stdout.
CompareVideosAndPrintResult(const std::string & test_label,int width,int height,const base::FilePath & captured_video_filename,const base::FilePath & reference_video_filename)180 bool CompareVideosAndPrintResult(
181 const std::string& test_label,
182 int width,
183 int height,
184 const base::FilePath& captured_video_filename,
185 const base::FilePath& reference_video_filename) {
186 base::FilePath path_to_analyzer = base::MakeAbsoluteFilePath(
187 GetBrowserDir().Append(kFrameAnalyzerExecutable));
188 base::FilePath path_to_compare_script = GetSourceDir().Append(
189 FILE_PATH_LITERAL("third_party/webrtc/rtc_tools/compare_videos.py"));
190
191 if (!base::PathExists(path_to_analyzer)) {
192 LOG(ERROR) << "Missing frame analyzer: should be in "
193 << path_to_analyzer.value()
194 << ". Try building the frame_analyzer target.";
195 return false;
196 }
197 if (!base::PathExists(path_to_compare_script)) {
198 LOG(ERROR) << "Missing video compare script: should be in "
199 << path_to_compare_script.value();
200 return false;
201 }
202
203 // Note: don't append switches to this command since it will mess up the
204 // -u in the python invocation!
205 base::CommandLine compare_command(base::CommandLine::NO_PROGRAM);
206 EXPECT_TRUE(GetPythonCommand(&compare_command));
207
208 compare_command.AppendArgPath(path_to_compare_script);
209 compare_command.AppendArg("--label=" + test_label);
210 compare_command.AppendArg("--ref_video");
211 compare_command.AppendArgPath(reference_video_filename);
212 compare_command.AppendArg("--test_video");
213 compare_command.AppendArgPath(captured_video_filename);
214 compare_command.AppendArg("--frame_analyzer");
215 compare_command.AppendArgPath(path_to_analyzer);
216 compare_command.AppendArg("--yuv_frame_width");
217 compare_command.AppendArg(base::NumberToString(width));
218 compare_command.AppendArg("--yuv_frame_height");
219 compare_command.AppendArg(base::NumberToString(height));
220
221 DVLOG(0) << "Running " << compare_command.GetCommandLineString();
222 std::string output;
223 bool ok = base::GetAppOutput(compare_command, &output);
224
225 // Print to stdout to ensure the perf numbers are parsed properly by the
226 // buildbot step. The tool should print a handful RESULT lines.
227 printf("Output was:\n\n%s\n", output.c_str());
228 bool has_result_lines = output.find("RESULT") != std::string::npos;
229 if (!ok || !has_result_lines) {
230 LOG(ERROR) << "Failed to compare videos; see output to see what "
231 << "the error was:\n\n"
232 << output;
233 return false;
234 }
235 // TODO(http://crbug.com/1874811): Enable this and drop the printf above
236 // when ready to switch to histogram sets.
237 // if (!test::WriteCompareVideosOutputAsHistogram(test_label, output))
238 // return false;
239
240 return true;
241 }
242
TestVideoQuality(const std::string & video_codec,bool prefer_hw_video_codec)243 void TestVideoQuality(const std::string& video_codec,
244 bool prefer_hw_video_codec) {
245 ASSERT_GE(TestTimeouts::test_launcher_timeout().InSeconds(), 150)
246 << "This is a long-running test; you must specify "
247 "--test-launcher-timeout to have a value of at least 150000.";
248 ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 150)
249 << "This is a long-running test; you must specify "
250 "--ui-test-action-max-timeout to have a value of at least 150000.";
251 ASSERT_LT(TestTimeouts::action_max_timeout(),
252 TestTimeouts::test_launcher_timeout())
253 << "action_max_timeout needs to be strictly-less-than "
254 "test_launcher_timeout";
255 ASSERT_TRUE(test::HasReferenceFilesInCheckout());
256 ASSERT_TRUE(embedded_test_server()->Start());
257
258 content::WebContents* left_tab =
259 OpenPageAndGetUserMediaInNewTabWithConstraints(
260 embedded_test_server()->GetURL(kMainWebrtcTestHtmlPage),
261 test_config_.constraints);
262 content::WebContents* right_tab =
263 OpenPageAndGetUserMediaInNewTabWithConstraints(
264 embedded_test_server()->GetURL(kCapturingWebrtcHtmlPage),
265 test_config_.constraints);
266
267 SetupPeerconnectionWithLocalStream(left_tab);
268 SetupPeerconnectionWithLocalStream(right_tab);
269
270 if (!video_codec.empty()) {
271 SetDefaultVideoCodec(left_tab, video_codec, prefer_hw_video_codec);
272 SetDefaultVideoCodec(right_tab, video_codec, prefer_hw_video_codec);
273 }
274 NegotiateCall(left_tab, right_tab);
275
276 // Poll slower here to avoid flooding the log with messages: capturing and
277 // sending frames take quite a bit of time.
278 int polling_interval_msec = 1000;
279
280 EXPECT_TRUE(test::PollingWaitUntil("doneFrameCapturing()", "done-capturing",
281 right_tab, polling_interval_msec));
282
283 HangUp(left_tab);
284
285 WriteCapturedWebmVideo(right_tab,
286 GetWorkingDir().Append(kCapturedWebmFileName));
287
288 // Shut everything down to avoid having the javascript race with the
289 // analysis tools. For instance, dont have console log printouts interleave
290 // with the RESULT lines from the analysis tools (crbug.com/323200).
291 chrome::CloseWebContents(browser(), left_tab, false);
292 chrome::CloseWebContents(browser(), right_tab, false);
293
294 RunWebmToI420Converter(GetWorkingDir().Append(kCapturedWebmFileName),
295 GetWorkingDir().Append(kCapturedYuvFileName),
296 test_config_.width, test_config_.height);
297
298 ASSERT_TRUE(CompareVideosAndPrintResult(
299 MakeLabel(test_config_.test_name, video_codec), test_config_.width,
300 test_config_.height, GetWorkingDir().Append(kCapturedYuvFileName),
301 test::GetReferenceFilesDir()
302 .Append(test_config_.reference_video)
303 .AddExtension(test::kYuvFileExtension)));
304 }
305
306 protected:
307 VideoQualityTestConfig test_config_;
308
GetWorkingDir()309 base::FilePath GetWorkingDir() { return temp_working_dir_.GetPath(); }
310
311 private:
GetSourceDir()312 base::FilePath GetSourceDir() {
313 base::FilePath source_dir;
314 base::PathService::Get(base::DIR_SOURCE_ROOT, &source_dir);
315 return source_dir;
316 }
317
GetBrowserDir()318 base::FilePath GetBrowserDir() {
319 base::FilePath browser_dir;
320 EXPECT_TRUE(base::PathService::Get(base::DIR_MODULE, &browser_dir));
321 return browser_dir;
322 }
323
324 std::unique_ptr<base::Environment> environment_;
325 base::FilePath webrtc_reference_video_y4m_;
326 base::ScopedTempDir temp_working_dir_;
327 };
328
329 INSTANTIATE_TEST_SUITE_P(WebRtcVideoQualityBrowserTests,
330 WebRtcVideoQualityBrowserTest,
331 testing::ValuesIn(kVideoConfigurations));
332
333 // WebRTC's frame_analyzer doesn't build from a Chromium's component build.
334 #if defined(COMPONENT_BUILD)
335 #define MAYBE_MANUAL_TestVideoQualityVp8 DISABLED_MANUAL_TestVideoQualityVp8
336 #else
337 #define MAYBE_MANUAL_TestVideoQualityVp8 MANUAL_TestVideoQualityVp8
338 #endif
IN_PROC_BROWSER_TEST_P(WebRtcVideoQualityBrowserTest,MAYBE_MANUAL_TestVideoQualityVp8)339 IN_PROC_BROWSER_TEST_P(WebRtcVideoQualityBrowserTest,
340 MAYBE_MANUAL_TestVideoQualityVp8) {
341 base::ScopedAllowBlockingForTesting allow_blocking;
342 TestVideoQuality("VP8", false /* prefer_hw_video_codec */);
343 }
344
345 // Flaky on windows and WebRTC's frame_analyzer doesn't build from a Chromium's
346 // component build.
347 // TODO(crbug.com/1008766): re-enable when flakiness is investigated, diagnosed
348 // and resolved.
349 #if defined(OS_WIN) || defined(COMPONENT_BUILD)
350 #define MAYBE_MANUAL_TestVideoQualityVp9 DISABLED_MANUAL_TestVideoQualityVp9
351 #else
352 #define MAYBE_MANUAL_TestVideoQualityVp9 MANUAL_TestVideoQualityVp9
353 #endif
IN_PROC_BROWSER_TEST_P(WebRtcVideoQualityBrowserTest,MAYBE_MANUAL_TestVideoQualityVp9)354 IN_PROC_BROWSER_TEST_P(WebRtcVideoQualityBrowserTest,
355 MAYBE_MANUAL_TestVideoQualityVp9) {
356 base::ScopedAllowBlockingForTesting allow_blocking;
357 TestVideoQuality("VP9", true /* prefer_hw_video_codec */);
358 }
359
360 #if BUILDFLAG(RTC_USE_H264)
361
362 // Flaky on mac (crbug.com/754684) and WebRTC's frame_analyzer doesn't build
363 // from a Chromium's component build.
364 #if defined(OS_MAC) || defined(COMPONENT_BUILD)
365 #define MAYBE_MANUAL_TestVideoQualityH264 DISABLED_MANUAL_TestVideoQualityH264
366 #else
367 #define MAYBE_MANUAL_TestVideoQualityH264 MANUAL_TestVideoQualityH264
368 #endif
369
IN_PROC_BROWSER_TEST_P(WebRtcVideoQualityBrowserTest,MAYBE_MANUAL_TestVideoQualityH264)370 IN_PROC_BROWSER_TEST_P(WebRtcVideoQualityBrowserTest,
371 MAYBE_MANUAL_TestVideoQualityH264) {
372 base::ScopedAllowBlockingForTesting allow_blocking;
373 // Only run test if run-time feature corresponding to |rtc_use_h264| is on.
374 if (!base::FeatureList::IsEnabled(
375 blink::features::kWebRtcH264WithOpenH264FFmpeg)) {
376 LOG(WARNING) << "Run-time feature WebRTC-H264WithOpenH264FFmpeg disabled. "
377 "Skipping WebRtcVideoQualityBrowserTest.MANUAL_TestVideoQualityH264 "
378 "(test \"OK\")";
379 return;
380 }
381 TestVideoQuality("H264", true /* prefer_hw_video_codec */);
382 }
383
384 #endif // BUILDFLAG(RTC_USE_H264)
385