1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "base/test/launcher/unit_test_launcher.h"
6
7 #include <map>
8 #include <memory>
9 #include <utility>
10
11 #include "base/base_switches.h"
12 #include "base/bind.h"
13 #include "base/callback_helpers.h"
14 #include "base/command_line.h"
15 #include "base/compiler_specific.h"
16 #include "base/debug/debugger.h"
17 #include "base/files/file_util.h"
18 #include "base/format_macros.h"
19 #include "base/location.h"
20 #include "base/logging.h"
21 #include "base/macros.h"
22 #include "base/message_loop/message_pump_type.h"
23 #include "base/sequence_checker.h"
24 #include "base/single_thread_task_runner.h"
25 #include "base/stl_util.h"
26 #include "base/strings/strcat.h"
27 #include "base/strings/string_number_conversions.h"
28 #include "base/strings/string_util.h"
29 #include "base/system/sys_info.h"
30 #include "base/task/single_thread_task_executor.h"
31 #include "base/test/launcher/test_launcher.h"
32 #include "base/test/test_switches.h"
33 #include "base/test/test_timeouts.h"
34 #include "base/threading/thread_checker.h"
35 #include "base/threading/thread_task_runner_handle.h"
36 #include "build/build_config.h"
37 #include "testing/gtest/include/gtest/gtest.h"
38
39 #if defined(OS_POSIX)
40 #include "base/files/file_descriptor_watcher_posix.h"
41 #endif
42
43 namespace base {
44
45 namespace {
46
47 // This constant controls how many tests are run in a single batch by default.
48 const size_t kDefaultTestBatchLimit = 10;
49
50 #if !defined(OS_ANDROID)
PrintUsage()51 void PrintUsage() {
52 fprintf(
53 stdout,
54 "Runs tests using the gtest framework, each batch of tests being\n"
55 "run in their own process. Supported command-line flags:\n"
56 "\n"
57 " Common flags:\n"
58 " --gtest_filter=...\n"
59 " Runs a subset of tests (see --gtest_help for more info).\n"
60 "\n"
61 " --help\n"
62 " Shows this message.\n"
63 "\n"
64 " --gtest_help\n"
65 " Shows the gtest help message.\n"
66 "\n"
67 " --test-launcher-jobs=N\n"
68 " Sets the number of parallel test jobs to N.\n"
69 "\n"
70 " --single-process-tests\n"
71 " Runs the tests and the launcher in the same process. Useful\n"
72 " for debugging a specific test in a debugger.\n"
73 "\n"
74 " Other flags:\n"
75 " --test-launcher-filter-file=PATH\n"
76 " Like --gtest_filter, but read the test filter from PATH.\n"
77 " Supports multiple filter paths separated by ';'.\n"
78 " One pattern per line; lines starting with '-' are exclusions.\n"
79 " See also //testing/buildbot/filters/README.md file.\n"
80 "\n"
81 " --test-launcher-batch-limit=N\n"
82 " Sets the limit of test batch to run in a single process to N.\n"
83 "\n"
84 " --test-launcher-debug-launcher\n"
85 " Disables autodetection of debuggers and similar tools,\n"
86 " making it possible to use them to debug launcher itself.\n"
87 "\n"
88 " --test-launcher-retry-limit=N\n"
89 " Sets the limit of test retries on failures to N.\n"
90 " --gtest-repeat=N\n"
91 " Forces the launcher to run every test N times. -1 is a special"
92 " value, causing the infinite amount of iterations."
93 " Repeated tests are run in parallel, unless the number of"
94 " iterations is infinite or --gtest-break-on-failure is specified"
95 " (see below)."
96 " Consider using --test_launcher-jobs flag to speed up the"
97 " parallel execution."
98 "\n"
99 " --gtest-break-on-failure\n"
100 " Stop running repeated tests as soon as one repeat of the test fails."
101 " This flag forces sequential repeats and prevents parallelised"
102 " execution."
103 "\n"
104 " --test-launcher-summary-output=PATH\n"
105 " Saves a JSON machine-readable summary of the run.\n"
106 "\n"
107 " --test-launcher-print-test-stdio=auto|always|never\n"
108 " Controls when full test output is printed.\n"
109 " auto means to print it when the test failed.\n"
110 "\n"
111 " --test-launcher-test-part-results-limit=N\n"
112 " Sets the limit of failed EXPECT/ASSERT entries in the xml and\n"
113 " JSON outputs per test to N (default N=10). Negative value \n"
114 " will disable this limit.\n"
115 "\n"
116 " --test-launcher-total-shards=N\n"
117 " Sets the total number of shards to N.\n"
118 "\n"
119 " --test-launcher-shard-index=N\n"
120 " Sets the shard index to run to N (from 0 to TOTAL - 1).\n"
121 "\n"
122 " --dont-use-job-objects\n"
123 " Avoids using job objects in Windows.\n"
124 "\n"
125 " --test-launcher-print-temp-leaks\n"
126 " Prints information about leaked files and/or directories in\n"
127 " child process's temporary directories (Windows and macOS).\n");
128 fflush(stdout);
129 }
130
GetSwitchValueAsInt(const std::string & switch_name,int * result)131 bool GetSwitchValueAsInt(const std::string& switch_name, int* result) {
132 if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name))
133 return true;
134
135 std::string switch_value =
136 CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name);
137 if (!StringToInt(switch_value, result) || *result < 0) {
138 LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value;
139 return false;
140 }
141
142 return true;
143 }
144 #endif
145
LaunchUnitTestsInternal(RunTestSuiteCallback run_test_suite,size_t parallel_jobs,int default_batch_limit,size_t retry_limit,bool use_job_objects,OnceClosure gtest_init)146 int LaunchUnitTestsInternal(RunTestSuiteCallback run_test_suite,
147 size_t parallel_jobs,
148 int default_batch_limit,
149 size_t retry_limit,
150 bool use_job_objects,
151 OnceClosure gtest_init) {
152 #if defined(OS_ANDROID)
153 // We can't easily fork on Android, just run the test suite directly.
154 return std::move(run_test_suite).Run();
155 #else
156 bool force_single_process = false;
157 if (CommandLine::ForCurrentProcess()->HasSwitch(
158 switches::kTestLauncherDebugLauncher)) {
159 fprintf(stdout, "Forcing test launcher debugging mode.\n");
160 fflush(stdout);
161 } else {
162 if (base::debug::BeingDebugged()) {
163 fprintf(stdout,
164 "Debugger detected, switching to single process mode.\n"
165 "Pass --test-launcher-debug-launcher to debug the launcher "
166 "itself.\n");
167 fflush(stdout);
168 force_single_process = true;
169 }
170 }
171
172 if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) ||
173 CommandLine::ForCurrentProcess()->HasSwitch(kGTestListTestsFlag) ||
174 CommandLine::ForCurrentProcess()->HasSwitch(
175 switches::kSingleProcessTests) ||
176 CommandLine::ForCurrentProcess()->HasSwitch(
177 switches::kTestChildProcess) ||
178 force_single_process) {
179 return std::move(run_test_suite).Run();
180 }
181
182 if (CommandLine::ForCurrentProcess()->HasSwitch(switches::kHelpFlag)) {
183 PrintUsage();
184 return 0;
185 }
186
187 TimeTicks start_time(TimeTicks::Now());
188
189 std::move(gtest_init).Run();
190 TestTimeouts::Initialize();
191
192 int batch_limit = default_batch_limit;
193 if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit))
194 return 1;
195
196 fprintf(stdout,
197 "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n"
198 "own process. For debugging a test inside a debugger, use the\n"
199 "--gtest_filter=<your_test_name> flag along with\n"
200 "--single-process-tests.\n");
201 fflush(stdout);
202
203 base::SingleThreadTaskExecutor executor(base::MessagePumpType::IO);
204 #if defined(OS_POSIX)
205 FileDescriptorWatcher file_descriptor_watcher(executor.task_runner());
206 #endif
207 use_job_objects =
208 use_job_objects &&
209 !CommandLine::ForCurrentProcess()->HasSwitch(kDontUseJobObjectFlag);
210
211 DefaultUnitTestPlatformDelegate platform_delegate;
212 UnitTestLauncherDelegate delegate(&platform_delegate, batch_limit,
213 use_job_objects);
214 TestLauncher launcher(&delegate, parallel_jobs, retry_limit);
215 bool success = launcher.Run();
216
217 fprintf(stdout, "Tests took %" PRId64 " seconds.\n",
218 (TimeTicks::Now() - start_time).InSeconds());
219 fflush(stdout);
220
221 return (success ? 0 : 1);
222 #endif
223 }
224
InitGoogleTestChar(int * argc,char ** argv)225 void InitGoogleTestChar(int* argc, char** argv) {
226 testing::InitGoogleTest(argc, argv);
227 }
228
229 #if defined(OS_WIN)
InitGoogleTestWChar(int * argc,wchar_t ** argv)230 void InitGoogleTestWChar(int* argc, wchar_t** argv) {
231 testing::InitGoogleTest(argc, argv);
232 }
233 #endif // defined(OS_WIN)
234
235 } // namespace
236
237 // Flag to avoid using job objects
238 const char kDontUseJobObjectFlag[] = "dont-use-job-objects";
239
LaunchUnitTests(int argc,char ** argv,RunTestSuiteCallback run_test_suite,size_t retry_limit)240 int LaunchUnitTests(int argc,
241 char** argv,
242 RunTestSuiteCallback run_test_suite,
243 size_t retry_limit) {
244 CommandLine::Init(argc, argv);
245 size_t parallel_jobs = NumParallelJobs(/*cores_per_job=*/1);
246 if (parallel_jobs == 0U) {
247 return 1;
248 }
249 return LaunchUnitTestsInternal(std::move(run_test_suite), parallel_jobs,
250 kDefaultTestBatchLimit, retry_limit, true,
251 BindOnce(&InitGoogleTestChar, &argc, argv));
252 }
253
LaunchUnitTestsSerially(int argc,char ** argv,RunTestSuiteCallback run_test_suite)254 int LaunchUnitTestsSerially(int argc,
255 char** argv,
256 RunTestSuiteCallback run_test_suite) {
257 CommandLine::Init(argc, argv);
258 return LaunchUnitTestsInternal(std::move(run_test_suite), 1U,
259 kDefaultTestBatchLimit, 1U, true,
260 BindOnce(&InitGoogleTestChar, &argc, argv));
261 }
262
LaunchUnitTestsWithOptions(int argc,char ** argv,size_t parallel_jobs,int default_batch_limit,bool use_job_objects,RunTestSuiteCallback run_test_suite)263 int LaunchUnitTestsWithOptions(int argc,
264 char** argv,
265 size_t parallel_jobs,
266 int default_batch_limit,
267 bool use_job_objects,
268 RunTestSuiteCallback run_test_suite) {
269 CommandLine::Init(argc, argv);
270 return LaunchUnitTestsInternal(std::move(run_test_suite), parallel_jobs,
271 default_batch_limit, 1U, use_job_objects,
272 BindOnce(&InitGoogleTestChar, &argc, argv));
273 }
274
275 #if defined(OS_WIN)
LaunchUnitTests(int argc,wchar_t ** argv,bool use_job_objects,RunTestSuiteCallback run_test_suite)276 int LaunchUnitTests(int argc,
277 wchar_t** argv,
278 bool use_job_objects,
279 RunTestSuiteCallback run_test_suite) {
280 // Windows CommandLine::Init ignores argv anyway.
281 CommandLine::Init(argc, NULL);
282 size_t parallel_jobs = NumParallelJobs(/*cores_per_job=*/1);
283 if (parallel_jobs == 0U) {
284 return 1;
285 }
286 return LaunchUnitTestsInternal(std::move(run_test_suite), parallel_jobs,
287 kDefaultTestBatchLimit, 1U, use_job_objects,
288 BindOnce(&InitGoogleTestWChar, &argc, argv));
289 }
290 #endif // defined(OS_WIN)
291
292 DefaultUnitTestPlatformDelegate::DefaultUnitTestPlatformDelegate() = default;
293
GetTests(std::vector<TestIdentifier> * output)294 bool DefaultUnitTestPlatformDelegate::GetTests(
295 std::vector<TestIdentifier>* output) {
296 *output = GetCompiledInTests();
297 return true;
298 }
299
CreateResultsFile(const base::FilePath & temp_dir,base::FilePath * path)300 bool DefaultUnitTestPlatformDelegate::CreateResultsFile(
301 const base::FilePath& temp_dir,
302 base::FilePath* path) {
303 if (!CreateTemporaryDirInDir(temp_dir, FilePath::StringType(), path))
304 return false;
305 *path = path->AppendASCII("test_results.xml");
306 return true;
307 }
308
CreateTemporaryFile(const base::FilePath & temp_dir,base::FilePath * path)309 bool DefaultUnitTestPlatformDelegate::CreateTemporaryFile(
310 const base::FilePath& temp_dir,
311 base::FilePath* path) {
312 if (temp_dir.empty())
313 return false;
314 return CreateTemporaryFileInDir(temp_dir, path);
315 }
316
GetCommandLineForChildGTestProcess(const std::vector<std::string> & test_names,const base::FilePath & output_file,const base::FilePath & flag_file)317 CommandLine DefaultUnitTestPlatformDelegate::GetCommandLineForChildGTestProcess(
318 const std::vector<std::string>& test_names,
319 const base::FilePath& output_file,
320 const base::FilePath& flag_file) {
321 CommandLine new_cmd_line(*CommandLine::ForCurrentProcess());
322
323 CHECK(base::PathExists(flag_file));
324
325 std::string long_flags(
326 StrCat({"--", kGTestFilterFlag, "=", JoinString(test_names, ":")}));
327 CHECK(WriteFile(flag_file, long_flags));
328
329 new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file);
330 new_cmd_line.AppendSwitchPath(kGTestFlagfileFlag, flag_file);
331 new_cmd_line.AppendSwitch(switches::kSingleProcessTests);
332
333 return new_cmd_line;
334 }
335
GetWrapperForChildGTestProcess()336 std::string DefaultUnitTestPlatformDelegate::GetWrapperForChildGTestProcess() {
337 return std::string();
338 }
339
UnitTestLauncherDelegate(UnitTestPlatformDelegate * platform_delegate,size_t batch_limit,bool use_job_objects)340 UnitTestLauncherDelegate::UnitTestLauncherDelegate(
341 UnitTestPlatformDelegate* platform_delegate,
342 size_t batch_limit,
343 bool use_job_objects)
344 : platform_delegate_(platform_delegate),
345 batch_limit_(batch_limit),
346 use_job_objects_(use_job_objects) {}
347
~UnitTestLauncherDelegate()348 UnitTestLauncherDelegate::~UnitTestLauncherDelegate() {
349 DCHECK(thread_checker_.CalledOnValidThread());
350 }
351
GetTests(std::vector<TestIdentifier> * output)352 bool UnitTestLauncherDelegate::GetTests(std::vector<TestIdentifier>* output) {
353 DCHECK(thread_checker_.CalledOnValidThread());
354 return platform_delegate_->GetTests(output);
355 }
356
GetCommandLine(const std::vector<std::string> & test_names,const FilePath & temp_dir,FilePath * output_file)357 CommandLine UnitTestLauncherDelegate::GetCommandLine(
358 const std::vector<std::string>& test_names,
359 const FilePath& temp_dir,
360 FilePath* output_file) {
361 CHECK(!test_names.empty());
362
363 // Create a dedicated temporary directory to store the xml result data
364 // per run to ensure clean state and make it possible to launch multiple
365 // processes in parallel.
366 CHECK(platform_delegate_->CreateResultsFile(temp_dir, output_file));
367 FilePath flag_file;
368 platform_delegate_->CreateTemporaryFile(temp_dir, &flag_file);
369
370 return CommandLine(platform_delegate_->GetCommandLineForChildGTestProcess(
371 test_names, *output_file, flag_file));
372 }
373
GetWrapper()374 std::string UnitTestLauncherDelegate::GetWrapper() {
375 return platform_delegate_->GetWrapperForChildGTestProcess();
376 }
377
GetLaunchOptions()378 int UnitTestLauncherDelegate::GetLaunchOptions() {
379 return use_job_objects_ ? TestLauncher::USE_JOB_OBJECTS : 0;
380 }
381
GetTimeout()382 TimeDelta UnitTestLauncherDelegate::GetTimeout() {
383 return TestTimeouts::test_launcher_timeout();
384 }
385
GetBatchSize()386 size_t UnitTestLauncherDelegate::GetBatchSize() {
387 return batch_limit_;
388 }
389
390 } // namespace base
391