1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef BASE_TEST_LAUNCHER_TEST_LAUNCHER_H_ 6 #define BASE_TEST_LAUNCHER_TEST_LAUNCHER_H_ 7 8 #include <stddef.h> 9 #include <stdint.h> 10 11 #include <memory> 12 #include <set> 13 #include <string> 14 #include <vector> 15 16 #include "base/command_line.h" 17 #include "base/compiler_specific.h" 18 #include "base/macros.h" 19 #include "base/process/launch.h" 20 #include "base/test/gtest_util.h" 21 #include "base/test/launcher/test_result.h" 22 #include "base/test/launcher/test_results_tracker.h" 23 #include "base/threading/thread_checker.h" 24 #include "base/time/time.h" 25 #include "base/timer/timer.h" 26 #include "build/build_config.h" 27 28 namespace base { 29 30 // Constants for GTest command-line flags. 31 extern const char kGTestFilterFlag[]; 32 extern const char kGTestFlagfileFlag[]; 33 extern const char kGTestHelpFlag[]; 34 extern const char kGTestListTestsFlag[]; 35 extern const char kGTestRepeatFlag[]; 36 extern const char kGTestRunDisabledTestsFlag[]; 37 extern const char kGTestOutputFlag[]; 38 extern const char kGTestShuffleFlag[]; 39 extern const char kGTestRandomSeedFlag[]; 40 extern const char kIsolatedScriptRunDisabledTestsFlag[]; 41 extern const char kIsolatedScriptTestFilterFlag[]; 42 extern const char kIsolatedScriptTestRepeatFlag[]; 43 44 // Interface for use with LaunchTests that abstracts away exact details 45 // which tests and how are run. 46 class TestLauncherDelegate { 47 public: 48 // Called to get names of tests available for running. The delegate 49 // must put the result in |output| and return true on success. 50 virtual bool GetTests(std::vector<TestIdentifier>* output) = 0; 51 52 // Additional delegate TestResult processing. ProcessTestResults(std::vector<TestResult> & test_results,TimeDelta elapsed_time)53 virtual void ProcessTestResults(std::vector<TestResult>& test_results, 54 TimeDelta elapsed_time) {} 55 56 // Called to get the command line for the specified tests. 57 // |output_file_| is populated with the path to the result file, and must 58 // be inside |temp_dir|. 59 virtual CommandLine GetCommandLine(const std::vector<std::string>& test_names, 60 const FilePath& temp_dir, 61 FilePath* output_file) = 0; 62 63 // Invoked when a test process exceeds its runtime, immediately before it is 64 // terminated. |command_line| is the command line used to launch the process. 65 // NOTE: this method is invoked on the thread the process is launched on. OnTestTimedOut(const CommandLine & cmd_line)66 virtual void OnTestTimedOut(const CommandLine& cmd_line) {} 67 68 // Returns the delegate specific wrapper for command line. 69 // If it is not empty, it is prepended to the final command line. 70 virtual std::string GetWrapper() = 0; 71 72 // Returns the delegate specific flags for launch options. 73 // The flags are specified in LaunchChildGTestProcessFlags. 74 virtual int GetLaunchOptions() = 0; 75 76 // Returns the delegate specific timeout per test. 77 virtual TimeDelta GetTimeout() = 0; 78 79 // Returns the delegate specific batch size. 80 virtual size_t GetBatchSize() = 0; 81 82 // Returns true if test should run. 83 virtual bool ShouldRunTest(const TestIdentifier& test); 84 85 protected: 86 virtual ~TestLauncherDelegate(); 87 }; 88 89 // Launches tests using a TestLauncherDelegate. 90 class TestLauncher { 91 public: 92 // Flags controlling behavior of LaunchChildGTestProcess. 93 enum LaunchChildGTestProcessFlags { 94 // Allows usage of job objects on Windows. Helps properly clean up child 95 // processes. 96 USE_JOB_OBJECTS = (1 << 0), 97 98 // Allows breakaway from job on Windows. May result in some child processes 99 // not being properly terminated after launcher dies if these processes 100 // fail to cooperate. 101 ALLOW_BREAKAWAY_FROM_JOB = (1 << 1), 102 }; 103 104 // Enum for subprocess stdio redirect. 105 enum StdioRedirect { AUTO, ALWAYS, NEVER }; 106 107 struct LaunchOptions { 108 LaunchOptions(); 109 LaunchOptions(const LaunchOptions& other); 110 ~LaunchOptions(); 111 112 int flags = 0; 113 // These mirror values in base::LaunchOptions, see it for details. 114 #if defined(OS_WIN) 115 base::LaunchOptions::Inherit inherit_mode = 116 base::LaunchOptions::Inherit::kSpecific; 117 base::HandlesToInheritVector handles_to_inherit; 118 #else 119 FileHandleMappingVector fds_to_remap; 120 #endif 121 }; 122 123 // Constructor. |parallel_jobs| is the limit of simultaneous parallel test 124 // jobs. |retry_limit| is the default limit of retries for bots or all tests. 125 TestLauncher(TestLauncherDelegate* launcher_delegate, 126 size_t parallel_jobs, 127 size_t retry_limit = 1U); 128 // virtual to mock in testing. 129 virtual ~TestLauncher(); 130 131 // Runs the launcher. Must be called at most once. 132 // command_line is null by default. 133 // if null, uses command line for current process. 134 bool Run(CommandLine* command_line = nullptr) WARN_UNUSED_RESULT; 135 136 // Launches a child process (assumed to be gtest-based binary) which runs 137 // tests indicated by |test_names|. 138 // |task_runner| is used to post results back to the launcher on the main 139 // thread. |task_temp_dir| is used for child process files such as user data, 140 // result file, and flag_file. |child_temp_dir|, if not empty, specifies a 141 // directory (within task_temp_dir) that the child process will use as its 142 // process-wide temporary directory. 143 // virtual to mock in testing. 144 virtual void LaunchChildGTestProcess( 145 scoped_refptr<TaskRunner> task_runner, 146 const std::vector<std::string>& test_names, 147 const FilePath& task_temp_dir, 148 const FilePath& child_temp_dir); 149 150 // Called when a test has finished running. 151 void OnTestFinished(const TestResult& result); 152 153 // Returns true if child test processes should have dedicated temporary 154 // directories. SupportsPerChildTempDirs()155 static constexpr bool SupportsPerChildTempDirs() { 156 #if defined(OS_WIN) 157 return true; 158 #else 159 // TODO(https://crbug.com/1038857): Enable for macOS, Linux, and Fuchsia. 160 return false; 161 #endif 162 } 163 164 private: 165 bool Init(CommandLine* command_line) WARN_UNUSED_RESULT; 166 167 // Gets tests from the delegate, and converts to TestInfo objects. 168 // Catches and logs uninstantiated parameterized tests. 169 // Returns false if delegate fails to return tests. 170 bool InitTests(); 171 172 // Some of the TestLauncherDelegate implementations don't call into gtest 173 // until they've already split into test-specific processes. This results 174 // in gtest's native shuffle implementation attempting to shuffle one test. 175 // Shuffling the list of tests in the test launcher (before the delegate 176 // gets involved) ensures that the entire shard is shuffled. 177 bool ShuffleTests(CommandLine* command_line); 178 179 // Move all PRE_ tests prior to the final test in order. 180 // Validate tests names. This includes no name conflict between tests 181 // without DISABLED_ prefix, and orphaned PRE_ tests. 182 // Add all tests and disabled tests names to result tracker. 183 // Filter Disabled tests if not flagged to run. 184 // Returns false if any violation is found. 185 bool ProcessAndValidateTests(); 186 187 // Runs all tests in current iteration. 188 void RunTests(); 189 190 // Print test names that almost match a filter (matches *<filter>*). 191 void PrintFuzzyMatchingTestNames(); 192 193 // Retry to run tests that failed during RunTests. 194 // Returns false if retry still fails or unable to start. 195 bool RunRetryTests(); 196 197 void CombinePositiveTestFilters(std::vector<std::string> filter_a, 198 std::vector<std::string> filter_b); 199 200 // Rest counters, retry tests list, and test result tracker. 201 void OnTestIterationStart(); 202 203 #if defined(OS_POSIX) 204 void OnShutdownPipeReadable(); 205 #endif 206 207 // Saves test results summary as JSON if requested from command line. 208 void MaybeSaveSummaryAsJSON(const std::vector<std::string>& additional_tags); 209 210 // Called when a test iteration is finished. 211 void OnTestIterationFinished(); 212 213 // Called by the delay timer when no output was made for a while. 214 void OnOutputTimeout(); 215 216 // Creates and starts a ThreadPoolInstance with |num_parallel_jobs| dedicated 217 // to foreground blocking tasks (corresponds to the traits used to launch and 218 // wait for child processes). virtual to mock in testing. 219 virtual void CreateAndStartThreadPool(int num_parallel_jobs); 220 221 // Callback to receive result of a test. 222 // |result_file| is a path to xml file written by child process. 223 // It contains information about test and failed 224 // EXPECT/ASSERT/DCHECK statements. Test launcher parses that 225 // file to get additional information about test run (status, 226 // error-messages, stack-traces and file/line for failures). 227 // |leaked_items| is the number of files and/or directories remaining in the 228 // child process's temporary directory upon its termination. 229 void ProcessTestResults(const std::vector<std::string>& test_names, 230 const FilePath& result_file, 231 const std::string& output, 232 TimeDelta elapsed_time, 233 int exit_code, 234 bool was_timeout, 235 int leaked_items); 236 237 std::vector<std::string> CollectTests(); 238 239 // Make sure we don't accidentally call the wrong methods e.g. on the worker 240 // pool thread. Should be the first member so that it's destroyed last: when 241 // destroying other members, especially the worker pool, we may check the code 242 // is running on the correct thread. 243 ThreadChecker thread_checker_; 244 245 TestLauncherDelegate* launcher_delegate_; 246 247 // Support for outer sharding, just like gtest does. 248 int32_t total_shards_; // Total number of outer shards, at least one. 249 int32_t shard_index_; // Index of shard the launcher is to run. 250 251 int cycles_; // Number of remaining test iterations, or -1 for infinite. 252 253 // Test filters (empty means no filter). 254 bool has_at_least_one_positive_filter_; 255 std::vector<std::string> positive_test_filter_; 256 std::vector<std::string> negative_test_filter_; 257 258 // Class to encapsulate gtest information. 259 class TestInfo; 260 261 // Tests to use (cached result of TestLauncherDelegate::GetTests). 262 std::vector<TestInfo> tests_; 263 264 // Threshold for number of broken tests. 265 size_t broken_threshold_; 266 267 // Number of tests started in this iteration. 268 size_t test_started_count_; 269 270 // Number of tests finished in this iteration. 271 size_t test_finished_count_; 272 273 // Number of tests successfully finished in this iteration. 274 size_t test_success_count_; 275 276 // Number of tests either timing out or having an unknown result, 277 // likely indicating a more systemic problem if widespread. 278 size_t test_broken_count_; 279 280 // How many retries are left. 281 size_t retries_left_; 282 283 // Maximum number of retries per iteration. 284 size_t retry_limit_; 285 286 // If true will not early exit nor skip retries even if too many tests are 287 // broken. 288 bool force_run_broken_tests_; 289 290 // Tests to retry in this iteration. 291 std::unordered_set<std::string> tests_to_retry_; 292 293 TestResultsTracker results_tracker_; 294 295 // Watchdog timer to make sure we do not go without output for too long. 296 DelayTimer watchdog_timer_; 297 298 // Number of jobs to run in parallel. 299 size_t parallel_jobs_; 300 301 // Switch to control tests stdio :{auto, always, never} 302 StdioRedirect print_test_stdio_; 303 304 // Skip disabled tests unless explicitly requested. 305 bool skip_diabled_tests_; 306 307 // Stop test iterations due to failure. 308 bool stop_on_failure_; 309 310 // Path to JSON summary result file. 311 FilePath summary_path_; 312 313 // Path to trace file. 314 FilePath trace_path_; 315 316 // redirect stdio of subprocess 317 bool redirect_stdio_; 318 319 // Number of times all tests should be repeated during each iteration. 320 // 1 if gtest_repeat is not specified or gtest_break_on_failure is specified. 321 // Otherwise it matches gtest_repeat value. 322 int repeats_per_iteration_ = 1; 323 324 DISALLOW_COPY_AND_ASSIGN(TestLauncher); 325 }; 326 327 // Return the number of parallel jobs to use, or 0U in case of error. 328 size_t NumParallelJobs(unsigned int cores_per_job); 329 330 // Extract part from |full_output| that applies to |result|. 331 std::string GetTestOutputSnippet(const TestResult& result, 332 const std::string& full_output); 333 334 } // namespace base 335 336 #endif // BASE_TEST_LAUNCHER_TEST_LAUNCHER_H_ 337