1<?php 2/** 3 4 * Created by JetBrains PhpStorm. 5 * User: alain_desilets 6 * Date: 2013-10-02 7 * Time: 2:12 PM 8 * To change this template use File | Settings | File Templates. 9 * 10 * This class is used to run phpunit tests and compare the list of failures 11 * and errors to those of a benchmark "normal" run. 12 * 13 * Use this class in situations where it's not practical for everyone to 14 * keep all the tests "in the green" at all time, and to only commit code 15 * that doesn't break any tests. 16 * 17 * With this class, you can tell if you have broken tests that were working 18 * previously, or if you have fixed tests that were broken before. 19 */ 20 21require_once(__DIR__ . '/../debug/Tracer.php'); 22 23class TestRunnerWithBaseline 24{ 25 26 private $baseline_log_fpath; 27 private $current_log_fpath; 28 private $output_fpath; 29 30 private $last_test_started = null; 31 32 private $logname_stem = 'phpunit-log'; 33 public $action = 'run'; // run|update_baseline 34 public $phpunit_options = ''; 35 public $help = false; 36 public $filter = ''; 37 public $diffs; 38 39 function __construct($baseline_log_fpath = null, $current_log_fpath = null, $output_fpath = null) 40 { 41 $this->baseline_log_fpath = $baseline_log_fpath; 42 $this->current_log_fpath = $current_log_fpath; 43 $this->output_fpath = $output_fpath; 44 } 45 46 function run() 47 { 48 global $tracer; 49 50 $this->config_from_cmdline_options(); 51 52 if ($this->help) { 53 $this->usage(); 54 } 55 56 $this->run_tests(); 57 58 $this->print_diffs_with_baseline(); 59 60 if ($this->action == 'update_baseline') { 61 $this->save_current_log_as_baseline(); 62 } 63 } 64 65 function run_tests() 66 { 67 global $tracer; 68 69 $cmd_line = "../../bin/phpunit --verbose"; 70 71 if ($this->phpunit_options != '') { 72 $cmd_line = "$cmd_line " . $this->phpunit_options; 73 } 74 75 if ($this->filter != '') { 76 $cmd_line = "$cmd_line --filter " . $this->filter; 77 } 78 79 $cmd_line = 'php ' . $cmd_line . " --log-json \"" . $this->logpath_current() . "\" ."; 80 81 $this->do_echo(" 82******************************************************************** 83* 84* Executing phpunit as: 85* 86* $cmd_line 87* 88******************************************************************** 89"); 90 91 if ($this->output_fpath == null) { 92 system($cmd_line); 93 } else { 94 $phpunit_output = []; 95 exec($cmd_line, $phpunit_output); 96 $this->do_echo(implode("\n", $phpunit_output)); 97 } 98 } 99 100 function print_diffs_with_baseline() 101 { 102 global $tracer; 103 104 $this->do_echo("\n\nChecking for differences with baseline test logs...\n\n"); 105 106 $baseline_issues; 107 if (file_exists($this->logpath_baseline())) { 108 $baseline_issues = $this->read_log_file($this->logpath_baseline()); 109 } else { 110 $this->do_echo("=== WARNING: No baseline file exists. Assuming empty baseline.\n\n"); 111 $baseline_issues = $this->make_empty_issues_list(); 112 } 113 114 $current_issues = $this->read_log_file($this->logpath_current()); 115 116 $this->diffs = $this->compare_two_test_runs($baseline_issues, $current_issues); 117 118 $nb_failures_introduced = count($this->diffs['failures_introduced']); 119 $nb_failures_fixed = count($this->diffs['failures_fixed']); 120 $nb_errors_introduced = count($this->diffs['errors_introduced']); 121 $nb_errors_fixed = count($this->diffs['errors_fixed']); 122 123 $total_diffs = 124 $nb_failures_introduced + $nb_errors_introduced + 125 $nb_failures_fixed + $nb_errors_fixed; 126 127 if ($total_diffs > 0) { 128 $this->do_echo("\n\nThere were $total_diffs differences with baseline.\n 129 130Below is a list of tests that differ from the baseline. 131See above details about each error or failure. 132"); 133 if ($nb_failures_introduced > 0) { 134 $this->do_echo("\nNb of new FAILURES: $nb_failures_introduced:\n"); 135 foreach ($this->diffs['failures_introduced'] as $an_issue) { 136 $this->do_echo(" $an_issue\n"); 137 } 138 } 139 140 if ($nb_errors_introduced > 0) { 141 $this->do_echo("\nNb of new ERRORS: $nb_errors_introduced:\n"); 142 foreach ($this->diffs['errors_introduced'] as $an_issue) { 143 $this->do_echo(" $an_issue\n"); 144 } 145 } 146 147 if ($nb_failures_fixed > 0) { 148 $this->do_echo("\nNb of newly FIXED FAILURES: $nb_failures_fixed:\n"); 149 foreach ($this->diffs['failures_fixed'] as $an_issue) { 150 $this->do_echo(" $an_issue\n"); 151 } 152 } 153 154 if ($nb_errors_fixed > 0) { 155 $this->do_echo("\nNb of newly FIXED ERRORS: $nb_errors_fixed:\n"); 156 foreach ($this->diffs['errors_fixed'] as $an_issue) { 157 $this->do_echo(" $an_issue\n"); 158 } 159 } 160 } else { 161 $this->do_echo("\n\nNo differences with baseline run. All is \"normal\".\n\n"); 162 } 163 164 $this->do_echo("\n\n"); 165 } 166 167 function logpath_current() 168 { 169 170 $path = __DIR__ . DIRECTORY_SEPARATOR . $this->logname_stem . ".current.json"; 171 if ($this->current_log_fpath != null) { 172 $path = $this->current_log_fpath; 173 } 174 return $path; 175 } 176 177 function logpath_baseline() 178 { 179 180 $path = __DIR__ . DIRECTORY_SEPARATOR . $this->logname_stem . ".baseline.json"; 181 if ($this->baseline_log_fpath != null) { 182 $path = $this->baseline_log_fpath; 183 } 184 return $path; 185 } 186 187 function ask_if_want_to_create_baseline() 188 { 189 $answer = $this->prompt_for( 190 "There is no baseline log. Would you like to log current failures and errors as the baseline?", 191 ['y', 'n'] 192 ); 193 if ($answer == 'y') { 194 $this->save_current_log_as_baseline(); 195 } 196 } 197 198 function process_phpunit_log_data($log_data) 199 { 200 global $tracer; 201 202 $issues = 203 [ 204 'errors' => [], 205 'failures' => [], 206 'pass' => [] 207 ]; 208 209 foreach ($log_data as $log_entry) { 210 $event = ''; 211 if (isset($log_entry['event'])) { 212 $event = $log_entry['event']; 213 } 214 215 if ($event != 'testStart' && $event != 'test') { 216 continue; 217 } 218 219 $test = ''; 220 if (isset($log_entry['test'])) { 221 $test = $log_entry['test']; 222 } 223 224 if ($event == 'testStart') { 225 $this->last_test_started = $test; 226 continue; 227 } else { 228 $this->last_test_started = null; 229 } 230 231 /* For some reason, sometimes an event=test entry does not have 232 a 'status' field. 233 Whenever that happens, it seems to be a sign of an error. 234 */ 235 $status = 'fail'; 236 if (isset($log_entry['status'])) { 237 $status = $log_entry['status']; 238 } 239 240 if ($status == 'fail') { 241 array_push($issues['failures'], $test); 242 } elseif ($status == 'error') { 243 array_push($issues['errors'], $test); 244 } elseif ($status == 'pass') { 245 array_push($issues['pass'], $test); 246 } 247 } 248 249 /* If a test was started by never ended, flag it as a failure */ 250 if ($this->last_test_started != null) { 251 if (! in_array($this->last_test_started, $issues['failures'])) { 252 array_push($issues['failures'], $this->last_test_started); 253 } 254 } 255 256 return $issues; 257 } 258 259 function compare_two_test_runs($baseline_issues, $current_issues) 260 { 261 global $tracer; 262 263 $diffs = ['failures_introduced' => [], 'failures_fixed' => [], 264 'errors_introduced' => [], 'errors_fixed' => []]; 265 266 $current_failures = $current_issues['failures']; 267 $current_pass = $current_issues['pass']; 268 $baseline_failures = $baseline_issues['failures']; 269 $baseline_errors = $baseline_issues['errors']; 270 foreach ($baseline_failures as $a_baseline_failure) { 271 if (in_array($a_baseline_failure, $current_pass)) { 272 array_push($diffs['failures_fixed'], $a_baseline_failure); 273 } 274 } 275 276 foreach ($current_failures as $a_current_failure) { 277 if (! in_array($a_current_failure, $baseline_failures) && ! in_array($a_current_failure, $baseline_errors)) { 278 array_push($diffs['failures_introduced'], $a_current_failure); 279 } 280 } 281 282 $baseline_errors = $baseline_issues['errors']; 283 $current_errors = $current_issues['errors']; 284 foreach ($baseline_errors as $a_baseline_error) { 285 if (in_array($a_baseline_error, $current_pass)) { 286 array_push($diffs['errors_fixed'], $a_baseline_error); 287 } 288 } 289 290 foreach ($current_errors as $a_current_error) { 291 if (! in_array($a_current_error, $baseline_errors) && ! in_array($a_current_error, $baseline_failures)) { 292 array_push($diffs['errors_introduced'], $a_current_error); 293 } 294 } 295 296 return $diffs; 297 } 298 299 function save_current_log_as_baseline() 300 { 301 if ($this->total_new_issues_found() > 0) { 302 $answer = $this->prompt_for( 303 "Some new failures and/or errors were introduced (see above for details).\n\nAre you SURE you want to save the current run as a baseline?\n", 304 ['y', 'n'] 305 ); 306 if ($answer == 'n') { 307 $this->do_echo("\nThe current run was NOT saved as the new baseline.\n"); 308 return; 309 } 310 } 311 312 $this->do_echo("\n\nSaving current phpunit log as the baseline.\n"); 313 copy($this->logpath_current(), $this->logpath_baseline()); 314 } 315 316 function prompt_for($prompt, $eligible_answers) 317 { 318 $prompt = "\n\n$prompt (" . implode('|', $eligible_answers) . ")\n> "; 319 $answer = null; 320 while ($answer == null) { 321 echo $prompt; 322 $tentative_answer = rtrim(fgets(STDIN)); 323 if (in_array($tentative_answer, $eligible_answers)) { 324 $answer = $tentative_answer; 325 } else { 326 $prompt = "\n\nSorry, '$tentative_answer' is not a valid answer.$prompt"; 327 } 328 } 329 330 $this->do_echo("\$answer='$answer'\n'"); 331 return $answer; 332 } 333 334 function read_log_file($log_file_path) 335 { 336 global $tracer; 337 338 $json_string = file_get_contents($log_file_path); 339 340 // The json string is actually a sequence of json arrays, but the 341 // sequence itself is not wrapped inside an array. 342 // 343 // Wrap all the json arrays into one before parsing the json. 344 // 345 $json_string = preg_replace('/}\s*{/', "},\n {", $json_string); 346 $json_string = "[\n $json_string\n]"; 347 348 $json_decoded = json_decode($json_string, true); 349 350 $issues = $this->process_phpunit_log_data($json_decoded); 351 352 return $issues; 353 } 354 355 function config_from_cmdline_options() 356 { 357 global $argv, $tracer; 358 359 $options = getopt('', ['action:', 'phpunit-options:', 'filter:', 'help']); 360 $options = $this->validate_cmdline_options($options); 361 362 if (isset($options['help'])) { 363 $this->help = true; 364 } 365 366 if (isset($options['action'])) { 367 $this->action = $options['action']; 368 } 369 370 if (isset($options['phpunit-options'])) { 371 $this->phpunit_options = $options['phpunit-options']; 372 } 373 374 if (isset($options['filter'])) { 375 $this->filter = $options['filter']; 376 } 377 } 378 379 function validate_cmdline_options($options) 380 { 381 global $tracer; 382 383 $action = ''; 384 if (isset($options['action'])) { 385 $action = $options['action']; 386 } 387 if ($action == 'update_baseline' && isset($options['phpunit-options'])) { 388 $this->usage("Cannot specify --phpunit-options with --action=update_baseline."); 389 } 390 391 $phpunit_options = ''; 392 if (isset($options['phpunit-options'])) { 393 $phpunit_options = $options['phpunit-options']; 394 } 395 if (preg_match('/--log-json/', $phpunit_options)) { 396 $this->usage("You cannot specify '--log-json' option in the '--phpunit-options' option."); 397 } 398 if (preg_match('/--filter/', $phpunit_options)) { 399 $this->usage("You cannot specify '--filter' option in the '--phpunit-options' option. Instead, the --filter option of {$GLOBALS['argv'][0]} directely (i.e., '{$GLOBALS['argv'][0]} --filter pattern')"); 400 } 401 402 403 404 return $options; 405 } 406 407 function usage($error_message = null) 408 { 409 global $argv; 410 411 $script_name = $argv[0]; 412 413 $help = "php $script_name options 414 415Run phpunit tests, and compare the list of errors and failures against 416a baseline. Only report tests that have either started or stopped 417failing. 418 419Options 420 421 --action run|update_baseline (Default: run) 422 run: 423 Run the tests and report diffs from baseline. 424 425 update_baseline 426 Run ALL the tests, and save the list of generated failures 427 and errors as the new baseline. 428 429 --filter pattern 430 Only run the test methods whose names match the pattern. 431 432 --phpunit-options options (Default: '') 433 Command line options to be passed to phpunit. 434 435 Those are ignored when --action=update_baseline. 436 437 Also, you cannot specify a --log-json option in those, as that would 438 interfere with the script's ability to log test results for comparison 439 against the baseline. 440 441"; 442 443 if ($error_message != null) { 444 $help = "ERROR: $error_message\n\n$help"; 445 } 446 447 exit("\n$help"); 448 } 449 450 function make_empty_issues_list() 451 { 452 $issues = 453 ['pass' => [], 'failures' => [], 'errors' => []]; 454 455 return $issues; 456 } 457 458 function total_new_issues_found() 459 { 460 global $tracer; 461 $total = count($this->diffs['errors_introduced']) + count($this->diffs['failures_introduced']); 462 463 $tracer->trace('total_new_issues_found', "** Returning \$total=$total"); 464 465 return $total; 466 } 467 468 private function do_echo($message) 469 { 470 if ($this->output_fpath == null) { 471 echo($message); 472 } else { 473 $fh_output = fopen($this->output_fpath, 'a'); 474 fwrite($fh_output, $message); 475 fclose($fh_output); 476 } 477 } 478} 479