1<?php
2/**
3
4 * Created by JetBrains PhpStorm.
5 * User: alain_desilets
6 * Date: 2013-10-02
7 * Time: 2:12 PM
8 * To change this template use File | Settings | File Templates.
9 *
10 * This class is used to run phpunit tests and compare the list of failures
11 * and errors to those of a benchmark "normal" run.
12 *
13 * Use this class in situations where it's not practical for everyone to
14 * keep all the tests "in the green" at all time, and to only commit code
15 * that doesn't break any tests.
16 *
17 * With this class, you can tell if you have broken tests that were working
18 * previously, or if you have fixed tests that were broken before.
19 */
20
21require_once(__DIR__ . '/../debug/Tracer.php');
22
23class TestRunnerWithBaseline
24{
25
26	private $baseline_log_fpath;
27	private $current_log_fpath;
28	private $output_fpath;
29
30	private $last_test_started = null;
31
32	private $logname_stem = 'phpunit-log';
33	public $action = 'run'; // run|update_baseline
34	public $phpunit_options = '';
35	public $help = false;
36	public $filter = '';
37	public $diffs;
38
39	function __construct($baseline_log_fpath = null, $current_log_fpath = null, $output_fpath = null)
40	{
41		$this->baseline_log_fpath = $baseline_log_fpath;
42		$this->current_log_fpath = $current_log_fpath;
43		$this->output_fpath = $output_fpath;
44	}
45
46	function run()
47	{
48		global $tracer;
49
50		$this->config_from_cmdline_options();
51
52		if ($this->help) {
53			$this->usage();
54		}
55
56		$this->run_tests();
57
58		$this->print_diffs_with_baseline();
59
60		if ($this->action == 'update_baseline') {
61			$this->save_current_log_as_baseline();
62		}
63	}
64
65	function run_tests()
66	{
67		global $tracer;
68
69		$cmd_line = "../../bin/phpunit --verbose";
70
71		if ($this->phpunit_options != '') {
72			$cmd_line = "$cmd_line " . $this->phpunit_options;
73		}
74
75		if ($this->filter != '') {
76			$cmd_line = "$cmd_line --filter " . $this->filter;
77		}
78
79		$cmd_line = 'php ' . $cmd_line . " --log-json \"" . $this->logpath_current() . "\" .";
80
81		$this->do_echo("
82********************************************************************
83*
84* Executing phpunit as:
85*
86*    $cmd_line
87*
88********************************************************************
89");
90
91		if ($this->output_fpath == null) {
92			system($cmd_line);
93		} else {
94			$phpunit_output = [];
95			exec($cmd_line, $phpunit_output);
96			$this->do_echo(implode("\n", $phpunit_output));
97		}
98	}
99
100	function print_diffs_with_baseline()
101	{
102		global $tracer;
103
104		$this->do_echo("\n\nChecking for differences with baseline test logs...\n\n");
105
106		$baseline_issues;
107		if (file_exists($this->logpath_baseline())) {
108			$baseline_issues = $this->read_log_file($this->logpath_baseline());
109		} else {
110			$this->do_echo("=== WARNING: No baseline file exists. Assuming empty baseline.\n\n");
111			$baseline_issues = $this->make_empty_issues_list();
112		}
113
114		$current_issues = $this->read_log_file($this->logpath_current());
115
116		$this->diffs = $this->compare_two_test_runs($baseline_issues, $current_issues);
117
118		$nb_failures_introduced = count($this->diffs['failures_introduced']);
119		$nb_failures_fixed = count($this->diffs['failures_fixed']);
120		$nb_errors_introduced = count($this->diffs['errors_introduced']);
121		$nb_errors_fixed = count($this->diffs['errors_fixed']);
122
123		$total_diffs =
124			$nb_failures_introduced + $nb_errors_introduced +
125				$nb_failures_fixed + $nb_errors_fixed;
126
127		if ($total_diffs > 0) {
128			$this->do_echo("\n\nThere were $total_diffs differences with baseline.\n
129
130Below is a list of tests that differ from the baseline.
131See above details about each error or failure.
132");
133			if ($nb_failures_introduced > 0) {
134				$this->do_echo("\nNb of new FAILURES: $nb_failures_introduced:\n");
135				foreach ($this->diffs['failures_introduced'] as $an_issue) {
136					$this->do_echo("   $an_issue\n");
137				}
138			}
139
140			if ($nb_errors_introduced > 0) {
141				$this->do_echo("\nNb of new ERRORS: $nb_errors_introduced:\n");
142				foreach ($this->diffs['errors_introduced'] as $an_issue) {
143					$this->do_echo("   $an_issue\n");
144				}
145			}
146
147			if ($nb_failures_fixed > 0) {
148				$this->do_echo("\nNb of newly FIXED FAILURES: $nb_failures_fixed:\n");
149				foreach ($this->diffs['failures_fixed'] as $an_issue) {
150					$this->do_echo("   $an_issue\n");
151				}
152			}
153
154			if ($nb_errors_fixed > 0) {
155				$this->do_echo("\nNb of newly FIXED ERRORS: $nb_errors_fixed:\n");
156				foreach ($this->diffs['errors_fixed'] as $an_issue) {
157					$this->do_echo("   $an_issue\n");
158				}
159			}
160		} else {
161			$this->do_echo("\n\nNo differences with baseline run. All is \"normal\".\n\n");
162		}
163
164		$this->do_echo("\n\n");
165	}
166
167	function logpath_current()
168	{
169
170		$path = __DIR__ . DIRECTORY_SEPARATOR . $this->logname_stem . ".current.json";
171		if ($this->current_log_fpath != null) {
172			$path = $this->current_log_fpath;
173		}
174		return $path;
175	}
176
177	function logpath_baseline()
178	{
179
180		$path = __DIR__ . DIRECTORY_SEPARATOR . $this->logname_stem . ".baseline.json";
181		if ($this->baseline_log_fpath != null) {
182			$path = $this->baseline_log_fpath;
183		}
184		return $path;
185	}
186
187	function ask_if_want_to_create_baseline()
188	{
189		$answer = $this->prompt_for(
190			"There is no baseline log. Would you like to log current failures and errors as the baseline?",
191			['y', 'n']
192		);
193		if ($answer == 'y') {
194			$this->save_current_log_as_baseline();
195		}
196	}
197
198	function process_phpunit_log_data($log_data)
199	{
200		global $tracer;
201
202		$issues =
203			[
204				'errors' => [],
205				'failures' => [],
206				'pass' => []
207			];
208
209		foreach ($log_data as $log_entry) {
210			$event = '';
211			if (isset($log_entry['event'])) {
212				$event = $log_entry['event'];
213			}
214
215			if ($event != 'testStart' && $event != 'test') {
216				continue;
217			}
218
219			$test = '';
220			if (isset($log_entry['test'])) {
221				$test = $log_entry['test'];
222			}
223
224			if ($event == 'testStart') {
225				$this->last_test_started = $test;
226				continue;
227			} else {
228				$this->last_test_started = null;
229			}
230
231			/* For some reason, sometimes an event=test entry does not have
232			   a 'status' field.
233			   Whenever that happens, it seems to be a sign of an error.
234			*/
235			$status = 'fail';
236			if (isset($log_entry['status'])) {
237				$status = $log_entry['status'];
238			}
239
240			if ($status == 'fail') {
241				array_push($issues['failures'], $test);
242			} elseif ($status == 'error') {
243				array_push($issues['errors'], $test);
244			} elseif ($status == 'pass') {
245				array_push($issues['pass'], $test);
246			}
247		}
248
249		/* If a test was started by never ended, flag it as a failure */
250		if ($this->last_test_started != null) {
251			if (! in_array($this->last_test_started, $issues['failures'])) {
252				array_push($issues['failures'], $this->last_test_started);
253			}
254		}
255
256		return $issues;
257	}
258
259	function compare_two_test_runs($baseline_issues, $current_issues)
260	{
261		global $tracer;
262
263		$diffs = ['failures_introduced' => [], 'failures_fixed' => [],
264			'errors_introduced' => [], 'errors_fixed' => []];
265
266		$current_failures = $current_issues['failures'];
267		$current_pass = $current_issues['pass'];
268		$baseline_failures = $baseline_issues['failures'];
269		$baseline_errors = $baseline_issues['errors'];
270		foreach ($baseline_failures as $a_baseline_failure) {
271			if (in_array($a_baseline_failure, $current_pass)) {
272				array_push($diffs['failures_fixed'], $a_baseline_failure);
273			}
274		}
275
276		foreach ($current_failures as $a_current_failure) {
277			if (! in_array($a_current_failure, $baseline_failures) && ! in_array($a_current_failure, $baseline_errors)) {
278				array_push($diffs['failures_introduced'], $a_current_failure);
279			}
280		}
281
282		$baseline_errors = $baseline_issues['errors'];
283		$current_errors = $current_issues['errors'];
284		foreach ($baseline_errors as $a_baseline_error) {
285			if (in_array($a_baseline_error, $current_pass)) {
286				array_push($diffs['errors_fixed'], $a_baseline_error);
287			}
288		}
289
290		foreach ($current_errors as $a_current_error) {
291			if (! in_array($a_current_error, $baseline_errors) && ! in_array($a_current_error, $baseline_failures)) {
292				array_push($diffs['errors_introduced'], $a_current_error);
293			}
294		}
295
296		return $diffs;
297	}
298
299	function save_current_log_as_baseline()
300	{
301		if ($this->total_new_issues_found() > 0) {
302			$answer = $this->prompt_for(
303				"Some new failures and/or errors were introduced (see above for details).\n\nAre you SURE you want to save the current run as a baseline?\n",
304				['y', 'n']
305			);
306			if ($answer == 'n') {
307				$this->do_echo("\nThe current run was NOT saved as the new baseline.\n");
308				return;
309			}
310		}
311
312		$this->do_echo("\n\nSaving current phpunit log as the baseline.\n");
313		copy($this->logpath_current(), $this->logpath_baseline());
314	}
315
316	function prompt_for($prompt, $eligible_answers)
317	{
318		$prompt = "\n\n$prompt (" . implode('|', $eligible_answers) . ")\n> ";
319		$answer = null;
320		while ($answer == null) {
321			echo $prompt;
322			$tentative_answer = rtrim(fgets(STDIN));
323			if (in_array($tentative_answer, $eligible_answers)) {
324				$answer = $tentative_answer;
325			} else {
326				$prompt = "\n\nSorry, '$tentative_answer' is not a valid answer.$prompt";
327			}
328		}
329
330		$this->do_echo("\$answer='$answer'\n'");
331		return $answer;
332	}
333
334	function read_log_file($log_file_path)
335	{
336		global $tracer;
337
338		$json_string = file_get_contents($log_file_path);
339
340		// The json string is actually a sequence of json arrays, but the
341		// sequence itself is not wrapped inside an array.
342		//
343		// Wrap all the json arrays into one before parsing the json.
344		//
345		$json_string = preg_replace('/}\s*{/', "},\n   {", $json_string);
346		$json_string = "[\n   $json_string\n]";
347
348		$json_decoded = json_decode($json_string, true);
349
350		$issues = $this->process_phpunit_log_data($json_decoded);
351
352		return $issues;
353	}
354
355	function config_from_cmdline_options()
356	{
357		global $argv, $tracer;
358
359		$options = getopt('', ['action:', 'phpunit-options:', 'filter:', 'help']);
360		$options = $this->validate_cmdline_options($options);
361
362		if (isset($options['help'])) {
363			$this->help = true;
364		}
365
366		if (isset($options['action'])) {
367			$this->action = $options['action'];
368		}
369
370		if (isset($options['phpunit-options'])) {
371			$this->phpunit_options = $options['phpunit-options'];
372		}
373
374		if (isset($options['filter'])) {
375			$this->filter = $options['filter'];
376		}
377	}
378
379	function validate_cmdline_options($options)
380	{
381		global $tracer;
382
383		$action = '';
384		if (isset($options['action'])) {
385			$action = $options['action'];
386		}
387		if ($action == 'update_baseline' && isset($options['phpunit-options'])) {
388			$this->usage("Cannot specify --phpunit-options with --action=update_baseline.");
389		}
390
391		$phpunit_options = '';
392		if (isset($options['phpunit-options'])) {
393			$phpunit_options = $options['phpunit-options'];
394		}
395		if (preg_match('/--log-json/', $phpunit_options)) {
396			$this->usage("You cannot specify '--log-json' option in the '--phpunit-options' option.");
397		}
398		if (preg_match('/--filter/', $phpunit_options)) {
399			$this->usage("You cannot specify '--filter' option in the '--phpunit-options' option. Instead, the --filter option of {$GLOBALS['argv'][0]} directely (i.e., '{$GLOBALS['argv'][0]} --filter pattern')");
400		}
401
402
403
404		return $options;
405	}
406
407	function usage($error_message = null)
408	{
409		global $argv;
410
411		$script_name = $argv[0];
412
413		$help = "php $script_name options
414
415Run phpunit tests, and compare the list of errors and failures against
416a baseline. Only report tests that have either started or stopped
417failing.
418
419Options
420
421   --action run|update_baseline (Default: run)
422        run:
423           Run the tests and report diffs from baseline.
424
425        update_baseline
426           Run ALL the tests, and save the list of generated failures
427           and errors as the new baseline.
428
429   --filter pattern
430         Only run the test methods whose names match the pattern.
431
432   --phpunit-options options (Default: '')
433        Command line options to be passed to phpunit.
434
435        Those are ignored when --action=update_baseline.
436
437        Also, you cannot specify a --log-json option in those, as that would
438        interfere with the script's ability to log test results for comparison
439        against the baseline.
440
441";
442
443		if ($error_message != null) {
444			$help = "ERROR: $error_message\n\n$help";
445		}
446
447		exit("\n$help");
448	}
449
450	function make_empty_issues_list()
451	{
452		$issues =
453			['pass' => [], 'failures' => [], 'errors' => []];
454
455		return $issues;
456	}
457
458	function total_new_issues_found()
459	{
460		global $tracer;
461		$total = count($this->diffs['errors_introduced']) + count($this->diffs['failures_introduced']);
462
463		$tracer->trace('total_new_issues_found', "** Returning \$total=$total");
464
465		return $total;
466	}
467
468	private function do_echo($message)
469	{
470		if ($this->output_fpath == null) {
471			echo($message);
472		} else {
473			$fh_output = fopen($this->output_fpath, 'a');
474			fwrite($fh_output, $message);
475			fclose($fh_output);
476		}
477	}
478}
479