1<?php
2// This file is part of Moodle - http://moodle.org/
3//
4// Moodle is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// Moodle is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12// GNU General Public License for more details.
13//
14// You should have received a copy of the GNU General Public License
15// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
16
17/**
18 * CLI tool with utilities to manage parallel Behat integration in Moodle
19 *
20 * All CLI utilities uses $CFG->behat_dataroot and $CFG->prefix_dataroot as
21 * $CFG->dataroot and $CFG->prefix
22 *
23 * @package    tool_behat
24 * @copyright  2012 David Monllaó
25 * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
26 */
27
28
29if (isset($_SERVER['REMOTE_ADDR'])) {
30    die(); // No access from web!.
31}
32
33define('BEHAT_UTIL', true);
34define('CLI_SCRIPT', true);
35define('NO_OUTPUT_BUFFERING', true);
36define('IGNORE_COMPONENT_CACHE', true);
37define('ABORT_AFTER_CONFIG', true);
38
39require_once(__DIR__ . '/../../../../lib/clilib.php');
40
41// CLI options.
42list($options, $unrecognized) = cli_get_params(
43    array(
44        'help'        => false,
45        'install'     => false,
46        'drop'        => false,
47        'enable'      => false,
48        'disable'     => false,
49        'diag'        => false,
50        'parallel'    => 0,
51        'maxruns'     => false,
52        'updatesteps' => false,
53        'fromrun'     => 1,
54        'torun'       => 0,
55        'optimize-runs' => '',
56        'add-core-features-to-theme' => false,
57        'axe'         => false,
58    ),
59    array(
60        'h' => 'help',
61        'j' => 'parallel',
62        'm' => 'maxruns',
63        'o' => 'optimize-runs',
64        'a' => 'add-core-features-to-theme',
65    )
66);
67
68// Checking util.php CLI script usage.
69$help = "
70Behat utilities to manage the test environment
71
72Usage:
73  php util.php [--install|--drop|--enable|--disable|--diag|--updatesteps|--axe|--help] [--parallel=value [--maxruns=value]]
74
75Options:
76--install      Installs the test environment for acceptance tests
77--drop         Drops the database tables and the dataroot contents
78--enable       Enables test environment and updates tests list
79--disable      Disables test environment
80--diag         Get behat test environment status code
81--updatesteps  Update feature step file.
82--axe          Include axe accessibility tests
83
84-j, --parallel Number of parallel behat run operation
85-m, --maxruns Max parallel processes to be executed at one time.
86-o, --optimize-runs Split features with specified tags in all parallel runs.
87-a, --add-core-features-to-theme Add all core features to specified theme's
88
89-h, --help     Print out this help
90
91Example from Moodle root directory:
92\$ php admin/tool/behat/cli/util.php --enable --parallel=4
93
94More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests
95";
96
97if (!empty($options['help'])) {
98    echo $help;
99    exit(0);
100}
101
102$cwd = getcwd();
103
104// If Behat parallel site is being initiliased, then define a param to be used to ignore single run install.
105if (!empty($options['parallel'])) {
106    define('BEHAT_PARALLEL_UTIL', true);
107}
108
109require_once(__DIR__ . '/../../../../config.php');
110require_once(__DIR__ . '/../../../../lib/behat/lib.php');
111require_once(__DIR__ . '/../../../../lib/behat/classes/behat_command.php');
112require_once(__DIR__ . '/../../../../lib/behat/classes/behat_config_manager.php');
113
114// Remove error handling overrides done in config.php. This is consistent with admin/tool/behat/cli/util_single_run.php.
115$CFG->debug = (E_ALL | E_STRICT);
116$CFG->debugdisplay = 1;
117error_reporting($CFG->debug);
118ini_set('display_errors', '1');
119ini_set('log_errors', '1');
120
121// Import the necessary libraries.
122require_once($CFG->libdir . '/setuplib.php');
123require_once($CFG->libdir . '/behat/classes/util.php');
124
125// For drop option check if parallel site.
126if ((empty($options['parallel'])) && ($options['drop']) || $options['updatesteps']) {
127    $options['parallel'] = behat_config_manager::get_behat_run_config_value('parallel');
128}
129
130// If not a parallel site then open single run.
131if (empty($options['parallel'])) {
132    // Set run config value for single run.
133    behat_config_manager::set_behat_run_config_value('singlerun', 1);
134
135    chdir(__DIR__);
136    // Check if behat is initialised, if not exit.
137    passthru("php util_single_run.php --diag", $status);
138    if ($status) {
139        exit ($status);
140    }
141    $cmd = commands_to_execute($options);
142    $processes = cli_execute_parallel(array($cmd), __DIR__);
143    $status = print_sequential_output($processes, false);
144    chdir($cwd);
145    exit($status);
146}
147
148// Default torun is maximum parallel runs.
149if (empty($options['torun'])) {
150    $options['torun'] = $options['parallel'];
151}
152
153$status = false;
154$cmds = commands_to_execute($options);
155
156// Start executing commands either sequential/parallel for options provided.
157if ($options['diag'] || $options['enable'] || $options['disable']) {
158    // Do it sequentially as it's fast and need to be displayed nicely.
159    foreach (array_chunk($cmds, 1, true) as $cmd) {
160        $processes = cli_execute_parallel($cmd, __DIR__);
161        print_sequential_output($processes);
162    }
163
164} else if ($options['drop']) {
165    $processes = cli_execute_parallel($cmds, __DIR__);
166    $exitcodes = print_combined_drop_output($processes);
167    foreach ($exitcodes as $exitcode) {
168        $status = (bool)$status || (bool)$exitcode;
169    }
170
171    // Remove run config file.
172    $behatrunconfigfile = behat_config_manager::get_behat_run_config_file_path();
173    if (file_exists($behatrunconfigfile)) {
174        if (!unlink($behatrunconfigfile)) {
175            behat_error(BEHAT_EXITCODE_PERMISSIONS, 'Can not delete behat run config file');
176        }
177    }
178
179    // Remove test file path.
180    if (file_exists(behat_util::get_test_file_path())) {
181        if (!unlink(behat_util::get_test_file_path())) {
182            behat_error(BEHAT_EXITCODE_PERMISSIONS, 'Can not delete test file enable info');
183        }
184    }
185
186} else if ($options['install']) {
187    // This is intensive compared to behat itself so run them in chunk if option maxruns not set.
188    if ($options['maxruns']) {
189        foreach (array_chunk($cmds, $options['maxruns'], true) as $chunk) {
190            $processes = cli_execute_parallel($chunk, __DIR__);
191            $exitcodes = print_combined_install_output($processes);
192            foreach ($exitcodes as $name => $exitcode) {
193                if ($exitcode != 0) {
194                    echo "Failed process [[$name]]" . PHP_EOL;
195                    echo $processes[$name]->getOutput();
196                    echo PHP_EOL;
197                    echo $processes[$name]->getErrorOutput();
198                    echo PHP_EOL . PHP_EOL;
199                }
200                $status = (bool)$status || (bool)$exitcode;
201            }
202        }
203    } else {
204        $processes = cli_execute_parallel($cmds, __DIR__);
205        $exitcodes = print_combined_install_output($processes);
206        foreach ($exitcodes as $name => $exitcode) {
207            if ($exitcode != 0) {
208                echo "Failed process [[$name]]" . PHP_EOL;
209                echo $processes[$name]->getOutput();
210                echo PHP_EOL;
211                echo $processes[$name]->getErrorOutput();
212                echo PHP_EOL . PHP_EOL;
213            }
214            $status = (bool)$status || (bool)$exitcode;
215        }
216    }
217
218} else if ($options['updatesteps']) {
219    // Rewrite config file to ensure we have all the features covered.
220    if (empty($options['parallel'])) {
221        behat_config_manager::update_config_file('', true, '', $options['add-core-features-to-theme'], false, false);
222    } else {
223        // Update config file, ensuring we have up-to-date behat.yml.
224        for ($i = $options['fromrun']; $i <= $options['torun']; $i++) {
225            $CFG->behatrunprocess = $i;
226
227            // Update config file for each run.
228            behat_config_manager::update_config_file('', true, $options['optimize-runs'], $options['add-core-features-to-theme'],
229                $options['parallel'], $i);
230        }
231        unset($CFG->behatrunprocess);
232    }
233
234    // Do it sequentially as it's fast and need to be displayed nicely.
235    foreach (array_chunk($cmds, 1, true) as $cmd) {
236        $processes = cli_execute_parallel($cmd, __DIR__);
237        print_sequential_output($processes);
238    }
239    exit(0);
240
241} else {
242    // We should never reach here.
243    echo $help;
244    exit(1);
245}
246
247// Ensure we have success status to show following information.
248if ($status) {
249    echo "Unknown failure $status" . PHP_EOL;
250    exit((int)$status);
251}
252
253// Show command o/p (only one per time).
254if ($options['install']) {
255    echo "Acceptance tests site installed for sites:".PHP_EOL;
256
257    // Display all sites which are installed/drop/diabled.
258    for ($i = $options['fromrun']; $i <= $options['torun']; $i++) {
259        if (empty($CFG->behat_parallel_run[$i - 1]['behat_wwwroot'])) {
260            echo $CFG->behat_wwwroot . "/" . BEHAT_PARALLEL_SITE_NAME . $i . PHP_EOL;
261        } else {
262            echo $CFG->behat_parallel_run[$i - 1]['behat_wwwroot'] . PHP_EOL;
263        }
264
265    }
266} else if ($options['drop']) {
267    echo "Acceptance tests site dropped for " . $options['parallel'] . " parallel sites" . PHP_EOL;
268
269} else if ($options['enable']) {
270    echo "Acceptance tests environment enabled on $CFG->behat_wwwroot, to run the tests use:" . PHP_EOL;
271    echo behat_command::get_behat_command(true, true);
272
273    // Save fromrun and to run information.
274    if (isset($options['fromrun'])) {
275        behat_config_manager::set_behat_run_config_value('fromrun', $options['fromrun']);
276    }
277
278    if (isset($options['torun'])) {
279        behat_config_manager::set_behat_run_config_value('torun', $options['torun']);
280    }
281    if (isset($options['parallel'])) {
282        behat_config_manager::set_behat_run_config_value('parallel', $options['parallel']);
283    }
284
285    echo PHP_EOL;
286
287} else if ($options['disable']) {
288    echo "Acceptance tests environment disabled for " . $options['parallel'] . " parallel sites" . PHP_EOL;
289
290} else if ($options['diag']) {
291    // Valid option, so nothing to do.
292} else {
293    echo $help;
294    chdir($cwd);
295    exit(1);
296}
297
298chdir($cwd);
299exit(0);
300
301/**
302 * Create commands to be executed for parallel run.
303 *
304 * @param array $options options provided by user.
305 * @return array commands to be executed.
306 */
307function commands_to_execute($options) {
308    $removeoptions = array('maxruns', 'fromrun', 'torun');
309    $cmds = array();
310    $extraoptions = $options;
311    $extra = "";
312
313    // Remove extra options not in util_single_run.php.
314    foreach ($removeoptions as $ro) {
315        $extraoptions[$ro] = null;
316        unset($extraoptions[$ro]);
317    }
318
319    foreach ($extraoptions as $option => $value) {
320        if ($options[$option]) {
321            $extra .= " --$option";
322            if ($value) {
323                $extra .= "=\"$value\"";
324            }
325        }
326    }
327
328    if (empty($options['parallel'])) {
329        $cmds = "php util_single_run.php " . $extra;
330    } else {
331        // Create commands which has to be executed for parallel site.
332        for ($i = $options['fromrun']; $i <= $options['torun']; $i++) {
333            $prefix = BEHAT_PARALLEL_SITE_NAME . $i;
334            $cmds[$prefix] = "php util_single_run.php " . $extra . " --run=" . $i . " 2>&1";
335        }
336    }
337    return $cmds;
338}
339
340/**
341 * Print drop output merging each run.
342 *
343 * @param array $processes list of processes.
344 * @return array exit codes of each process.
345 */
346function print_combined_drop_output($processes) {
347    $exitcodes = array();
348    $maxdotsonline = 70;
349    $remainingprintlen = $maxdotsonline;
350    $progresscount = 0;
351    echo "Dropping tables:" . PHP_EOL;
352
353    while (count($exitcodes) != count($processes)) {
354        usleep(10000);
355        foreach ($processes as $name => $process) {
356            if ($process->isRunning()) {
357                $op = $process->getIncrementalOutput();
358                if (trim($op)) {
359                    $update = preg_filter('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', '$1', $op);
360                    $strlentoprint = strlen($update);
361
362                    // If not enough dots printed on line then just print.
363                    if ($strlentoprint < $remainingprintlen) {
364                        echo $update;
365                        $remainingprintlen = $remainingprintlen - $strlentoprint;
366                    } else if ($strlentoprint == $remainingprintlen) {
367                        $progresscount += $maxdotsonline;
368                        echo $update . " " . $progresscount . PHP_EOL;
369                        $remainingprintlen = $maxdotsonline;
370                    } else {
371                        while ($part = substr($update, 0, $remainingprintlen) > 0) {
372                            $progresscount += $maxdotsonline;
373                            echo $part . " " . $progresscount . PHP_EOL;
374                            $update = substr($update, $remainingprintlen);
375                            $remainingprintlen = $maxdotsonline;
376                        }
377                    }
378                }
379            } else {
380                // Process exited.
381                $process->clearOutput();
382                $exitcodes[$name] = $process->getExitCode();
383            }
384        }
385    }
386
387    echo PHP_EOL;
388    return $exitcodes;
389}
390
391/**
392 * Print install output merging each run.
393 *
394 * @param array $processes list of processes.
395 * @return array exit codes of each process.
396 */
397function print_combined_install_output($processes) {
398    $exitcodes = array();
399    $line = array();
400
401    // Check what best we can do to accommodate  all parallel run o/p on single line.
402    // Windows command line has length of 80 chars, so default we will try fit o/p in 80 chars.
403    if (defined('BEHAT_MAX_CMD_LINE_OUTPUT') && BEHAT_MAX_CMD_LINE_OUTPUT) {
404        $lengthofprocessline = (int)max(10, BEHAT_MAX_CMD_LINE_OUTPUT / count($processes));
405    } else {
406        $lengthofprocessline = (int)max(10, 80 / count($processes));
407    }
408
409    echo "Installing behat site for " . count($processes) . " parallel behat run" . PHP_EOL;
410
411    // Show process name in first row.
412    foreach ($processes as $name => $process) {
413        // If we don't have enough space to show full run name then show runX.
414        if ($lengthofprocessline < strlen($name) + 2) {
415            $name = substr($name, -5);
416        }
417        // One extra padding as we are adding | separator for rest of the data.
418        $line[$name] = str_pad('[' . $name . '] ', $lengthofprocessline + 1);
419    }
420    ksort($line);
421    $tableheader = array_keys($line);
422    echo implode("", $line) . PHP_EOL;
423
424    // Now print o/p from each process.
425    while (count($exitcodes) != count($processes)) {
426        usleep(50000);
427        $poutput = array();
428        // Create child process.
429        foreach ($processes as $name => $process) {
430            if ($process->isRunning()) {
431                $output = $process->getIncrementalOutput();
432                if (trim($output)) {
433                    $poutput[$name] = explode(PHP_EOL, $output);
434                }
435            } else {
436                // Process exited.
437                $exitcodes[$name] = $process->getExitCode();
438            }
439        }
440        ksort($poutput);
441
442        // Get max depth of o/p before displaying.
443        $maxdepth = 0;
444        foreach ($poutput as $pout) {
445            $pdepth = count($pout);
446            $maxdepth = $pdepth >= $maxdepth ? $pdepth : $maxdepth;
447        }
448
449        // Iterate over each process to get line to print.
450        for ($i = 0; $i <= $maxdepth; $i++) {
451            $pline = "";
452            foreach ($tableheader as $name) {
453                $po = empty($poutput[$name][$i]) ? "" : substr($poutput[$name][$i], 0, $lengthofprocessline - 1);
454                $po = str_pad($po, $lengthofprocessline);
455                $pline .= "|". $po;
456            }
457            if (trim(str_replace("|", "", $pline))) {
458                echo $pline . PHP_EOL;
459            }
460        }
461        unset($poutput);
462        $poutput = null;
463
464    }
465    echo PHP_EOL;
466    return $exitcodes;
467}
468
469/**
470 * Print install output merging showing one run at a time.
471 * If any process fail then exit.
472 *
473 * @param array $processes list of processes.
474 * @param bool $showprefix show prefix.
475 * @return bool exitcode.
476 */
477function print_sequential_output($processes, $showprefix = true) {
478    $status = false;
479    foreach ($processes as $name => $process) {
480        $shownname = false;
481        while ($process->isRunning()) {
482            $op = $process->getIncrementalOutput();
483            if (trim($op)) {
484                // Show name of the run once for sequential.
485                if ($showprefix && !$shownname) {
486                    echo '[' . $name . '] ';
487                    $shownname = true;
488                }
489                echo $op;
490            }
491        }
492        // If any error then exit.
493        $exitcode = $process->getExitCode();
494        if ($exitcode != 0) {
495            exit($exitcode);
496        }
497        $status = $status || (bool)$exitcode;
498    }
499    return $status;
500}
501