1#!/usr/bin/env perl
2# -*- cperl -*-
3
4# Copyright (c) 2004, 2014, Oracle and/or its affiliates.
5# Copyright (c) 2009, 2020, MariaDB Corporation
6#
7# This program is free software; you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation; version 2 of the License.
10#
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14# GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License
17# along with this program; if not, write to the Free Software
18# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1335  USA
19
20#
21##############################################################################
22#
23#  mysql-test-run.pl
24#
25#  Tool used for executing a suite of .test files
26#
27#  See the "MySQL Test framework manual" for more information
28#  https://mariadb.com/kb/en/library/mysqltest/
29#
30#
31##############################################################################
32
33use strict;
34use warnings;
35
36BEGIN {
37  # Check that mysql-test-run.pl is started from mysql-test/
38  unless ( -f "mysql-test-run.pl" )
39  {
40    print "**** ERROR **** ",
41      "You must start mysql-test-run from the mysql-test/ directory\n";
42    exit(1);
43  }
44  # Check that lib exist
45  unless ( -d "lib/" )
46  {
47    print "**** ERROR **** ",
48      "Could not find the lib/ directory \n";
49    exit(1);
50  }
51
52  # Check backward compatibility support
53  # By setting the environment variable MTR_VERSION
54  # it's possible to use a previous version of
55  # mysql-test-run.pl
56  my $version= $ENV{MTR_VERSION} || 2;
57  if ( $version == 1 )
58  {
59    print "=======================================================\n";
60    print "  WARNING: Using mysql-test-run.pl version 1!  \n";
61    print "=======================================================\n";
62    # Should use exec() here on *nix but this appears not to work on Windows
63    exit(system($^X, "lib/v1/mysql-test-run.pl", @ARGV) >> 8);
64  }
65  elsif ( $version == 2 )
66  {
67    # This is the current version, just continue
68    ;
69  }
70  else
71  {
72    print "ERROR: Version $version of mysql-test-run does not exist!\n";
73    exit(1);
74  }
75}
76
77use lib "lib";
78
79use Cwd ;
80use Cwd 'realpath';
81use Getopt::Long;
82use My::File::Path; # Patched version of File::Path
83use File::Basename;
84use File::Copy;
85use File::Find;
86use File::Temp qw/tempdir/;
87use File::Spec::Functions qw/splitdir rel2abs/;
88use My::Platform;
89use My::SafeProcess;
90use My::ConfigFactory;
91use My::Options;
92use My::Tee;
93use My::Find;
94use My::SysInfo;
95use My::CoreDump;
96use My::Debugger;
97use mtr_cases;
98use mtr_report;
99use mtr_match;
100use mtr_unique;
101use mtr_results;
102use IO::Socket::INET;
103use IO::Select;
104
105require "mtr_process.pl";
106require "mtr_io.pl";
107require "mtr_gprof.pl";
108require "mtr_misc.pl";
109
110my $opt_valgrind;
111my $valgrind_reports= 0;
112
113$SIG{INT}= sub { mtr_error("Got ^C signal"); };
114$SIG{HUP}= sub { mtr_error("Hangup detected on controlling terminal"); };
115
116our $mysql_version_id;
117my $mysql_version_extra;
118our $glob_mysql_test_dir;
119our $basedir;
120our $bindir;
121
122our $path_charsetsdir;
123our $path_client_bindir;
124our $path_client_libdir;
125our $path_language;
126
127our $path_current_testlog;
128our $path_testlog;
129
130our $default_vardir;
131our $opt_vardir;                # Path to use for var/ dir
132our $plugindir;
133our $opt_xml_report;            # XML output
134
135my $path_vardir_trace;          # unix formatted opt_vardir for trace files
136my $opt_tmpdir;                 # Path to use for tmp/ dir
137my $opt_tmpdir_pid;
138
139my $opt_start;
140my $opt_start_dirty;
141my $opt_start_exit;
142my $start_only;
143my $file_wsrep_provider;
144my $extra_path;
145my $mariabackup_path;
146my $mariabackup_exe;
147my $garbd_exe;
148
149our @global_suppressions;
150
151END {
152  if ( defined $opt_tmpdir_pid and $opt_tmpdir_pid == $$ )
153  {
154    if (!$opt_start_exit)
155    {
156      # Remove the tempdir this process has created
157      mtr_verbose("Removing tmpdir $opt_tmpdir");
158      rmtree($opt_tmpdir);
159    }
160    else
161    {
162      mtr_warning("tmpdir $opt_tmpdir should be removed after the server has finished");
163    }
164  }
165}
166
167sub env_or_val($$) { defined $ENV{$_[0]} ? $ENV{$_[0]} : $_[1] }
168
169my $path_config_file;           # The generated config file, var/my.cnf
170
171# Visual Studio produces executables in different sub-directories based on the
172# configuration used to build them.  To make life easier, an environment
173# variable or command-line option may be specified to control which set of
174# executables will be used by the test suite.
175our $multiconfig = $ENV{'MTR_VS_CONFIG'};
176
177my @DEFAULT_SUITES= qw(
178    main-
179    archive-
180    binlog-
181    binlog_encryption-
182    csv-
183    compat/oracle-
184    compat/maxdb-
185    encryption-
186    federated-
187    funcs_1-
188    funcs_2-
189    gcol-
190    handler-
191    heap-
192    innodb-
193    innodb_fts-
194    innodb_gis-
195    innodb_zip-
196    json-
197    maria-
198    mariabackup-
199    multi_source-
200    optimizer_unfixed_bugs-
201    parts-
202    perfschema-
203    plugins-
204    roles-
205    rpl-
206    stress-
207    sys_vars-
208    sql_sequence-
209    unit-
210    vcol-
211    versioning-
212  );
213my $opt_suites;
214
215our $opt_verbose= 0;  # Verbose output, enable with --verbose
216our $exe_patch;
217our $exe_mysql;
218our $exe_mysql_plugin;
219our $exe_mysqladmin;
220our $exe_mysqltest;
221our $exe_libtool;
222our $exe_mysql_embedded;
223
224our $opt_big_test= 0;
225our $opt_staging_run= 0;
226
227our @opt_combinations;
228
229our @opt_extra_mysqld_opt;
230our @opt_mysqld_envs;
231
232my $opt_stress;
233my $opt_tail_lines= 20;
234
235my $opt_dry_run;
236
237my $opt_compress;
238my $opt_ssl;
239my $opt_skip_ssl;
240my @opt_skip_test_list;
241our $opt_ssl_supported;
242my $opt_ps_protocol;
243my $opt_sp_protocol;
244my $opt_cursor_protocol;
245my $opt_view_protocol;
246my $opt_non_blocking_api;
247
248our $opt_debug;
249my $debug_d= "d,*";
250my $opt_debug_common;
251our $opt_debug_server;
252our @opt_cases;                  # The test cases names in argv
253our $opt_embedded_server;
254
255# Options used when connecting to an already running server
256my %opts_extern;
257sub using_extern { return (keys %opts_extern > 0);};
258
259our $opt_fast= 0;
260our $opt_force= 0;
261our $opt_mem= $ENV{'MTR_MEM'};
262our $opt_clean_vardir= $ENV{'MTR_CLEAN_VARDIR'};
263
264our $opt_gcov;
265our $opt_gprof;
266our %gprof_dirs;
267
268my $config; # The currently running config
269my $current_config_name; # The currently running config file template
270
271our @opt_experimentals;
272our $experimental_test_cases= [];
273
274our $baseport;
275# $opt_build_thread may later be set from $opt_port_base
276my $opt_build_thread= $ENV{'MTR_BUILD_THREAD'} || "auto";
277my $opt_port_base= $ENV{'MTR_PORT_BASE'} || "auto";
278my $build_thread= 0;
279
280my $opt_record;
281
282our $opt_resfile= $ENV{'MTR_RESULT_FILE'} || 0;
283
284my $opt_skip_core;
285
286our $opt_check_testcases= 1;
287my $opt_mark_progress;
288my $opt_max_connections;
289our $opt_report_times= 0;
290
291my $opt_sleep;
292
293our $opt_retry= 1;
294our $opt_retry_failure= env_or_val(MTR_RETRY_FAILURE => 2);
295our $opt_testcase_timeout= $ENV{MTR_TESTCASE_TIMEOUT} ||  15; # minutes
296our $opt_suite_timeout   = $ENV{MTR_SUITE_TIMEOUT}    || 360; # minutes
297our $opt_shutdown_timeout= $ENV{MTR_SHUTDOWN_TIMEOUT} ||  10; # seconds
298our $opt_start_timeout   = $ENV{MTR_START_TIMEOUT}    || 180; # seconds
299
300sub suite_timeout { return $opt_suite_timeout * 60; };
301
302my $opt_wait_all;
303my $opt_user_args;
304my $opt_repeat= 1;
305my $opt_reorder= 1;
306my $opt_force_restart= 0;
307
308our $opt_user = "root";
309
310my %mysqld_logs;
311my $opt_debug_sync_timeout= 300; # Default timeout for WAIT_FOR actions.
312my $warn_seconds = 60;
313
314my $rebootstrap_re= '--innodb[-_](?:page[-_]size|checksum[-_]algorithm|undo[-_]tablespaces|log[-_]group[-_]home[-_]dir|data[-_]home[-_]dir)|data[-_]file[-_]path|force_rebootstrap';
315
316sub testcase_timeout ($) { return $opt_testcase_timeout * 60; }
317sub check_timeout ($) { return testcase_timeout($_[0]); }
318
319our $opt_warnings= 1;
320
321our %mysqld_variables;
322our @optional_plugins;
323
324my $source_dist=  -d "../sql";
325
326my $opt_max_save_core= env_or_val(MTR_MAX_SAVE_CORE => 5);
327my $opt_max_save_datadir= env_or_val(MTR_MAX_SAVE_DATADIR => 20);
328my $opt_max_test_fail= env_or_val(MTR_MAX_TEST_FAIL => 10);
329my $opt_core_on_failure= 0;
330
331my $opt_parallel= $ENV{MTR_PARALLEL} || 1;
332my $opt_port_group_size = $ENV{MTR_PORT_GROUP_SIZE} || 20;
333
334# lock file to stop tests
335my $opt_stop_file= $ENV{MTR_STOP_FILE};
336# print messages when test suite is stopped (for buildbot)
337my $opt_stop_keep_alive= $ENV{MTR_STOP_KEEP_ALIVE};
338
339select(STDOUT);
340$| = 1; # Automatically flush STDOUT
341
342main();
343
344sub have_wsrep() {
345  my $wsrep_on= $mysqld_variables{'wsrep-on'};
346  return defined $wsrep_on
347}
348
349sub have_wsrep_provider() {
350  return $file_wsrep_provider ne "";
351}
352
353sub have_mariabackup() {
354  return $mariabackup_path ne "";
355}
356
357sub have_garbd() {
358  return $garbd_exe ne "";
359}
360
361sub check_wsrep_version() {
362  if ($My::SafeProcess::wsrep_check_version ne "") {
363    system($My::SafeProcess::wsrep_check_version);
364    return ($? >> 8) == 0;
365  }
366  else {
367    return 0;
368  }
369}
370
371sub wsrep_version_message() {
372  if ($My::SafeProcess::wsrep_check_version ne "") {
373     my $output= `$My::SafeProcess::wsrep_check_version -p`;
374     if (($? >> 8) == 0) {
375        $output =~ s/\s+\z//;
376        return "Wsrep provider version mismatch (".$output.")";
377     }
378     else {
379        return "Galera library does not contain a version symbol";
380     }
381  }
382  else {
383     return "Unable to find a wsrep version check utility";
384  }
385}
386
387sub which($) { return `sh -c "command -v $_[0]"` }
388
389sub check_garbd_support() {
390  if (defined $ENV{'MTR_GARBD_EXE'}) {
391    if (mtr_file_exists($ENV{'MTR_GARBD_EXE'}) ne "") {
392      $garbd_exe= $ENV{'MTR_GARBD_EXE'};
393    } else {
394      mtr_error("MTR_GARBD_EXE env set to an invalid path");
395    }
396  }
397  else {
398    my $wsrep_path= dirname($file_wsrep_provider);
399    $garbd_exe=
400      mtr_file_exists($wsrep_path."/garb/garbd",
401                      $wsrep_path."/../../bin/garb/garbd");
402    if ($garbd_exe ne "") {
403      $ENV{MTR_GARBD_EXE}= $garbd_exe;
404    }
405  }
406}
407
408sub check_wsrep_support() {
409  $garbd_exe= "";
410  if (have_wsrep()) {
411    mtr_report(" - binaries built with wsrep patch");
412
413    # ADD scripts to $PATH to that wsrep_sst_* can be found
414    my ($spath) = grep { -f "$_/wsrep_sst_rsync"; } "$bindir/scripts", $path_client_bindir;
415    mtr_error("No SST scripts") unless $spath;
416    my $separator= (IS_WINDOWS) ? ';' : ':';
417    $ENV{PATH}="$spath$separator$ENV{PATH}";
418
419    # ADD mysql client library path to path so that wsrep_notify_cmd can find mysql
420    # client for loading the tables. (Don't assume each machine has mysql install)
421    my ($cpath) = grep { -f "$_/mysql"; } "$bindir/scripts", $path_client_bindir;
422    mtr_error("No scritps") unless $cpath;
423    $ENV{PATH}="$cpath$separator$ENV{PATH}" unless $cpath eq $spath;
424
425    # ADD my_print_defaults script path to path so that SST scripts can find it
426    my $my_print_defaults_exe=
427      mtr_exe_maybe_exists(
428        "$bindir/extra/my_print_defaults",
429        "$bindir/extra/$multiconfig/my_print_defaults",
430        "$path_client_bindir/my_print_defaults");
431    my $epath= "";
432    if ($my_print_defaults_exe ne "") {
433       $epath= dirname($my_print_defaults_exe);
434    }
435    mtr_error("No my_print_defaults") unless $epath;
436    $ENV{PATH}="$epath$separator$ENV{PATH}" unless ($epath eq $spath) or
437                                                   ($epath eq $cpath);
438
439    $extra_path= $epath;
440
441    if (!IS_WINDOWS) {
442      if (which("socat")) {
443        $ENV{MTR_GALERA_TFMT}="socat";
444      } elsif (which("nc")) {
445        $ENV{MTR_GALERA_TFMT}="nc";
446      }
447    }
448
449    # Check whether WSREP_PROVIDER environment variable is set.
450    if (defined $ENV{'WSREP_PROVIDER'}) {
451      $file_wsrep_provider= "";
452      if ($ENV{'WSREP_PROVIDER'} ne "none") {
453        if (mtr_file_exists($ENV{'WSREP_PROVIDER'}) ne "") {
454          $file_wsrep_provider= $ENV{'WSREP_PROVIDER'};
455        } else {
456          mtr_error("WSREP_PROVIDER env set to an invalid path");
457        }
458        check_garbd_support();
459      }
460      # WSREP_PROVIDER is valid; set to a valid path or "none").
461      mtr_verbose("WSREP_PROVIDER env set to $ENV{'WSREP_PROVIDER'}");
462    } else {
463      # WSREP_PROVIDER env not defined. Lets try to locate the wsrep provider
464      # library.
465      $file_wsrep_provider=
466        mtr_file_exists("/usr/lib64/galera-3/libgalera_smm.so",
467                        "/usr/lib64/galera/libgalera_smm.so",
468                        "/usr/lib/galera-3/libgalera_smm.so",
469                        "/usr/lib/galera/libgalera_smm.so");
470      if ($file_wsrep_provider ne "") {
471        # wsrep provider library found !
472        mtr_verbose("wsrep provider library found : $file_wsrep_provider");
473        $ENV{'WSREP_PROVIDER'}= $file_wsrep_provider;
474        check_garbd_support();
475      } else {
476        mtr_verbose("Could not find wsrep provider library, setting it to 'none'");
477        $ENV{'WSREP_PROVIDER'}= "none";
478      }
479    }
480  } else {
481    $file_wsrep_provider= "";
482    $extra_path= "";
483  }
484}
485
486sub check_mariabackup_support() {
487  $mariabackup_path= "";
488  $mariabackup_exe=
489    mtr_exe_maybe_exists(
490      "$bindir/extra/mariabackup$multiconfig/mariabackup",
491      "$path_client_bindir/mariabackup");
492  if ($mariabackup_exe ne "") {
493    my $bpath= dirname($mariabackup_exe);
494    my $separator= (IS_WINDOWS) ? ';' : ':';
495    $ENV{PATH}="$bpath$separator$ENV{PATH}" unless $bpath eq $extra_path;
496
497    $mariabackup_path= $bpath;
498
499    $ENV{XTRABACKUP}= $mariabackup_exe;
500
501    $ENV{XBSTREAM}= mtr_exe_maybe_exists(
502      "$bindir/extra/mariabackup/$multiconfig/mbstream",
503      "$path_client_bindir/mbstream");
504
505    $ENV{INNOBACKUPEX}= "$mariabackup_exe --innobackupex";
506  }
507}
508
509sub main {
510  $ENV{MTR_PERL}=$^X;
511
512  # Default, verbosity on
513  report_option('verbose', 0);
514
515  # This is needed for test log evaluation in "gen-build-status-page"
516  # in all cases where the calling tool does not log the commands
517  # directly before it executes them, like "make test-force-pl" in RPM builds.
518  mtr_report("Logging: $0 ", join(" ", @ARGV));
519
520  command_line_setup();
521
522  # --help will not reach here, so now it's safe to assume we have binaries
523  My::SafeProcess::find_bin();
524
525  print "vardir: $opt_vardir\n";
526  initialize_servers();
527  init_timers();
528
529  unless (IS_WINDOWS) {
530    binmode(STDOUT,":via(My::Tee)") or die "binmode(STDOUT, :via(My::Tee)):$!";
531    binmode(STDERR,":via(My::Tee)") or die "binmode(STDERR, :via(My::Tee)):$!";
532  }
533
534  mtr_report("Checking supported features...");
535
536  executable_setup();
537
538  # --debug[-common] implies we run debug server
539  $opt_debug_server= 1 if $opt_debug || $opt_debug_common;
540
541  if (using_extern())
542  {
543    # Connect to the running mysqld and find out what it supports
544    collect_mysqld_features_from_running_server();
545  }
546  else
547  {
548    # Run the mysqld to find out what features are available
549    collect_mysqld_features();
550  }
551  check_ssl_support();
552  check_debug_support();
553  check_wsrep_support();
554  check_mariabackup_support();
555
556  if (!$opt_suites) {
557    $opt_suites= join ',', collect_default_suites(@DEFAULT_SUITES);
558  }
559  mtr_report("Using suites: $opt_suites") unless @opt_cases;
560
561  mtr_report("Collecting tests...");
562  my $tests= collect_test_cases($opt_reorder, $opt_suites, \@opt_cases, \@opt_skip_test_list);
563  mark_time_used('collect');
564
565  mysql_install_db(default_mysqld(), "$opt_vardir/install.db") unless using_extern();
566
567  if ($opt_dry_run)
568  {
569    for (@$tests) {
570      print $_->fullname(), "\n";
571    }
572    exit 0;
573  }
574
575  if ($opt_gcov) {
576    system './dgcov.pl --purge';
577  }
578
579  #######################################################################
580  my $num_tests= $mtr_report::tests_total= @$tests;
581  if ( $opt_parallel eq "auto" ) {
582    # Try to find a suitable value for number of workers
583    if (IS_WINDOWS)
584    {
585      $opt_parallel= $ENV{NUMBER_OF_PROCESSORS} || 1;
586    }
587    else
588    {
589      my $sys_info= My::SysInfo->new();
590      $opt_parallel= $sys_info->num_cpus()+int($sys_info->min_bogomips()/500)-4;
591    }
592    my $max_par= $ENV{MTR_MAX_PARALLEL} || 8;
593    $opt_parallel= $max_par if ($opt_parallel > $max_par);
594    $opt_parallel= $num_tests if ($opt_parallel > $num_tests);
595    $opt_parallel= 1 if ($opt_parallel < 1);
596    mtr_report("Using parallel: $opt_parallel");
597  }
598  $ENV{MTR_PARALLEL} = $opt_parallel;
599
600  if ($opt_parallel > 1 && ($opt_start_exit || $opt_stress)) {
601    mtr_warning("Parallel cannot be used with --start-and-exit or --stress\n" .
602               "Setting parallel to 1");
603    $opt_parallel= 1;
604  }
605
606  # Create server socket on any free port
607  my $server = new IO::Socket::INET
608    (
609     LocalAddr => 'localhost',
610     Proto => 'tcp',
611     Listen => $opt_parallel,
612    );
613  mtr_error("Could not create testcase server port: $!") unless $server;
614  my $server_port = $server->sockport();
615
616  if ($opt_resfile) {
617    resfile_init("$opt_vardir/mtr-results.txt");
618    print_global_resfile();
619  }
620
621  # Create child processes
622  my %children;
623  for my $child_num (1..$opt_parallel){
624    my $child_pid= My::SafeProcess::Base::_safe_fork();
625    if ($child_pid == 0){
626      $server= undef; # Close the server port in child
627      $tests= {}; # Don't need the tests list in child
628
629      # Use subdir of var and tmp unless only one worker
630      if ($opt_parallel > 1) {
631	set_vardir("$opt_vardir/$child_num");
632	$opt_tmpdir= "$opt_tmpdir/$child_num";
633      }
634
635      init_timers();
636      run_worker($server_port, $child_num);
637      exit(1);
638    }
639
640    $children{$child_pid}= 1;
641  }
642  #######################################################################
643
644  mtr_report();
645  mtr_print_thick_line();
646  mtr_print_header($opt_parallel > 1);
647
648  mark_time_used('init');
649
650  my ($prefix, $fail, $completed, $extra_warnings)=
651    run_test_server($server, $tests, $opt_parallel);
652
653  exit(0) if $opt_start_exit;
654
655  # Send Ctrl-C to any children still running
656  kill("INT", keys(%children));
657
658  if (!IS_WINDOWS) {
659    # Wait for children to exit
660    foreach my $pid (keys %children)
661    {
662      my $ret_pid= waitpid($pid, 0);
663      if ($ret_pid != $pid){
664        mtr_report("Unknown process $ret_pid exited");
665      }
666      else {
667        delete $children{$ret_pid};
668      }
669    }
670  }
671
672  if ( not @$completed ) {
673    mtr_error("Test suite aborted");
674  }
675
676  if ( @$completed != $num_tests){
677
678    # Not all tests completed, failure
679    mtr_report();
680    mtr_report("Only ", int(@$completed), " of $num_tests completed.");
681  }
682
683  if ($opt_valgrind) {
684    # Create minimalistic "test" for the reporting
685    my $tinfo = My::Test->new
686      (
687       suite          => { name => 'valgrind', },
688       name           => 'valgrind_report',
689      );
690    # Set dummy worker id to align report with normal tests
691    $tinfo->{worker} = 0 if $opt_parallel > 1;
692    if ($valgrind_reports) {
693      $tinfo->{result}= 'MTR_RES_FAILED';
694      $tinfo->{comment}= "Valgrind reported failures at shutdown, see above";
695      $tinfo->{failures}= 1;
696    } else {
697      $tinfo->{result}= 'MTR_RES_PASSED';
698    }
699    mtr_report_test($tinfo);
700    push @$completed, $tinfo;
701    ++$num_tests
702  }
703
704  mtr_print_line();
705
706  print_total_times($opt_parallel) if $opt_report_times;
707  mtr_report_stats($prefix, $fail, $completed, $extra_warnings);
708
709  if ($opt_gcov) {
710    mtr_report("Running dgcov");
711    system "./dgcov.pl --generate > $opt_vardir/last_changes.dgcov";
712  }
713
714  if ( @$completed != $num_tests)
715  {
716    mtr_error("Not all tests completed (only ". scalar(@$completed) .
717              " of $num_tests)");
718  }
719
720  remove_vardir_subs() if $opt_clean_vardir;
721
722  exit(0);
723}
724
725
726sub run_test_server ($$$) {
727  my ($server, $tests, $childs) = @_;
728
729  my $num_saved_cores= 0;  # Number of core files saved in vardir/log/ so far.
730  my $num_saved_datadir= 0;  # Number of datadirs saved in vardir/log/ so far.
731  my $num_failed_test= 0; # Number of tests failed so far
732  my $test_failure= 0;    # Set true if test suite failed
733  my $extra_warnings= []; # Warnings found during server shutdowns
734
735  my $completed= [];
736  my %running;
737  my $result;
738  my $exe_mysqld= find_mysqld($bindir) || ""; # Used as hint to CoreDump
739
740  my $suite_timeout= start_timer(suite_timeout());
741
742  my $s= IO::Select->new();
743  $s->add($server);
744  while (1) {
745    if ($opt_stop_file)
746    {
747      if (mtr_wait_lock_file($opt_stop_file, $opt_stop_keep_alive))
748      {
749        # We were waiting so restart timer process
750        my $suite_timeout= start_timer(suite_timeout());
751      }
752    }
753
754    mark_time_used('admin');
755    my @ready = $s->can_read(1); # Wake up once every second
756    mark_time_idle();
757    foreach my $sock (@ready) {
758      if ($sock == $server) {
759	# New client connected
760	my $child= $sock->accept();
761	mtr_verbose("Client connected");
762	$s->add($child);
763	print $child "HELLO\n";
764      }
765      else {
766	my $line= <$sock>;
767	if (!defined $line) {
768	  # Client disconnected
769	  mtr_verbose("Child closed socket");
770	  $s->remove($sock);
771	  $sock->close;
772	  if (--$childs == 0){
773	    return ("Completed", $test_failure, $completed, $extra_warnings);
774	  }
775	  next;
776	}
777	chomp($line);
778
779	if ($line eq 'TESTRESULT'){
780	  $result= My::Test::read_test($sock);
781
782	  # Report test status
783	  mtr_report_test($result);
784
785	  if ( $result->is_failed() ) {
786
787	    # Save the workers "savedir" in var/log
788	    my $worker_savedir= $result->{savedir};
789	    my $worker_savename= basename($worker_savedir);
790	    my $savedir= "$opt_vardir/log/$worker_savename";
791
792            # Move any core files from e.g. mysqltest
793            foreach my $coref (glob("core*"), glob("*.dmp"))
794            {
795              mtr_report(" - found '$coref', moving it to '$worker_savedir'");
796              move($coref, $worker_savedir);
797            }
798
799            find(
800            {
801              no_chdir => 1,
802              wanted => sub
803              {
804                my $core_file= $File::Find::name;
805                my $core_name= basename($core_file);
806
807                # Name beginning with core, not ending in .gz
808                if (($core_name =~ /^core/ and $core_name !~ /\.gz$/)
809                    or (IS_WINDOWS and $core_name =~ /\.dmp$/))
810                {
811                  # Ending with .dmp
812                  mtr_report(" - found '$core_name'",
813                             "($num_saved_cores/$opt_max_save_core)");
814
815                  My::CoreDump->show($core_file, $exe_mysqld, $opt_parallel);
816
817                  # Limit number of core files saved
818                  if ($num_saved_cores >= $opt_max_save_core)
819                  {
820                    mtr_report(" - deleting it, already saved",
821                               "$opt_max_save_core");
822                    unlink("$core_file");
823                  }
824                  else
825                  {
826                    mtr_compress_file($core_file) unless @opt_cases;
827                    ++$num_saved_cores;
828                  }
829                }
830              }
831            },
832            $worker_savedir);
833
834	    if ($num_saved_datadir >= $opt_max_save_datadir)
835	    {
836	      mtr_report(" - skipping '$worker_savedir/'");
837	      rmtree($worker_savedir);
838	    }
839            else
840            {
841	      mtr_report(" - saving '$worker_savedir/' to '$savedir/'");
842	      rename($worker_savedir, $savedir);
843	      $num_saved_datadir++;
844	    }
845	    resfile_print_test();
846	    $num_failed_test++ unless ($result->{retries} ||
847                                       $result->{exp_fail});
848
849            $test_failure= 1;
850	    if ( !$opt_force ) {
851	      # Test has failed, force is off
852	      push(@$completed, $result);
853	      if ($result->{'dont_kill_server'})
854              {
855	        print $sock "BYE\n";
856	        next;
857              }
858	      return ("Failure", 1, $completed, $extra_warnings);
859	    }
860	    elsif ($opt_max_test_fail > 0 and
861		   $num_failed_test >= $opt_max_test_fail) {
862	      push(@$completed, $result);
863	      mtr_report("Too many tests($num_failed_test) failed!",
864			 "Terminating...");
865	      return ("Too many failed", 1, $completed, $extra_warnings);
866	    }
867	  }
868
869	  resfile_print_test();
870	  # Retry test run after test failure
871	  my $retries= $result->{retries} || 2;
872	  my $test_has_failed= $result->{failures} || 0;
873	  if ($test_has_failed and $retries <= $opt_retry){
874	    # Test should be run one more time unless it has failed
875	    # too many times already
876	    my $tname= $result->{name};
877	    my $failures= $result->{failures};
878	    if ($opt_retry > 1 and $failures >= $opt_retry_failure){
879	      mtr_report("\nTest $tname has failed $failures times,",
880			 "no more retries!\n");
881	    }
882	    else {
883	      mtr_report("\nRetrying test $tname, ".
884			 "attempt($retries/$opt_retry)...\n");
885              #saving the log file as filename.failed in case of retry
886              if ( $result->is_failed() ) {
887                my $worker_logdir= $result->{savedir};
888                my $log_file_name=dirname($worker_logdir)."/".$result->{shortname}.".log";
889
890                if (-e $log_file_name) {
891                  $result->{'logfile-failed'} = mtr_lastlinesfromfile($log_file_name, 20);
892                } else {
893                  $result->{'logfile-failed'} = "";
894                }
895
896                rename $log_file_name, $log_file_name.".failed";
897              }
898            {
899              local @$result{'retries', 'result'};
900              delete $result->{result};
901              $result->{retries}= $retries+1;
902              $result->write_test($sock, 'TESTCASE');
903            }
904            push(@$completed, $result);
905	      next;
906	    }
907	  }
908
909	  # Repeat test $opt_repeat number of times
910	  my $repeat= $result->{repeat} || 1;
911	  if ($repeat < $opt_repeat)
912	  {
913	    $result->{retries}= 0;
914	    $result->{rep_failures}++ if $result->{failures};
915	    $result->{failures}= 0;
916	    delete($result->{result});
917	    $result->{repeat}= $repeat+1;
918	    $result->write_test($sock, 'TESTCASE');
919	    next;
920	  }
921
922	  # Remove from list of running
923	  mtr_error("'", $result->{name},"' is not known to be running")
924	    unless delete $running{$result->key()};
925
926	  # Save result in completed list
927	  push(@$completed, $result);
928
929	}
930	elsif ($line eq 'START'){
931	  ; # Send first test
932	}
933	elsif ($line eq 'WARNINGS'){
934          my $fake_test= My::Test::read_test($sock);
935          my $test_list= join (" ", @{$fake_test->{testnames}});
936          push @$extra_warnings, $test_list;
937          my $report= $fake_test->{'warnings'};
938          mtr_report("***Warnings generated in error logs during shutdown ".
939                     "after running tests: $test_list\n\n$report");
940          $test_failure= 1;
941          if ( !$opt_force ) {
942            # Test failure due to warnings, force is off
943            return ("Warnings in log", 1, $completed, $extra_warnings);
944          }
945          next;
946        }
947	elsif ($line =~ /^SPENT/) {
948	  add_total_times($line);
949	}
950	elsif ($line eq 'VALGREP' && $opt_valgrind) {
951	  $valgrind_reports= 1;
952	}
953	else {
954	  mtr_error("Unknown response: '$line' from client");
955	}
956
957	# Find next test to schedule
958	# - Try to use same configuration as worker used last time
959
960	my $next;
961	my $second_best;
962	for(my $i= 0; $i <= @$tests; $i++)
963	{
964	  my $t= $tests->[$i];
965
966	  last unless defined $t;
967
968	  if (run_testcase_check_skip_test($t)){
969	    # Move the test to completed list
970	    #mtr_report("skip - Moving test $i to completed");
971	    push(@$completed, splice(@$tests, $i, 1));
972
973	    # Since the test at pos $i was taken away, next
974	    # test will also be at $i -> redo
975	    redo;
976	  }
977
978	  # From secondary choices, we prefer to pick a 'long-running' test if
979          # possible; this helps avoid getting stuck with a few of those at the
980          # end of high --parallel runs, with most workers being idle.
981	  if (!defined $second_best ||
982              ($t->{'long_test'} && !($tests->[$second_best]{'long_test'}))){
983	    #mtr_report("Setting second_best to $i");
984	    $second_best= $i;
985	  }
986
987	  # Smart allocation of next test within this thread.
988
989	  if ($opt_reorder and $opt_parallel > 1 and defined $result)
990	  {
991	    my $wid= $result->{worker};
992	    # Reserved for other thread, try next
993	    next if (defined $t->{reserved} and $t->{reserved} != $wid);
994	    if (! defined $t->{reserved})
995	    {
996	      # Force-restart not relevant when comparing *next* test
997	      $t->{criteria} =~ s/force-restart$/no-restart/;
998	      my $criteria= $t->{criteria};
999	      # Reserve similar tests for this worker, but not too many
1000	      my $maxres= (@$tests - $i) / $opt_parallel + 1;
1001	      for (my $j= $i+1; $j <= $i + $maxres; $j++)
1002	      {
1003		my $tt= $tests->[$j];
1004		last unless defined $tt;
1005		last if $tt->{criteria} ne $criteria;
1006		$tt->{reserved}= $wid;
1007	      }
1008	    }
1009	  }
1010
1011	  # At this point we have found next suitable test
1012	  $next= splice(@$tests, $i, 1);
1013	  last;
1014	}
1015
1016	# Use second best choice if no other test has been found
1017	if (!$next and defined $second_best){
1018	  #mtr_report("Take second best choice $second_best");
1019	  mtr_error("Internal error, second best too large($second_best)")
1020	    if $second_best >  $#$tests;
1021	  $next= splice(@$tests, $second_best, 1);
1022	  delete $next->{reserved};
1023	}
1024
1025	if ($next) {
1026	  # We don't need this any more
1027	  delete $next->{criteria};
1028	  $next->write_test($sock, 'TESTCASE');
1029	  $running{$next->key()}= $next;
1030	}
1031	else {
1032	  # No more test, tell child to exit
1033	  #mtr_report("Saying BYE to child");
1034	  print $sock "BYE\n";
1035	}
1036      }
1037    }
1038
1039    # ----------------------------------------------------
1040    # Check if test suite timer expired
1041    # ----------------------------------------------------
1042    if ( has_expired($suite_timeout) )
1043    {
1044      mtr_report("Test suite timeout! Terminating...");
1045      return ("Timeout", 1, $completed, $extra_warnings);
1046    }
1047  }
1048}
1049
1050
1051sub run_worker ($) {
1052  my ($server_port, $thread_num)= @_;
1053
1054  $SIG{INT}= sub { exit(1); };
1055  $SIG{HUP}= sub { exit(1); };
1056
1057  # Connect to server
1058  my $server = new IO::Socket::INET
1059    (
1060     PeerAddr => 'localhost',
1061     PeerPort => $server_port,
1062     Proto    => 'tcp'
1063    );
1064  mtr_error("Could not connect to server at port $server_port: $!")
1065    unless $server;
1066
1067  # --------------------------------------------------------------------------
1068  # Set worker name
1069  # --------------------------------------------------------------------------
1070  report_option('name',"worker[$thread_num]");
1071
1072  # --------------------------------------------------------------------------
1073  # Set different ports per thread
1074  # --------------------------------------------------------------------------
1075  set_build_thread_ports($thread_num);
1076
1077  # --------------------------------------------------------------------------
1078  # Turn off verbosity in workers, unless explicitly specified
1079  # --------------------------------------------------------------------------
1080  report_option('verbose', undef) if ($opt_verbose == 0);
1081
1082  environment_setup();
1083
1084  # Read hello from server which it will send when shared
1085  # resources have been setup
1086  my $hello= <$server>;
1087
1088  setup_vardir();
1089  check_running_as_root();
1090
1091  if ( using_extern() ) {
1092    create_config_file_for_extern(%opts_extern);
1093  }
1094
1095  # Ask server for first test
1096  print $server "START\n";
1097
1098  mark_time_used('init');
1099
1100  while (my $line= <$server>){
1101    chomp($line);
1102    if ($line eq 'TESTCASE'){
1103      my $test= My::Test::read_test($server);
1104
1105      # Clear comment and logfile, to avoid
1106      # reusing them from previous test
1107      delete($test->{'comment'});
1108      delete($test->{'logfile'});
1109
1110      # A sanity check. Should this happen often we need to look at it.
1111      if (defined $test->{reserved} && $test->{reserved} != $thread_num) {
1112	my $tres= $test->{reserved};
1113	mtr_warning("Test reserved for w$tres picked up by w$thread_num");
1114      }
1115      $test->{worker} = $thread_num if $opt_parallel > 1;
1116
1117      run_testcase($test, $server);
1118      #$test->{result}= 'MTR_RES_PASSED';
1119      # Send it back, now with results set
1120      $test->write_test($server, 'TESTRESULT');
1121      mark_time_used('restart');
1122    }
1123    elsif ($line eq 'BYE'){
1124      mtr_report("Server said BYE");
1125      # We need to gracefully shut down the servers to see any
1126      # Valgrind memory leak errors etc. since last server restart.
1127      if ($opt_warnings) {
1128        stop_servers(reverse all_servers());
1129        if(check_warnings_post_shutdown($server)) {
1130          # Warnings appeared in log file(s) during final server shutdown.
1131          exit(1);
1132        }
1133      }
1134      else {
1135        stop_all_servers($opt_shutdown_timeout);
1136      }
1137      mark_time_used('restart');
1138      my $valgrind_reports= 0;
1139      if ($opt_valgrind) {
1140        $valgrind_reports= valgrind_exit_reports();
1141	print $server "VALGREP\n" if $valgrind_reports;
1142      }
1143      if ( $opt_gprof ) {
1144	gprof_collect (find_mysqld($bindir), keys %gprof_dirs);
1145      }
1146      mark_time_used('admin');
1147      print_times_used($server, $thread_num);
1148      exit($valgrind_reports);
1149    }
1150    else {
1151      mtr_error("Could not understand server, '$line'");
1152    }
1153  }
1154
1155  stop_all_servers();
1156
1157  exit(1);
1158}
1159
1160
1161sub ignore_option {
1162  my ($opt, $value)= @_;
1163  mtr_report("Ignoring option '$opt'");
1164}
1165
1166
1167
1168# Setup any paths that are $opt_vardir related
1169sub set_vardir {
1170  ($opt_vardir)= @_;
1171
1172  $path_vardir_trace= $opt_vardir;
1173  # Chop off any "c:", DBUG likes a unix path ex: c:/src/... => /src/...
1174  $path_vardir_trace=~ s/^\w://;
1175
1176  # Location of my.cnf that all clients use
1177  $path_config_file= "$opt_vardir/my.cnf";
1178
1179  $path_testlog=         "$opt_vardir/log/mysqltest.log";
1180  $path_current_testlog= "$opt_vardir/log/current_test";
1181
1182}
1183
1184
1185sub print_global_resfile {
1186  resfile_global("start_time", isotime $^T);
1187  resfile_global("user_id", $<);
1188  resfile_global("embedded-server", $opt_embedded_server ? 1 : 0);
1189  resfile_global("ps-protocol", $opt_ps_protocol ? 1 : 0);
1190  resfile_global("sp-protocol", $opt_sp_protocol ? 1 : 0);
1191  resfile_global("view-protocol", $opt_view_protocol ? 1 : 0);
1192  resfile_global("cursor-protocol", $opt_cursor_protocol ? 1 : 0);
1193  resfile_global("ssl", $opt_ssl ? 1 : 0);
1194  resfile_global("compress", $opt_compress ? 1 : 0);
1195  resfile_global("parallel", $opt_parallel);
1196  resfile_global("check-testcases", $opt_check_testcases ? 1 : 0);
1197  resfile_global("mysqld", \@opt_extra_mysqld_opt);
1198  resfile_global("debug", $opt_debug ? 1 : 0);
1199  resfile_global("gcov", $opt_gcov ? 1 : 0);
1200  resfile_global("gprof", $opt_gprof ? 1 : 0);
1201  resfile_global("mem", $opt_mem);
1202  resfile_global("tmpdir", $opt_tmpdir);
1203  resfile_global("vardir", $opt_vardir);
1204  resfile_global("fast", $opt_fast ? 1 : 0);
1205  resfile_global("force-restart", $opt_force_restart ? 1 : 0);
1206  resfile_global("reorder", $opt_reorder ? 1 : 0);
1207  resfile_global("sleep", $opt_sleep);
1208  resfile_global("repeat", $opt_repeat);
1209  resfile_global("user", $opt_user);
1210  resfile_global("testcase-timeout", $opt_testcase_timeout);
1211  resfile_global("suite-timeout", $opt_suite_timeout);
1212  resfile_global("shutdown-timeout", $opt_shutdown_timeout ? 1 : 0);
1213  resfile_global("warnings", $opt_warnings ? 1 : 0);
1214  resfile_global("max-connections", $opt_max_connections);
1215  resfile_global("product", "MySQL");
1216  resfile_global("xml-report", $opt_xml_report);
1217  # Somewhat hacky code to convert numeric version back to dot notation
1218  my $v1= int($mysql_version_id / 10000);
1219  my $v2= int(($mysql_version_id % 10000)/100);
1220  my $v3= $mysql_version_id % 100;
1221  resfile_global("version", "$v1.$v2.$v3");
1222}
1223
1224
1225
1226sub command_line_setup {
1227  my $opt_comment;
1228  my $opt_usage;
1229  my $opt_list_options;
1230
1231  # Read the command line options
1232  # Note: Keep list in sync with usage at end of this file
1233  Getopt::Long::Configure("pass_through");
1234  my %options=(
1235             # Control what engine/variation to run
1236             'embedded-server'          => \$opt_embedded_server,
1237             'ps-protocol'              => \$opt_ps_protocol,
1238             'sp-protocol'              => \$opt_sp_protocol,
1239             'view-protocol'            => \$opt_view_protocol,
1240             'cursor-protocol'          => \$opt_cursor_protocol,
1241             'non-blocking-api'         => \$opt_non_blocking_api,
1242             'ssl|with-openssl'         => \$opt_ssl,
1243             'skip-ssl'                 => \$opt_skip_ssl,
1244             'compress'                 => \$opt_compress,
1245             'vs-config=s'              => \$multiconfig,
1246
1247	     # Max number of parallel threads to use
1248	     'parallel=s'               => \$opt_parallel,
1249
1250             # Config file to use as template for all tests
1251	     'defaults-file=s'          => \&collect_option,
1252	     # Extra config file to append to all generated configs
1253	     'defaults-extra-file=s'    => \&collect_option,
1254
1255             # Control what test suites or cases to run
1256             'force+'                   => \$opt_force,
1257             'suite|suites=s'           => \$opt_suites,
1258             'skip-rpl'                 => \&collect_option,
1259             'skip-test=s'              => \&collect_option,
1260             'do-test=s'                => \&collect_option,
1261             'start-from=s'             => \&collect_option,
1262             'big-test+'                => \$opt_big_test,
1263	     'combination=s'            => \@opt_combinations,
1264             'experimental=s'           => \@opt_experimentals,
1265	     # skip-im is deprecated and silently ignored
1266	     'skip-im'                  => \&ignore_option,
1267             'staging-run'              => \$opt_staging_run,
1268
1269             # Specify ports
1270	     'build-thread|mtr-build-thread=i' => \$opt_build_thread,
1271	     'port-base|mtr-port-base=i'       => \$opt_port_base,
1272	     'port-group-size=s'        => \$opt_port_group_size,
1273
1274             # Test case authoring
1275             'record'                   => \$opt_record,
1276             'check-testcases!'         => \$opt_check_testcases,
1277             'mark-progress'            => \$opt_mark_progress,
1278
1279             # Extra options used when starting mysqld
1280             'mysqld=s'                 => \@opt_extra_mysqld_opt,
1281             'mysqld-env=s'             => \@opt_mysqld_envs,
1282
1283             # Run test on running server
1284             'extern=s'                  => \%opts_extern, # Append to hash
1285
1286             # Debugging
1287             'debug'                    => \$opt_debug,
1288             'debug-common'             => \$opt_debug_common,
1289             'debug-server'             => \$opt_debug_server,
1290             'max-save-core=i'          => \$opt_max_save_core,
1291             'max-save-datadir=i'       => \$opt_max_save_datadir,
1292             'max-test-fail=i'          => \$opt_max_test_fail,
1293             'core-on-failure'          => \$opt_core_on_failure,
1294
1295             # Coverage, profiling etc
1296             'gcov'                     => \$opt_gcov,
1297             'gprof'                    => \$opt_gprof,
1298	     'debug-sync-timeout=i'     => \$opt_debug_sync_timeout,
1299
1300	     # Directories
1301             'tmpdir=s'                 => \$opt_tmpdir,
1302             'vardir=s'                 => \$opt_vardir,
1303             'mem'                      => \$opt_mem,
1304	     'clean-vardir'             => \$opt_clean_vardir,
1305             'client-bindir=s'          => \$path_client_bindir,
1306             'client-libdir=s'          => \$path_client_libdir,
1307
1308             # Misc
1309             'comment=s'                => \$opt_comment,
1310             'fast'                     => \$opt_fast,
1311	     'force-restart'            => \$opt_force_restart,
1312             'reorder!'                 => \$opt_reorder,
1313             'enable-disabled'          => \&collect_option,
1314             'verbose+'                 => \$opt_verbose,
1315             'verbose-restart'          => \&report_option,
1316             'sleep=i'                  => \$opt_sleep,
1317             'start-dirty'              => \$opt_start_dirty,
1318             'start-and-exit'           => \$opt_start_exit,
1319             'start'                    => \$opt_start,
1320	     'user-args'                => \$opt_user_args,
1321             'wait-all'                 => \$opt_wait_all,
1322	     'print-testcases'          => \&collect_option,
1323	     'repeat=i'                 => \$opt_repeat,
1324	     'retry=i'                  => \$opt_retry,
1325	     'retry-failure=i'          => \$opt_retry_failure,
1326             'timer!'                   => \&report_option,
1327             'user=s'                   => \$opt_user,
1328             'testcase-timeout=i'       => \$opt_testcase_timeout,
1329             'suite-timeout=i'          => \$opt_suite_timeout,
1330             'shutdown-timeout=i'       => \$opt_shutdown_timeout,
1331             'warnings!'                => \$opt_warnings,
1332	     'timestamp'                => \&report_option,
1333	     'timediff'                 => \&report_option,
1334             'stop-file=s'              => \$opt_stop_file,
1335             'stop-keep-alive=i'        => \$opt_stop_keep_alive,
1336	     'max-connections=i'        => \$opt_max_connections,
1337	     'report-times'             => \$opt_report_times,
1338	     'result-file'              => \$opt_resfile,
1339	     'stress=s'                 => \$opt_stress,
1340	     'tail-lines=i'             => \$opt_tail_lines,
1341             'dry-run'                  => \$opt_dry_run,
1342
1343             'help|h'                   => \$opt_usage,
1344	     # list-options is internal, not listed in help
1345	     'list-options'             => \$opt_list_options,
1346             'skip-test-list=s'         => \@opt_skip_test_list,
1347             'xml-report=s'             => \$opt_xml_report,
1348
1349             My::Debugger::options()
1350           );
1351
1352  # fix options (that take an optional argument and *only* after = sign
1353  @ARGV = My::Debugger::fix_options(@ARGV);
1354  GetOptions(%options) or usage("Can't read options");
1355  usage("") if $opt_usage;
1356  list_options(\%options) if $opt_list_options;
1357
1358  # --------------------------------------------------------------------------
1359  # Setup verbosity
1360  # --------------------------------------------------------------------------
1361  if ($opt_verbose != 0){
1362    report_option('verbose', $opt_verbose);
1363  }
1364
1365  # Negative values aren't meaningful on integer options
1366  foreach(grep(/=i$/, keys %options))
1367  {
1368    if (defined ${$options{$_}} &&
1369        do { no warnings "numeric"; int ${$options{$_}} < 0})
1370    {
1371      my $v= (split /=/)[0];
1372      die("$v doesn't accept a negative value:");
1373    }
1374  }
1375
1376  # Find the absolute path to the test directory
1377  $glob_mysql_test_dir= cwd();
1378  if ($glob_mysql_test_dir =~ / /)
1379  {
1380    die("Working directory \"$glob_mysql_test_dir\" contains space\n".
1381	"Bailing out, cannot function properly with space in path");
1382  }
1383  if (IS_CYGWIN)
1384  {
1385    # Use mixed path format i.e c:/path/to/
1386    $glob_mysql_test_dir= mixed_path($glob_mysql_test_dir);
1387  }
1388
1389  # In most cases, the base directory we find everything relative to,
1390  # is the parent directory of the "mysql-test" directory. For source
1391  # distributions, TAR binary distributions and some other packages.
1392  $basedir= dirname($glob_mysql_test_dir);
1393
1394  # In the RPM case, binaries and libraries are installed in the
1395  # default system locations, instead of having our own private base
1396  # directory. And we install "/usr/share/mysql-test". Moving up one
1397  # more directory relative to "mysql-test" gives us a usable base
1398  # directory for RPM installs.
1399  if ( ! $source_dist and ! -d "$basedir/bin" )
1400  {
1401    $basedir= dirname($basedir);
1402  }
1403  # For .deb, it's like RPM, but installed in /usr/share/mysql/mysql-test.
1404  # So move up one more directory level yet.
1405  if ( ! $source_dist and ! -d "$basedir/bin" )
1406  {
1407    $basedir= dirname($basedir);
1408  }
1409
1410  # Respect MTR_BINDIR variable, which is typically set in to the
1411  # build directory in out-of-source builds.
1412  $bindir=$ENV{MTR_BINDIR}||$basedir;
1413
1414  fix_vs_config_dir();
1415
1416  # Look for the client binaries directory
1417  if ($path_client_bindir)
1418  {
1419    # --client-bindir=path set on command line, check that the path exists
1420    $path_client_bindir= mtr_path_exists($path_client_bindir);
1421  }
1422  else
1423  {
1424    $path_client_bindir= mtr_path_exists("$bindir/client_release",
1425					 "$bindir/client_debug",
1426					 "$bindir/client/$multiconfig",
1427					 "$bindir/client$multiconfig",
1428					 "$bindir/client",
1429					 "$bindir/bin");
1430  }
1431
1432  # Look for language files and charsetsdir, use same share
1433  $path_language=   mtr_path_exists("$bindir/share/mariadb",
1434                                    "$bindir/share/mysql",
1435                                    "$bindir/sql/share",
1436                                    "$bindir/share");
1437  my $path_share= $path_language;
1438  $path_charsetsdir =   mtr_path_exists("$basedir/share/mariadb/charsets",
1439                                    "$basedir/share/mysql/charsets",
1440                                    "$basedir/sql/share/charsets",
1441                                    "$basedir/share/charsets");
1442  if ( $opt_comment )
1443  {
1444    mtr_report();
1445    mtr_print_thick_line('#');
1446    mtr_report("# $opt_comment");
1447    mtr_print_thick_line('#');
1448  }
1449
1450  if ( @opt_experimentals )
1451  {
1452    # $^O on Windows considered not generic enough
1453    my $plat= (IS_WINDOWS) ? 'windows' : $^O;
1454
1455    # read the list of experimental test cases from the files specified on
1456    # the command line
1457    $experimental_test_cases = [];
1458    foreach my $exp_file (@opt_experimentals)
1459    {
1460      open(FILE, "<", $exp_file)
1461	or mtr_error("Can't read experimental file: $exp_file");
1462      mtr_report("Using experimental file: $exp_file");
1463      while(<FILE>) {
1464	chomp;
1465	# remove comments (# foo) at the beginning of the line, or after a
1466	# blank at the end of the line
1467	s/(\s+|^)#.*$//;
1468	# If @ platform specifier given, use this entry only if it contains
1469	# @<platform> or @!<xxx> where xxx != platform
1470	if (/\@.*/)
1471	{
1472	  next if (/\@!$plat/);
1473	  next unless (/\@$plat/ or /\@!/);
1474	  # Then remove @ and everything after it
1475	  s/\@.*$//;
1476	}
1477	# remove whitespace
1478	s/^\s+//;
1479	s/\s+$//;
1480	# if nothing left, don't need to remember this line
1481	if ( $_ eq "" ) {
1482	  next;
1483	}
1484	# remember what is left as the name of another test case that should be
1485	# treated as experimental
1486	print " - $_\n";
1487	push @$experimental_test_cases, $_;
1488      }
1489      close FILE;
1490    }
1491  }
1492
1493  foreach my $arg ( @ARGV )
1494  {
1495    if ( $arg =~ /^--skip-/ )
1496    {
1497      push(@opt_extra_mysqld_opt, $arg);
1498    }
1499    elsif ( $arg =~ /^--$/ )
1500    {
1501      # It is an effect of setting 'pass_through' in option processing
1502      # that the lone '--' separating options from arguments survives,
1503      # simply ignore it.
1504    }
1505    elsif ( $arg =~ /^-/ )
1506    {
1507      usage("Invalid option \"$arg\"");
1508    }
1509    else
1510    {
1511      push(@opt_cases, $arg);
1512    }
1513  }
1514
1515  if ( @opt_cases )
1516  {
1517    # Run big tests if explicitly specified on command line
1518    $opt_big_test= 1;
1519  }
1520
1521  # --------------------------------------------------------------------------
1522  # Find out type of logging that are being used
1523  # --------------------------------------------------------------------------
1524  foreach my $arg ( @opt_extra_mysqld_opt )
1525  {
1526    if ( $arg =~ /binlog[-_]format=(\S+)/ )
1527    {
1528      # Save this for collect phase
1529      collect_option('binlog-format', $1);
1530      mtr_report("Using binlog format '$1'");
1531    }
1532  }
1533
1534
1535  # --------------------------------------------------------------------------
1536  # Find out default storage engine being used(if any)
1537  # --------------------------------------------------------------------------
1538  foreach my $arg ( @opt_extra_mysqld_opt )
1539  {
1540    if ( $arg =~ /default-storage-engine=(\S+)/ )
1541    {
1542      # Save this for collect phase
1543      collect_option('default-storage-engine', $1);
1544      mtr_report("Using default engine '$1'")
1545    }
1546  }
1547
1548  if (IS_WINDOWS and defined $opt_mem) {
1549    mtr_report("--mem not supported on Windows, ignored");
1550    $opt_mem= undef;
1551  }
1552
1553  if ($opt_port_base ne "auto")
1554  {
1555    if (my $rem= $opt_port_base % 10)
1556    {
1557      mtr_warning ("Port base $opt_port_base rounded down to multiple of 10");
1558      $opt_port_base-= $rem;
1559    }
1560    $opt_build_thread= $opt_port_base / 10 - 1000;
1561  }
1562
1563  # --------------------------------------------------------------------------
1564  # Check if we should speed up tests by trying to run on tmpfs
1565  # --------------------------------------------------------------------------
1566  if ( defined $opt_mem)
1567  {
1568    mtr_error("Can't use --mem and --vardir at the same time ")
1569      if $opt_vardir;
1570    mtr_error("Can't use --mem and --tmpdir at the same time ")
1571      if $opt_tmpdir;
1572
1573    # Search through list of locations that are known
1574    # to be "fast disks" to find a suitable location
1575    my @tmpfs_locations= ("/run/shm", "/dev/shm", "/tmp");
1576
1577    # Use $ENV{'MTR_MEM'} as first location to look (if defined)
1578    unshift(@tmpfs_locations, $ENV{'MTR_MEM'}) if defined $ENV{'MTR_MEM'};
1579
1580    foreach my $fs (@tmpfs_locations)
1581    {
1582      if ( -d $fs && ! -l $fs  && -w $fs )
1583      {
1584	my $template= "var_${opt_build_thread}_XXXX";
1585	$opt_mem= tempdir( $template, DIR => $fs, CLEANUP => 0);
1586	last;
1587      }
1588    }
1589  }
1590
1591  # --------------------------------------------------------------------------
1592  # Set the "var/" directory, the base for everything else
1593  # --------------------------------------------------------------------------
1594  my $vardir_location= (defined $ENV{MTR_BINDIR}
1595                          ? "$ENV{MTR_BINDIR}/mysql-test"
1596                          : $glob_mysql_test_dir);
1597  $vardir_location= realpath $vardir_location unless IS_WINDOWS;
1598  $default_vardir= "$vardir_location/var";
1599
1600  if ( ! $opt_vardir )
1601  {
1602    $opt_vardir= $default_vardir;
1603  }
1604
1605  # We make the path absolute, as the server will do a chdir() before usage
1606  unless ( $opt_vardir =~ m,^/, or
1607           (IS_WINDOWS and $opt_vardir =~ m,^[a-z]:[/\\],i) )
1608  {
1609    # Make absolute path, relative test dir
1610    $opt_vardir= "$glob_mysql_test_dir/$opt_vardir";
1611  }
1612
1613  set_vardir($opt_vardir);
1614
1615  # --------------------------------------------------------------------------
1616  # Set the "tmp" directory
1617  # --------------------------------------------------------------------------
1618  if ( ! $opt_tmpdir )
1619  {
1620    $opt_tmpdir=       "$opt_vardir/tmp" unless $opt_tmpdir;
1621
1622    if (check_socket_path_length("$opt_tmpdir/mysql_testsocket.sock"))
1623    {
1624      mtr_report("Too long tmpdir path '$opt_tmpdir'",
1625		 " creating a shorter one...");
1626
1627      # Create temporary directory in standard location for temporary files
1628      $opt_tmpdir= tempdir( TMPDIR => 1, CLEANUP => 0 );
1629      mtr_report(" - using tmpdir: '$opt_tmpdir'\n");
1630
1631      # Remember pid that created dir so it's removed by correct process
1632      $opt_tmpdir_pid= $$;
1633    }
1634  }
1635  $opt_tmpdir =~ s,/+$,,;       # Remove ending slash if any
1636
1637  # --------------------------------------------------------------------------
1638  # fast option
1639  # --------------------------------------------------------------------------
1640  if ($opt_fast){
1641    $opt_shutdown_timeout= 0; # Kill processes instead of nice shutdown
1642  }
1643
1644  # --------------------------------------------------------------------------
1645  # Check parallel value
1646  # --------------------------------------------------------------------------
1647  if ($opt_parallel ne "auto" && $opt_parallel < 1)
1648  {
1649    mtr_error("0 or negative parallel value makes no sense, use 'auto' or positive number");
1650  }
1651
1652  # --------------------------------------------------------------------------
1653  # Record flag
1654  # --------------------------------------------------------------------------
1655  if ( $opt_record and ! @opt_cases )
1656  {
1657    mtr_error("Will not run in record mode without a specific test case");
1658  }
1659
1660  if ( $opt_record ) {
1661    # Use only one worker with --record
1662    $opt_parallel= 1;
1663  }
1664
1665  # --------------------------------------------------------------------------
1666  # Embedded server flag
1667  # --------------------------------------------------------------------------
1668  if ( $opt_embedded_server )
1669  {
1670    $opt_skip_ssl= 1;              # Turn off use of SSL
1671
1672    # Turn off use of bin log
1673    push(@opt_extra_mysqld_opt, "--skip-log-bin");
1674
1675    if ( using_extern() )
1676    {
1677      mtr_error("Can't use --extern with --embedded-server");
1678    }
1679  }
1680
1681  # --------------------------------------------------------------------------
1682  # Big test and staging_run flags
1683  # --------------------------------------------------------------------------
1684   if ( $opt_big_test )
1685   {
1686     $ENV{'BIG_TEST'}= 1;
1687   }
1688  $ENV{'STAGING_RUN'}= $opt_staging_run;
1689
1690  # --------------------------------------------------------------------------
1691  # Gcov flag
1692  # --------------------------------------------------------------------------
1693  if ( ($opt_gcov or $opt_gprof) and ! $source_dist )
1694  {
1695    mtr_error("Coverage test needs the source - please use source dist");
1696  }
1697
1698  # --------------------------------------------------------------------------
1699  # Modified behavior with --start options
1700  # --------------------------------------------------------------------------
1701  if ($opt_start or $opt_start_dirty or $opt_start_exit) {
1702    collect_option ('quick-collect', 1);
1703    $start_only= 1;
1704  }
1705  if ($opt_debug)
1706  {
1707    $opt_testcase_timeout= 7 * 24 * 60;
1708    $opt_suite_timeout= 7 * 24 * 60;
1709    $opt_retry= 1;
1710    $opt_retry_failure= 1;
1711  }
1712
1713  # --------------------------------------------------------------------------
1714  # Check use of user-args
1715  # --------------------------------------------------------------------------
1716
1717  if ($opt_user_args) {
1718    mtr_error("--user-args only valid with --start options")
1719      unless $start_only;
1720    mtr_error("--user-args cannot be combined with named suites or tests")
1721      if $opt_suites || @opt_cases;
1722  }
1723
1724  # --------------------------------------------------------------------------
1725  # Check use of wait-all
1726  # --------------------------------------------------------------------------
1727
1728  if ($opt_wait_all && ! $start_only)
1729  {
1730    mtr_error("--wait-all can only be used with --start options");
1731  }
1732
1733  # --------------------------------------------------------------------------
1734  # Gather stress-test options and modify behavior
1735  # --------------------------------------------------------------------------
1736
1737  if ($opt_stress)
1738  {
1739    $opt_stress=~ s/,/ /g;
1740    $opt_user_args= 1;
1741    mtr_error("--stress cannot be combined with named ordinary suites or tests")
1742      if $opt_suites || @opt_cases;
1743    $opt_suites="stress";
1744    @opt_cases= ("wrapper");
1745    $ENV{MST_OPTIONS}= $opt_stress;
1746  }
1747
1748  # --------------------------------------------------------------------------
1749  # Check timeout arguments
1750  # --------------------------------------------------------------------------
1751
1752  mtr_error("Invalid value '$opt_testcase_timeout' supplied ".
1753	    "for option --testcase-timeout")
1754    if ($opt_testcase_timeout <= 0);
1755  mtr_error("Invalid value '$opt_suite_timeout' supplied ".
1756	    "for option --testsuite-timeout")
1757    if ($opt_suite_timeout <= 0);
1758
1759  if ($opt_debug_common)
1760  {
1761    $opt_debug= 1;
1762    $debug_d= "d,query,info,error,enter,exit";
1763  }
1764}
1765
1766
1767#
1768# To make it easier for different devs to work on the same host,
1769# an environment variable can be used to control all ports. A small
1770# number is to be used, 0 - 16 or similar.
1771#
1772# Note the MASTER_MYPORT has to be set the same in all 4.x and 5.x
1773# versions of this script, else a 4.0 test run might conflict with a
1774# 5.1 test run, even if different MTR_BUILD_THREAD is used. This means
1775# all port numbers might not be used in this version of the script.
1776#
1777# Also note the limitation of ports we are allowed to hand out. This
1778# differs between operating systems and configuration, see
1779# http://www.ncftp.com/ncftpd/doc/misc/ephemeral_ports.html
1780# But a fairly safe range seems to be 5001 - 32767
1781#
1782sub set_build_thread_ports($) {
1783  my $thread= shift || 0;
1784
1785  if ( lc($opt_build_thread) eq 'auto' ) {
1786    my $found_free = 0;
1787    $build_thread = 300;	# Start attempts from here
1788    my $build_thread_upper = $build_thread + ($opt_parallel > 1500
1789                                              ? 3000
1790                                              : 2 * $opt_parallel) + 300;
1791    while (! $found_free)
1792    {
1793      $build_thread= mtr_get_unique_id($build_thread, $build_thread_upper);
1794      if ( !defined $build_thread ) {
1795        mtr_error("Could not get a unique build thread id");
1796      }
1797      $found_free= check_ports_free($build_thread);
1798      # If not free, release and try from next number
1799      if (! $found_free) {
1800        mtr_release_unique_id();
1801        $build_thread++;
1802      }
1803    }
1804  }
1805  else
1806  {
1807    $build_thread = $opt_build_thread + $thread - 1;
1808    if (! check_ports_free($build_thread)) {
1809      # Some port was not free(which one has already been printed)
1810      mtr_error("Some port(s) was not free")
1811    }
1812  }
1813  $ENV{MTR_BUILD_THREAD}= $build_thread;
1814
1815  # Calculate baseport
1816  $baseport= $build_thread * $opt_port_group_size + 10000;
1817  if ( $baseport < 5001 or $baseport + $opt_port_group_size >= 32767 )
1818  {
1819    mtr_error("MTR_BUILD_THREAD number results in a port",
1820              "outside 5001 - 32767",
1821              "($baseport - $baseport + $opt_port_group_size)");
1822  }
1823
1824  mtr_report("Using MTR_BUILD_THREAD $build_thread,",
1825	     "with reserved ports $baseport..".($baseport+($opt_port_group_size-1)));
1826
1827}
1828
1829
1830sub collect_mysqld_features {
1831  #
1832  # Execute "mysqld --no-defaults --help --verbose" to get a
1833  # list of all features and settings
1834  #
1835  # --no-defaults and --skip-grant-tables are to avoid loading
1836  # system-wide configs and plugins
1837  #
1838  # --datadir must exist, mysqld will chdir into it
1839  #
1840  my $args;
1841  mtr_init_args(\$args);
1842  mtr_add_arg($args, "--no-defaults");
1843  mtr_add_arg($args, "--datadir=.");
1844  mtr_add_arg($args, "--basedir=%s", $basedir);
1845  mtr_add_arg($args, "--lc-messages-dir=%s", $path_language);
1846  mtr_add_arg($args, "--skip-grant-tables");
1847  mtr_add_arg($args, "--log-warnings=0");
1848  mtr_add_arg($args, "--log-slow-admin-statements=0");
1849  mtr_add_arg($args, "--log-queries-not-using-indexes=0");
1850  mtr_add_arg($args, "--log-slow-slave-statements=0");
1851  mtr_add_arg($args, "--verbose");
1852  mtr_add_arg($args, "--help");
1853
1854  my $exe_mysqld= find_mysqld($bindir);
1855  my $cmd= join(" ", $exe_mysqld, @$args);
1856
1857  mtr_verbose("cmd: $cmd");
1858
1859  my $list= `$cmd`;
1860
1861  # to simplify the parsing, we'll merge all nicely formatted --help texts
1862  $list =~ s/\n {22}(\S)/ $1/g;
1863
1864  my @list= split '\n', $list;
1865
1866  $mysql_version_id= 0;
1867  while (defined(my $line = shift @list)){
1868     if ($line =~ /^\Q$exe_mysqld\E\s+Ver\s(\d+)\.(\d+)\.(\d+)(\S*)/ ) {
1869      $mysql_version_id= $1*10000 + $2*100 + $3;
1870      mtr_report("MariaDB Version $1.$2.$3$4");
1871      last;
1872    }
1873  }
1874  mtr_error("Could not find version of MariaDB")
1875     unless $mysql_version_id > 0;
1876
1877  for (@list)
1878  {
1879    # first part of the help - command-line options.
1880    if (/Copyright/ .. /^-{30,}/) {
1881      # here we want to detect all not mandatory plugins
1882      # they are listed in the --help output as
1883      #  --archive[=name]
1884      # Enable or disable ARCHIVE plugin. Possible values are ON, OFF,
1885      # FORCE (don't start if the plugin fails to load),
1886      # FORCE_PLUS_PERMANENT (like FORCE, but the plugin can not be uninstalled).
1887      push @optional_plugins, $1
1888        if /^  --([-a-z0-9]+)\[=name\] +Enable or disable \w+ plugin. One of: ON, OFF, FORCE/;
1889      next;
1890    }
1891
1892    last if /^$/; # then goes a list of variables, it ends with an empty line
1893
1894    # Put a variable into hash
1895    /^([\S]+)[ \t]+(.*?)\r?$/ or die "Could not parse mysqld --help: $_\n";
1896    $mysqld_variables{$1}= $2;
1897  }
1898  mtr_error("Could not find variabes list") unless %mysqld_variables;
1899}
1900
1901
1902
1903sub collect_mysqld_features_from_running_server ()
1904{
1905  my $mysql= mtr_exe_exists("$path_client_bindir/mysql");
1906
1907  my $args;
1908  mtr_init_args(\$args);
1909
1910  mtr_add_arg($args, "--no-defaults");
1911  mtr_add_arg($args, "--user=%s", $opt_user);
1912
1913  while (my ($option, $value)= each( %opts_extern )) {
1914    mtr_add_arg($args, "--$option=$value");
1915  }
1916
1917  mtr_add_arg($args, "--silent"); # Tab separated output
1918  mtr_add_arg($args, "-e \"use mysql; SHOW VARIABLES\"");
1919  my $cmd= "$mysql " . join(' ', @$args);
1920  mtr_verbose("cmd: $cmd");
1921
1922  my $list = `$cmd` or
1923    mtr_error("Could not connect to extern server using command: '$cmd'");
1924  foreach my $line (split('\n', $list ))
1925  {
1926    # Put variables into hash
1927    if ( $line =~ /^([\S]+)[ \t]+(.*?)\r?$/ )
1928    {
1929      my $name= $1;
1930      my $value=$2;
1931      $name =~ s/_/-/g;
1932      # print "$name=\"$value\"\n";
1933      $mysqld_variables{$name}= $value;
1934    }
1935  }
1936
1937  # Parse version
1938  my $version_str= $mysqld_variables{'version'};
1939  if ( $version_str =~ /^([0-9]*)\.([0-9]*)\.([0-9]*)([^\s]*)/ )
1940  {
1941    #print "Major: $1 Minor: $2 Build: $3\n";
1942    $mysql_version_id= $1*10000 + $2*100 + $3;
1943    #print "mysql_version_id: $mysql_version_id\n";
1944    mtr_report("MariaDB Version $1.$2.$3");
1945    $mysql_version_extra= $4;
1946  }
1947  mtr_error("Could not find version of MariaDBL") unless $mysql_version_id;
1948}
1949
1950sub find_mysqld {
1951
1952  my ($mysqld_basedir)= $ENV{MTR_BINDIR}|| @_;
1953
1954  my @mysqld_names= ("mysqld", "mysqld-max-nt", "mysqld-max",
1955		     "mysqld-nt");
1956
1957  if ( $opt_debug_server ){
1958    # Put mysqld-debug first in the list of binaries to look for
1959    mtr_verbose("Adding mysqld-debug first in list of binaries to look for");
1960    unshift(@mysqld_names, "mysqld-debug");
1961  }
1962
1963  return my_find_bin($bindir,
1964		     ["sql", "libexec", "sbin", "bin"],
1965		     [@mysqld_names]);
1966}
1967
1968
1969sub executable_setup () {
1970
1971  $exe_patch='patch' if `patch -v`;
1972
1973  # Look for the client binaries
1974  $exe_mysqladmin=     mtr_exe_exists("$path_client_bindir/mysqladmin");
1975  $exe_mysql=          mtr_exe_exists("$path_client_bindir/mysql");
1976  $exe_mysql_plugin=   mtr_exe_exists("$path_client_bindir/mysql_plugin");
1977
1978  $exe_mysql_embedded= mtr_exe_maybe_exists("$bindir/libmysqld/examples/mysql_embedded");
1979
1980  # Look for mysqltest executable
1981  if ( $opt_embedded_server )
1982  {
1983    $exe_mysqltest=
1984      mtr_exe_exists("$bindir/libmysqld/examples$multiconfig/mysqltest_embedded",
1985                     "$path_client_bindir/mysqltest_embedded");
1986  }
1987  else
1988  {
1989    if ( defined $ENV{'MYSQL_TEST'} )
1990    {
1991      $exe_mysqltest=$ENV{'MYSQL_TEST'};
1992      print "===========================================================\n";
1993      print "WARNING:The mysqltest binary is fetched from $exe_mysqltest\n";
1994      print "===========================================================\n";
1995    }
1996    else
1997    {
1998      $exe_mysqltest= mtr_exe_exists("$path_client_bindir/mysqltest");
1999    }
2000  }
2001
2002}
2003
2004
2005sub client_debug_arg($$) {
2006  my ($args, $client_name)= @_;
2007
2008  # Workaround for Bug #50627: drop any debug opt
2009  return if $client_name =~ /^mysqlbinlog/;
2010
2011  if ( $opt_debug ) {
2012    mtr_add_arg($args,
2013		"--loose-debug=$debug_d:t:A,%s/log/%s.trace",
2014		$path_vardir_trace, $client_name)
2015  }
2016}
2017
2018
2019sub client_arguments ($;$) {
2020 my $client_name= shift;
2021  my $group_suffix= shift;
2022  my $client_exe= mtr_exe_exists("$path_client_bindir/$client_name");
2023
2024  my $args;
2025  mtr_init_args(\$args);
2026  mtr_add_arg($args, "--defaults-file=%s", $path_config_file);
2027  if (defined($group_suffix)) {
2028    mtr_add_arg($args, "--defaults-group-suffix=%s", $group_suffix);
2029    client_debug_arg($args, "$client_name-$group_suffix");
2030  }
2031  else
2032  {
2033    client_debug_arg($args, $client_name);
2034  }
2035  return mtr_args2str($client_exe, @$args);
2036}
2037
2038
2039sub mysqlbinlog_arguments () {
2040  my $exe= mtr_exe_exists("$path_client_bindir/mysqlbinlog");
2041
2042  my $args;
2043  mtr_init_args(\$args);
2044  mtr_add_arg($args, "--defaults-file=%s", $path_config_file);
2045  mtr_add_arg($args, "--local-load=%s", $opt_tmpdir);
2046  client_debug_arg($args, "mysqlbinlog");
2047  return mtr_args2str($exe, @$args);
2048}
2049
2050
2051sub mysqlslap_arguments () {
2052  my $exe= mtr_exe_maybe_exists("$path_client_bindir/mysqlslap");
2053  if ( $exe eq "" ) {
2054    # mysqlap was not found
2055
2056    if (defined $mysql_version_id and $mysql_version_id >= 50100 ) {
2057      mtr_error("Could not find the mysqlslap binary");
2058    }
2059    return ""; # Don't care about mysqlslap
2060  }
2061
2062  my $args;
2063  mtr_init_args(\$args);
2064  mtr_add_arg($args, "--defaults-file=%s", $path_config_file);
2065  client_debug_arg($args, "mysqlslap");
2066  return mtr_args2str($exe, @$args);
2067}
2068
2069
2070sub mysqldump_arguments ($) {
2071  my($group_suffix) = @_;
2072  my $exe= mtr_exe_exists("$path_client_bindir/mysqldump");
2073
2074  my $args;
2075  mtr_init_args(\$args);
2076  mtr_add_arg($args, "--defaults-file=%s", $path_config_file);
2077  mtr_add_arg($args, "--defaults-group-suffix=%s", $group_suffix);
2078  client_debug_arg($args, "mysqldump-$group_suffix");
2079  return mtr_args2str($exe, @$args);
2080}
2081
2082
2083sub mysql_client_test_arguments(){
2084  my $exe;
2085  # mysql_client_test executable may _not_ exist
2086  if ( $opt_embedded_server ) {
2087    $exe= mtr_exe_maybe_exists(
2088            "$bindir/libmysqld/examples$multiconfig/mysql_client_test_embedded",
2089		"$bindir/bin/mysql_client_test_embedded");
2090  } else {
2091    $exe= mtr_exe_maybe_exists("$bindir/tests$multiconfig/mysql_client_test",
2092			       "$bindir/bin/mysql_client_test");
2093  }
2094
2095  my $args;
2096  mtr_init_args(\$args);
2097  mtr_add_arg($args, "--defaults-file=%s", $path_config_file);
2098  mtr_add_arg($args, "--testcase");
2099  mtr_add_arg($args, "--vardir=$opt_vardir");
2100  client_debug_arg($args,"mysql_client_test");
2101  my $ret=mtr_args2str($exe, @$args);
2102  return $ret;
2103}
2104
2105sub tool_arguments ($$) {
2106  my($sedir, $tool_name) = @_;
2107  my $exe= my_find_bin($bindir,
2108		       [$sedir, "bin"],
2109		       $tool_name);
2110
2111  my $args;
2112  mtr_init_args(\$args);
2113  client_debug_arg($args, $tool_name);
2114  return mtr_args2str($exe, @$args);
2115}
2116
2117# This is not used to actually start a mysqld server, just to allow test
2118# scripts to run the mysqld binary to test invalid server startup options.
2119sub mysqld_client_arguments () {
2120  my $default_mysqld= default_mysqld();
2121  my $exe = find_mysqld($bindir);
2122  my $args;
2123  mtr_init_args(\$args);
2124  mtr_add_arg($args, "--no-defaults");
2125  mtr_add_arg($args, "--basedir=%s", $basedir);
2126  mtr_add_arg($args, "--character-sets-dir=%s", $default_mysqld->value("character-sets-dir"));
2127  mtr_add_arg($args, "--language=%s", $default_mysqld->value("language"));
2128  return mtr_args2str($exe, @$args);
2129}
2130
2131
2132sub have_maria_support () {
2133  my $maria_var= $mysqld_variables{'aria-recover-options'};
2134  return defined $maria_var;
2135}
2136
2137
2138sub environment_setup {
2139
2140  umask(022);
2141
2142  $ENV{'USE_RUNNING_SERVER'}= using_extern();
2143
2144  my @ld_library_paths;
2145
2146  if ($path_client_libdir)
2147  {
2148    # Use the --client-libdir passed on commandline
2149    push(@ld_library_paths, "$path_client_libdir");
2150  }
2151  else
2152  {
2153    # Setup LD_LIBRARY_PATH so the libraries from this distro/clone
2154    # are used in favor of the system installed ones
2155    if ( $source_dist )
2156    {
2157      push(@ld_library_paths, "$basedir/libmysql/.libs/",
2158	   "$basedir/libmysql_r/.libs/",
2159	   "$basedir/zlib/.libs/");
2160      if ($^O eq "darwin")
2161      {
2162        # it is MAC OS and we have to add dynamic libraries paths
2163        push @ld_library_paths, grep {<$_/*.dylib>}
2164          (<$bindir/storage/*/.libs/>,<$bindir/plugin/*/.libs/>,
2165          <$bindir/plugin/*/*/.libs/>,<$bindir/storage/*/*/.libs>);
2166      }
2167    }
2168    else
2169    {
2170      push(@ld_library_paths, "$basedir/lib", "$basedir/lib/mysql");
2171    }
2172  }
2173
2174  $ENV{'LD_LIBRARY_PATH'}= join(":", @ld_library_paths,
2175				$ENV{'LD_LIBRARY_PATH'} ?
2176				split(':', $ENV{'LD_LIBRARY_PATH'}) : ());
2177
2178  My::Debugger::pre_setup();
2179
2180  mtr_debug("LD_LIBRARY_PATH: $ENV{'LD_LIBRARY_PATH'}");
2181
2182  $ENV{'DYLD_LIBRARY_PATH'}= join(":", @ld_library_paths,
2183				  $ENV{'DYLD_LIBRARY_PATH'} ?
2184				  split(':', $ENV{'DYLD_LIBRARY_PATH'}) : ());
2185  mtr_debug("DYLD_LIBRARY_PATH: $ENV{'DYLD_LIBRARY_PATH'}");
2186
2187  # The environment variable used for shared libs on AIX
2188  $ENV{'SHLIB_PATH'}= join(":", @ld_library_paths,
2189                           $ENV{'SHLIB_PATH'} ?
2190                           split(':', $ENV{'SHLIB_PATH'}) : ());
2191  mtr_debug("SHLIB_PATH: $ENV{'SHLIB_PATH'}");
2192
2193  # The environment variable used for shared libs on hp-ux
2194  $ENV{'LIBPATH'}= join(":", @ld_library_paths,
2195                        $ENV{'LIBPATH'} ?
2196                        split(':', $ENV{'LIBPATH'}) : ());
2197  mtr_debug("LIBPATH: $ENV{'LIBPATH'}");
2198
2199  $ENV{'UMASK'}=              "0660"; # The octal *string*
2200  $ENV{'UMASK_DIR'}=          "0770"; # The octal *string*
2201
2202  #
2203  # MariaDB tests can produce output in various character sets
2204  # (especially, ctype_xxx.test). To avoid confusing Perl
2205  # with output which is incompatible with the current locale
2206  # settings, we reset the current values of LC_ALL and LC_CTYPE to "C".
2207  # For details, please see
2208  # Bug#27636 tests fails if LC_* variables set to *_*.UTF-8
2209  #
2210  $ENV{'LC_ALL'}=             "C";
2211  $ENV{'LC_CTYPE'}=           "C";
2212
2213  $ENV{'LC_COLLATE'}=         "C";
2214  $ENV{'MYSQL_TEST_DIR'}=     $glob_mysql_test_dir;
2215  $ENV{'DEFAULT_MASTER_PORT'}= $mysqld_variables{'port'};
2216  $ENV{'MYSQL_TMP_DIR'}=      $opt_tmpdir;
2217  $ENV{'MYSQLTEST_VARDIR'}=   $opt_vardir;
2218  $ENV{'MYSQL_BINDIR'}=       $bindir;
2219  $ENV{'MYSQL_SHAREDIR'}=     $path_language;
2220  $ENV{'MYSQL_CHARSETSDIR'}=  $path_charsetsdir;
2221
2222  if (IS_WINDOWS)
2223  {
2224    $ENV{'SECURE_LOAD_PATH'}= $glob_mysql_test_dir."\\std_data";
2225  }
2226  else
2227  {
2228    $ENV{'SECURE_LOAD_PATH'}= $glob_mysql_test_dir."/std_data";
2229  }
2230
2231  #
2232  # Some stupid^H^H^H^H^H^Hignorant network providers set up "wildcard DNS"
2233  # servers that return some given web server address for any lookup of a
2234  # non-existent host name. This confuses test cases that want to test the
2235  # behaviour when connecting to a non-existing host, so we need to be able
2236  # to disable those tests when DNS is broken.
2237  #
2238  $ENV{HAVE_BROKEN_DNS}= defined(gethostbyname('invalid_hostname'));
2239
2240  # ----------------------------------------------------
2241  # mysql clients
2242  # ----------------------------------------------------
2243  $ENV{'MYSQL_CHECK'}=              client_arguments("mysqlcheck");
2244  $ENV{'MYSQL_DUMP'}=               mysqldump_arguments(".1");
2245  $ENV{'MYSQL_DUMP_SLAVE'}=         mysqldump_arguments(".2");
2246  $ENV{'MYSQL_SLAP'}=               mysqlslap_arguments();
2247  $ENV{'MYSQL_IMPORT'}=             client_arguments("mysqlimport");
2248  $ENV{'MYSQL_SHOW'}=               client_arguments("mysqlshow");
2249  $ENV{'MYSQL_BINLOG'}=             mysqlbinlog_arguments();
2250  $ENV{'MYSQL'}=                    client_arguments("mysql");
2251  $ENV{'MYSQL_SLAVE'}=              client_arguments("mysql", ".2");
2252  $ENV{'MYSQL_UPGRADE'}=            client_arguments("mysql_upgrade");
2253  $ENV{'MYSQLADMIN'}=               client_arguments("mysqladmin");
2254  $ENV{'MYSQL_CLIENT_TEST'}=        mysql_client_test_arguments();
2255  $ENV{'EXE_MYSQL'}=                $exe_mysql;
2256  $ENV{'MYSQL_PLUGIN'}=             $exe_mysql_plugin;
2257  $ENV{'MYSQL_EMBEDDED'}=           $exe_mysql_embedded;
2258
2259  my $client_config_exe=
2260    mtr_exe_maybe_exists(
2261        "$bindir/libmariadb/mariadb_config$multiconfig/mariadb_config",
2262               "$bindir/bin/mariadb_config");
2263  if ($client_config_exe)
2264  {
2265    my $tls_info= `$client_config_exe --tlsinfo`;
2266    ($ENV{CLIENT_TLS_LIBRARY},$ENV{CLIENT_TLS_LIBRARY_VERSION})=
2267      split(/ /, $tls_info, 2);
2268  }
2269  my $exe_mysqld= find_mysqld($basedir);
2270  $ENV{'MYSQLD'}= $exe_mysqld;
2271  my $extra_opts= join (" ", @opt_extra_mysqld_opt);
2272  $ENV{'MYSQLD_CMD'}= "$exe_mysqld --defaults-group-suffix=.1 ".
2273    "--defaults-file=$path_config_file $extra_opts";
2274
2275  # ----------------------------------------------------
2276  # bug25714 executable may _not_ exist in
2277  # some versions, test using it should be skipped
2278  # ----------------------------------------------------
2279  my $exe_bug25714=
2280      mtr_exe_maybe_exists("$bindir/tests$multiconfig/bug25714");
2281  $ENV{'MYSQL_BUG25714'}=  native_path($exe_bug25714);
2282
2283  # ----------------------------------------------------
2284  # mysql_fix_privilege_tables.sql
2285  # ----------------------------------------------------
2286  my $file_mysql_fix_privilege_tables=
2287    mtr_file_exists("$bindir/scripts/mysql_fix_privilege_tables.sql",
2288		    "$bindir/share/mysql_fix_privilege_tables.sql",
2289		    "$bindir/share/mariadb/mysql_fix_privilege_tables.sql",
2290		    "$bindir/share/mysql/mysql_fix_privilege_tables.sql");
2291  $ENV{'MYSQL_FIX_PRIVILEGE_TABLES'}=  $file_mysql_fix_privilege_tables;
2292
2293  # ----------------------------------------------------
2294  # my_print_defaults
2295  # ----------------------------------------------------
2296  my $exe_my_print_defaults=
2297    mtr_exe_exists("$bindir/extra$multiconfig/my_print_defaults",
2298		   "$path_client_bindir/my_print_defaults");
2299  $ENV{'MYSQL_MY_PRINT_DEFAULTS'}= native_path($exe_my_print_defaults);
2300
2301  # ----------------------------------------------------
2302  # myisam tools
2303  # ----------------------------------------------------
2304  $ENV{'MYISAMLOG'}= tool_arguments("storage/myisam", "myisamlog", );
2305  $ENV{'MYISAMCHK'}= tool_arguments("storage/myisam", "myisamchk");
2306  $ENV{'MYISAMPACK'}= tool_arguments("storage/myisam", "myisampack");
2307  $ENV{'MYISAM_FTDUMP'}= tool_arguments("storage/myisam", "myisam_ftdump");
2308
2309  # ----------------------------------------------------
2310  # aria tools
2311  # ----------------------------------------------------
2312  if (have_maria_support())
2313  {
2314    $ENV{'MARIA_CHK'}= tool_arguments("storage/maria", "aria_chk");
2315    $ENV{'MARIA_PACK'}= tool_arguments("storage/maria", "aria_pack");
2316  }
2317
2318  # ----------------------------------------------------
2319  # mysqlhotcopy
2320  # ----------------------------------------------------
2321  my $mysqlhotcopy=
2322    mtr_pl_maybe_exists("$bindir/scripts/mysqlhotcopy") ||
2323    mtr_pl_maybe_exists("$path_client_bindir/mysqlhotcopy");
2324  if ($mysqlhotcopy)
2325  {
2326    $ENV{'MYSQLHOTCOPY'}= $mysqlhotcopy;
2327  }
2328
2329  # ----------------------------------------------------
2330  # perror
2331  # ----------------------------------------------------
2332  my $exe_perror= mtr_exe_exists("$bindir/extra$multiconfig/perror",
2333				 "$path_client_bindir/perror");
2334  $ENV{'MY_PERROR'}= native_path($exe_perror);
2335
2336  # ----------------------------------------------------
2337  # mysql_tzinfo_to_sql
2338  # ----------------------------------------------------
2339  my $exe_mysql_tzinfo_to_sql= mtr_exe_exists("$basedir/sql$multiconfig/mysql_tzinfo_to_sql",
2340                                 "$path_client_bindir/mysql_tzinfo_to_sql",
2341                                 "$bindir/sql$multiconfig/mysql_tzinfo_to_sql");
2342  $ENV{'MYSQL_TZINFO_TO_SQL'}= native_path($exe_mysql_tzinfo_to_sql);
2343
2344  # ----------------------------------------------------
2345  # replace
2346  # ----------------------------------------------------
2347  my $exe_replace= mtr_exe_exists(vs_config_dirs('extra', 'replace'),
2348                                 "$basedir/extra/replace",
2349                                 "$bindir/extra$multiconfig/replace",
2350                                 "$path_client_bindir/replace");
2351  $ENV{'REPLACE'}= native_path($exe_replace);
2352
2353  # ----------------------------------------------------
2354  # innochecksum
2355  # ----------------------------------------------------
2356  my $exe_innochecksum=
2357    mtr_exe_maybe_exists("$bindir/extra$multiconfig/innochecksum",
2358		         "$path_client_bindir/innochecksum");
2359  if ($exe_innochecksum)
2360  {
2361    $ENV{'INNOCHECKSUM'}= native_path($exe_innochecksum);
2362  }
2363
2364  # Add dir of this perl to aid mysqltest in finding perl
2365  my $perldir= dirname($^X);
2366  my $pathsep= ":";
2367  $pathsep= ";" if IS_WINDOWS && ! IS_CYGWIN;
2368  $ENV{'PATH'}= "$ENV{'PATH'}".$pathsep.$perldir;
2369}
2370
2371
2372sub remove_vardir_subs() {
2373  foreach my $sdir ( glob("$opt_vardir/*") ) {
2374    mtr_verbose("Removing subdir $sdir");
2375    rmtree($sdir);
2376  }
2377}
2378
2379#
2380# Remove var and any directories in var/ created by previous
2381# tests
2382#
2383sub remove_stale_vardir () {
2384
2385  mtr_report("Removing old var directory...");
2386
2387  # Safety!
2388  mtr_error("No, don't remove the vardir when running with --extern")
2389    if using_extern();
2390
2391  mtr_verbose("opt_vardir: $opt_vardir");
2392  if ( $opt_vardir eq $default_vardir )
2393  {
2394    #
2395    # Running with "var" in mysql-test dir
2396    #
2397    if ( -l $opt_vardir)
2398    {
2399      # var is a symlink
2400
2401      if ( $opt_mem )
2402      {
2403	# Remove the directory which the link points at
2404	mtr_verbose("Removing " . readlink($opt_vardir));
2405	rmtree(readlink($opt_vardir));
2406
2407	# Remove the "var" symlink
2408	mtr_verbose("unlink($opt_vardir)");
2409	unlink($opt_vardir);
2410      }
2411      else
2412      {
2413	# Some users creates a soft link in mysql-test/var to another area
2414	# - allow it, but remove all files in it
2415
2416	mtr_report(" - WARNING: Using the 'mysql-test/var' symlink");
2417
2418	# Make sure the directory where it points exist
2419        if (! -d readlink($opt_vardir))
2420        {
2421          mtr_report("The destination for symlink $opt_vardir does not exist; Removing it and creating a new var directory");
2422          unlink($opt_vardir);
2423        }
2424	remove_vardir_subs();
2425      }
2426    }
2427    else
2428    {
2429      # Remove the entire "var" dir
2430      mtr_verbose("Removing $opt_vardir/");
2431      rmtree("$opt_vardir/");
2432    }
2433
2434    if ( $opt_mem )
2435    {
2436      # A symlink from var/ to $opt_mem will be set up
2437      # remove the $opt_mem dir to assure the symlink
2438      # won't point at an old directory
2439      mtr_verbose("Removing $opt_mem");
2440      rmtree($opt_mem);
2441    }
2442
2443  }
2444  else
2445  {
2446    #
2447    # Running with "var" in some other place
2448    #
2449
2450    # Don't remove the var/ dir in mysql-test dir as it may be in
2451    # use by another mysql-test-run run with --vardir
2452    # mtr_verbose("Removing $default_vardir");
2453    # rmtree($default_vardir);
2454
2455    # Remove the "var" dir
2456    mtr_verbose("Removing $opt_vardir/");
2457    rmtree("$opt_vardir/");
2458  }
2459  # Remove the "tmp" dir
2460  mtr_verbose("Removing $opt_tmpdir/");
2461  rmtree("$opt_tmpdir/");
2462}
2463
2464sub set_plugin_var($) {
2465  local $_ = $_[0];
2466  s/\.\w+$//;
2467  $ENV{"\U${_}_SO"} = $_[0];
2468}
2469
2470#
2471# Create var and the directories needed in var
2472#
2473sub setup_vardir() {
2474  mtr_report("Creating var directory '$opt_vardir'...");
2475
2476  if ( $opt_vardir eq $default_vardir )
2477  {
2478    #
2479    # Running with "var" in mysql-test dir
2480    #
2481    if ( -l $opt_vardir )
2482    {
2483      #  it's a symlink
2484
2485      # Make sure the directory where it points exist
2486      if (! -d readlink($opt_vardir))
2487      {
2488        mtr_report("The destination for symlink $opt_vardir does not exist; Removing it and creating a new var directory");
2489        unlink($opt_vardir);
2490      }
2491    }
2492    elsif ( $opt_mem )
2493    {
2494      # Runinng with "var" as a link to some "memory" location, normally tmpfs
2495      mtr_verbose("Creating $opt_mem");
2496      mkpath($opt_mem);
2497
2498      mtr_report(" - symlinking 'var' to '$opt_mem'");
2499      symlink($opt_mem, $opt_vardir);
2500    }
2501  }
2502
2503  if ( ! -d $opt_vardir )
2504  {
2505    mtr_verbose("Creating $opt_vardir");
2506    mkpath($opt_vardir);
2507  }
2508
2509  # Ensure a proper error message if vardir couldn't be created
2510  unless ( -d $opt_vardir and -w $opt_vardir )
2511  {
2512    mtr_error("Writable 'var' directory is needed, use the " .
2513	      "'--vardir=<path>' option");
2514  }
2515
2516  mkpath("$opt_vardir/log");
2517  mkpath("$opt_vardir/run");
2518
2519  # Create var/tmp and tmp - they might be different
2520  mkpath("$opt_vardir/tmp");
2521  mkpath($opt_tmpdir) if ($opt_tmpdir ne "$opt_vardir/tmp");
2522
2523  # On some operating systems, there is a limit to the length of a
2524  # UNIX domain socket's path far below PATH_MAX.
2525  # Don't allow that to happen
2526  if (check_socket_path_length("$opt_tmpdir/testsocket.sock")){
2527    mtr_error("Socket path '$opt_tmpdir' too long, it would be ",
2528	      "truncated and thus not possible to use for connection to ",
2529	      "MariaDB Server. Set a shorter with --tmpdir=<path> option");
2530  }
2531
2532  # copy all files from std_data into var/std_data
2533  # and make them world readable
2534  copytree("$glob_mysql_test_dir/std_data", "$opt_vardir/std_data", "0022");
2535
2536  # create a plugin dir and copy or symlink plugins into it
2537  if ($source_dist)
2538  {
2539    $plugindir="$opt_vardir/plugins";
2540    mkpath($plugindir);
2541    if (IS_WINDOWS)
2542    {
2543      if (!$opt_embedded_server)
2544      {
2545        for (<$bindir/storage/*$multiconfig/*.dll>,
2546             <$bindir/plugin/*$multiconfig/*.dll>,
2547             <$bindir/libmariadb$multiconfig/*.dll>,
2548             <$bindir/sql$multiconfig/*.dll>)
2549        {
2550          my $pname=basename($_);
2551          copy rel2abs($_), "$plugindir/$pname";
2552          set_plugin_var($pname);
2553        }
2554      }
2555    }
2556    else
2557    {
2558      my $opt_use_copy= 1;
2559      if (symlink "$opt_vardir/run", "$plugindir/symlink_test")
2560      {
2561        $opt_use_copy= 0;
2562        unlink "$plugindir/symlink_test";
2563      }
2564
2565      for (<$bindir/storage/*$multiconfig/*.so>,
2566           <$bindir/plugin/*$multiconfig/*.so>,
2567           <$bindir/libmariadb/plugins/*/*.so>,
2568           <$bindir/libmariadb/$multiconfig/*.so>,
2569           <$bindir/sql$multiconfig/*.so>)
2570      {
2571        my $pname=basename($_);
2572        if ($opt_use_copy)
2573        {
2574          copy rel2abs($_), "$plugindir/$pname";
2575        }
2576        else
2577        {
2578          symlink rel2abs($_), "$plugindir/$pname";
2579        }
2580        set_plugin_var($pname);
2581      }
2582    }
2583  }
2584  else
2585  {
2586    $plugindir= $mysqld_variables{'plugin-dir'} || '.';
2587    # hm, what paths work for debs and for rpms ?
2588    for (<$bindir/lib64/mysql/plugin/*.so>,
2589         <$bindir/lib/mysql/plugin/*.so>,
2590         <$bindir/lib64/mariadb/plugin/*.so>,
2591         <$bindir/lib/mariadb/plugin/*.so>,
2592         <$bindir/lib/plugin/*.so>,             # bintar
2593         <$bindir/lib/plugin/*.dll>)
2594    {
2595      my $pname=basename($_);
2596      set_plugin_var($pname);
2597    }
2598  }
2599
2600  # Remove old log files
2601  foreach my $name (glob("r/*.progress r/*.log r/*.warnings"))
2602  {
2603    unlink($name);
2604  }
2605}
2606
2607
2608#
2609# Check if running as root
2610# i.e a file can be read regardless what mode we set it to
2611#
2612sub  check_running_as_root () {
2613  my $test_file= "$opt_vardir/test_running_as_root.txt";
2614  mtr_tofile($test_file, "MySQL");
2615  chmod(oct("0000"), $test_file);
2616
2617  my $result="";
2618  if (open(FILE,"<",$test_file))
2619  {
2620    $result= join('', <FILE>);
2621    close FILE;
2622  }
2623
2624  # Some filesystems( for example CIFS) allows reading a file
2625  # although mode was set to 0000, but in that case a stat on
2626  # the file will not return 0000
2627  my $file_mode= (stat($test_file))[2] & 07777;
2628
2629  mtr_verbose("result: $result, file_mode: $file_mode");
2630  if ($result eq "MySQL" && $file_mode == 0)
2631  {
2632    mtr_warning("running this script as _root_ will cause some " .
2633                "tests to be skipped");
2634    $ENV{'MYSQL_TEST_ROOT'}= "1";
2635  }
2636
2637  chmod(oct("0755"), $test_file);
2638  unlink($test_file);
2639}
2640
2641
2642sub check_ssl_support {
2643  if ($opt_skip_ssl)
2644  {
2645    mtr_report(" - skipping SSL");
2646    $opt_ssl_supported= 0;
2647    $opt_ssl= 0;
2648    return;
2649  }
2650
2651  if ( ! $mysqld_variables{'ssl'} )
2652  {
2653    if ( $opt_ssl)
2654    {
2655      mtr_error("Couldn't find support for SSL");
2656      return;
2657    }
2658    mtr_report(" - skipping SSL, mysqld not compiled with SSL");
2659    $opt_ssl_supported= 0;
2660    $opt_ssl= 0;
2661    return;
2662  }
2663  mtr_report(" - SSL connections supported");
2664  $opt_ssl_supported= 1;
2665}
2666
2667sub check_debug_support {
2668  if (defined $mysqld_variables{'debug-dbug'})
2669  {
2670    mtr_report(" - binaries are debug compiled");
2671  }
2672  elsif ($opt_debug_server)
2673  {
2674    mtr_error("Can't use --debug[-server], binary does not support it");
2675  }
2676}
2677
2678
2679#
2680# Helper function to find the correct value for the opt_vs_config
2681# if it was not set explicitly.
2682#
2683# the configuration with the most recent build dir in sql/ is selected.
2684#
2685# note: looking for all BuildLog.htm files everywhere in the tree with the
2686# help of File::Find would be possibly more precise, but it is also
2687# many times slower. Thus we are only looking at the server, client
2688# executables, and plugins - that is, something that can affect the test suite
2689#
2690sub fix_vs_config_dir () {
2691  return $multiconfig="/$multiconfig" if $multiconfig;
2692
2693  my $modified = 1e30;
2694  $multiconfig="";
2695
2696
2697  for (<$bindir/sql/*/mysqld.exe>,
2698      <$bindir/sql/*/mysqld>
2699  ) { #/
2700    if (-M $_ < $modified)
2701    {
2702      $modified = -M _;
2703      $multiconfig = basename(dirname($_));
2704    }
2705  }
2706
2707  mtr_report("VS config: $multiconfig");
2708  $multiconfig="/$multiconfig" if $multiconfig;
2709}
2710
2711
2712#
2713# Helper function to handle configuration-based subdirectories which Visual
2714# Studio uses for storing binaries.  If opt_vs_config is set, this returns
2715# a path based on that setting; if not, it returns paths for the default
2716# /release/ and /debug/ subdirectories.
2717#
2718# $exe can be undefined, if the directory itself will be used
2719#
2720sub vs_config_dirs ($$) {
2721  my ($path_part, $exe) = @_;
2722
2723  $exe = "" if not defined $exe;
2724
2725  # Don't look in these dirs when not on windows
2726  return () unless IS_WINDOWS;
2727
2728  if ($multiconfig)
2729  {
2730    return ("$basedir/$path_part/$multiconfig/$exe");
2731  }
2732
2733  return ("$basedir/$path_part/release/$exe",
2734          "$basedir/$path_part/relwithdebinfo/$exe",
2735          "$basedir/$path_part/debug/$exe");
2736}
2737
2738sub mysql_server_start($) {
2739  my ($mysqld, $tinfo) = @_;
2740
2741  if ( $mysqld->{proc} )
2742  {
2743    # Already started
2744
2745    # Write start of testcase to log file
2746    mark_log($mysqld->value('log-error'), $tinfo);
2747
2748    return;
2749  }
2750
2751  my $datadir= $mysqld->value('datadir');
2752  if (not $opt_start_dirty)
2753  {
2754
2755    my @options= ('log-bin', 'relay-log');
2756    foreach my $option_name ( @options )  {
2757      next unless $mysqld->option($option_name);
2758
2759      my $file_name= $mysqld->value($option_name);
2760      next unless
2761        defined $file_name and
2762          -e $file_name;
2763
2764      mtr_debug(" -removing '$file_name'");
2765      unlink($file_name) or die ("unable to remove file '$file_name'");
2766    }
2767
2768    if (-d $datadir ) {
2769      mtr_verbose(" - removing '$datadir'");
2770      rmtree($datadir);
2771    }
2772  }
2773
2774  my $mysqld_basedir= $mysqld->value('basedir');
2775  my $extra_opts= get_extra_opts($mysqld, $tinfo);
2776
2777  if ( $basedir eq $mysqld_basedir )
2778  {
2779    if (! $opt_start_dirty)	# If dirty, keep possibly grown system db
2780    {
2781      # Some InnoDB options are incompatible with the default bootstrap.
2782      # If they are used, re-bootstrap
2783      my @rebootstrap_opts;
2784      @rebootstrap_opts = grep {/$rebootstrap_re/o} @$extra_opts if $extra_opts;
2785      if (@rebootstrap_opts)
2786      {
2787        mtr_verbose("Re-bootstrap with @rebootstrap_opts");
2788        mysql_install_db($mysqld, undef, \@rebootstrap_opts);
2789      }
2790      else {
2791        # Copy datadir from installed system db
2792        my $path= ($opt_parallel == 1) ? "$opt_vardir" : "$opt_vardir/..";
2793        my $install_db= "$path/install.db";
2794        copytree($install_db, $datadir) if -d $install_db;
2795        mtr_error("Failed to copy system db to '$datadir'") unless -d $datadir;
2796      }
2797    }
2798  }
2799  else
2800  {
2801    mysql_install_db($mysqld); # For versional testing
2802
2803    mtr_error("Failed to install system db to '$datadir'")
2804      unless -d $datadir;
2805
2806  }
2807
2808  # Create the servers tmpdir
2809  my $tmpdir= $mysqld->value('tmpdir');
2810  mkpath($tmpdir) unless -d $tmpdir;
2811
2812  # Write start of testcase to log file
2813  mark_log($mysqld->value('log-error'), $tinfo);
2814
2815  # Run <tname>-master.sh
2816  if ($mysqld->option('#!run-master-sh') and
2817      defined $tinfo->{master_sh} and
2818      run_system('/bin/sh ' . $tinfo->{master_sh}) )
2819  {
2820    $tinfo->{'comment'}= "Failed to execute '$tinfo->{master_sh}'";
2821    return 1;
2822  }
2823
2824  # Run <tname>-slave.sh
2825  if ($mysqld->option('#!run-slave-sh') and
2826      defined $tinfo->{slave_sh} and
2827      run_system('/bin/sh ' . $tinfo->{slave_sh}))
2828  {
2829    $tinfo->{'comment'}= "Failed to execute '$tinfo->{slave_sh}'";
2830    return 1;
2831  }
2832
2833  if (!$opt_embedded_server)
2834  {
2835    mysqld_start($mysqld, $extra_opts) or
2836      mtr_error("Failed to start mysqld ".$mysqld->name()." with command "
2837        . $ENV{MYSQLD_LAST_CMD});
2838
2839    # Save this test case information, so next can examine it
2840    $mysqld->{'started_tinfo'}= $tinfo;
2841  }
2842
2843  # If wsrep is on, we need to wait until the first
2844  # server starts and bootstraps the cluster before
2845  # starting other servers. The bootsrap server in the
2846  # configuration should always be the first which has
2847  # wsrep_on=ON
2848  if (wsrep_on($mysqld) && wsrep_is_bootstrap_server($mysqld))
2849  {
2850    mtr_verbose("Waiting for wsrep bootstrap server to start");
2851    if ($mysqld->{WAIT}->($mysqld))
2852    {
2853      return 1;
2854    }
2855  }
2856}
2857
2858sub mysql_server_wait {
2859  my ($mysqld, $tinfo) = @_;
2860  my $expect_file= "$opt_vardir/tmp/".$mysqld->name().".expect";
2861
2862  if (!sleep_until_file_created($mysqld->value('pid-file'), $expect_file,
2863                                $opt_start_timeout, $mysqld->{'proc'},
2864                                $warn_seconds))
2865  {
2866    $tinfo->{comment}= "Failed to start ".$mysqld->name() . "\n";
2867    return 1;
2868  }
2869
2870  if (wsrep_on($mysqld))
2871  {
2872    mtr_verbose("Waiting for wsrep server " . $mysqld->name() . " to be ready");
2873    if (!wait_wsrep_ready($tinfo, $mysqld))
2874    {
2875      return 1;
2876    }
2877  }
2878  return 0;
2879}
2880
2881sub create_config_file_for_extern {
2882  my %opts=
2883    (
2884     socket     => '/tmp/mysqld.sock',
2885     port       => 3306,
2886     user       => $opt_user,
2887     password   => '',
2888     @_
2889    );
2890
2891  mtr_report("Creating my.cnf file for extern server...");
2892  my $F= IO::File->new($path_config_file, "w")
2893    or mtr_error("Can't write to $path_config_file: $!");
2894
2895  print $F "[client]\n";
2896  while (my ($option, $value)= each( %opts )) {
2897    print $F "$option= $value\n";
2898    mtr_report(" $option= $value");
2899  }
2900
2901  print $F <<EOF
2902
2903# binlog reads from [client] and [mysqlbinlog]
2904[mysqlbinlog]
2905character-sets-dir= $path_charsetsdir
2906local-load= $opt_tmpdir
2907
2908EOF
2909;
2910
2911  $F= undef; # Close file
2912}
2913
2914
2915#
2916# Kill processes left from previous runs, normally
2917# there should be none so make sure to warn
2918# if there is one
2919#
2920sub kill_leftovers ($) {
2921  my $rundir= shift;
2922  return unless ( -d $rundir );
2923
2924  mtr_report("Checking leftover processes...");
2925
2926  # Scan the "run" directory for process id's to kill
2927  opendir(RUNDIR, $rundir)
2928    or mtr_error("kill_leftovers, can't open dir \"$rundir\": $!");
2929  while ( my $elem= readdir(RUNDIR) )
2930  {
2931    # Only read pid from files that end with .pid
2932    if ( $elem =~ /.*[.]pid$/ )
2933    {
2934      my $pidfile= "$rundir/$elem";
2935      next unless -f $pidfile;
2936      my $pid= mtr_fromfile($pidfile);
2937      unlink($pidfile);
2938      unless ($pid=~ /^(\d+)/){
2939	# The pid was not a valid number
2940	mtr_warning("Got invalid pid '$pid' from '$elem'");
2941	next;
2942      }
2943      mtr_report(" - found old pid $pid in '$elem', killing it...");
2944
2945      my $ret= kill("KILL", $pid);
2946      if ($ret == 0) {
2947	mtr_report("   process did not exist!");
2948	next;
2949      }
2950
2951      my $check_counter= 100;
2952      while ($ret > 0 and $check_counter--) {
2953	mtr_milli_sleep(100);
2954	$ret= kill(0, $pid);
2955      }
2956      mtr_report($check_counter ? "   ok!" : "   failed!");
2957    }
2958    else
2959    {
2960      mtr_warning("Found non pid file '$elem' in '$rundir'")
2961	if -f "$rundir/$elem";
2962    }
2963  }
2964  closedir(RUNDIR);
2965}
2966
2967#
2968# Check that all the ports that are going to
2969# be used are free
2970#
2971sub check_ports_free ($)
2972{
2973  my $bthread= shift;
2974  my $portbase = $bthread * $opt_port_group_size + 10000;
2975  for ($portbase..$portbase+($opt_port_group_size-1)){
2976    if (mtr_ping_port($_)){
2977      mtr_report(" - 'localhost:$_' was not free");
2978      return 0; # One port was not free
2979    }
2980  }
2981
2982  return 1; # All ports free
2983}
2984
2985
2986sub initialize_servers {
2987
2988  if ( using_extern() )
2989  {
2990    # Running against an already started server, if the specified
2991    # vardir does not already exist it should be created
2992    if ( ! -d $opt_vardir )
2993    {
2994      setup_vardir();
2995    }
2996    else
2997    {
2998      mtr_verbose("No need to create '$opt_vardir' it already exists");
2999    }
3000  }
3001  else
3002  {
3003    # Kill leftovers from previous run
3004    # using any pidfiles found in var/run
3005    kill_leftovers("$opt_vardir/run");
3006
3007    if ( ! $opt_start_dirty )
3008    {
3009      remove_stale_vardir();
3010      setup_vardir();
3011    }
3012  }
3013}
3014
3015
3016#
3017# Remove all newline characters expect after semicolon
3018#
3019sub sql_to_bootstrap {
3020  my ($sql) = @_;
3021  my @lines= split(/\n/, $sql);
3022  my $result= "\n";
3023  my $delimiter= ';';
3024
3025  foreach my $line (@lines) {
3026
3027    # Change current delimiter if line starts with "delimiter"
3028    if ( $line =~ /^delimiter (.*)/ ) {
3029      my $new= $1;
3030      # Remove old delimiter from end of new
3031      $new=~ s/\Q$delimiter\E$//;
3032      $delimiter = $new;
3033      mtr_debug("changed delimiter to $delimiter");
3034      # No need to add the delimiter to result
3035      next;
3036    }
3037
3038    # Add newline if line ends with $delimiter
3039    # and convert the current delimiter to semicolon
3040    if ( $line =~ /\Q$delimiter\E$/ ){
3041      $line =~ s/\Q$delimiter\E$/;/;
3042      $result.= "$line\n";
3043      mtr_debug("Added default delimiter");
3044      next;
3045    }
3046
3047    # Remove comments starting with --
3048    if ( $line =~ /^\s*--/ ) {
3049      mtr_debug("Discarded $line");
3050      next;
3051    }
3052
3053    # Replace @HOSTNAME with localhost
3054    $line=~ s/\'\@HOSTNAME\@\'/localhost/;
3055
3056    # Default, just add the line without newline
3057    # but with a space as separator
3058    $result.= "$line ";
3059
3060  }
3061  return $result;
3062}
3063
3064
3065sub default_mysqld {
3066  # Generate new config file from template
3067  environment_setup();
3068  my $config= My::ConfigFactory->new_config
3069    ( {
3070       basedir         => $basedir,
3071       testdir         => $glob_mysql_test_dir,
3072       template_path   => "include/default_my.cnf",
3073       vardir          => $opt_vardir,
3074       tmpdir          => $opt_tmpdir,
3075       baseport        => 0,
3076       user            => $opt_user,
3077       password        => '',
3078      }
3079    );
3080
3081  my $mysqld= $config->group('mysqld.1')
3082    or mtr_error("Couldn't find mysqld.1 in default config");
3083  return $mysqld;
3084}
3085
3086
3087sub mysql_install_db {
3088  my ($mysqld, $datadir, $extra_opts)= @_;
3089
3090  my $install_datadir= $datadir || $mysqld->value('datadir');
3091  my $install_basedir= $mysqld->value('basedir');
3092  my $install_lang= $mysqld->value('lc-messages-dir');
3093  my $install_chsdir= $mysqld->value('character-sets-dir');
3094
3095  mtr_report("Installing system database...");
3096
3097  my $args;
3098  mtr_init_args(\$args);
3099  mtr_add_arg($args, "--no-defaults");
3100  mtr_add_arg($args, "--disable-getopt-prefix-matching");
3101  mtr_add_arg($args, "--bootstrap");
3102  mtr_add_arg($args, "--basedir=%s", $install_basedir);
3103  mtr_add_arg($args, "--datadir=%s", $install_datadir);
3104  mtr_add_arg($args, "--plugin-dir=%s", $plugindir);
3105  mtr_add_arg($args, "--default-storage-engine=myisam");
3106  mtr_add_arg($args, "--loose-skip-plugin-$_") for @optional_plugins;
3107  # starting from 10.0 bootstrap scripts require InnoDB
3108  mtr_add_arg($args, "--loose-innodb");
3109  mtr_add_arg($args, "--loose-innodb-log-file-size=5M");
3110  mtr_add_arg($args, "--disable-sync-frm");
3111  mtr_add_arg($args, "--tmpdir=%s", "$opt_vardir/tmp/");
3112  mtr_add_arg($args, "--core-file");
3113  mtr_add_arg($args, "--console");
3114  mtr_add_arg($args, "--character-set-server=latin1");
3115
3116  if ( $opt_debug )
3117  {
3118    mtr_add_arg($args, "--debug-dbug=$debug_d:t:i:A,%s/log/bootstrap.trace",
3119		$path_vardir_trace);
3120  }
3121
3122  mtr_add_arg($args, "--lc-messages-dir=%s", $install_lang);
3123  mtr_add_arg($args, "--character-sets-dir=%s", $install_chsdir);
3124
3125  # InnoDB arguments that affect file location and sizes may
3126  # need to be given to the bootstrap process as well as the
3127  # server process.
3128  foreach my $extra_opt ( @opt_extra_mysqld_opt ) {
3129    if ($extra_opt =~ /--innodb/) {
3130      mtr_add_arg($args, $extra_opt);
3131    }
3132  }
3133
3134  # If DISABLE_GRANT_OPTIONS is defined when the server is compiled (e.g.,
3135  # configure --disable-grant-options), mysqld will not recognize the
3136  # --bootstrap or --skip-grant-tables options.  The user can set
3137  # MYSQLD_BOOTSTRAP to the full path to a mysqld which does accept
3138  # --bootstrap, to accommodate this.
3139  my $exe_mysqld_bootstrap =
3140    $ENV{'MYSQLD_BOOTSTRAP'} || find_mysqld($install_basedir);
3141
3142  # ----------------------------------------------------------------------
3143  # export MYSQLD_BOOTSTRAP_CMD variable containing <path>/mysqld <args>
3144  # ----------------------------------------------------------------------
3145  $ENV{'MYSQLD_BOOTSTRAP_CMD'}= "$exe_mysqld_bootstrap " . join(" ", @$args)
3146    unless defined $ENV{'MYSQLD_BOOTSTRAP_CMD'};
3147
3148  # Extra options can come not only from the command line, but also
3149  # from option files or combinations. We want them on a command line
3150  # that is executed now, because otherwise the datadir might be
3151  # incompatible with the test settings, but not on the general
3152  # $MYSQLD_BOOTSTRAP_CMD line
3153  foreach my $extra_opt ( @$extra_opts ) {
3154    mtr_add_arg($args, $extra_opt);
3155  }
3156
3157  # ----------------------------------------------------------------------
3158  # Create the bootstrap.sql file
3159  # ----------------------------------------------------------------------
3160  my $bootstrap_sql_file= "$opt_vardir/log/bootstrap.sql";
3161  $ENV{'MYSQL_BOOTSTRAP_SQL_FILE'}= $bootstrap_sql_file;
3162
3163  if (! -e $bootstrap_sql_file)
3164  {
3165    My::Debugger::setup_boot_args(\$args, \$exe_mysqld_bootstrap, $bootstrap_sql_file);
3166
3167    my $path_sql= my_find_file($install_basedir,
3168             ["mysql", "sql/share", "share/mariadb",
3169              "share/mysql", "share", "scripts"],
3170             "mysql_system_tables.sql",
3171             NOT_REQUIRED);
3172
3173    if (-f $path_sql )
3174    {
3175      my $sql_dir= dirname($path_sql);
3176      # Use the mysql database for system tables
3177      mtr_tofile($bootstrap_sql_file, "use mysql;\n");
3178
3179      # Add the offical mysql system tables
3180      # for a production system
3181      mtr_appendfile_to_file("$sql_dir/mysql_system_tables.sql",
3182           $bootstrap_sql_file);
3183
3184      my $gis_sp_path = $source_dist ? "$bindir/scripts" : $sql_dir;
3185      mtr_appendfile_to_file("$gis_sp_path/maria_add_gis_sp_bootstrap.sql",
3186           $bootstrap_sql_file);
3187
3188      # Add the performance tables
3189      # for a production system
3190      mtr_appendfile_to_file("$sql_dir/mysql_performance_tables.sql",
3191                            $bootstrap_sql_file);
3192
3193      # Add the mysql system tables initial data
3194      # for a production system
3195      mtr_appendfile_to_file("$sql_dir/mysql_system_tables_data.sql",
3196           $bootstrap_sql_file);
3197
3198      # Add test data for timezone - this is just a subset, on a real
3199      # system these tables will be populated either by mysql_tzinfo_to_sql
3200      # or by downloading the timezone table package from our website
3201      mtr_appendfile_to_file("$sql_dir/mysql_test_data_timezone.sql",
3202           $bootstrap_sql_file);
3203
3204      # Fill help tables, just an empty file when running from bk repo
3205      # but will be replaced by a real fill_help_tables.sql when
3206      # building the source dist
3207      mtr_appendfile_to_file("$sql_dir/fill_help_tables.sql",
3208           $bootstrap_sql_file);
3209
3210      # Create test database
3211      mtr_appendfile_to_file("$sql_dir/mysql_test_db.sql",
3212                            $bootstrap_sql_file);
3213
3214      # mysql.gtid_slave_pos was created in InnoDB, but many tests
3215      # run without InnoDB. Alter it to MyISAM now
3216      mtr_tofile($bootstrap_sql_file, "ALTER TABLE gtid_slave_pos ENGINE=MyISAM;\n");
3217    }
3218    else
3219    {
3220      # Install db from init_db.sql that exist in early 5.1 and 5.0
3221      # versions of MySQL
3222      my $init_file= "$install_basedir/mysql-test/lib/init_db.sql";
3223      mtr_report(" - from '$init_file'");
3224      my $text= mtr_grab_file($init_file) or
3225        mtr_error("Can't open '$init_file': $!");
3226
3227      mtr_tofile($bootstrap_sql_file,
3228           sql_to_bootstrap($text));
3229    }
3230
3231    # Remove anonymous users
3232    mtr_tofile($bootstrap_sql_file,
3233         "DELETE FROM mysql.user where user= '';\n");
3234
3235    # Create mtr database
3236    mtr_tofile($bootstrap_sql_file,
3237         "CREATE DATABASE mtr CHARSET=latin1;\n");
3238
3239    # Add help tables and data for warning detection and supression
3240    mtr_tofile($bootstrap_sql_file,
3241               sql_to_bootstrap(mtr_grab_file("include/mtr_warnings.sql")));
3242
3243    # Add procedures for checking server is restored after testcase
3244    mtr_tofile($bootstrap_sql_file,
3245               sql_to_bootstrap(mtr_grab_file("include/mtr_check.sql")));
3246  }
3247
3248  # Log bootstrap command
3249  my $path_bootstrap_log= "$opt_vardir/log/bootstrap.log";
3250  mtr_tofile($path_bootstrap_log,
3251	     "$exe_mysqld_bootstrap " . join(" ", @$args) . "\n");
3252
3253  # Create directories mysql
3254  mkpath("$install_datadir/mysql");
3255
3256  if ( My::SafeProcess->run
3257       (
3258	name          => "bootstrap",
3259	path          => $exe_mysqld_bootstrap,
3260	args          => \$args,
3261	input         => $bootstrap_sql_file,
3262	output        => $path_bootstrap_log,
3263	error         => $path_bootstrap_log,
3264	append        => 1,
3265	verbose       => $opt_verbose,
3266       ) != 0)
3267  {
3268    my $data= mtr_grab_file($path_bootstrap_log);
3269    mtr_error("Error executing mysqld --bootstrap\n" .
3270              "Could not install system database from $bootstrap_sql_file\n" .
3271	      "The $path_bootstrap_log file contains:\n$data\n");
3272  }
3273}
3274
3275
3276sub run_testcase_check_skip_test($)
3277{
3278  my ($tinfo)= @_;
3279
3280  # ----------------------------------------------------------------------
3281  # Skip some tests silently
3282  # ----------------------------------------------------------------------
3283
3284  my $start_from= $mtr_cases::start_from;
3285  if ( $start_from )
3286  {
3287    if ($tinfo->{'name'} eq $start_from ||
3288        $tinfo->{'shortname'} eq $start_from)
3289    {
3290      ## Found parting test. Run this test and all tests after this one
3291      $mtr_cases::start_from= "";
3292    }
3293    else
3294    {
3295      $tinfo->{'result'}= 'MTR_RES_SKIPPED';
3296      return 1;
3297    }
3298  }
3299
3300  # ----------------------------------------------------------------------
3301  # If marked to skip, just print out and return.
3302  # Note that a test case not marked as 'skip' can still be
3303  # skipped later, because of the test case itself in cooperation
3304  # with the mysqltest program tells us so.
3305  # ----------------------------------------------------------------------
3306
3307  if ( $tinfo->{'skip'} )
3308  {
3309    mtr_report_test_skipped($tinfo) unless $start_only;
3310    return 1;
3311  }
3312
3313  return 0;
3314}
3315
3316
3317sub run_query {
3318  my ($tinfo, $mysqld, $query)= @_;
3319
3320  my $args;
3321  mtr_init_args(\$args);
3322  mtr_add_arg($args, "--defaults-file=%s", $path_config_file);
3323  mtr_add_arg($args, "--defaults-group-suffix=%s", $mysqld->after('mysqld'));
3324
3325  mtr_add_arg($args, "-e %s", $query);
3326
3327  my $res= My::SafeProcess->run
3328    (
3329     name          => "run_query -> ".$mysqld->name(),
3330     path          => $exe_mysql,
3331     args          => \$args,
3332     output        => '/dev/null',
3333     error         => '/dev/null'
3334    );
3335
3336  return $res
3337}
3338
3339
3340sub do_before_run_mysqltest($)
3341{
3342  my $tinfo= shift;
3343  my $resfile= $tinfo->{result_file};
3344  return unless defined $resfile;
3345
3346  # Remove old files produced by mysqltest
3347  die "unsupported result file name $resfile, stoping" unless
3348         $resfile =~ /^(.*?)((?:,\w+)*)\.(rdiff|result|result~)$/;
3349  my ($base_file, $suites, $ext)= ($1, $2, $3);
3350  # if the result file is a diff, make a proper result file
3351  if ($ext eq 'rdiff') {
3352    my $base_result = $tinfo->{base_result};
3353    my $resdir= dirname($resfile);
3354    # we'll use a separate extension for generated result files
3355    # to be able to distinguish them from manually created
3356    # version-controlled results, and to ignore them in git.
3357    my $dest = "$base_file$suites.result~";
3358    my @cmd = ($exe_patch);
3359    if ($^O eq "MSWin32") {
3360      push @cmd, '--binary';
3361    }
3362    push @cmd, (qw/-r - -f -s -o/, $dest, $base_result, $resfile);
3363    if (-w $resdir) {
3364      # don't rebuild a file if it's up to date
3365      unless (-e $dest and -M $dest < -M $resfile
3366                       and -M $dest < -M $base_result) {
3367        run_system(@cmd);
3368      }
3369    } else {
3370      $cmd[-3] = $dest = $opt_tmpdir . '/' . basename($dest);
3371      run_system(@cmd);
3372    }
3373    $tinfo->{result_file} = $dest;
3374  }
3375
3376  unlink("$base_file.reject");
3377  unlink("$base_file.progress");
3378  unlink("$base_file.log");
3379  unlink("$base_file.warnings");
3380}
3381
3382
3383#
3384# Check all server for sideffects
3385#
3386# RETURN VALUE
3387#  0 ok
3388#  1 Check failed
3389#  >1 Fatal errro
3390
3391sub check_testcase($$)
3392{
3393  my ($tinfo, $mode)= @_;
3394  my $tname= $tinfo->{name};
3395
3396  # Start the mysqltest processes in parallel to save time
3397  # also makes it possible to wait for any process to exit during the check
3398  my %started;
3399  foreach my $mysqld ( mysqlds() )
3400  {
3401    # Skip if server has been restarted with additional options
3402    if ( defined $mysqld->{'proc'} && ! exists $mysqld->{'restart_opts'} )
3403    {
3404      my $proc= start_check_testcase($tinfo, $mode, $mysqld);
3405      $started{$proc->pid()}= $proc;
3406    }
3407  }
3408
3409  # Return immediately if no check proceess was started
3410  return 0 unless ( keys %started );
3411
3412  my $timeout= start_timer(check_timeout($tinfo));
3413
3414  while (1){
3415    my $result;
3416    my $proc= My::SafeProcess->wait_any_timeout($timeout);
3417    mtr_report("Got $proc");
3418
3419    if ( delete $started{$proc->pid()} ) {
3420
3421      my $err_file= $proc->user_data();
3422      my $base_file= mtr_match_extension($err_file, "err"); # Trim extension
3423
3424      # One check testcase process returned
3425      my $res= $proc->exit_status();
3426
3427      if ( $res == 0){
3428	# Check completed without problem
3429
3430	# Remove the .err file the check generated
3431	unlink($err_file);
3432
3433	# Remove the .result file the check generated
3434	if ( $mode eq 'after' ){
3435	  unlink("$base_file.result");
3436	}
3437
3438	if ( keys(%started) == 0){
3439	  # All checks completed
3440	  mark_time_used('check');
3441	  return 0;
3442	}
3443	# Wait for next process to exit
3444	next;
3445      }
3446      else
3447      {
3448	if ( $mode eq "after" and $res == 1 )
3449	{
3450	  # Test failed, grab the report mysqltest has created
3451	  my $report= mtr_grab_file($err_file);
3452	  $tinfo->{check}.=
3453	    "\nMTR's internal check of the test case '$tname' failed.
3454This means that the test case does not preserve the state that existed
3455before the test case was executed.  Most likely the test case did not
3456do a proper clean-up. It could also be caused by the previous test run
3457by this thread, if the server wasn't restarted.
3458This is the diff of the states of the servers before and after the
3459test case was executed:\n";
3460	  $tinfo->{check}.= $report;
3461
3462	  # Check failed, mark the test case with that info
3463	  $tinfo->{'check_testcase_failed'}= 1;
3464	  $result= 1;
3465	}
3466	elsif ( $res )
3467	{
3468	  my $report= mtr_grab_file($err_file);
3469	  $tinfo->{comment}.=
3470	    "Could not execute 'check-testcase' $mode ".
3471	      "testcase '$tname' (res: $res):\n";
3472	  $tinfo->{comment}.= $report;
3473
3474	  $result= 2;
3475	}
3476        else
3477        {
3478          # Remove the .result file the check generated
3479          unlink("$base_file.result");
3480        }
3481	# Remove the .err file the check generated
3482	unlink($err_file);
3483
3484      }
3485    }
3486    elsif ( $proc->{timeout} ) {
3487      $tinfo->{comment}.= "Timeout for 'check-testcase' expired after "
3488	.check_timeout($tinfo)." seconds";
3489      $result= 4;
3490    }
3491    else {
3492      # Unknown process returned, most likley a crash, abort everything
3493      $tinfo->{comment}=
3494	"The server $proc crashed while running ".
3495	"'check testcase $mode test'".
3496	get_log_from_proc($proc, $tinfo->{name});
3497      $result= 3;
3498    }
3499
3500    # Kill any check processes still running
3501    map($_->kill(), values(%started));
3502
3503    mtr_warning("Check-testcase failed, this could also be caused by the" .
3504		" previous test run by this worker thread")
3505      if $result > 1 && $mode eq "before";
3506    mark_time_used('check');
3507
3508    return $result;
3509  }
3510
3511  mtr_error("INTERNAL_ERROR: check_testcase");
3512}
3513
3514
3515# Start run mysqltest on one server
3516#
3517# RETURN VALUE
3518#  0 OK
3519#  1 Check failed
3520#
3521sub start_run_one ($$) {
3522  my ($mysqld, $run)= @_;
3523
3524  my $name= "$run-".$mysqld->name();
3525
3526  my $args;
3527  mtr_init_args(\$args);
3528
3529  mtr_add_arg($args, "--defaults-file=%s", $path_config_file);
3530  mtr_add_arg($args, "--defaults-group-suffix=%s", $mysqld->after('mysqld'));
3531
3532  mtr_add_arg($args, "--silent");
3533  mtr_add_arg($args, "--test-file=%s", "include/$run.test");
3534
3535  my $errfile= "$opt_vardir/tmp/$name.err";
3536  my $proc= My::SafeProcess->new
3537    (
3538     name          => $name,
3539     path          => $exe_mysqltest,
3540     error         => $errfile,
3541     output        => $errfile,
3542     args          => \$args,
3543     user_data     => $errfile,
3544     verbose       => $opt_verbose,
3545    );
3546  mtr_verbose("Started $proc");
3547  return $proc;
3548}
3549
3550
3551#
3552# Run script on all servers, collect results
3553#
3554# RETURN VALUE
3555#  0 ok
3556#  1 Failure
3557
3558sub run_on_all($$)
3559{
3560  my ($tinfo, $run)= @_;
3561
3562  # Start the mysqltest processes in parallel to save time
3563  # also makes it possible to wait for any process to exit during the check
3564  # and to have a timeout process
3565  my %started;
3566  foreach my $mysqld ( mysqlds() )
3567  {
3568    if ( defined $mysqld->{'proc'} )
3569    {
3570      my $proc= start_run_one($mysqld, $run);
3571      $started{$proc->pid()}= $proc;
3572    }
3573  }
3574
3575  # Return immediately if no check proceess was started
3576  return 0 unless ( keys %started );
3577
3578  my $timeout= start_timer(check_timeout($tinfo));
3579
3580  while (1){
3581    my $result;
3582    my $proc= My::SafeProcess->wait_any_timeout($timeout);
3583    mtr_report("Got $proc");
3584
3585    if ( delete $started{$proc->pid()} ) {
3586
3587      # One mysqltest process returned
3588      my $err_file= $proc->user_data();
3589      my $res= $proc->exit_status();
3590
3591      # Append the report from .err file
3592      $tinfo->{comment}.= " == $err_file ==\n";
3593      $tinfo->{comment}.= mtr_grab_file($err_file);
3594      $tinfo->{comment}.= "\n";
3595
3596      # Remove the .err file
3597      unlink($err_file);
3598
3599      if ( keys(%started) == 0){
3600	# All completed
3601	return 0;
3602      }
3603
3604      # Wait for next process to exit
3605      next;
3606    }
3607    elsif ($proc->{timeout}) {
3608      $tinfo->{comment}.= "Timeout for '$run' expired after "
3609	.check_timeout($tinfo)." seconds";
3610    }
3611    else {
3612      # Unknown process returned, most likley a crash, abort everything
3613      $tinfo->{comment}.=
3614	"The server $proc crashed while running '$run'".
3615	get_log_from_proc($proc, $tinfo->{name});
3616    }
3617
3618    # Kill any check processes still running
3619    map($_->kill(), values(%started));
3620
3621    return 1;
3622  }
3623  mtr_error("INTERNAL_ERROR: run_on_all");
3624}
3625
3626
3627sub mark_log {
3628  my ($log, $tinfo)= @_;
3629  my $log_msg= "CURRENT_TEST: $tinfo->{name}\n";
3630  pre_write_errorlog($log, $tinfo->{name});
3631  mtr_tofile($log, $log_msg);
3632}
3633
3634
3635sub find_testcase_skipped_reason($)
3636{
3637  my ($tinfo)= @_;
3638
3639  # Set default message
3640  $tinfo->{'comment'}= "Detected by testcase(no log file)";
3641
3642  # Open the test log file
3643  my $F= IO::File->new($path_current_testlog)
3644    or return;
3645  my $reason;
3646
3647  while ( my $line= <$F> )
3648  {
3649    # Look for "reason: <reason for skipping test>"
3650    if ( $line =~ /reason: (.*)/ )
3651    {
3652      $reason= $1;
3653    }
3654  }
3655
3656  if ( ! $reason )
3657  {
3658    mtr_warning("Could not find reason for skipping test in $path_current_testlog");
3659    $reason= "Detected by testcase(reason unknown) ";
3660  }
3661  $tinfo->{'comment'}= $reason;
3662}
3663
3664
3665sub find_analyze_request
3666{
3667  # Open the test log file
3668  my $F= IO::File->new($path_current_testlog)
3669    or return;
3670  my $analyze;
3671
3672  while ( my $line= <$F> )
3673  {
3674    # Look for "reason: <reason for skipping test>"
3675    if ( $line =~ /analyze: (.*)/ )
3676    {
3677      $analyze= $1;
3678    }
3679  }
3680
3681  return $analyze;
3682}
3683
3684# The test can leave a file in var/tmp/ to signal
3685# that all servers should be restarted
3686sub restart_forced_by_test($)
3687{
3688  my $file = shift;
3689  my $restart = 0;
3690  foreach my $mysqld ( mysqlds() )
3691  {
3692    my $datadir = $mysqld->value('datadir');
3693    my $force_restart_file = "$datadir/mtr/$file";
3694    if ( -f $force_restart_file )
3695    {
3696      mtr_verbose("Restart of servers forced by test");
3697      $restart = 1;
3698      last;
3699    }
3700  }
3701  return $restart;
3702}
3703
3704# Return timezone value of tinfo or default value
3705sub timezone {
3706  my ($tinfo)= @_;
3707  local $_ = $tinfo->{timezone};
3708  return 'DEFAULT' unless defined $_;
3709  no warnings 'uninitialized';
3710  s/\$\{(\w+)\}/$ENV{$1}/ge;
3711  s/\$(\w+)/$ENV{$1}/ge;
3712  $_;
3713}
3714
3715sub mycnf_create {
3716  my ($config) = @_;
3717  my $res;
3718
3719  foreach my $group ($config->option_groups()) {
3720    $res .= "[$group->{name}]\n";
3721
3722    foreach my $option ($group->options()) {
3723      $res .= $option->name();
3724      my $value= $option->value();
3725      if (defined $value) {
3726	$res .= "=$value";
3727      }
3728      $res .= "\n";
3729    }
3730    $res .= "\n";
3731  }
3732  $res;
3733}
3734
3735sub config_files($) {
3736  my ($tinfo) = @_;
3737  (
3738    'my.cnf' => \&mycnf_create,
3739    $tinfo->{suite}->config_files()
3740  );
3741}
3742
3743sub _like   { return $config ? $config->like($_[0]) : (); }
3744sub mysqlds { return _like('mysqld\.'); }
3745
3746sub fix_servers($) {
3747  my ($tinfo) = @_;
3748  return () unless $config;
3749  my %servers = (
3750    qr/mysqld\./ => {
3751      SORT => 300,
3752      START => \&mysql_server_start,
3753      WAIT => \&mysql_server_wait,
3754    },
3755    $tinfo->{suite}->servers()
3756  );
3757  for ($config->groups()) {
3758    while (my ($re,$prop) = each %servers) {
3759      @$_{keys %$prop} = values %$prop if $_->{name} =~ /^$re/;
3760    }
3761  }
3762}
3763
3764sub all_servers {
3765  return unless $config;
3766  ( sort { $a->{SORT} <=> $b->{SORT} }
3767       grep { defined $_->{SORT} } $config->groups() );
3768}
3769
3770# Storage for changed environment variables
3771our %old_env;
3772
3773sub resfile_report_test ($) {
3774  my $tinfo=  shift;
3775
3776  resfile_new_test();
3777
3778  resfile_test_info("name", $tinfo->{name});
3779  resfile_test_info("variation", $tinfo->{combination})
3780    if $tinfo->{combination};
3781  resfile_test_info("start_time", isotime time);
3782}
3783
3784
3785#
3786# Run a single test case
3787#
3788# RETURN VALUE
3789#  0 OK
3790#  > 0 failure
3791#
3792
3793sub run_testcase ($$) {
3794  my ($tinfo, $server_socket)= @_;
3795  my $print_freq=20;
3796
3797  mtr_verbose("Running test:", $tinfo->{name});
3798  $ENV{'MTR_TEST_NAME'} = $tinfo->{name};
3799  resfile_report_test($tinfo) if $opt_resfile;
3800
3801  for my $key (grep { /^MTR_COMBINATION/ } keys %ENV)
3802  {
3803    delete $ENV{$key};
3804  }
3805
3806  if (ref $tinfo->{combinations} eq 'ARRAY')
3807  {
3808    for (my $i = 0; $i < @{$tinfo->{combinations}}; ++$i )
3809    {
3810      my $combination = $tinfo->{combinations}->[$i];
3811      # Allow only alphanumerics plus _ - + . in combination names,
3812      # or anything beginning with -- (the latter comes from --combination)
3813      if ($combination && $combination !~ /^\w[-\w\.\+]*$/
3814                      && $combination !~ /^--/)
3815      {
3816        mtr_error("Combination '$combination' contains illegal characters");
3817      }
3818      $ENV{"MTR_COMBINATION_". uc(${combination})} = 1;
3819    }
3820    $ENV{"MTR_COMBINATIONS"} = join(',', @{$tinfo->{combinations}});
3821  }
3822  elsif (exists $tinfo->{combinations})
3823  {
3824    die 'Unexpected type of $tinfo->{combinations}';
3825  }
3826
3827  # -------------------------------------------------------
3828  # Init variables that can change between each test case
3829  # -------------------------------------------------------
3830  my $timezone= timezone($tinfo);
3831  if ($timezone ne 'DEFAULT') {
3832    $ENV{'TZ'}= $timezone;
3833  } else {
3834    delete($ENV{'TZ'});
3835  }
3836  $ENV{MTR_SUITE_DIR} = $tinfo->{suite}->{dir};
3837  mtr_verbose("Setting timezone: $timezone");
3838
3839  if ( ! using_extern() )
3840  {
3841    my @restart= servers_need_restart($tinfo);
3842    if ( @restart != 0) {
3843      # Remember that we restarted for this test case (count restarts)
3844      $tinfo->{'restarted'}= 1;
3845      stop_servers(reverse @restart);
3846      if ($opt_warnings) {
3847        check_warnings_post_shutdown($server_socket);
3848      }
3849    }
3850
3851    if ( started(all_servers()) == 0 )
3852    {
3853
3854      # Remove old datadirs
3855      clean_datadir() unless $opt_start_dirty;
3856
3857      # Restore old ENV
3858      while (my ($option, $value)= each( %old_env )) {
3859	if (defined $value){
3860	  mtr_verbose("Restoring $option to $value");
3861	  $ENV{$option}= $value;
3862
3863	} else {
3864	  mtr_verbose("Removing $option");
3865	  delete($ENV{$option});
3866	}
3867      }
3868      %old_env= ();
3869
3870      mtr_verbose("Generating my.cnf from '$tinfo->{template_path}'");
3871
3872      # Generate new config file from template
3873      $config= My::ConfigFactory->new_config
3874	( {
3875           testname        => $tinfo->{name},
3876	   basedir         => $basedir,
3877	   testdir         => $glob_mysql_test_dir,
3878	   template_path   => $tinfo->{template_path},
3879	   extra_template_path => $tinfo->{extra_template_path},
3880	   vardir          => $opt_vardir,
3881	   tmpdir          => $opt_tmpdir,
3882	   baseport        => $baseport,
3883	   user            => $opt_user,
3884	   password        => '',
3885	   ssl             => $opt_ssl_supported,
3886	   embedded        => $opt_embedded_server,
3887	  }
3888	);
3889
3890      fix_servers($tinfo);
3891
3892      # Write config files:
3893      my %config_files = config_files($tinfo);
3894      while (my ($file, $generate) = each %config_files) {
3895        next unless $generate;
3896        my ($path) = "$opt_vardir/$file";
3897        open (F, '>', $path) or die "Could not open '$path': $!";
3898        print F &$generate($config);
3899        close F;
3900      }
3901
3902      # Remember current config so a restart can occur when a test need
3903      # to use a different one
3904      $current_config_name= $tinfo->{template_path};
3905
3906      #
3907      # Set variables in the ENV section
3908      #
3909      foreach my $option ($config->options_in_group("ENV"))
3910      {
3911        my ($name, $val)= ($option->name(), $option->value());
3912
3913	# Save old value to restore it before next time
3914	$old_env{$name}= $ENV{$name};
3915
3916        unless (defined $val) {
3917          mtr_warning("Uninitialized value for ", $name,
3918            ", group [ENV], file ", $current_config_name);
3919        } else {
3920          mtr_verbose($name, "=", $val);
3921          $ENV{$name}= $val;
3922        }
3923      }
3924    }
3925
3926    # Write start of testcase to log
3927    mark_log($path_current_testlog, $tinfo);
3928
3929    # Make sure the safe_process also exits from now on
3930    if ($opt_start_exit) {
3931      My::SafeProcess->start_exit();
3932    }
3933
3934    if (start_servers($tinfo))
3935    {
3936      report_failure_and_restart($tinfo);
3937      unlink $path_current_testlog;
3938      return 1;
3939    }
3940  }
3941  mark_time_used('restart');
3942
3943  # --------------------------------------------------------------------
3944  # If --start or --start-dirty given, stop here to let user manually
3945  # run tests
3946  # If --wait-all is also given, do the same, but don't die if one
3947  # server exits
3948  # ----------------------------------------------------------------------
3949
3950  if ( $start_only )
3951  {
3952    mtr_print("\nStarted", started(all_servers()));
3953    mtr_print("Using config for test", $tinfo->{name});
3954    mtr_print("Port and socket path for server(s):");
3955    foreach my $mysqld ( mysqlds() )
3956    {
3957      mtr_print ($mysqld->name() . "  " . $mysqld->value('port') .
3958	      "  " . $mysqld->value('socket'));
3959    }
3960    if ( $opt_start_exit )
3961    {
3962      mtr_print("Server(s) started, not waiting for them to finish");
3963      if (IS_WINDOWS)
3964      {
3965	POSIX::_exit(0);	# exit hangs here in ActiveState Perl
3966      }
3967      else
3968      {
3969	exit(0);
3970      }
3971    }
3972    mtr_print("Waiting for server(s) to exit...");
3973    if ( $opt_wait_all ) {
3974      My::SafeProcess->wait_all();
3975      mtr_print( "All servers exited" );
3976      exit(1);
3977    }
3978    else {
3979      my $proc= My::SafeProcess->wait_any();
3980      if ( grep($proc eq $_, started(all_servers())) )
3981      {
3982        mtr_print("Server $proc died");
3983        exit(1);
3984      }
3985      mtr_print("Unknown process $proc died");
3986      exit(1);
3987    }
3988  }
3989
3990  my $test_timeout= start_timer(testcase_timeout($tinfo));
3991
3992  do_before_run_mysqltest($tinfo);
3993
3994  mark_time_used('admin');
3995
3996  if ( $opt_check_testcases and check_testcase($tinfo, "before") ){
3997    # Failed to record state of server or server crashed
3998    report_failure_and_restart($tinfo);
3999
4000    return 1;
4001  }
4002
4003  my $test= $tinfo->{suite}->start_test($tinfo);
4004  # Set to a list of processes we have to keep waiting (expectedly died servers)
4005  my %keep_waiting_proc = ();
4006  my $print_timeout= start_timer($print_freq * 60);
4007
4008  while (1)
4009  {
4010    my $proc;
4011    if (%keep_waiting_proc)
4012    {
4013      # Any other process exited?
4014      $proc = My::SafeProcess->check_any();
4015      if ($proc)
4016      {
4017	mtr_verbose ("Found exited process $proc");
4018      }
4019      else
4020      {
4021	# Also check if timer has expired, if so cancel waiting
4022	if ( has_expired($test_timeout) )
4023	{
4024	  %keep_waiting_proc = ();
4025	}
4026      }
4027    }
4028    if (!%keep_waiting_proc && !$proc)
4029    {
4030      if ($test_timeout > $print_timeout)
4031      {
4032        $proc= My::SafeProcess->wait_any_timeout($print_timeout);
4033        if ($proc->{timeout})
4034        {
4035          #print out that the test is still on
4036          mtr_print("Test still running: $tinfo->{name}");
4037          #reset the timer
4038          $print_timeout= start_timer($print_freq * 60);
4039          next;
4040        }
4041      }
4042      else
4043      {
4044        $proc= My::SafeProcess->wait_any_timeout($test_timeout);
4045      }
4046    }
4047
4048    if ($proc and $proc eq $test) # mysqltest itself exited
4049    {
4050      my $res= $test->exit_status();
4051
4052      if (($res == 0 or $res == 62) and $opt_warnings and check_warnings($tinfo) )
4053      {
4054        # If test case suceeded, but it has produced unexpected
4055        # warnings, continue with $res == 1;
4056        # but if the test was skipped, it should remain skipped
4057        $res= 1 if $res == 0;
4058        resfile_output($tinfo->{'warnings'}) if $opt_resfile;
4059      }
4060
4061      if ( $res == 0 )
4062      {
4063        my $check_res;
4064	if ( restart_forced_by_test('force_restart') )
4065        {
4066          stop_all_servers($opt_shutdown_timeout);
4067        }
4068        elsif ( $opt_check_testcases and
4069                $check_res= check_testcase($tinfo, "after"))
4070        {
4071          if ($check_res == 1) {
4072            # Test case had sideeffects, not fatal error, just continue
4073            if ($opt_warnings) {
4074              # Checking error logs for warnings, so need to stop server
4075              # gracefully so that memory leaks etc. can be properly detected.
4076              stop_servers(reverse all_servers());
4077              check_warnings_post_shutdown($server_socket);
4078              # Even if we got warnings here, we should not fail this
4079              # particular test, as the warnings may be caused by an earlier
4080              # test.
4081            } else {
4082              # Not checking warnings, so can do a hard shutdown.
4083              stop_all_servers($opt_shutdown_timeout);
4084            }
4085            mtr_report("Resuming tests...\n");
4086            resfile_output($tinfo->{'check'}) if $opt_resfile;
4087          }
4088          else {
4089            # Test case check failed fatally, probably a server crashed
4090            report_failure_and_restart($tinfo);
4091            return 1;
4092          }
4093        }
4094        mtr_report_test_passed($tinfo);
4095      }
4096      elsif ( $res == 62 )
4097      {
4098        # Testcase itself tell us to skip this one
4099        $tinfo->{skip_detected_by_test}= 1;
4100        # Try to get reason from test log file
4101        find_testcase_skipped_reason($tinfo);
4102        mtr_report_test_skipped($tinfo);
4103        # Restart if skipped due to missing perl, it may have had side effects
4104	if ( restart_forced_by_test('force_restart_if_skipped') ||
4105             $tinfo->{'comment'} =~ /^perl not found/ )
4106        {
4107          stop_all_servers($opt_shutdown_timeout);
4108        }
4109      }
4110      elsif ( $res == 65 )
4111      {
4112        # Testprogram killed by signal
4113        $tinfo->{comment}=
4114          "testprogram crashed(returned code $res)";
4115        report_failure_and_restart($tinfo);
4116      }
4117      elsif ( $res == 1 )
4118      {
4119        # Check if the test tool requests that
4120        # an analyze script should be run
4121        my $analyze= find_analyze_request();
4122        if ($analyze){
4123          run_on_all($tinfo, "analyze-$analyze");
4124        }
4125
4126        # Wait a bit and see if a server died, if so report that instead
4127        mtr_milli_sleep(100);
4128        my $srvproc= My::SafeProcess::check_any();
4129        if ($srvproc && grep($srvproc eq $_, started(all_servers()))) {
4130          $proc= $srvproc;
4131          goto SRVDIED;
4132        }
4133
4134        # Test case failure reported by mysqltest
4135        report_failure_and_restart($tinfo);
4136      }
4137      else
4138      {
4139        # mysqltest failed, probably crashed
4140        $tinfo->{comment}=
4141          "mysqltest failed with unexpected return code $res\n";
4142        report_failure_and_restart($tinfo);
4143      }
4144
4145      # Save info from this testcase run to mysqltest.log
4146      if( -f $path_current_testlog)
4147      {
4148        if ($opt_resfile && $res && $res != 62) {
4149          resfile_output_file($path_current_testlog);
4150        }
4151        mtr_appendfile_to_file($path_current_testlog, $path_testlog);
4152        unlink($path_current_testlog);
4153      }
4154
4155      return ($res == 62) ? 0 : $res;
4156    }
4157
4158    if ($proc)
4159    {
4160      # It was not mysqltest that exited, add to a wait-to-be-started-again list.
4161      $keep_waiting_proc{$proc} = 1;
4162    }
4163
4164    mtr_verbose("Got " . join(",", keys(%keep_waiting_proc)));
4165
4166    mark_time_used('test');
4167    foreach my $wait_for_proc (keys(%keep_waiting_proc)) {
4168      # ----------------------------------------------------
4169      # Check if it was an expected crash
4170      # ----------------------------------------------------
4171      my @mysqld = grep($wait_for_proc eq $_->{proc}, mysqlds());
4172      goto SRVDIED unless @mysqld;
4173      my $check_crash = check_expected_crash_and_restart($mysqld[0]);
4174      if ($check_crash == 0) # unexpected exit/crash of $wait_for_proc
4175      {
4176        $proc= $mysqld[0]->{proc};
4177        goto SRVDIED;
4178      }
4179      elsif ($check_crash == 1) # $wait_for_proc was started again by check_expected_crash_and_restart()
4180      {
4181        delete $keep_waiting_proc{$wait_for_proc};
4182      }
4183      elsif ($check_crash == 2) # we must keep waiting
4184      {
4185        # do nothing
4186      }
4187    }
4188
4189    next;
4190
4191  SRVDIED:
4192    # ----------------------------------------------------
4193    # Stop the test case timer
4194    # ----------------------------------------------------
4195    $test_timeout= 0;
4196
4197    # ----------------------------------------------------
4198    # Check if it was a server that died
4199    # ----------------------------------------------------
4200    if ( grep($proc eq $_, started(all_servers())) )
4201    {
4202      # Server failed, probably crashed
4203      $tinfo->{comment}=
4204	"Server $proc failed during test run" .
4205	get_log_from_proc($proc, $tinfo->{name});
4206
4207      # ----------------------------------------------------
4208      # It's not mysqltest that has exited, kill it
4209      # ----------------------------------------------------
4210      $test->kill();
4211
4212      report_failure_and_restart($tinfo);
4213      return 1;
4214    }
4215
4216    # Try to dump core for mysqltest and all servers
4217    foreach my $proc ($test, started(all_servers()))
4218    {
4219      mtr_print("Trying to dump core for $proc");
4220      if ($proc->dump_core())
4221      {
4222	$proc->wait_one(20);
4223      }
4224    }
4225
4226    # ----------------------------------------------------
4227    # It's not mysqltest that has exited, kill it
4228    # ----------------------------------------------------
4229    $test->kill();
4230
4231    # ----------------------------------------------------
4232    # Check if testcase timer expired
4233    # ----------------------------------------------------
4234    if ( $proc->{timeout} )
4235    {
4236      my $log_file_name= $opt_vardir."/log/".$tinfo->{shortname}.".log";
4237      $tinfo->{comment}=
4238        "Test case timeout after ".testcase_timeout($tinfo).
4239	  " seconds\n\n";
4240      # Add 20 last executed commands from test case log file
4241      if  (-e $log_file_name)
4242      {
4243        $tinfo->{comment}.=
4244	   "== $log_file_name == \n".
4245	     mtr_lastlinesfromfile($log_file_name, 20)."\n";
4246      }
4247      $tinfo->{'timeout'}= testcase_timeout($tinfo); # Mark as timeout
4248      run_on_all($tinfo, 'analyze-timeout');
4249
4250      report_failure_and_restart($tinfo);
4251      return 1;
4252    }
4253
4254    mtr_error("Unhandled process $proc exited");
4255  }
4256  mtr_error("Should never come here");
4257}
4258
4259
4260# Keep track of last position in mysqld error log where we scanned for
4261# warnings, so we can attribute any warnings found to the correct test
4262# suite or server restart.
4263our $last_warning_position= { };
4264
4265# Called just before a mysqld server is started or a testcase is run,
4266# to keep track of which tests have been run since last restart, and
4267# of when the error log is reset.
4268#
4269# Second argument $test_name is test name, or undef for server restart.
4270sub pre_write_errorlog {
4271  my ($error_log, $test_name)= @_;
4272
4273  if (! -e $error_log) {
4274    # If the error log is moved away, reset the warning parse position.
4275    delete $last_warning_position->{$error_log};
4276  }
4277
4278  if (defined($test_name)) {
4279    $last_warning_position->{$error_log}{test_names}= []
4280      unless exists($last_warning_position->{$error_log}{test_names});
4281    push @{$last_warning_position->{$error_log}{test_names}}, $test_name;
4282  } else {
4283    # Server restart, so clear the list of tests run since last restart.
4284    # (except the last one (if any), which is the test about to be run).
4285    if (defined($last_warning_position->{$error_log}{test_names}) &&
4286        @{$last_warning_position->{$error_log}{test_names}}) {
4287      $last_warning_position->{$error_log}{test_names}=
4288        [$last_warning_position->{$error_log}{test_names}[-1]];
4289    } else {
4290      $last_warning_position->{$error_log}{test_names}= [];
4291    }
4292  }
4293}
4294
4295# Extract server log from after the last occurrence of named test
4296# Return as an array of lines
4297#
4298
4299sub extract_server_log ($$) {
4300  my ($error_log, $tname) = @_;
4301
4302  return unless $error_log;
4303
4304  # Open the servers .err log file and read all lines
4305  # belonging to current test into @lines
4306  my $Ferr = IO::File->new($error_log)
4307    or mtr_error("Could not open file '$error_log' for reading: $!");
4308
4309  my @lines;
4310  my $found_test= 0;		# Set once we've found the log of this test
4311  while ( my $line = <$Ferr> )
4312  {
4313    if ($found_test)
4314    {
4315      # If test wasn't last after all, discard what we found, test again.
4316      if ( $line =~ /^CURRENT_TEST:/)
4317      {
4318	@lines= ();
4319	$found_test= $line =~ /^CURRENT_TEST: $tname/;
4320      }
4321      else
4322      {
4323	push(@lines, $line);
4324	if (scalar(@lines) > 1000000) {
4325	  $Ferr = undef;
4326	  mtr_warning("Too much log in $error_log, bailing out from extracting");
4327	  return ();
4328	}
4329      }
4330    }
4331    else
4332    {
4333      # Search for beginning of test, until found
4334      $found_test= 1 if ($line =~ /^CURRENT_TEST: $tname/);
4335    }
4336  }
4337  $Ferr = undef; # Close error log file
4338
4339  return @lines;
4340}
4341
4342# Get log from server identified from its $proc object, from named test
4343# Return as a single string
4344#
4345
4346sub get_log_from_proc ($$) {
4347  my ($proc, $name)= @_;
4348  my $srv_log= "";
4349
4350  foreach my $mysqld (all_servers()) {
4351    if ($mysqld->{proc} eq $proc) {
4352      my @srv_lines= extract_server_log($mysqld->if_exist('log-error'), $name);
4353      $srv_log= "\nServer log from this test:\n" .
4354	"----------SERVER LOG START-----------\n". join ("", @srv_lines) .
4355	"----------SERVER LOG END-------------\n";
4356      last;
4357    }
4358  }
4359  return $srv_log;
4360}
4361
4362#
4363# Perform a rough examination of the servers
4364# error log and write all lines that look
4365# suspicious into $error_log.warnings
4366#
4367
4368sub extract_warning_lines ($$) {
4369  my ($error_log, $append) = @_;
4370
4371  # Open the servers .err log file and read all lines
4372  # belonging to current tets into @lines
4373  my $Ferr = IO::File->new($error_log)
4374    or return [];
4375  my $last_pos= $last_warning_position->{$error_log}{seek_pos};
4376  $Ferr->seek($last_pos, 0) if defined($last_pos);
4377  # If the seek fails, we will parse the whole error log, at least we will not
4378  # miss any warnings.
4379
4380  my @lines= <$Ferr>;
4381  $last_warning_position->{$error_log}{seek_pos}= $Ferr->tell();
4382  $Ferr = undef; # Close error log file
4383
4384  # mysql_client_test.test sends a COM_DEBUG packet to the server
4385  # to provoke a safemalloc leak report, ignore any warnings
4386  # between "Begin/end safemalloc memory dump"
4387  if ( grep(/Begin safemalloc memory dump:/, @lines) > 0)
4388  {
4389    my $discard_lines= 1;
4390    foreach my $line ( @lines )
4391    {
4392      if ($line =~ /Begin safemalloc memory dump:/){
4393	$discard_lines = 1;
4394      } elsif ($line =~ /End safemalloc memory dump./){
4395	$discard_lines = 0;
4396      }
4397
4398      if ($discard_lines){
4399	$line = "ignored";
4400      }
4401    }
4402  }
4403
4404  # Write all suspicious lines to $error_log.warnings file
4405  my $warning_log = "$error_log.warnings";
4406  my $Fwarn = IO::File->new($warning_log, $append ? "a+" : "w")
4407    or die("Could not open file '$warning_log' for writing: $!");
4408  if (!$append)
4409  {
4410    print $Fwarn "Suspicious lines from $error_log\n";
4411  }
4412
4413  my @patterns =
4414    (
4415     qr/^Warning|mysqld: Warning|\[Warning\]/,
4416     qr/^Error:|\[ERROR\]/,
4417     qr/^==\d+==\s+\S/, # valgrind errors
4418     qr/InnoDB: Warning|InnoDB: Error/,
4419     qr/^safe_mutex:|allocated at line/,
4420     qr/missing DBUG_RETURN/,
4421     qr/Attempting backtrace/,
4422     qr/Assertion .* failed/,
4423     qr/Sanitizer/,
4424     qr/runtime error:/,
4425    );
4426  # These are taken from the include/mtr_warnings.sql global suppression
4427  # list. They occur delayed, so they can be parsed during shutdown rather
4428  # than during the per-test check.
4429  #
4430  # ToDo: having the warning suppressions inside the mysqld we are trying to
4431  # check is in any case horrible. We should change it to all be done here
4432  # within the Perl code, which is both simpler, easier, faster, and more
4433  # robust. We could still have individual test cases put in suppressions by
4434  # parsing statically or by writing dynamically to a CSV table read by the
4435  # Perl code.
4436  my @antipatterns =
4437    (
4438     @global_suppressions,
4439     qr/error .*connecting to master/,
4440     qr/InnoDB: Error: in ALTER TABLE `test`.`t[12]`/,
4441     qr/InnoDB: Error: table `test`.`t[12]` .*does not exist in the InnoDB internal/,
4442     qr/InnoDB: Warning: a long semaphore wait:/,
4443     qr/InnoDB: Dumping buffer pool.*/,
4444     qr/InnoDB: Buffer pool.*/,
4445     qr/InnoDB: Warning: Writer thread is waiting this semaphore:/,
4446     qr/InnoDB: innodb_open_files .* should not be greater than/,
4447     qr/Slave: Unknown table 't1' .* 1051/,
4448     qr/Slave SQL:.*(Internal MariaDB error code: [[:digit:]]+|Query:.*)/,
4449     qr/slave SQL thread aborted/,
4450     qr/unknown option '--loose[-_]/,
4451     qr/unknown variable 'loose[-_]/,
4452     #qr/Invalid .*old.* table or database name/,
4453     qr/Now setting lower_case_table_names to [02]/,
4454     qr/Setting lower_case_table_names=2/,
4455     qr/You have forced lower_case_table_names to 0/,
4456     qr/deprecated/,
4457     qr/Slave SQL thread retried transaction/,
4458     qr/Slave \(additional info\)/,
4459     qr/Incorrect information in file/,
4460     qr/Slave I\/O: Get master SERVER_ID failed with error:.*/,
4461     qr/Slave I\/O: Get master clock failed with error:.*/,
4462     qr/Slave I\/O: Get master COLLATION_SERVER failed with error:.*/,
4463     qr/Slave I\/O: Get master TIME_ZONE failed with error:.*/,
4464     qr/Slave I\/O: Get master \@\@GLOBAL.gtid_domain_id failed with error:.*/,
4465     qr/Slave I\/O: Setting \@slave_connect_state failed with error:.*/,
4466     qr/Slave I\/O: Setting \@slave_gtid_strict_mode failed with error:.*/,
4467     qr/Slave I\/O: Setting \@slave_gtid_ignore_duplicates failed with error:.*/,
4468     qr/Slave I\/O: Setting \@slave_until_gtid failed with error:.*/,
4469     qr/Slave I\/O: Get master GTID position failed with error:.*/,
4470     qr/Slave I\/O: error reconnecting to master '.*' - retry-time: [1-3]  retries/,
4471     qr/Slave I\/0: Master command COM_BINLOG_DUMP failed/,
4472     qr/Error reading packet/,
4473     qr/Lost connection to MariaDB server at 'reading initial communication packet'/,
4474     qr/Failed on request_dump/,
4475     qr/Slave: Can't drop database.* database doesn't exist/,
4476     qr/Slave: Operation DROP USER failed for 'create_rout_db'/,
4477     qr|Checking table:   '\..mtr.test_suppressions'|,
4478     qr|Table \./test/bug53592 has a primary key in InnoDB data dictionary, but not in|,
4479     qr|Table '\..mtr.test_suppressions' is marked as crashed and should be repaired|,
4480     qr|Table 'test_suppressions' is marked as crashed and should be repaired|,
4481     qr|Can't open shared library|,
4482     qr|Couldn't load plugin named .*EXAMPLE.*|,
4483     qr|InnoDB: Error: table 'test/bug39438'|,
4484     qr| entry '.*' ignored in --skip-name-resolve mode|,
4485     qr|mysqld got signal 6|,
4486     qr|Error while setting value 'pool-of-threads' to 'thread_handling'|,
4487     qr|Access denied for user|,
4488     qr|Aborted connection|,
4489     qr|table.*is full|,
4490     qr|Linux Native AIO|, # warning that aio does not work on /dev/shm
4491     qr|InnoDB: io_setup\(\) attempt|,
4492     qr|InnoDB: io_setup\(\) failed with EAGAIN|,
4493     qr|setrlimit could not change the size of core files to 'infinity';|,
4494     qr|feedback plugin: failed to retrieve the MAC address|,
4495     qr|Plugin 'FEEDBACK' init function returned error|,
4496     qr|Plugin 'FEEDBACK' registration as a INFORMATION SCHEMA failed|,
4497     qr|'log-bin-use-v1-row-events' is MySQL .* compatible option|,
4498     qr|InnoDB: Setting thread \d+ nice to \d+ failed, current nice \d+, errno 13|, # setpriority() fails under valgrind
4499     qr|Failed to setup SSL|,
4500     qr|SSL error: Failed to set ciphers to use|,
4501     qr/Plugin 'InnoDB' will be forced to shutdown/,
4502     qr|Could not increase number of max_open_files to more than|,
4503     qr|Changed limits: max_open_files|,
4504     qr/InnoDB: Error table encrypted but encryption service not available.*/,
4505     qr/InnoDB: Could not find a valid tablespace file for*/,
4506     qr/InnoDB: Tablespace open failed for*/,
4507     qr/InnoDB: Failed to find tablespace for table*/,
4508     qr/InnoDB: Space */,
4509     qr|InnoDB: You may have to recover from a backup|,
4510     qr|InnoDB: It is also possible that your operatingsystem has corrupted its own file cache|,
4511     qr|InnoDB: and rebooting your computer removes the error|,
4512     qr|InnoDB: If the corrupt page is an index page you can also try to|,
4513     qr|nnoDB: fix the corruption by dumping, dropping, and reimporting|,
4514     qr|InnoDB: the corrupt table. You can use CHECK|,
4515     qr|InnoDB: TABLE to scan your table for corruption|,
4516     qr/InnoDB: See also */,
4517     qr/InnoDB: Cannot open .*ib_buffer_pool.* for reading: No such file or directory*/,
4518     qr/InnoDB: Table .*mysql.*innodb_table_stats.* not found./,
4519     qr/InnoDB: User stopword table .* does not exist./,
4520     qr/Detected table cache mutex contention at instance .* waits. Additional table cache instance cannot be activated: consider raising table_open_cache_instances. Number of active instances/
4521    );
4522
4523  my $matched_lines= [];
4524  LINE: foreach my $line ( @lines )
4525  {
4526    PAT: foreach my $pat ( @patterns )
4527    {
4528      if ( $line =~ /$pat/ )
4529      {
4530        foreach my $apat (@antipatterns)
4531        {
4532          next LINE if $line =~ $apat;
4533        }
4534	print $Fwarn $line;
4535        push @$matched_lines, $line;
4536	last PAT;
4537      }
4538    }
4539  }
4540  $Fwarn = undef; # Close file
4541
4542  if (scalar(@$matched_lines) > 0 &&
4543      defined($last_warning_position->{$error_log}{test_names})) {
4544    return ($last_warning_position->{$error_log}{test_names}, $matched_lines);
4545  } else {
4546    return ([], $matched_lines);
4547  }
4548}
4549
4550
4551# Run include/check-warnings.test
4552#
4553# RETURN VALUE
4554#  0 OK
4555#  1 Check failed
4556#
4557sub start_check_warnings ($$) {
4558  my $tinfo=    shift;
4559  my $mysqld=   shift;
4560
4561  my $name= "warnings-".$mysqld->name();
4562
4563  my $log_error= $mysqld->value('log-error');
4564  # To be communicated to the test
4565  $ENV{MTR_LOG_ERROR}= $log_error;
4566  extract_warning_lines($log_error, 0);
4567
4568  my $args;
4569  mtr_init_args(\$args);
4570
4571  mtr_add_arg($args, "--defaults-file=%s", $path_config_file);
4572  mtr_add_arg($args, "--defaults-group-suffix=%s", $mysqld->after('mysqld'));
4573  mtr_add_arg($args, "--test-file=%s", "include/check-warnings.test");
4574
4575  if ( $opt_embedded_server )
4576  {
4577
4578    # Get the args needed for the embedded server
4579    # and append them to args prefixed
4580    # with --sever-arg=
4581
4582    my $mysqld=  $config->group('embedded')
4583      or mtr_error("Could not get [embedded] section");
4584
4585    my $mysqld_args;
4586    mtr_init_args(\$mysqld_args);
4587    my $extra_opts= get_extra_opts($mysqld, $tinfo);
4588    mysqld_arguments($mysqld_args, $mysqld, $extra_opts);
4589    mtr_add_arg($args, "--server-arg=%s", $_) for @$mysqld_args;
4590  }
4591
4592  my $errfile= "$opt_vardir/tmp/$name.err";
4593  my $proc= My::SafeProcess->new
4594    (
4595     name          => $name,
4596     path          => $exe_mysqltest,
4597     error         => $errfile,
4598     output        => $errfile,
4599     args          => \$args,
4600     user_data     => [$errfile, $mysqld],
4601     verbose       => $opt_verbose,
4602    );
4603  mtr_verbose("Started $proc");
4604  return $proc;
4605}
4606
4607
4608#
4609# Loop through our list of processes and check the error log
4610# for unexpected errors and warnings
4611#
4612sub check_warnings ($) {
4613  my ($tinfo)= @_;
4614  my $res= 0;
4615
4616  my $tname= $tinfo->{name};
4617
4618  # Clear previous warnings
4619  delete($tinfo->{warnings});
4620
4621  # Start the mysqltest processes in parallel to save time
4622  # also makes it possible to wait for any process to exit during the check
4623  my %started;
4624  foreach my $mysqld ( mysqlds() )
4625  {
4626    if ( defined $mysqld->{'proc'} )
4627    {
4628      my $proc= start_check_warnings($tinfo, $mysqld);
4629      $started{$proc->pid()}= $proc;
4630    }
4631  }
4632
4633  # Return immediately if no check proceess was started
4634  return 0 unless ( keys %started );
4635
4636  my $timeout= start_timer(check_timeout($tinfo));
4637
4638  my $result= 0;
4639  while (1){
4640    my $proc= My::SafeProcess->wait_any_timeout($timeout);
4641    mtr_report("Got $proc");
4642
4643    if ( delete $started{$proc->pid()} ) {
4644      # One check warning process returned
4645      my $res= $proc->exit_status();
4646      my ($err_file, $mysqld)= @{$proc->user_data()};
4647
4648      if ( $res == 0 or $res == 62 ){
4649
4650	if ( $res == 0 ) {
4651	  # Check completed with problem
4652	  my $report= mtr_grab_file($err_file);
4653	  # Log to var/log/warnings file
4654	  mtr_tofile("$opt_vardir/log/warnings",
4655		     $tname."\n".$report);
4656
4657	  $tinfo->{'warnings'}.= $report;
4658	  $result= 1;
4659	}
4660
4661	if ( $res == 62 ) {
4662	  # Test case was ok and called "skip"
4663	  # Remove the .err file the check generated
4664	  unlink($err_file);
4665	}
4666
4667	if ( keys(%started) == 0){
4668	  # All checks completed
4669	  mark_time_used('ch-warn');
4670	  return $result;
4671	}
4672	# Wait for next process to exit
4673	next;
4674      }
4675      else
4676      {
4677	my $report= mtr_grab_file($err_file);
4678	$tinfo->{comment}.=
4679	  "Could not execute 'check-warnings' for ".
4680	    "testcase '$tname' (res: $res) server: '".
4681              $mysqld->name() .":\n";
4682	$tinfo->{comment}.= $report;
4683
4684	$result= 2;
4685      }
4686    }
4687    elsif ( $proc->{timeout} ) {
4688      $tinfo->{comment}.= "Timeout for 'check warnings' expired after "
4689	.check_timeout($tinfo)." seconds";
4690      $result= 4;
4691    }
4692    else {
4693      # Unknown process returned, most likley a crash, abort everything
4694      $tinfo->{comment}=
4695	"The server $proc crashed while running 'check warnings'".
4696	get_log_from_proc($proc, $tinfo->{name});
4697      $result= 3;
4698    }
4699
4700    # Kill any check processes still running
4701    map($_->kill(), values(%started));
4702
4703    mark_time_used('ch-warn');
4704    return $result;
4705  }
4706
4707  mtr_error("INTERNAL_ERROR: check_warnings");
4708}
4709
4710# Check for warnings generated during shutdown of a mysqld server.
4711# If any, report them to master server, and return true; else just return
4712# false.
4713
4714sub check_warnings_post_shutdown {
4715  my ($server_socket)= @_;
4716  my $testname_hash= { };
4717  my $report= '';
4718  foreach my $mysqld ( mysqlds())
4719  {
4720    my ($testlist, $match_lines)=
4721        extract_warning_lines($mysqld->value('log-error'), 1);
4722    $testname_hash->{$_}= 1 for @$testlist;
4723    $report.= join('', @$match_lines);
4724  }
4725  my @warning_tests= keys(%$testname_hash);
4726  if (@warning_tests) {
4727    my $fake_test= My::Test->new(testnames => \@warning_tests);
4728    $fake_test->{'warnings'}= $report;
4729    $fake_test->write_test($server_socket, 'WARNINGS');
4730  }
4731}
4732
4733#
4734# Check for the file indicating expected crash and restart it.
4735#
4736sub check_expected_crash_and_restart {
4737  my $mysqld = shift;
4738
4739  # Check if crash expected by looking at the .expect file
4740  # in var/tmp
4741  my $expect_file= "$opt_vardir/tmp/".$mysqld->name().".expect";
4742  if ( -f $expect_file )
4743  {
4744    mtr_verbose("Crash was expected, file '$expect_file' exists");
4745
4746    for (my $waits = 0;  $waits < 50;  mtr_milli_sleep(100), $waits++)
4747    {
4748      # Race condition seen on Windows: try again until file not empty
4749      next if -z $expect_file;
4750      # If last line in expect file starts with "wait"
4751      # sleep a little and try again, thus allowing the
4752      # test script to control when the server should start
4753      # up again. Keep trying for up to 5s at a time.
4754      my $last_line= mtr_lastlinesfromfile($expect_file, 1);
4755      if ($last_line =~ /^wait/ )
4756      {
4757        mtr_verbose("Test says wait before restart") if $waits == 0;
4758        next;
4759      }
4760
4761      # Ignore any partial or unknown command
4762      next unless $last_line =~ /^restart/;
4763      # If last line begins "restart:", the rest of the line is read as
4764      # extra command line options to add to the restarted mysqld.
4765      # Anything other than 'wait' or 'restart:' (with a colon) will
4766      # result in a restart with original mysqld options.
4767      if ($last_line =~ /restart:(.+)/) {
4768        my @rest_opt= split(' ', $1);
4769        $mysqld->{'restart_opts'}= \@rest_opt;
4770      } else {
4771        delete $mysqld->{'restart_opts'};
4772      }
4773      unlink($expect_file);
4774
4775      # Start server with same settings as last time
4776      return mysqld_start($mysqld, $mysqld->{'started_opts'});
4777    }
4778    # Loop ran through: we should keep waiting after a re-check
4779    return 2;
4780  }
4781
4782  # Not an expected crash
4783  return 0;
4784}
4785
4786
4787# Remove all files and subdirectories of a directory
4788sub clean_dir {
4789  my ($dir)= @_;
4790  mtr_verbose("clean_dir: $dir");
4791  finddepth(
4792	  { no_chdir => 1,
4793	    wanted => sub {
4794	      if (-d $_){
4795		# A dir
4796		if ($_ eq $dir){
4797		  # The dir to clean
4798		  return;
4799		} else {
4800		  mtr_verbose("rmdir: '$_'");
4801		  rmdir($_) or mtr_warning("rmdir($_) failed: $!");
4802		}
4803	      } else {
4804		# Hopefully a file
4805		mtr_verbose("unlink: '$_'");
4806		unlink($_) or mtr_warning("unlink($_) failed: $!");
4807	      }
4808	    }
4809	  },
4810	    $dir);
4811}
4812
4813
4814sub clean_datadir {
4815  mtr_verbose("Cleaning datadirs...");
4816
4817  if (started(all_servers()) != 0){
4818    mtr_error("Trying to clean datadir before all servers stopped");
4819  }
4820
4821  for (all_servers())
4822  {
4823    my $dir= "$opt_vardir/".$_->{name};
4824    mtr_verbose(" - removing '$dir'");
4825    rmtree($dir);
4826  }
4827
4828  # Remove all files in tmp and var/tmp
4829  clean_dir("$opt_vardir/tmp");
4830  if ($opt_tmpdir ne "$opt_vardir/tmp"){
4831    clean_dir($opt_tmpdir);
4832  }
4833}
4834
4835
4836#
4837# Save datadir before it's removed
4838#
4839sub save_datadir_after_failure($$) {
4840  my ($dir, $savedir)= @_;
4841
4842  mtr_report(" - saving '$dir'");
4843  my $dir_name= basename($dir);
4844  rename("$dir", "$savedir/$dir_name");
4845}
4846
4847
4848sub after_failure ($) {
4849  my ($tinfo)= @_;
4850
4851  mtr_report("Saving datadirs...");
4852
4853  my $save_dir= "$opt_vardir/log/";
4854  $save_dir.= $tinfo->{name};
4855  # Add combination name if any
4856  $save_dir.= '-' . join(',', sort @{$tinfo->{combinations}})
4857    if defined $tinfo->{combinations};
4858
4859  # Save savedir  path for server
4860  $tinfo->{savedir}= $save_dir;
4861
4862  mkpath($save_dir) if ! -d $save_dir;
4863
4864  #
4865  # Create a log of files in vardir (good for buildbot)
4866  #
4867  if (!IS_WINDOWS)
4868  {
4869    my $Flog= IO::File->new("$opt_vardir/log/files.log", "w");
4870    if ($Flog)
4871    {
4872      print $Flog scalar(`/bin/ls -Rl $opt_vardir/*`);
4873      close($Flog);
4874    }
4875  }
4876
4877  # Save the used config files
4878  my %config_files = config_files($tinfo);
4879  while (my ($file, $generate) = each %config_files) {
4880    copy("$opt_vardir/$file", $save_dir);
4881  }
4882
4883  # Copy the tmp dir
4884  copytree("$opt_vardir/tmp/", "$save_dir/tmp/");
4885
4886  foreach (all_servers()) {
4887    my $dir= "$opt_vardir/".$_->{name};
4888    save_datadir_after_failure($dir, $save_dir);
4889  }
4890}
4891
4892
4893sub report_failure_and_restart ($) {
4894  my $tinfo= shift;
4895
4896  if ($opt_valgrind && ($tinfo->{'warnings'} || $tinfo->{'timeout'}) &&
4897      $opt_core_on_failure == 0)
4898  {
4899    # In these cases we may want valgrind report from normal termination
4900    $tinfo->{'dont_kill_server'}= 1;
4901  }
4902  # Shutdown properly if not to be killed (for valgrind)
4903  stop_all_servers($tinfo->{'dont_kill_server'} ? $opt_shutdown_timeout : 0);
4904
4905  $tinfo->{'result'}= 'MTR_RES_FAILED';
4906
4907  my $test_failures= $tinfo->{'failures'} || 0;
4908  $tinfo->{'failures'}=  $test_failures + 1;
4909
4910
4911  if ( $tinfo->{comment} )
4912  {
4913    # The test failure has been detected by mysql-test-run.pl
4914    # when starting the servers or due to other error, the reason for
4915    # failing the test is saved in "comment"
4916    ;
4917  }
4918
4919  if ( !defined $tinfo->{logfile} )
4920  {
4921    my $logfile= $path_current_testlog;
4922    if ( defined $logfile )
4923    {
4924      if ( -f $logfile )
4925      {
4926	# Test failure was detected by test tool and its report
4927	# about what failed has been saved to file. Save the report
4928	# in tinfo
4929	$tinfo->{logfile}= mtr_fromfile($logfile);
4930	# If no newlines in the test log:
4931	# (it will contain the CURRENT_TEST written by mtr, so is not empty)
4932	if ($tinfo->{logfile} !~ /\n/)
4933	{
4934	  # Show how far it got before suddenly failing
4935	  $tinfo->{comment}.= "mysqltest failed but provided no output\n";
4936	  my $log_file_name= $opt_vardir."/log/".$tinfo->{shortname}.".log";
4937	  if (-e $log_file_name) {
4938	    $tinfo->{comment}.=
4939	      "The result from queries just before the failure was:".
4940	      "\n< snip >\n".
4941	      mtr_lastlinesfromfile($log_file_name, $opt_tail_lines)."\n";
4942	  }
4943	}
4944      }
4945      else
4946      {
4947	# The test tool report didn't exist, display an
4948	# error message
4949	$tinfo->{logfile}= "Could not open test tool report '$logfile'";
4950      }
4951    }
4952  }
4953
4954  after_failure($tinfo);
4955
4956  mtr_report_test($tinfo);
4957
4958}
4959
4960
4961sub run_system(@) {
4962  mtr_verbose("Running '$_[0]'");
4963  my $ret= system(@_) >> 8;
4964  return $ret;
4965}
4966
4967
4968sub mysqld_stop {
4969  my $mysqld= shift or die "usage: mysqld_stop(<mysqld>)";
4970
4971  my $args;
4972  mtr_init_args(\$args);
4973
4974  mtr_add_arg($args, "--no-defaults");
4975  mtr_add_arg($args, "--character-sets-dir=%s", $mysqld->value('character-sets-dir'));
4976  mtr_add_arg($args, "--user=%s", $opt_user);
4977  mtr_add_arg($args, "--password=");
4978  mtr_add_arg($args, "--port=%d", $mysqld->value('port'));
4979  mtr_add_arg($args, "--host=%s", $mysqld->value('#host'));
4980  mtr_add_arg($args, "--connect_timeout=20");
4981  mtr_add_arg($args, "--protocol=tcp");
4982
4983  mtr_add_arg($args, "shutdown");
4984
4985  My::SafeProcess->run
4986    (
4987     name          => "mysqladmin shutdown ".$mysqld->name(),
4988     path          => $exe_mysqladmin,
4989     args          => \$args,
4990     error         => "$opt_vardir/log/mysqladmin.err",
4991
4992    );
4993}
4994
4995
4996sub mysqld_arguments ($$$) {
4997  my $args=              shift;
4998  my $mysqld=            shift;
4999  my $extra_opts=        shift;
5000
5001  my @defaults = grep(/^--defaults-file=/, @$extra_opts);
5002  if (@defaults > 0) {
5003    mtr_add_arg($args, pop(@defaults))
5004  }
5005  else {
5006    mtr_add_arg($args, "--defaults-file=%s",  $path_config_file);
5007  }
5008
5009  # When mysqld is run by a root user(euid is 0), it will fail
5010  # to start unless we specify what user to run as, see BUG#30630
5011  my $euid= $>;
5012  if (!IS_WINDOWS and $euid == 0 and
5013      (grep(/^--user/, @$extra_opts)) == 0) {
5014    mtr_add_arg($args, "--user=root");
5015  }
5016
5017  if (!using_extern() and !$opt_user_args)
5018  {
5019    # Turn on logging to file
5020    mtr_add_arg($args, "%s--log-output=file");
5021  }
5022
5023  # Check if "extra_opt" contains --log-bin
5024  my $skip_binlog= not grep /^--(loose-)?log-bin/, @$extra_opts;
5025
5026  my $found_skip_core= 0;
5027  foreach my $arg ( @$extra_opts )
5028  {
5029    # Skip --defaults-file option since it's handled above.
5030    next if $arg =~ /^--defaults-file/;
5031
5032    # Allow --skip-core-file to be set in <testname>-[master|slave].opt file
5033    if ($arg eq "--skip-core-file")
5034    {
5035      $found_skip_core= 1;
5036    }
5037    elsif ($skip_binlog and mtr_match_prefix($arg, "--binlog-format"))
5038    {
5039      ; # Dont add --binlog-format when running without binlog
5040    }
5041    elsif ($arg eq "--loose-skip-log-bin" and
5042           $mysqld->option("log-slave-updates"))
5043    {
5044      ; # Dont add --skip-log-bin when mysqld have --log-slave-updates in config
5045    }
5046    else
5047    {
5048      mtr_add_arg($args, "%s", $arg);
5049    }
5050  }
5051  $opt_skip_core = $found_skip_core;
5052  if ( !$found_skip_core && !$opt_user_args )
5053  {
5054    mtr_add_arg($args, "%s", "--core-file");
5055  }
5056
5057  # Enable the debug sync facility, set default wait timeout.
5058  # Facility stays disabled if timeout value is zero.
5059  mtr_add_arg($args, "--loose-debug-sync-timeout=%s",
5060              $opt_debug_sync_timeout) unless $opt_user_args;
5061
5062  return $args;
5063}
5064
5065
5066
5067sub mysqld_start ($$) {
5068  my $mysqld=            shift;
5069  my $extra_opts=        shift;
5070
5071  mtr_verbose(My::Options::toStr("mysqld_start", @$extra_opts));
5072
5073  my $exe= find_mysqld($mysqld->value('basedir'));
5074
5075  mtr_error("Internal error: mysqld should never be started for embedded")
5076    if $opt_embedded_server;
5077
5078  my $args;
5079  mtr_init_args(\$args);
5080
5081  mtr_add_arg($args, "--defaults-group-suffix=%s", $mysqld->after('mysqld'));
5082
5083  # Add any additional options from an in-test restart
5084  my @all_opts= @$extra_opts;
5085  if (exists $mysqld->{'restart_opts'}) {
5086    push (@all_opts, @{$mysqld->{'restart_opts'}});
5087    mtr_verbose(My::Options::toStr("mysqld_start restart",
5088				   @{$mysqld->{'restart_opts'}}));
5089  }
5090  mysqld_arguments($args,$mysqld,\@all_opts);
5091
5092  if ( $opt_debug )
5093  {
5094    mtr_add_arg($args, "--debug-dbug=$debug_d:t:i:A,%s/log/%s.trace",
5095		$path_vardir_trace, $mysqld->name());
5096  }
5097
5098
5099  # Command line for mysqld started for *this particular test*.
5100  # Differs from "generic" MYSQLD_CMD by including all command line
5101  # options from *.opt and *.combination files.
5102  $ENV{'MYSQLD_LAST_CMD'}= "$exe  @$args";
5103  my $oldexe= $exe;
5104
5105  My::Debugger::setup_args(\$args, \$exe, $mysqld->name());
5106  $ENV{'VALGRIND_TEST'}= $opt_valgrind = int(($exe || '') eq 'valgrind');
5107
5108  # Remove the old pidfile if any
5109  unlink($mysqld->value('pid-file'));
5110
5111  my $output= $mysqld->value('log-error');
5112
5113  if ( $opt_valgrind and $opt_debug )
5114  {
5115    # When both --valgrind and --debug is selected, send
5116    # all output to the trace file, making it possible to
5117    # see the exact location where valgrind complains
5118
5119    # Write a message about this to the normal log file
5120    my $trace_name= "$opt_vardir/log/".$mysqld->name().".trace";
5121    mtr_tofile($output,
5122               "NOTE: When running with --valgrind --debug the output from ",
5123	       "mysqld (where the valgrind messages shows up) is stored ",
5124	       "together with the trace file to make it ",
5125	       "easier to find the exact position of valgrind errors.",
5126	       "See trace file $trace_name.\n");
5127    $output= $trace_name;
5128
5129  }
5130  # Remember this log file for valgrind error report search
5131  $mysqld_logs{$output}= 1 if $opt_valgrind;
5132  # Remember data dir for gmon.out files if using gprof
5133  $gprof_dirs{$mysqld->value('datadir')}= 1 if $opt_gprof;
5134
5135  if ( defined $exe )
5136  {
5137    mtr_tofile($output, "\$ $exe @$args\n");
5138    pre_write_errorlog($output);
5139    $mysqld->{'proc'}= My::SafeProcess->new
5140      (
5141       name          => $mysqld->name(),
5142       path          => $exe,
5143       args          => \$args,
5144       output        => $output,
5145       error         => $output,
5146       append        => 1,
5147       verbose       => $opt_verbose,
5148       nocore        => $opt_skip_core,
5149       host          => undef,
5150       shutdown      => sub { mysqld_stop($mysqld) },
5151       envs          => \@opt_mysqld_envs,
5152      );
5153    mtr_verbose("Started $mysqld->{proc}");
5154  }
5155
5156  $mysqld->{'started_opts'}= $extra_opts;
5157
5158  my $expect_file= "$opt_vardir/tmp/".$mysqld->name().".expect";
5159  return $oldexe eq ($exe || '') ||
5160         sleep_until_file_created($mysqld->value('pid-file'), $expect_file,
5161                     $opt_start_timeout, $mysqld->{'proc'}, $warn_seconds);
5162}
5163
5164
5165sub stop_all_servers () {
5166  my $shutdown_timeout = $_[0] or 0;
5167
5168  mtr_verbose("Stopping all servers...");
5169
5170  # Kill all started servers
5171  My::SafeProcess::shutdown($shutdown_timeout,
5172			    started(all_servers()));
5173
5174  # Remove pidfiles
5175  foreach my $server ( all_servers() )
5176  {
5177    my $pid_file= $server->if_exist('pid-file');
5178    unlink($pid_file) if defined $pid_file;
5179  }
5180
5181  # Mark servers as stopped
5182  map($_->{proc}= undef, all_servers());
5183
5184}
5185
5186
5187# Find out if server should be restarted for this test
5188sub server_need_restart {
5189  my ($tinfo, $server)= @_;
5190
5191  if ( using_extern() )
5192  {
5193    mtr_verbose_restart($server, "no restart for --extern server");
5194    return 0;
5195  }
5196
5197  if ( $tinfo->{'force_restart'} ) {
5198    mtr_verbose_restart($server, "forced in .opt file");
5199    return 1;
5200  }
5201
5202  if ( $opt_force_restart ) {
5203    mtr_verbose_restart($server, "forced restart turned on");
5204    return 1;
5205  }
5206
5207  if ( $tinfo->{template_path} ne $current_config_name)
5208  {
5209    mtr_verbose_restart($server, "using different config file");
5210    return 1;
5211  }
5212
5213  if ( $tinfo->{'master_sh'}  || $tinfo->{'slave_sh'} )
5214  {
5215    mtr_verbose_restart($server, "sh script to run");
5216    return 1;
5217  }
5218
5219  if ( ! started($server) )
5220  {
5221    mtr_verbose_restart($server, "not started");
5222    return 1;
5223  }
5224
5225  my $started_tinfo= $server->{'started_tinfo'};
5226  if ( defined $started_tinfo )
5227  {
5228
5229    # Check if timezone of  test that server was started
5230    # with differs from timezone of next test
5231    if ( timezone($started_tinfo) ne timezone($tinfo) )
5232    {
5233      mtr_verbose_restart($server, "different timezone");
5234      return 1;
5235    }
5236  }
5237
5238  if ($server->name() =~ /^mysqld\./)
5239  {
5240
5241    # Check that running process was started with same options
5242    # as the current test requires
5243    my $extra_opts= get_extra_opts($server, $tinfo);
5244    my $started_opts= $server->{'started_opts'};
5245
5246    # Also, always restart if server had been restarted with additional
5247    # options within test.
5248    if (!My::Options::same($started_opts, $extra_opts) ||
5249        exists $server->{'restart_opts'})
5250    {
5251      delete $server->{'restart_opts'};
5252      my $use_dynamic_option_switch= 0;
5253      my $restart_opts = delete $server->{'restart_opts'} || [];
5254      if (!$use_dynamic_option_switch)
5255      {
5256	mtr_verbose_restart($server, "running with different options '" .
5257			    join(" ", @{$extra_opts}) . "' != '" .
5258			    join(" ", @{$started_opts}, @{$restart_opts}) . "'" );
5259	return 1;
5260      }
5261
5262      mtr_verbose(My::Options::toStr("started_opts", @$started_opts));
5263      mtr_verbose(My::Options::toStr("extra_opts", @$extra_opts));
5264
5265      # Get diff and check if dynamic switch is possible
5266      my @diff_opts= My::Options::diff($started_opts, $extra_opts);
5267      mtr_verbose(My::Options::toStr("diff_opts", @diff_opts));
5268
5269      my $query= My::Options::toSQL(@diff_opts);
5270      mtr_verbose("Attempting dynamic switch '$query'");
5271      if (run_query($tinfo, $server, $query)){
5272	mtr_verbose("Restart: running with different options '" .
5273		    join(" ", @{$extra_opts}) . "' != '" .
5274		    join(" ", @{$started_opts}) . "'" );
5275	return 1;
5276      }
5277
5278      # Remember the dynamically set options
5279      $server->{'started_opts'}= $extra_opts;
5280    }
5281  }
5282
5283  # Default, no restart
5284  return 0;
5285}
5286
5287
5288sub servers_need_restart($) {
5289  my ($tinfo)= @_;
5290  return grep { server_need_restart($tinfo, $_); } all_servers();
5291}
5292
5293
5294
5295############################################
5296
5297############################################
5298
5299#
5300# Filter a list of servers and return the SafeProcess
5301# for only those that are started or stopped
5302#
5303sub started { return grep(defined $_, map($_->{proc}, @_));  }
5304sub stopped { return grep(!defined $_, map($_->{proc}, @_)); }
5305
5306
5307sub get_extra_opts {
5308  # No extra options if --user-args
5309  return \@opt_extra_mysqld_opt if $opt_user_args;
5310
5311  my ($mysqld, $tinfo)= @_;
5312
5313  my $opts=
5314    $mysqld->option("#!use-slave-opt") ?
5315      $tinfo->{slave_opt} : $tinfo->{master_opt};
5316
5317  # Expand environment variables
5318  foreach my $opt ( @$opts )
5319  {
5320    no warnings 'uninitialized';
5321    $opt =~ s/\$\{(\w+)\}/$ENV{$1}/ge;
5322    $opt =~ s/\$(\w+)/$ENV{$1}/ge;
5323  }
5324  return $opts;
5325}
5326
5327
5328sub stop_servers($$) {
5329  my (@servers)= @_;
5330
5331  mtr_report("Stopping ", started(@servers));
5332
5333  My::SafeProcess::shutdown($opt_shutdown_timeout,
5334                             started(@servers));
5335
5336  foreach my $server (@servers)
5337  {
5338    # Mark server as stopped
5339    $server->{proc}= undef;
5340
5341    # Forget history
5342    delete $server->{'started_tinfo'};
5343    delete $server->{'started_opts'};
5344    delete $server->{'started_cnf'};
5345  }
5346}
5347
5348#
5349# run_query_output
5350#
5351# Run a query against a server using mysql client. The output of
5352# the query will be written into outfile.
5353#
5354sub run_query_output {
5355  my ($mysqld, $query, $outfile)= @_;
5356  my $args;
5357
5358  mtr_init_args(\$args);
5359  mtr_add_arg($args, "--defaults-file=%s", $path_config_file);
5360  mtr_add_arg($args, "--defaults-group-suffix=%s", $mysqld->after('mysqld'));
5361  mtr_add_arg($args, "--silent");
5362  mtr_add_arg($args, "--execute=%s", $query);
5363
5364  my $res= My::SafeProcess->run
5365  (
5366    name          => "run_query_output -> ".$mysqld->name(),
5367    path          => $exe_mysql,
5368    args          => \$args,
5369    output        => $outfile,
5370    error         => $outfile
5371  );
5372
5373  return $res
5374}
5375
5376
5377#
5378# wsrep_wait_ready
5379#
5380# Wait until the server has been joined to the cluster and is
5381# ready for operation.
5382#
5383# RETURN
5384# 1 Server is ready
5385# 0 Server didn't transition to ready state within start timeout
5386#
5387sub wait_wsrep_ready($$) {
5388  my ($tinfo, $mysqld)= @_;
5389
5390  my $sleeptime= 100; # Milliseconds
5391  my $loops= ($opt_start_timeout * 1000) / $sleeptime;
5392
5393  my $name= $mysqld->name();
5394  my $outfile= "$opt_vardir/tmp/$name.wsrep_ready";
5395  my $query= "SET SESSION wsrep_sync_wait = 0;
5396              SELECT VARIABLE_NAME, VARIABLE_VALUE
5397              FROM INFORMATION_SCHEMA.GLOBAL_STATUS
5398              WHERE VARIABLE_NAME = 'wsrep_ready'";
5399
5400  for (my $loop= 1; $loop <= $loops; $loop++)
5401  {
5402    # Careful... if MTR runs with option 'verbose' then the
5403    # file contains also SafeProcess verbose output
5404    if (run_query_output($mysqld, $query, $outfile) == 0 &&
5405        mtr_grab_file($outfile) =~ /WSREP_READY\s+ON/)
5406    {
5407      unlink($outfile);
5408      return 1;
5409    }
5410    mtr_milli_sleep($sleeptime);
5411  }
5412
5413  $tinfo->{logfile}= "WSREP did not transition to state READY";
5414  return 0;
5415}
5416
5417#
5418# wsrep_is_bootstrap_server
5419#
5420# Check if the server is the first one to be started in the
5421# cluster.
5422#
5423# RETURN
5424# 1 The server is a bootstrap server
5425# 0 The server is not a bootstrap server
5426#
5427sub wsrep_is_bootstrap_server($) {
5428  my $mysqld= shift;
5429
5430  my $cluster_address= $mysqld->if_exist('wsrep-cluster-address') ||
5431                       $mysqld->if_exist('wsrep_cluster_address');
5432  if (defined $cluster_address)
5433  {
5434    return $cluster_address eq "gcomm://" || $cluster_address eq "'gcomm://'";
5435  }
5436  return 0;
5437}
5438
5439#
5440# wsrep_on
5441#
5442# Check if wsrep has been enabled for a server.
5443#
5444# RETURN
5445# 1 Wsrep has been enabled
5446# 0 Wsrep is not enabled
5447#
5448sub wsrep_on($) {
5449  my $mysqld= shift;
5450  #check if wsrep_on=  is set in configuration
5451  if ($mysqld->if_exist('wsrep-on')) {
5452    my $on= "".$mysqld->value('wsrep-on');
5453    if ($on eq "1" || $on eq "ON") {
5454      return 1;
5455    }
5456  }
5457  return 0;
5458}
5459
5460
5461#
5462# start_servers
5463#
5464# Start servers not already started
5465#
5466# RETURN
5467#  0 OK
5468#  1 Start failed
5469#
5470sub start_servers($) {
5471  my ($tinfo)= @_;
5472
5473  for (all_servers()) {
5474    $_->{START}->($_, $tinfo) if $_->{START};
5475  }
5476
5477  for (all_servers()) {
5478    next unless $_->{WAIT} and started($_);
5479    if ($_->{WAIT}->($_, $tinfo)) {
5480      $tinfo->{comment}= "Failed to start ".$_->name() . "\n";
5481      return 1;
5482    }
5483  }
5484  return 0;
5485}
5486
5487
5488#
5489# Run include/check-testcase.test
5490# Before a testcase, run in record mode and save result file to var/tmp
5491# After testcase, run and compare with the recorded file, they should be equal!
5492#
5493# RETURN VALUE
5494#  The newly started process
5495#
5496sub start_check_testcase ($$$) {
5497  my $tinfo=    shift;
5498  my $mode=     shift;
5499  my $mysqld=   shift;
5500
5501  my $name= "check-".$mysqld->name();
5502  # Replace dots in name with underscore to avoid that mysqltest
5503  # misinterpret's what the filename extension is :(
5504  $name=~ s/\./_/g;
5505
5506  my $args;
5507  mtr_init_args(\$args);
5508
5509  mtr_add_arg($args, "--defaults-file=%s", $path_config_file);
5510  mtr_add_arg($args, "--defaults-group-suffix=%s", $mysqld->after('mysqld'));
5511  mtr_add_arg($args, "--result-file=%s", "$opt_vardir/tmp/$name.result");
5512  mtr_add_arg($args, "--test-file=%s", "include/check-testcase.test");
5513  mtr_add_arg($args, "--verbose");
5514
5515  if ( $mode eq "before" )
5516  {
5517    mtr_add_arg($args, "--record");
5518  }
5519  my $errfile= "$opt_vardir/tmp/$name.err";
5520  my $proc= My::SafeProcess->new
5521    (
5522     name          => $name,
5523     path          => $exe_mysqltest,
5524     error         => $errfile,
5525     output        => $errfile,
5526     args          => \$args,
5527     user_data     => $errfile,
5528     verbose       => $opt_verbose,
5529    );
5530
5531  mtr_report("Started $proc");
5532  return $proc;
5533}
5534
5535
5536sub run_mysqltest ($) {
5537  my $proc= start_mysqltest(@_);
5538  $proc->wait();
5539}
5540
5541
5542sub start_mysqltest ($) {
5543  my ($tinfo)= @_;
5544  my $exe= $exe_mysqltest;
5545  my $args;
5546
5547  mark_time_used('admin');
5548
5549  mtr_init_args(\$args);
5550
5551  mtr_add_arg($args, "--defaults-file=%s", $path_config_file);
5552  mtr_add_arg($args, "--silent");
5553  mtr_add_arg($args, "--tmpdir=%s", $opt_tmpdir);
5554  mtr_add_arg($args, "--character-sets-dir=%s", $path_charsetsdir);
5555  mtr_add_arg($args, "--logdir=%s/log", $opt_vardir);
5556
5557  # Log line number and time  for each line in .test file
5558  mtr_add_arg($args, "--mark-progress")
5559    if $opt_mark_progress;
5560
5561  mtr_add_arg($args, "--database=test");
5562
5563  if ( $opt_ps_protocol )
5564  {
5565    mtr_add_arg($args, "--ps-protocol");
5566  }
5567
5568  if ( $opt_sp_protocol )
5569  {
5570    mtr_add_arg($args, "--sp-protocol");
5571  }
5572
5573  if ( $opt_view_protocol )
5574  {
5575    mtr_add_arg($args, "--view-protocol");
5576  }
5577
5578  if ( $opt_cursor_protocol )
5579  {
5580    mtr_add_arg($args, "--cursor-protocol");
5581  }
5582
5583  if ( $opt_non_blocking_api )
5584  {
5585    mtr_add_arg($args, "--non-blocking-api");
5586  }
5587
5588  mtr_add_arg($args, "--timer-file=%s/log/timer", $opt_vardir);
5589
5590  if ( $opt_compress )
5591  {
5592    mtr_add_arg($args, "--compress");
5593  }
5594
5595  if ( $opt_sleep )
5596  {
5597    mtr_add_arg($args, "--sleep=%d", $opt_sleep);
5598  }
5599
5600  if ( $opt_ssl )
5601  {
5602    # Turn on SSL for _all_ test cases if option --ssl was used
5603    mtr_add_arg($args, "--ssl");
5604  }
5605
5606  if ( $opt_max_connections ) {
5607    mtr_add_arg($args, "--max-connections=%d", $opt_max_connections);
5608  }
5609
5610  if ( $opt_embedded_server )
5611  {
5612
5613    # Get the args needed for the embedded server
5614    # and append them to args prefixed
5615    # with --sever-arg=
5616
5617    my $mysqld=  $config->group('embedded')
5618      or mtr_error("Could not get [embedded] section");
5619
5620    my $mysqld_args;
5621    mtr_init_args(\$mysqld_args);
5622    my $extra_opts= get_extra_opts($mysqld, $tinfo);
5623    mysqld_arguments($mysqld_args, $mysqld, $extra_opts);
5624    mtr_add_arg($args, "--server-arg=%s", $_) for @$mysqld_args;
5625  }
5626
5627  # ----------------------------------------------------------------------
5628  # export MYSQL_TEST variable containing <path>/mysqltest <args>
5629  # ----------------------------------------------------------------------
5630  $ENV{'MYSQL_TEST'}= mtr_args2str($exe_mysqltest, @$args);
5631
5632  if ($opt_force > 1)
5633  {
5634    mtr_add_arg($args, "--continue-on-error");
5635  }
5636
5637  my $suite = $tinfo->{suite};
5638  if ($suite->{parent}) {
5639    mtr_add_arg($args, "--overlay-dir=%s/", $suite->{dir});
5640    mtr_add_arg($args, "--suite-dir=%s/", $suite->{parent}->{dir});
5641  } else {
5642    mtr_add_arg($args, "--suite-dir=%s/", $suite->{dir});
5643  }
5644
5645  mtr_add_arg($args, "--test-file=%s", $tinfo->{'path'});
5646
5647  # Number of lines of resut to include in failure report
5648  mtr_add_arg($args, "--tail-lines=%d", $opt_tail_lines);
5649
5650  if ( defined $tinfo->{'result_file'} ) {
5651    mtr_add_arg($args, "--result-file=%s", $tinfo->{'result_file'});
5652  }
5653
5654  client_debug_arg($args, "mysqltest");
5655
5656  if ( $opt_record )
5657  {
5658    mtr_add_arg($args, "--record");
5659
5660    # When recording to a non existing result file
5661    # the name of that file is in "record_file"
5662    if ( defined $tinfo->{'record_file'} ) {
5663      mtr_add_arg($args, "--result-file=%s", $tinfo->{record_file});
5664    }
5665  }
5666
5667  My::Debugger::setup_client_args(\$args, \$exe);
5668
5669  my $proc= My::SafeProcess->new
5670    (
5671     name          => "mysqltest",
5672     path          => $exe,
5673     args          => \$args,
5674     append        => 1,
5675     error         => $path_current_testlog,
5676     verbose       => $opt_verbose,
5677    );
5678  mtr_verbose("Started $proc");
5679  return $proc;
5680}
5681
5682#
5683# Search server logs for valgrind reports printed at mysqld termination
5684#
5685sub valgrind_exit_reports() {
5686  my $found_err= 0;
5687
5688  foreach my $log_file (keys %mysqld_logs)
5689  {
5690    my @culprits= ();
5691    my $valgrind_rep= "";
5692    my $found_report= 0;
5693    my $err_in_report= 0;
5694
5695    my $LOGF = IO::File->new($log_file)
5696      or mtr_error("Could not open file '$log_file' for reading: $!");
5697
5698    while ( my $line = <$LOGF> )
5699    {
5700      if ($line =~ /^CURRENT_TEST: (.+)$/)
5701      {
5702        my $testname= $1;
5703        # If we have a report, report it if needed and start new list of tests
5704        if ($found_report)
5705        {
5706          if ($err_in_report)
5707          {
5708            mtr_print ("Valgrind report from $log_file after tests:\n",
5709                        @culprits);
5710            mtr_print_line();
5711            print ("$valgrind_rep\n");
5712            $found_err= 1;
5713            $err_in_report= 0;
5714          }
5715          # Make ready to collect new report
5716          @culprits= ();
5717          $found_report= 0;
5718          $valgrind_rep= "";
5719        }
5720        push (@culprits, $testname);
5721        next;
5722      }
5723      # This line marks the start of a valgrind report
5724      $found_report= 1 if $line =~ /^==\d+== .* SUMMARY:/;
5725
5726      if ($found_report) {
5727        $line=~ s/^==\d+== //;
5728        $valgrind_rep .= $line;
5729        $err_in_report= 1 if $line =~ /ERROR SUMMARY: [1-9]/;
5730        $err_in_report= 1 if $line =~ /definitely lost: [1-9]/;
5731        $err_in_report= 1 if $line =~ /possibly lost: [1-9]/;
5732      }
5733    }
5734
5735    $LOGF= undef;
5736
5737    if ($err_in_report) {
5738      mtr_print ("Valgrind report from $log_file after tests:\n", @culprits);
5739      mtr_print_line();
5740      print ("$valgrind_rep\n");
5741      $found_err= 1;
5742    }
5743  }
5744
5745  return $found_err;
5746}
5747
5748#
5749# Usage
5750#
5751sub usage ($) {
5752  my ($message)= @_;
5753
5754  if ( $message )
5755  {
5756    print STDERR "$message\n";
5757    print STDERR "For full list of options, use $0 --help\n";
5758    exit;
5759  }
5760
5761  local $"= ','; # for @DEFAULT_SUITES below
5762
5763  print <<HERE . My::Debugger::help() . <<HERE;
5764
5765$0 [ OPTIONS ] [ TESTCASE ]
5766
5767Where test case can be specified as:
5768
5769testcase[.test]         Runs the test case named 'testcase' from all suits
5770path-to-testcase
5771[suite.]testcase[,combination]
5772
5773Examples:
5774
5775alias
5776main.alias              'main' is the name of the main test suite
5777rpl.rpl_invoked_features,mix,innodb
5778suite/rpl/t/rpl.rpl_invoked_features
5779
5780Options to control what engine/variation to run:
5781
5782  embedded-server       Use the embedded server, i.e. no mysqld daemons
5783  ps-protocol           Use the binary protocol between client and server
5784  cursor-protocol       Use the cursor protocol between client and server
5785                        (implies --ps-protocol)
5786  view-protocol         Create a view to execute all non updating queries
5787  sp-protocol           Create a stored procedure to execute all queries
5788  non-blocking-api      Use the non-blocking client API
5789  compress              Use the compressed protocol between client and server
5790  ssl                   Use ssl protocol between client and server
5791  skip-ssl              Don't start server with support for ssl connections
5792  vs-config             Visual Studio configuration used to create executables
5793                        (default: MTR_VS_CONFIG environment variable)
5794  parallel=#            How many parallel test should be run
5795  defaults-file=<config template> Use fixed config template for all
5796                        tests
5797  defaults-extra-file=<config template> Extra config template to add to
5798                        all generated configs
5799  combination=<opt>     Use at least twice to run tests with specified
5800                        options to mysqld
5801  dry-run               Don't run any tests, print the list of tests
5802                        that were selected for execution
5803
5804Options to control directories to use
5805  tmpdir=DIR            The directory where temporary files are stored
5806                        (default: ./var/tmp).
5807  vardir=DIR            The directory where files generated from the test run
5808                        is stored (default: ./var). Specifying a ramdisk or
5809                        tmpfs will speed up tests.
5810  mem[=DIR]             Run testsuite in "memory" using tmpfs or ramdisk
5811                        Attempts to use DIR first if specified else
5812                        uses a builtin list of standard locations
5813                        for tmpfs (/run/shm, /dev/shm, /tmp)
5814                        The option can also be set using environment
5815                        variable MTR_MEM=[DIR]
5816  clean-vardir          Clean vardir if tests were successful and if
5817                        running in "memory". Otherwise this option is ignored
5818  client-bindir=PATH    Path to the directory where client binaries are located
5819  client-libdir=PATH    Path to the directory where client libraries are located
5820
5821
5822Options to control what test suites or cases to run
5823
5824  force                 Continue after a failure. When specified once, a
5825                        failure in a test file will abort this test file, and
5826                        the execution will continue from the next test file.
5827                        When specified twice, execution will continue executing
5828                        the failed test file from the next command.
5829  do-test=PREFIX or REGEX
5830                        Run test cases which name are prefixed with PREFIX
5831                        or fulfills REGEX
5832  skip-test=PREFIX or REGEX
5833                        Skip test cases which name are prefixed with PREFIX
5834                        or fulfills REGEX
5835  start-from=PREFIX     Run test cases starting test prefixed with PREFIX where
5836                        prefix may be suite.testname or just testname
5837  suite[s]=NAME1,..,NAMEN
5838                        Collect tests in suites from the comma separated
5839                        list of suite names.
5840                        The default is: "@DEFAULT_SUITES"
5841  skip-rpl              Skip the replication test cases.
5842  big-test              Also run tests marked as "big". Repeat this option
5843                        twice to run only "big" tests.
5844  staging-run           Run a limited number of tests (no slow tests). Used
5845                        for running staging trees with valgrind.
5846  enable-disabled       Run also tests marked as disabled
5847  print-testcases       Don't run the tests but print details about all the
5848                        selected tests, in the order they would be run.
5849  skip-test-list=FILE   Skip the tests listed in FILE. Each line in the file
5850                        is an entry and should be formatted as:
5851                        <TESTNAME> : <COMMENT>
5852  force-restart         Always restart servers between tests. This makes it
5853                        easier to see from which test warnings may come from.
5854
5855Options that specify ports
5856
5857  mtr-port-base=#       Base for port numbers, ports from this number to
5858  port-base=#           number+9 are reserved. Should be divisible by 10;
5859                        if not it will be rounded down. May be set with
5860                        environment variable MTR_PORT_BASE. If this value is
5861                        set and is not "auto", it overrides build-thread.
5862  mtr-build-thread=#    Specify unique number to calculate port number(s) from.
5863  build-thread=#        Can be set in environment variable MTR_BUILD_THREAD.
5864                        Set  MTR_BUILD_THREAD="auto" to automatically aquire
5865                        a build thread id that is unique to current host
5866  port-group-size=N     Reserve groups of TCP ports of size N for each MTR thread
5867
5868
5869Options for test case authoring
5870
5871  record TESTNAME       (Re)genereate the result file for TESTNAME
5872  check-testcases       Check testcases for sideeffects
5873  mark-progress         Log line number and elapsed time to <testname>.progress
5874
5875Options that pass on options (these may be repeated)
5876
5877  mysqld=ARGS           Specify additional arguments to "mysqld"
5878  mysqld-env=VAR=VAL    Specify additional environment settings for "mysqld"
5879
5880Options to run test on running server
5881
5882  extern option=value   Run only the tests against an already started server
5883                        the options to use for connection to the extern server
5884                        must be specified using name-value pair notation
5885                        For example:
5886                         ./$0 --extern socket=/tmp/mysqld.sock
5887
5888Options for debugging the product
5889
5890  debug                 Dump trace output for all servers and client programs
5891  debug-common          Same as debug, but sets 'd' debug flags to
5892                        "query,info,error,enter,exit"
5893  debug-server          Use debug version of server, but without turning on
5894                        tracing
5895  max-save-core         Limit the number of core files saved (to avoid filling
5896                        up disks for heavily crashing server). Defaults to
5897                        $opt_max_save_core. Set its default with
5898                        MTR_MAX_SAVE_CORE
5899  max-save-datadir      Limit the number of datadir saved (to avoid filling
5900                        up disks for heavily crashing server). Defaults to
5901                        $opt_max_save_datadir. Set its default with
5902                        MTR_MAX_SAVE_DATADIR
5903  max-test-fail         Limit the number of test failures before aborting
5904                        the current test run. Defaults to
5905                        $opt_max_test_fail, set to 0 for no limit. Set
5906                        it's default with MTR_MAX_TEST_FAIL
5907  core-in-failure	Generate a core even if run server is run with valgrind
5908HERE
5909
5910Misc options
5911  user=USER             User for connecting to mysqld(default: $opt_user)
5912  comment=STR           Write STR to the output
5913  timer                 Show test case execution time.
5914  verbose               More verbose output(use multiple times for even more)
5915  verbose-restart       Write when and why servers are restarted
5916  start                 Only initialize and start the servers, using the
5917                        startup settings for the first specified test case
5918                        Example:
5919                         $0 --start alias &
5920  start-and-exit        Same as --start, but mysql-test-run terminates and
5921                        leaves just the server running
5922  start-dirty           Only start the servers (without initialization) for
5923                        the first specified test case
5924  user-args             In combination with start* and no test name, drops
5925                        arguments to mysqld except those specified with
5926                        --mysqld (if any)
5927  wait-all              If --start or --start-dirty option is used, wait for all
5928                        servers to exit before finishing the process
5929  fast                  Run as fast as possible, don't wait for servers
5930                        to shutdown etc.
5931  force-restart         Always restart servers between tests
5932  parallel=N            Run tests in N parallel threads (default 1)
5933                        Use parallel=auto for auto-setting of N
5934  repeat=N              Run each test N number of times
5935  retry=N               Retry tests that fail up to N times (default $opt_retry).
5936                        Retries are also limited by the maximum number of
5937                        failures before stopping, set with the --retry-failure
5938                        option
5939  retry-failure=N       When using the --retry option to retry failed tests,
5940                        stop when N failures have occurred (default $opt_retry_failure)
5941  reorder               Reorder tests to get fewer server restarts
5942  help                  Get this help text
5943
5944  testcase-timeout=MINUTES Max test case run time (default $opt_testcase_timeout)
5945  suite-timeout=MINUTES Max test suite run time (default $opt_suite_timeout)
5946  shutdown-timeout=SECONDS Max number of seconds to wait for server shutdown
5947                        before killing servers (default $opt_shutdown_timeout)
5948  warnings              Scan the log files for warnings. Use --nowarnings
5949                        to turn off.
5950
5951  stop-file=file        (also MTR_STOP_FILE environment variable) if this
5952                        file detected mysql test will not start new tests
5953                        until the file will be removed.
5954  stop-keep-alive=sec   (also MTR_STOP_KEEP_ALIVE environment variable)
5955                        works with stop-file, print messages every sec
5956                        seconds when mysql test is waiting to removing
5957                        the file (for buildbot)
5958
5959  sleep=SECONDS         Passed to mysqltest, will be used as fixed sleep time
5960  debug-sync-timeout=NUM Set default timeout for WAIT_FOR debug sync
5961                        actions. Disable facility with NUM=0.
5962  gcov                  Collect coverage information after the test.
5963                        The result is a gcov file per source and header file.
5964  gprof                 Collect profiling information using gprof.
5965  experimental=<file>   Refer to list of tests considered experimental;
5966                        failures will be marked exp-fail instead of fail.
5967  timestamp             Print timestamp before each test report line
5968  timediff              With --timestamp, also print time passed since
5969                        *previous* test started
5970  max-connections=N     Max number of open connection to server in mysqltest
5971  report-times          Report how much time has been spent on different
5972                        phases of test execution.
5973  stress=ARGS           Run stress test, providing options to
5974                        mysql-stress-test.pl. Options are separated by comma.
5975  xml-report=<file>     Output jUnit xml file of the results.
5976  tail-lines=N          Number of lines of the result to include in a failure
5977                        report.
5978
5979Some options that control enabling a feature for normal test runs,
5980can be turned off by prepending 'no' to the option, e.g. --notimer.
5981This applies to reorder, timer, check-testcases and warnings.
5982
5983HERE
5984  exit(1);
5985
5986}
5987
5988sub list_options ($) {
5989  my $hash= shift;
5990
5991  for (keys %$hash) {
5992    s/([:=].*|[+!])$//;
5993    s/\|/\n--/g;
5994    print "--$_\n";
5995  }
5996
5997  exit(1);
5998}
5999