1#!/usr/bin/perl 2 3# Copyright (c) 2005, 2021, Oracle and/or its affiliates. 4# 5# This program is free software; you can redistribute it and/or modify 6# it under the terms of the GNU General Public License, version 2.0, 7# as published by the Free Software Foundation. 8# 9# This program is also distributed with certain software (including 10# but not limited to OpenSSL) that is licensed under separate terms, 11# as designated in a particular file or component or in included license 12# documentation. The authors of MySQL hereby grant you an additional 13# permission to link the program and your derivative works with the 14# separately licensed software that they have included with MySQL. 15# 16# This program is distributed in the hope that it will be useful, 17# but WITHOUT ANY WARRANTY; without even the implied warranty of 18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19# GNU General Public License, version 2.0, for more details. 20# 21# You should have received a copy of the GNU General Public License 22# along with this program; if not, write to the Free Software 23# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 24 25# ====================================================================== 26# MySQL server stress test system 27# ====================================================================== 28# 29########################################################################## 30# 31# SCENARIOS AND REQUIREMENTS 32# 33# The system should perform stress testing of MySQL server with 34# following requirements and basic scenarios: 35# 36# Basic requirements: 37# 38# Design of stress script should allow one: 39# 40# - to use for stress testing mysqltest binary as test engine 41# - to use for stress testing both regular test suite and any 42# additional test suites (e.g. mysql-test-extra-5.0) 43# - to specify files with lists of tests both for initialization of 44# stress db and for further testing itself 45# - to define number of threads that will be concurrently used in testing 46# - to define limitations for test run. e.g. number of tests or loops 47# for execution or duration of testing, delay between test executions, etc. 48# - to get readable log file which can be used for identification of 49# errors arose during testing 50# 51# Basic scenarios: 52# 53# * It should be possible to run stress script in standalone mode 54# which will allow to create various scenarios of stress workloads: 55# 56# simple ones: 57# 58# box #1: 59# - one instance of script with list of tests #1 60# 61# and more advanced ones: 62# 63# box #1: 64# - one instance of script with list of tests #1 65# - another instance of script with list of tests #2 66# box #2: 67# - one instance of script with list of tests #3 68# - another instance of script with list of tests #4 69# that will recreate whole database to back it to clean 70# state 71# 72# One kind of such complex scenarios maybe continued testing 73# when we want to run stress tests from many boxes with various 74# lists of tests that will last very long time. And in such case 75# we need some wrapper for MySQL server that will restart it in 76# case of crashes. 77# 78# * It should be possible to run stress script in ad-hoc mode from 79# shell or perl versions of mysql-test-run. This allows developers 80# to reproduce and debug errors that was found in continued stress 81# testing 82# 83# 2009-01-28 OBN Additions and modifications per WL#4685 84# 85######################################################################## 86 87use Config; 88 89if (!defined($Config{useithreads})) 90{ 91 die <<EOF; 92It is unable to run threaded version of stress test on this system 93due to disabled ithreads. Please check that installed perl binary 94was built with support of ithreads. 95EOF 96} 97 98use threads; 99use threads::shared; 100 101use IO::Socket; 102use Sys::Hostname; 103use File::Copy; 104use File::Spec; 105use File::Find; 106use File::Basename; 107use File::Path; 108use Cwd; 109 110use Data::Dumper; 111use Getopt::Long; 112 113my $stress_suite_version="1.0"; 114 115$|=1; 116 117$opt_server_host=""; 118$opt_server_logs_dir=""; 119$opt_help=""; 120$opt_server_port=""; 121$opt_server_socket=""; 122$opt_server_user=""; 123$opt_server_password=""; 124$opt_server_database=""; 125$opt_cleanup=""; 126$opt_verbose=""; 127$opt_log_error_details=""; 128 129 130$opt_suite="main"; 131$opt_stress_suite_basedir=""; 132$opt_stress_basedir=""; 133$opt_stress_datadir=""; 134$opt_test_suffix=""; 135 136$opt_stress_mode="random"; 137 138$opt_loop_count=0; 139$opt_test_count=0; 140$opt_test_duration=0; 141# OBN: Changing abort-on-error default to -1 (for WL-4626/4685): -1 means no abort 142$opt_abort_on_error=-1; 143$opt_sleep_time = 0; 144$opt_threads=1; 145$pid_file="mysql_stress_test.pid"; 146$opt_mysqltest= ($^O =~ /mswin32/i) ? "mysqltest.exe" : "mysqltest"; 147$opt_check_tests_file=""; 148# OBM adding a setting for 'max-connect-retries=20' the default of 500 is to high 149@mysqltest_args=("--silent", "-v", "--max-connect-retries=20"); 150 151# Client ip address 152$client_ip=inet_ntoa((gethostbyname(hostname()))[4]); 153$client_ip=~ s/\.//g; 154 155%tests_files=(client => {mtime => 0, data => []}, 156 initdb => {mtime => 0, data => []}); 157 158# Error codes and sub-strings with corresponding severity 159# 160# S1 - Critical errors - cause immediately abort of testing. These errors 161# could be caused by server crash or impossibility 162# of test execution. 163# 164# S2 - Serious errors - these errors are bugs for sure as it knowns that 165# they shouldn't appear during stress testing 166# 167# S3 - Unknown errors - Errors were returned but we don't know what they are 168# so script can't determine if they are OK or not 169# 170# S4 - Non-seriuos errros - these errors could be caused by fact that 171# we execute simultaneously statements that 172# affect tests executed by other threads 173 174%error_strings = ( 'Failed in mysql_real_connect()' => S1, 175 'Can\'t connect' => S1, 176 'not found (Errcode: 2)' => S1, 177 'does not exist' => S1, 178 'Could not open connection \'default\' after \d+ attempts' => S1, 179 'wrong errno ' => S3, 180 'Result length mismatch' => S4, 181 'Result content mismatch' => S4); 182 183%error_codes = ( 1012 => S2, 1015 => S2, 1021 => S2, 184 1027 => S2, 1037 => S2, 1038 => S2, 185 1039 => S2, 1040 => S2, 1046 => S2, 186 1053 => S2, 1180 => S2, 1181 => S2, 187 1203 => S2, 1205 => S4, 1206 => S2, 188 1207 => S2, 1213 => S4, 1223 => S2, 189 2002 => S1, 2003 => S1, 2006 => S1, 190 2013 => S1 191 ); 192 193share(%test_counters); 194%test_counters=( loop_count => 0, test_count=>0); 195 196share($exiting); 197$exiting=0; 198 199# OBN Code and 'set_exit_code' function added by ES to set an exit code based on the error category returned 200# in combination with the --abort-on-error value see WL#4685) 201use constant ABORT_MAKEWEIGHT => 20; 202share($gExitCode); 203$gExitCode = 0; # global exit code 204sub set_exit_code { 205 my $severity = shift; 206 my $code = 0; 207 if ( $severity =~ /^S(\d+)/ ) { 208 $severity = $1; 209 $code = 11 - $severity; # S1=10, S2=9, ... -- as per WL 210 } 211 else { 212 # we know how we call the sub: severity should be S<num>; so, we should never be here... 213 print STDERR "Unknown severity format: $severity; setting to S1\n"; 214 $severity = 1; 215 } 216 $abort = 0; 217 if ( $severity <= $opt_abort_on_error ) { 218 # the test finished with a failure severe enough to abort. We are adding the 'abort flag' to the exit code 219 $code += ABORT_MAKEWEIGHT; 220 # but are not exiting just yet -- we need to update global exit code first 221 $abort = 1; 222 } 223 lock $gExitCode; # we can use lock here because the script uses threads anyway 224 $gExitCode = $code if $code > $gExitCode; 225 kill INT, $$ if $abort; # this is just a way to call sig_INT_handler: it will set exiting flag, which should do the rest 226} 227 228share($test_counters_lock); 229$test_counters_lock=0; 230share($log_file_lock); 231$log_file_lock=0; 232 233$SIG{INT}= \&sig_INT_handler; 234$SIG{TERM}= \&sig_TERM_handler; 235 236 237GetOptions("server-host=s", "server-logs-dir=s", "server-port=s", 238 "server-socket=s", "server-user=s", "server-password=s", 239 "server-database=s", 240 "stress-suite-basedir=s", "suite=s", "stress-init-file:s", 241 "stress-tests-file:s", "stress-basedir=s", "stress-mode=s", 242 "stress-datadir=s", 243 "threads=s", "sleep-time=s", "loop-count=i", "test-count=i", 244 "test-duration=i", "test-suffix=s", "check-tests-file", 245 "verbose", "log-error-details", "cleanup", "mysqltest=s", 246 # OBN: (changing 'abort-on-error' to numberic for WL-4626/4685) 247 "abort-on-error=i" => \$opt_abort_on_error, "help") || usage(1); 248 249usage(0) if ($opt_help); 250 251#$opt_abort_on_error=1; 252 253$test_dirname=get_timestamp(); 254$test_dirname.="-$opt_test_suffix" if ($opt_test_suffix ne ''); 255 256print <<EOF; 257############################################################# 258 CONFIGURATION STAGE 259############################################################# 260EOF 261 262if ($opt_stress_basedir eq '' || $opt_stress_suite_basedir eq '' || 263 $opt_server_logs_dir eq '') 264{ 265 die <<EOF; 266 267Options --stress-basedir, --stress-suite-basedir and --server-logs-dir are 268required. Please use these options to specify proper basedir for 269client, test suite and location of server logs. 270 271stress-basedir: '$opt_stress_basedir' 272stress-suite-basedir: '$opt_stress_suite_basedir' 273server-logs-dir: '$opt_server_logs_dir' 274 275EOF 276} 277 278#Workaround for case when we got relative but not absolute path 279$opt_stress_basedir=File::Spec->rel2abs($opt_stress_basedir); 280$opt_stress_suite_basedir=File::Spec->rel2abs($opt_stress_suite_basedir); 281$opt_server_logs_dir=File::Spec->rel2abs($opt_server_logs_dir); 282 283if ($opt_stress_datadir ne '') 284{ 285 $opt_stress_datadir=File::Spec->rel2abs($opt_stress_datadir); 286} 287 288if (! -d "$opt_stress_basedir") 289{ 290 die <<EOF; 291 292Directory '$opt_stress_basedir' does not exist. 293Use --stress-basedir option to specify proper basedir for client 294 295EOF 296} 297 298if (!-d $opt_stress_suite_basedir) 299{ 300 die <<EOF; 301 302Directory '$opt_stress_suite_basedir' does not exist. 303Use --stress-suite-basedir option to specify proper basedir for test suite 304 305EOF 306} 307 308$test_dataset_dir=$opt_stress_suite_basedir; 309if ($opt_stress_datadir ne '') 310{ 311 if (-d $opt_stress_datadir) 312 { 313 $test_dataset_dir=$opt_stress_datadir; 314 315 } 316 else 317 { 318 die <<EOF; 319Directory '$opt_stress_datadir' not exists. Please specify proper one 320with --stress-datadir option. 321EOF 322 } 323} 324 325if ($^O =~ /mswin32/i) 326{ 327 $test_dataset_dir=~ s/\\/\\\\/g; 328} 329else 330{ 331 $test_dataset_dir.="/"; 332} 333 334 335 336if (!-d $opt_server_logs_dir) 337{ 338 die <<EOF; 339 340Directory server-logs-dir '$opt_server_logs_dir' does not exist. 341Use --server-logs-dir option to specify proper directory for storing 342logs 343 344EOF 345} 346else 347{ 348 #Create sub-directory for test session logs 349 mkpath(File::Spec->catdir($opt_server_logs_dir, $test_dirname), 0, 0755); 350 #Define filename of global session log file 351 $stress_log_file=File::Spec->catfile($opt_server_logs_dir, $test_dirname, 352 "mysql-stress-test.log"); 353} 354 355if ($opt_suite ne '' && $opt_suite ne 'main' && $opt_suite ne 'default') 356{ 357 $test_suite_dir=File::Spec->catdir($opt_stress_suite_basedir, "suite", $opt_suite); 358} 359else 360{ 361 $test_suite_dir= $opt_stress_suite_basedir; 362} 363 364if (!-d $test_suite_dir) 365{ 366 die <<EOF 367 368Directory '$test_suite_dir' does not exist. 369Use --suite options to specify proper dir for test suite 370 371EOF 372} 373 374$test_suite_t_path=File::Spec->catdir($test_suite_dir,'t'); 375$test_suite_r_path=File::Spec->catdir($test_suite_dir,'r'); 376 377foreach my $suite_dir ($test_suite_t_path, $test_suite_r_path) 378{ 379 if (!-d $suite_dir) 380 { 381 die <<EOF; 382 383Directory '$suite_dir' does not exist. 384Please ensure that you specified proper source location for 385test/result files with --stress-suite-basedir option and name 386of test suite with --suite option 387 388EOF 389 } 390} 391 392$test_t_path=File::Spec->catdir($opt_stress_basedir,'t'); 393$test_r_path=File::Spec->catdir($opt_stress_basedir,'r'); 394 395foreach $test_dir ($test_t_path, $test_r_path) 396{ 397 if (-d $test_dir) 398 { 399 if ($opt_cleanup) 400 { 401 #Delete existing 't', 'r', 'r/*' subfolders in $stress_basedir 402 rmtree("$test_dir", 0, 0); 403 print "Cleanup $test_dir\n"; 404 } 405 else 406 { 407 die <<EOF; 408Directory '$test_dir' already exist. 409Please ensure that you specified proper location of working dir 410for current test run with --stress-basedir option or in case of staled 411directories use --cleanup option to remove ones 412EOF 413 } 414 } 415 #Create empty 't', 'r' subfolders that will be filled later 416 mkpath("$test_dir", 0, 0777); 417} 418 419if (!defined($opt_stress_tests_file) && !defined($opt_stress_init_file)) 420{ 421 die <<EOF; 422You should run stress script either with --stress-tests-file or with 423--stress-init-file otions. See help for details. 424EOF 425} 426 427if (defined($opt_stress_tests_file)) 428{ 429 if ($opt_stress_tests_file eq '') 430 { 431 #Default location of file with set of tests for current test run 432 $tests_files{client}->{filename}= File::Spec->catfile($opt_stress_suite_basedir, 433 "testslist_client.txt"); 434 } 435 else 436 { 437 $tests_files{client}->{filename}= $opt_stress_tests_file; 438 } 439 440 if (!-f $tests_files{client}->{filename}) 441 { 442 die <<EOF; 443 444File '$tests_files{client}->{filename}' with list of tests not exists. 445Please ensure that this file exists, readable or specify another one with 446--stress-tests-file option. 447 448EOF 449 } 450} 451 452if (defined($opt_stress_init_file)) 453{ 454 if ($opt_stress_init_file eq '') 455 { 456 #Default location of file with set of tests for current test run 457 $tests_files{initdb}->{filename}= File::Spec->catfile($opt_stress_suite_basedir, 458 "testslist_initdb.txt"); 459 } 460 else 461 { 462 $tests_files{initdb}->{filename}= $opt_stress_init_file; 463 } 464 465 if (!-f $tests_files{initdb}->{filename}) 466 { 467 die <<EOF; 468 469File '$tests_files{initdb}->{filename}' with list of tests for initialization of database 470for stress test not exists. 471Please ensure that this file exists, readable or specify another one with 472--stress-init-file option. 473 474EOF 475 } 476} 477 478if ($opt_stress_mode !~ /^(random|seq)$/) 479{ 480 die <<EOF 481Was specified wrong --stress-mode. Correct values 'random' and 'seq'. 482EOF 483} 484 485if (open(TEST, "$opt_mysqltest -V |")) 486{ 487 $mysqltest_version=join("",<TEST>); 488 close(TEST); 489 print "FOUND MYSQLTEST BINARY: ", $mysqltest_version,"\n"; 490} 491else 492{ 493 die <<EOF; 494ERROR: mysqltest binary $opt_mysqltest not found $!. 495You must either specify file location explicitly using --mysqltest 496option, or make sure path to mysqltest binary is listed 497in your PATH environment variable. 498EOF 499} 500 501# 502#Adding mysql server specific command line options for mysqltest binary 503# 504$opt_server_host= $opt_server_host ? $opt_server_host : "localhost"; 505$opt_server_port= $opt_server_port ? $opt_server_port : "3306"; 506$opt_server_user= $opt_server_user ? $opt_server_user : "root"; 507$opt_server_socket= $opt_server_socket ? $opt_server_socket : "/tmp/mysql.sock"; 508$opt_server_database= $opt_server_database ? $opt_server_database : "test"; 509 510unshift @mysqltest_args, "--host=$opt_server_host"; 511unshift @mysqltest_args, "--port=$opt_server_port"; 512unshift @mysqltest_args, "--user=$opt_server_user"; 513unshift @mysqltest_args, "--password=$opt_server_password"; 514unshift @mysqltest_args, "--socket=$opt_server_socket"; 515unshift @mysqltest_args, "--database=$opt_server_database"; 516 517#Export variables that could be used in tests 518$ENV{MYSQL_TEST_DIR}=$test_dataset_dir; 519$ENV{MASTER_MYPORT}=$opt_server_port; 520$ENV{MASTER_MYSOCK}=$opt_server_socket; 521 522print <<EOF; 523TEST-SUITE-BASEDIR: $opt_stress_suite_basedir 524SUITE: $opt_suite 525TEST-BASE-DIR: $opt_stress_basedir 526TEST-DATADIR: $test_dataset_dir 527SERVER-LOGS-DIR: $opt_server_logs_dir 528 529THREADS: $opt_threads 530TEST-MODE: $opt_stress_mode 531 532EOF 533 534#------------------------------------------------------------------------------- 535#At this stage we've already checked all needed pathes/files 536#and ready to start the test 537#------------------------------------------------------------------------------- 538 539if (defined($opt_stress_tests_file) || defined($opt_stress_init_file)) 540{ 541 print <<EOF; 542############################################################# 543 PREPARATION STAGE 544############################################################# 545EOF 546 547 #Copy Test files from network share to 't' folder 548 print "\nCopying Test files from $test_suite_t_path to $test_t_path folder..."; 549 find({wanted=>\©_test_files, bydepth=>1}, "$test_suite_t_path"); 550 print "Done\n"; 551 552 #$test_r_path/r0 dir reserved for initdb 553 $count_start= defined($opt_stress_init_file) ? 0 : 1; 554 555 our $r_folder=''; 556 print "\nCreating 'r' folder and copying Protocol files to each 'r#' sub-folder..."; 557 for($count=$count_start; $count <= $opt_threads; $count++) 558 { 559 $r_folder = File::Spec->catdir($test_r_path, "r".$count); 560 mkpath("$r_folder", 0, 0777); 561 562 find(\©_result_files,"$test_suite_r_path"); 563 } 564 print "Done\n\n"; 565} 566 567if (defined($opt_stress_init_file)) 568{ 569 print <<EOF; 570############################################################# 571 INITIALIZATION STAGE 572############################################################# 573EOF 574 575 #Set limits for stress db initialization 576 %limits=(loop_count => 1, test_count => undef); 577 578 #Read list of tests from $opt_stress_init_file 579 read_tests_names($tests_files{initdb}); 580 test_loop($client_ip, 0, 'seq', $tests_files{initdb}); 581 #print Dumper($tests_files{initdb}),"\n"; 582 print <<EOF; 583 584Done initialization of stress database by tests from 585$tests_files{initdb}->{filename} file. 586 587EOF 588} 589 590if (defined($opt_stress_tests_file)) 591{ 592 print <<EOF; 593############################################################# 594 STRESS TEST RUNNING STAGE 595############################################################# 596EOF 597 598 $exiting=0; 599 #Read list of tests from $opt_stress_tests_file 600 read_tests_names($tests_files{client}); 601 602 #Reset current counter and set limits 603 %test_counters=( loop_count => 0, test_count=>0); 604 %limits=(loop_count => $opt_loop_count, test_count => $opt_test_count); 605 606 if (($opt_loop_count && $opt_threads > $opt_loop_count) || 607 ($opt_test_count && $opt_threads > $opt_test_count)) 608 { 609 warn <<EOF; 610 611WARNING: Possible inaccuracies in number of executed loops or 612 tests because number of threads bigger than number of 613 loops or tests: 614 615 Threads will be started: $opt_threads 616 Loops will be executed: $opt_loop_count 617 Tests will be executed: $opt_test_count 618 619EOF 620 } 621 622 #Create threads (number depending on the variable ) 623 for ($id=1; $id<=$opt_threads && !$exiting; $id++) 624 { 625 $thrd[$id] = threads->create("test_loop", $client_ip, $id, 626 $opt_stress_mode, $tests_files{client}); 627 628 print "main: Thread ID $id TID ",$thrd[$id]->tid," started\n"; 629 select(undef, undef, undef, 0.5); 630 } 631 632 if ($opt_test_duration) 633 { 634 # OBN - At this point we need to wait for the duration of the test, hoever 635 # we need to be able to quit if an 'abort-on-error' condition has happend 636 # with one of the children (WL#4685). Using solution by ES and replacing 637 # the 'sleep' command with a loop checking the abort condition every second 638 639 foreach ( 1..$opt_test_duration ) { 640 last if $exiting; 641 sleep 1; 642 } 643 kill INT, $$; #Interrupt child threads 644 } 645 646 #Let other threads to process INT signal 647 sleep(1); 648 649 for ($id=1; $id<=$opt_threads;$id++) 650 { 651 if (defined($thrd[$id])) 652 { 653 $thrd[$id]->join(); 654 } 655 } 656 print "EXIT\n"; 657} 658 659exit $gExitCode; # ES WL#4685: script should return a meaningful exit code 660 661sub test_init 662{ 663 my ($env)=@_; 664 665 $env->{session_id}=$env->{ip}."_".$env->{thread_id}; 666 $env->{r_folder}='r'.$env->{thread_id}; 667 $env->{screen_logs}=File::Spec->catdir($opt_server_logs_dir, $test_dirname, 668 "screen_logs", $env->{session_id}); 669 $env->{reject_logs}=File::Spec->catdir($opt_server_logs_dir, $test_dirname, 670 "reject_logs", $env->{session_id}); 671 672 mkpath($env->{screen_logs}, 0, 0755) unless (-d $env->{screen_logs}); 673 mkpath($env->{reject_logs}, 0, 0755) unless (-d $env->{reject_logs}); 674 675 $env->{session_log}= File::Spec->catfile($env->{screen_logs}, $env->{session_id}.".log"); 676} 677 678sub test_execute 679{ 680 my $env= shift; 681 my $test_name= shift; 682 683 my $g_start= ""; 684 my $g_end= ""; 685 my $mysqltest_cmd= ""; 686 my @mysqltest_test_args=(); 687 my @stderr=(); 688 689 #Get time stamp 690 $g_start = get_timestamp(); 691 $env->{errors}={}; 692 @{$env->{test_status}}=(); 693 694 my $test_file= $test_name.".test"; 695 my $result_file= $test_name.".result"; 696 my $reject_file = $test_name.'.reject'; 697 my $output_file = $env->{session_id}.'_'.$test_name.'_'.$g_start."_".$env->{test_count}.'.txt'; 698 699 my $test_filename = File::Spec->catfile($test_t_path, $test_file); 700 my $result_filename = File::Spec->catdir($test_r_path, $env->{r_folder}, $result_file); 701 my $reject_filename = File::Spec->catdir($test_r_path, $env->{r_folder}, $reject_file); 702 my $output_filename = File::Spec->catfile($env->{screen_logs}, $output_file); 703 704 705 push @mysqltest_test_args, "--basedir=$opt_stress_suite_basedir/", 706 "--tmpdir=$opt_stress_basedir", 707 "-x $test_filename", 708 "-R $result_filename", 709 "2>$output_filename"; 710 711 $cmd= "$opt_mysqltest --no-defaults ".join(" ", @mysqltest_args)." ". 712 join(" ", @mysqltest_test_args); 713 714 system($cmd); 715 716 $exit_value = $? >> 8; 717 $signal_num = $? & 127; 718 $dumped_core = $? & 128; 719 720 my $tid= threads->self->tid; 721 722 if (-s $output_filename > 0) 723 { 724 #Read stderr for further analysis 725 open (STDERR_LOG, $output_filename) or 726 warn "Can't open file $output_filename"; 727 @stderr=<STDERR_LOG>; 728 close(STDERR_LOG); 729 730 if ($opt_verbose) 731 { 732 $session_debug_file="$opt_stress_basedir/error$tid.txt"; 733 734 stress_log($session_debug_file, 735 "Something wrong happened during execution of this command line:"); 736 stress_log($session_debug_file, "MYSQLTEST CMD - $cmd"); 737 stress_log($session_debug_file, "STDERR:".join("",@stderr)); 738 739 stress_log($session_debug_file, "EXIT STATUS:\n1. EXIT: $exit_value \n". 740 "2. SIGNAL: $signal_num\n". 741 "3. CORE: $dumped_core\n"); 742 } 743 } 744 745 #If something wrong trying to analyse stderr 746 if ($exit_value || $signal_num) 747 { 748 if (@stderr) 749 { 750 foreach my $line (@stderr) 751 { 752 #FIXME: we should handle case when for one sub-string/code 753 # we have several different error messages 754 # Now for both codes/substrings we assume that 755 # first found message will represent error 756 757 #Check line for error codes 758 if (($err_msg, $err_code)= $line=~/failed: ((\d+):.+?$)/) 759 { 760 if (!exists($error_codes{$err_code})) 761 { 762 # OBN Changing severity level to S4 from S3 as S3 now reserved 763 # for the case where the error is unknown (for WL#4626/4685 764 $severity="S4"; 765 $err_code=0; 766 } 767 else 768 { 769 $severity=$error_codes{$err_code}; 770 } 771 772 if (!exists($env->{errors}->{$severity}->{$err_code})) 773 { 774 $env->{errors}->{$severity}->{$err_code}=[0, $err_msg]; 775 } 776 $env->{errors}->{$severity}->{$err_code}->[0]++; 777 $env->{errors}->{$severity}->{total}++; 778 } 779 780 #Check line for error patterns 781 foreach $err_string (keys %error_strings) 782 { 783 $pattern= quotemeta $err_string; 784 if ($line =~ /$pattern/i) 785 { 786 my $severity= $error_strings{$err_string}; 787 if (!exists($env->{errors}->{$severity}->{$err_string})) 788 { 789 $env->{errors}->{$severity}->{$err_string}=[0, $line]; 790 } 791 $env->{errors}->{$severity}->{$err_string}->[0]++; 792 $env->{errors}->{$severity}->{total}++; 793 } 794 } 795 } 796 } 797 else 798 { 799 $env->{errors}->{S3}->{'Unknown error'}= 800 [1,"Unknown error. Nothing was output to STDERR"]; 801 $env->{errors}->{S3}->{total}=1; 802 } 803 } 804 805 # 806 #FIXME: Here we can perform further analysis of recognized 807 # error codes 808 # 809 810 foreach my $severity (sort {$a cmp $b} keys %{$env->{errors}}) 811 { 812 my $total=$env->{errors}->{$severity}->{total}; 813 if ($total) 814 { 815 push @{$env->{test_status}}, "Severity $severity: $total"; 816 $env->{errors}->{total}=+$total; 817 set_exit_code($severity); 818 } 819 } 820 821 #FIXME: Should we take into account $exit_value here? 822 # Now we assume that all stringified errors(i.e. errors without 823 # error codes) which are not exist in %error_string structure 824 # are OK 825 if (!$env->{errors}->{total}) 826 { 827 push @{$env->{test_status}},"No Errors. Test Passed OK"; 828 } 829 830 log_session_errors($env, $test_file); 831 832 #OBN Removing the case of S1 and abort-on-error as that is now set 833 # inside the set_exit_code function (for WL#4626/4685) 834 #if (!$exiting && ($signal_num == 2 || $signal_num == 15 || 835 # ($opt_abort_on_error && $env->{errors}->{S1} > 0))) 836 if (!$exiting && ($signal_num == 2 || $signal_num == 15)) 837 { 838 #mysqltest was interrupted with INT or TERM signals 839 #so we assume that we should cancel testing and exit 840 $exiting=1; 841 # OBN - Adjusted text to exclude case of S1 and abort-on-error that 842 # was mentioned (for WL#4626/4685) 843 print STDERR<<EOF; 844WARNING: 845 mysqltest was interrupted with INT or TERM signals so we assume that 846 we should cancel testing and exit. Please check log file for this thread 847 in $stress_log_file or 848 inspect below output of the last test case executed with mysqltest to 849 find out cause of error. 850 851 Output of mysqltest: 852 @stderr 853 854EOF 855 } 856 857 if (-e $reject_filename) 858 { 859 move_to_logs($env->{reject_logs}, $reject_filename, $reject_file); 860 } 861 862 if (-e $output_filename) 863 { 864 move_to_logs($env->{screen_logs}, $output_filename, $output_file); 865 } 866 867} 868 869sub test_loop 870{ 871 my %client_env=(); 872 my $test_name=""; 873 874 # KEY for session identification: IP-THREAD_ID 875 $client_env{ip} = shift; 876 $client_env{thread_id} = shift; 877 878 $client_env{mode} = shift; 879 $client_env{tests_file}=shift; 880 881 $client_env{test_seq_idx}=0; 882 883 #Initialize session variables 884 test_init(\%client_env); 885 886LOOP: 887 888 while(!$exiting) 889 { 890 if ($opt_check_tests_file) 891 { 892 #Check if tests_file was modified and reread it in this case 893 read_tests_names($client_env{tests_file}, 0); 894 } 895 896 { 897 lock($test_counters_lock); 898 899 if (($limits{loop_count} && $limits{loop_count} <= $test_counters{loop_count}*1) || 900 ($limits{test_count} && $limits{test_count} <= $test_counters{test_count}*1) ) 901 { 902 $exiting=1; 903 next LOOP; 904 } 905 } 906 907 #Get random file name 908 if (($test_name = get_test(\%client_env)) ne '') 909 { 910 { 911 lock($test_counters_lock); 912 913 #Save current counters values 914 $client_env{loop_count}=$test_counters{loop_count}; 915 $client_env{test_count}=$test_counters{test_count}; 916 } 917 #Run test and analyze results 918 test_execute(\%client_env, $test_name); 919 920 print "test_loop[".$limits{loop_count}.":". 921 $limits{test_count}." ". 922 $client_env{loop_count}.":". 923 $client_env{test_count}."]:". 924 " TID ".$client_env{thread_id}. 925 " test: '$test_name' ". 926 " Errors: ".join(" ",@{$client_env{test_status}}). 927 ( $exiting ? " (thread aborting)" : "" )."\n"; 928 } 929 930 # OBN - At this point we need to wait until the 'wait' time between test 931 # executions passes (in case it is specifed) passes, hoever we need 932 # to be able to quit and break out of the test if an 'abort-on-error' 933 # condition has happend with one of the other children (WL#4685). 934 # Using solution by ES and replacing the 'sleep' command with a loop 935 # checking the abort condition every second 936 937 if ( $opt_sleep_time ) { 938 foreach ( 1..$opt_sleep_time ) { 939 last if $exiting; 940 sleep 1; 941 } 942 } 943 } 944} 945 946sub move_to_logs ($$$) 947{ 948 my $path_to_logs = shift; 949 my $src_file = shift; 950 my $random_filename = shift; 951 952 my $dst_file = File::Spec->catfile($path_to_logs, $random_filename); 953 954 move ($src_file, $dst_file) or warn<<EOF; 955ERROR: move_to_logs: File $src_file cannot be moved to $dst_file: $! 956EOF 957} 958 959sub copy_test_files () 960{ 961 if (/\.test$/) 962 { 963 $src_file = $File::Find::name; 964 #print "## $File::Find::topdir - $File::Find::dir - $src_file\n"; 965 966 if ($File::Find::topdir eq $File::Find::dir && $src_file !~ /SCCS/) 967 { 968 $test_filename = basename($src_file); 969 $dst_file = File::Spec->catfile($test_t_path, $test_filename); 970 971 copy($src_file, $dst_file) or die "ERROR: copy_test_files: File cannot be copied. $!"; 972 } 973 } 974} 975 976sub copy_result_files () 977{ 978 if (/\.result$/) 979 { 980 $src_file = $File::Find::name; 981 982 if ($File::Find::topdir eq $File::Find::dir && $src_file !~ /SCCS/) 983 { 984 $result_filename = basename($src_file) ; 985 $dst_file = File::Spec->catfile($r_folder, $result_filename); 986 987 copy($src_file, $dst_file) or die "ERROR: copy_result_files: File cannot be copied. $!"; 988 } 989 } 990} 991 992sub get_timestamp 993{ 994 my ($sec,$min,$hour,$mday,$mon,$year,$wday,$ydat,$isdst) = localtime(); 995 996 return sprintf("%04d%02d%02d%02d%02d%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec); 997} 998 999sub read_tests_names 1000{ 1001 my $tests_file = shift; 1002 my $force_load = shift; 1003 1004 if ($force_load || ( (stat($tests_file->{filename}))[9] != $tests_file->{mtime}) ) 1005 { 1006 open (TEST, $tests_file->{filename}) || die ("Could not open file <". 1007 $tests_file->{filename}."> $!"); 1008 @{$tests_file->{data}}= grep {!/^[#\r\n]|^$/} map { s/[\r\n]//g; $_ } <TEST>; 1009 1010 close (TEST); 1011 $tests_file->{mtime}=(stat(_))[9]; 1012 } 1013} 1014 1015sub get_random_test 1016{ 1017 my $envt=shift; 1018 my $tests= $envt->{tests_file}->{data}; 1019 1020 my $random = int(rand(@{$tests})); 1021 my $test = $tests->[$random]; 1022 1023 return $test; 1024} 1025 1026sub get_next_test 1027{ 1028 my $envt=shift; 1029 my $test; 1030 1031 if (@{$envt->{tests_file}->{data}}) 1032 { 1033 $test=${$envt->{tests_file}->{data}}[$envt->{test_seq_idx}]; 1034 $envt->{test_seq_idx}++; 1035 } 1036 1037 #If we reach bound of array, reset seq index and increment loop counter 1038 if ($envt->{test_seq_idx} == scalar(@{$envt->{tests_file}->{data}})) 1039 { 1040 $envt->{test_seq_idx}=0; 1041 { 1042 lock($test_counters_lock); 1043 $test_counters{loop_count}++; 1044 } 1045 } 1046 1047 return $test; 1048} 1049 1050sub get_test 1051{ 1052 my $envt=shift; 1053 1054 { 1055 lock($test_counters_lock); 1056 $test_counters{test_count}++; 1057 } 1058 1059 if ($envt->{mode} eq 'seq') 1060 { 1061 return get_next_test($envt); 1062 } 1063 elsif ($envt->{mode} eq 'random') 1064 { 1065 return get_random_test($envt); 1066 } 1067} 1068 1069sub stress_log 1070{ 1071 my ($log_file, $line)=@_; 1072 1073 { 1074 open(SLOG,">>$log_file") or warn "Error during opening log file $log_file"; 1075 print SLOG $line,"\n"; 1076 close(SLOG); 1077 } 1078} 1079 1080sub log_session_errors 1081{ 1082 my ($env, $test_name) = @_; 1083 my $line=''; 1084 1085 { 1086 lock ($log_file_lock); 1087 1088 #header in the begining of log file 1089 if (!-e $stress_log_file) 1090 { 1091 stress_log($stress_log_file, 1092 "TestID TID Suite TestFileName Found Errors"); 1093 stress_log($stress_log_file, 1094 "======================================================="); 1095 } 1096 1097 $line=sprintf('%6d %3d %10s %20s %s', $env->{test_count}, threads->self->tid, 1098 $opt_suite, $test_name, 1099 join(",", @{$env->{test_status}})); 1100 1101 stress_log($stress_log_file, $line); 1102 #stress_log_with_lock($stress_log_file, "\n"); 1103 1104 if ($opt_log_error_details) 1105 { 1106 foreach $severity (sort {$a cmp $b} keys %{$env->{errors}}) 1107 { 1108 stress_log($stress_log_file, ""); 1109 foreach $error (keys %{$env->{errors}->{$severity}}) 1110 { 1111 if ($error ne 'total') 1112 { 1113 stress_log($stress_log_file, "$severity: Count:". 1114 $env->{errors}->{$severity}->{$error}->[0]. 1115 " Error:". $env->{errors}->{$severity}->{$error}->[1]); 1116 } 1117 } 1118 } 1119 } 1120 } 1121} 1122 1123sub sig_INT_handler 1124{ 1125 $SIG{INT}= \&sig_INT_handler; 1126 $exiting=1; 1127 print STDERR "$$: Got INT signal-------------------------------------------\n"; 1128 1129} 1130 1131sub sig_TERM_handler 1132{ 1133 $SIG{TERM}= \&sig_TERM_handler; 1134 $exiting=1; 1135 print STDERR "$$: Got TERM signal\n"; 1136} 1137 1138sub usage 1139{ 1140 my $retcode= shift; 1141 print <<EOF; 1142 1143The MySQL Stress suite Ver $stress_suite_version 1144 1145mysql-stress-test.pl --stress-basedir=<dir> --stress-suite-basedir=<dir> --server-logs-dir=<dir> 1146 1147--server-host 1148--server-port 1149--server-socket 1150--server-user 1151--server-password 1152--server-logs-dir 1153 Directory where all clients session logs will be stored. Usually 1154 this is shared directory associated with server that used 1155 in testing 1156 1157 Required option. 1158 1159--stress-suite-basedir=<dir> 1160 Directory that has r/ t/ subfolders with test/result files 1161 which will be used for testing. Also by default we are looking 1162 in this directory for 'stress-tests.txt' file which contains 1163 list of tests. It is possible to specify other location of this 1164 file with --stress-tests-file option. 1165 1166 Required option. 1167 1168--stress-basedir=<dir> 1169 Working directory for this test run. This directory will be used 1170 as temporary location for results tracking during testing 1171 1172 Required option. 1173 1174--stress-datadir=<dir> 1175 Location of data files used which will be used in testing. 1176 By default we search for these files in <dir>/data where dir 1177 is value of --stress-suite-basedir option. 1178 1179--stress-init-file[=/path/to/file with tests for initialization of stress db] 1180 Using of this option allows to perform initialization of database 1181 by execution of test files. List of tests will be taken either from 1182 specified file or if it omited from default file 'stress-init.txt' 1183 located in <--stress-suite-basedir/--suite> dir 1184 1185--stress-tests-file[=/path/to/file with tests] 1186 Using of this option allows to run stress test itself. Tests for testing 1187 will be taken either from specified file or if it omited from default 1188 file 'stress-tests.txt' located in <--stress-suite-basedir/--suite> dir 1189 1190--stress-mode= [random|seq] 1191 There are two possible modes which affect order of selecting tests 1192 from the list: 1193 - in random mode tests will be selected in random order 1194 - in seq mode each thread will execute tests in the loop one by one as 1195 they specified in the list file. 1196 1197--sleep-time=<time in seconds> 1198 Delay between test execution. Could be usefull in continued testsing 1199 when one of instance of stress script perform periodical cleanup or 1200 recreating of some database objects 1201 1202--threads=#number of threads 1203 Define number of threads 1204 1205--check-tests-file 1206 Check file with list of tests. If file was modified it will force to 1207 reread list of tests. Could be usefull in continued testing for 1208 adding/removing tests without script interruption 1209 1210--mysqltest=/path/to/mysqltest binary 1211 1212--verbose 1213 1214--cleanup 1215 Force to clean up working directory (specified with --stress-basedir) 1216 1217--abort-on-error=<number> 1218 Causes the script to abort if an error with severity <= number was encounterd 1219 1220--log-error-details 1221 Enable errors details in the global error log file. (Default: off) 1222 1223--test-count=<number of executed tests before we have to exit> 1224--loop-count=<number of executed loops in sequential mode before we have to exit> 1225--test-duration=<number of seconds that stress test should run> 1226 1227Example of tool usage: 1228 1229perl mysql-stress-test.pl \ 1230--stress-suite-basedir=/opt/qa/mysql-test-extra-5.0/mysql-test \ 1231--stress-basedir=/opt/qa/test \ 1232--server-logs-dir=/opt/qa/logs \ 1233--test-count=20 \ 1234--stress-tests-file=innodb-tests.txt \ 1235--stress-init-file=innodb-init.txt \ 1236--threads=5 \ 1237--suite=funcs_1 \ 1238--mysqltest=/opt/mysql/mysql-5.0/client/mysqltest \ 1239--server-user=root \ 1240--server-database=test \ 1241--cleanup \ 1242 1243EOF 1244exit($retcode); 1245} 1246 1247 1248