1#!/usr/bin/perl -w 2#---------------------------------------------------------------------- 3# 4# pg_regress_multi.pl - Test runner for Citus 5# 6# Portions Copyright (c) Citus Data, Inc. 7# Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group 8# Portions Copyright (c) 1994, Regents of the University of California 9# 10# src/test/regress/pg_regress_multi.pl 11# 12#---------------------------------------------------------------------- 13 14use strict; 15use warnings; 16 17use Fcntl; 18use Getopt::Long; 19use File::Basename; 20use File::Spec::Functions; 21use File::Path qw(make_path remove_tree); 22use Config; 23use POSIX qw( WNOHANG mkfifo ); 24use Cwd 'abs_path'; 25 26my $regressdir = (File::Spec->splitpath(__FILE__))[1]; 27 28 29sub Usage() 30{ 31 print "pg_regress_multi - Citus test runner\n"; 32 print "\n"; 33 print "Usage:\n"; 34 print " pg_regress_multi [MULTI OPTIONS] -- [PG REGRESS OPTS]\n"; 35 print "\n"; 36 print "Multi Options:\n"; 37 print " --isolationtester Run isolationtester tests instead of plain tests\n"; 38 print " --vanillatest Run postgres tests with citus loaded as shared preload library\n"; 39 print " --bindir Path to postgres binary directory\n"; 40 print " --libdir Path to postgres library directory\n"; 41 print " --postgres-builddir Path to postgres build directory\n"; 42 print " --postgres-srcdir Path to postgres build directory\n"; 43 print " --pgxsdir Path to the PGXS directory\n"; 44 print " --load-extension Extensions to install in all nodes\n"; 45 print " --server-option Config option to pass to the server\n"; 46 print " --valgrind Run server via valgrind\n"; 47 print " --valgrind-path Path to the valgrind executable\n"; 48 print " --valgrind-log-file Path to the write valgrind logs\n"; 49 print " --pg_ctl-timeout Timeout for pg_ctl\n"; 50 print " --connection-timeout Timeout for connecting to worker nodes\n"; 51 print " --mitmproxy Start a mitmproxy for one of the workers\n"; 52 exit 1; 53} 54 55my $TMP_CHECKDIR = 'tmp_check'; 56my $TMP_BINDIR = 'tmp-bin'; 57my $MASTERDIR = 'master'; 58my $MASTER_FOLLOWERDIR = 'master-follower'; 59 60# Option parsing 61my $isolationtester = 0; 62my $vanillatest = 0; 63my $followercluster = 0; 64my $bindir = ""; 65my $libdir = undef; 66my $pgxsdir = ""; 67my $postgresBuilddir = ""; 68my $postgresSrcdir = ""; 69my $majorversion = ""; 70my $synchronousReplication = ""; 71my @extensions = (); 72my @userPgOptions = (); 73my %fdws = (); 74my %fdwServers = (); 75my %functions = (); 76my $valgrind = 0; 77my $valgrindPath = "valgrind"; 78my $valgrindLogFile = "valgrind_test_log.txt"; 79my $pgCtlTimeout = undef; 80my $connectionTimeout = 5000; 81my $useMitmproxy = 0; 82my $mitmFifoPath = catfile($TMP_CHECKDIR, "mitmproxy.fifo"); 83my $conninfo = ""; 84my $publicWorker1Host = "localhost"; 85my $publicWorker2Host = "localhost"; 86 87my $serversAreShutdown = "TRUE"; 88my $usingWindows = 0; 89my $mitmPid = 0; 90 91if ($Config{osname} eq "MSWin32") 92{ 93 $usingWindows = 1; 94}; 95 96GetOptions( 97 'isolationtester' => \$isolationtester, 98 'vanillatest' => \$vanillatest, 99 'follower-cluster' => \$followercluster, 100 'bindir=s' => \$bindir, 101 'libdir=s' => \$libdir, 102 'pgxsdir=s' => \$pgxsdir, 103 'postgres-builddir=s' => \$postgresBuilddir, 104 'postgres-srcdir=s' => \$postgresSrcdir, 105 'majorversion=s' => \$majorversion, 106 'load-extension=s' => \@extensions, 107 'server-option=s' => \@userPgOptions, 108 'valgrind' => \$valgrind, 109 'valgrind-path=s' => \$valgrindPath, 110 'valgrind-log-file=s' => \$valgrindLogFile, 111 'pg_ctl-timeout=s' => \$pgCtlTimeout, 112 'connection-timeout=s' => \$connectionTimeout, 113 'mitmproxy' => \$useMitmproxy, 114 'conninfo=s' => \$conninfo, 115 'worker-1-public-hostname=s' => \$publicWorker1Host, 116 'worker-2-public-hostname=s' => \$publicWorker2Host, 117 'help' => sub { Usage() }); 118 119my $fixopen = "$bindir/postgres.fixopen"; 120my @pg_ctl_args = (); 121if (-e $fixopen) 122{ 123 push(@pg_ctl_args, "-p"); 124 push(@pg_ctl_args, $fixopen); 125} 126 127# Update environment to include [DY]LD_LIBRARY_PATH/LIBDIR/etc - 128# pointing to the libdir - that's required so the right version of 129# libpq, citus et al is being picked up. 130# 131# XXX: There's some issues with el capitan's SIP here, causing 132# DYLD_LIBRARY_PATH not being inherited if SIP is enabled. That's a 133# known problem, present in postgres itself as well. 134if (defined $libdir) 135{ 136 $ENV{LD_LIBRARY_PATH} = "$libdir:".($ENV{LD_LIBRARY_PATH} || ''); 137 $ENV{DYLD_LIBRARY_PATH} = "$libdir:".($ENV{DYLD_LIBRARY_PATH} || ''); 138 $ENV{LIBPATH} = "$libdir:".($ENV{LIBPATH} || ''); 139 $ENV{PATH} = "$libdir:".($ENV{PATH} || ''); 140} 141 142# Put $bindir to the end of PATH. We want to prefer system binaries by 143# default (as e.g. new libpq and old psql can cause issues), but still 144# want to find binaries if they're not in PATH. 145if (defined $bindir) 146{ 147 $ENV{PATH} = ($ENV{PATH} || '').":$bindir"; 148} 149 150 151# Most people are used to unified diffs these days, rather than the 152# context diffs pg_regress defaults to. Change default to avoid 153# everyone having to (re-)learn how to change that setting. Also add 154# a bit more context to make it easier to locate failed test sections. 155# 156# Also, ignore whitespace, without this the diffs on windows are unreadable 157$ENV{PG_REGRESS_DIFF_OPTS} = '-dU10 -w'; 158 159my $plainRegress = ""; 160my $isolationRegress = ""; 161my $pgConfig = ""; 162 163if ($usingWindows) 164{ 165 $plainRegress = "$bindir\\pg_regress.exe"; 166 $isolationRegress = "$bindir\\pg_isolation_regress.exe"; 167 $pgConfig = "$bindir\\pg_config.exe"; 168} 169else 170{ 171 $plainRegress = "$pgxsdir/src/test/regress/pg_regress"; 172 $isolationRegress = "${postgresBuilddir}/src/test/isolation/pg_isolation_regress"; 173 $pgConfig = "$bindir/pg_config"; 174 175 if (-x "$pgxsdir/src/test/isolation/pg_isolation_regress") 176 { 177 $isolationRegress = "$pgxsdir/src/test/isolation/pg_isolation_regress"; 178 } 179} 180 181if ($isolationtester && ! -f "$isolationRegress") 182{ 183 die <<"MESSAGE"; 184 185isolationtester not found at $isolationRegress. 186 187isolationtester tests can only be run when source (detected as ${postgresSrcdir}) 188and build (detected as ${postgresBuilddir}) directory corresponding to $bindir 189are present. 190 191Additionally isolationtester in src/test/isolation needs to be built, 192which it is not by default if tests have not been run. If the build 193directory is present locally 194"make -C ${postgresBuilddir} all" should do the trick. 195MESSAGE 196} 197 198my $vanillaRegress = catfile("${postgresBuilddir}", "src", "test", "regress", "pg_regress"); 199my $vanillaSchedule = catfile(dirname("${pgxsdir}"), "regress", "parallel_schedule"); 200 201if ($vanillatest && ! (-f "$vanillaRegress" or -f "$vanillaSchedule")) 202{ 203 die <<"MESSAGE"; 204 205pg_regress (for vanilla tests) not found at $vanillaRegress. 206 207Vanilla tests can only be run when source (detected as ${postgresSrcdir}) 208and build (detected as ${postgresBuilddir}) directory corresponding to $bindir 209are present. 210MESSAGE 211} 212 213if ($useMitmproxy) 214{ 215 system("mitmdump --version") == 0 or die "make sure mitmdump is on PATH"; 216} 217 218# If pgCtlTimeout is defined, we will set related environment variable. 219# This is generally used with valgrind because valgrind starts slow and we 220# need to increase timeout. 221if (defined $pgCtlTimeout) 222{ 223 $ENV{PGCTLTIMEOUT} = "$pgCtlTimeout"; 224} 225 226# We don't want valgrind to run pg_ctl itself, as that'd trigger a lot 227# of spurious OS failures, e.g. in bash. So instead we have to replace 228# the postgres binary with a wrapper that exec's valgrind, which in 229# turn then executes postgres. That's unfortunately at the moment the 230# only reliable way to do this. 231sub replace_postgres 232{ 233 if (-e catfile("$bindir", "postgres.orig")) 234 { 235 print "wrapper exists\n"; 236 } 237 else 238 { 239 print "moving $bindir/postgres to $bindir/postgres.orig\n"; 240 rename catfile("$bindir", "postgres"), catfile("$bindir", "postgres.orig") 241 or die "Could not move postgres out of the way"; 242 } 243 244 sysopen my $fh, catfile("$bindir", "postgres"), O_CREAT|O_TRUNC|O_RDWR, 0700 245 or die "Could not create postgres wrapper at $bindir/postgres"; 246 print $fh <<"END"; 247#!/bin/bash 248exec $valgrindPath \\ 249 --quiet \\ 250 --suppressions=${postgresSrcdir}/src/tools/valgrind.supp \\ 251 --trace-children=yes --track-origins=yes --read-var-info=no \\ 252 --leak-check=no \\ 253 --error-markers=VALGRINDERROR-BEGIN,VALGRINDERROR-END \\ 254 --max-stackframe=16000000 \\ 255 --log-file=$valgrindLogFile \\ 256 --fullpath-after=/ \\ 257 $bindir/postgres.orig \\ 258 "\$@" 259END 260 close $fh; 261} 262 263sub write_settings_to_postgres_conf 264{ 265 my ($pgOptions, $pgConfigPath) = @_; 266 open(my $fd, ">>", $pgConfigPath); 267 268 foreach (@$pgOptions) 269 { 270 print $fd "$_\n"; 271 } 272 273 close $fd; 274} 275 276# revert changes replace_postgres() performed 277sub revert_replace_postgres 278{ 279 if (-e catfile("$bindir", "postgres.orig")) 280 { 281 print "wrapper exists, removing\n"; 282 print "moving $bindir/postgres.orig to $bindir/postgres\n"; 283 rename catfile("$bindir", "postgres.orig"), catfile("$bindir", "postgres") 284 or die "Could not move postgres back"; 285 } 286} 287 288# always want to call initdb under normal postgres, so revert from a 289# partial run, even if we're now not using valgrind. 290revert_replace_postgres(); 291 292my $host = "localhost"; 293my $user = "postgres"; 294my $dbname = "postgres"; 295 296# n.b. previously this was on port 57640, which caused issues because that's in the 297# ephemeral port range, it was sometimes in the TIME_WAIT state which prevented us from 298# binding to it. 9060 is now used because it will never be used for client connections, 299# and there don't appear to be any other applications on this port that developers are 300# likely to be running. 301my $mitmPort = 9060; 302 303# Set some default configuration options 304my $masterPort = 57636; 305 306my $workerCount = 2; 307my @workerHosts = (); 308my @workerPorts = (); 309 310if ( $conninfo ) 311{ 312 my %convals = split /=|\s/, $conninfo; 313 if (exists $convals{user}) 314 { 315 $user = $convals{user}; 316 } 317 if (exists $convals{host}) 318 { 319 $host = $convals{host}; 320 } 321 if (exists $convals{port}) 322 { 323 $masterPort = $convals{port}; 324 } 325 if (exists $convals{dbname}) 326 { 327 $dbname = $convals{dbname}; 328 } 329 330 open my $in, '<', "bin/normalize.sed" or die "Cannot open normalize.sed file\n"; 331 open my $out, '>', "bin/normalize_modified.sed" or die "Cannot open normalize_modified.sed file\n"; 332 333 while ( <$in> ) 334 { 335 print $out $_; 336 } 337 338 close $in; 339 340 341 print $out "\n"; 342 print $out "s/\\bdbname=regression\\b/dbname=<db>/g\n"; 343 print $out "s/\\bdbname=$dbname\\b/dbname=<db>/g\n"; 344 print $out "s/\\b$user\\b/<user>/g\n"; 345 print $out "s/\\bpostgres\\b/<user>/g\n"; 346 print $out "s/\\blocalhost\\b/<host>/g\n"; 347 print $out "s/\\b$host\\b/<host>/g\n"; 348 print $out "s/\\b576[0-9][0-9]\\b/xxxxx/g\n"; 349 print $out "s/", substr("$masterPort", 0, length("$masterPort")-2), "[0-9][0-9]/xxxxx/g\n"; 350 351 352 my $worker1host = `psql "$conninfo" -qtAX -c "SELECT nodename FROM pg_dist_node ORDER BY nodeid LIMIT 1;"`; 353 my $worker1port = `psql "$conninfo" -qtAX -c "SELECT nodeport FROM pg_dist_node ORDER BY nodeid LIMIT 1;"`; 354 my $worker2host = `psql "$conninfo" -qtAX -c "SELECT nodename FROM pg_dist_node ORDER BY nodeid OFFSET 1 LIMIT 1;"`; 355 my $worker2port = `psql "$conninfo" -qtAX -c "SELECT nodeport FROM pg_dist_node ORDER BY nodeid OFFSET 1 LIMIT 1;"`; 356 357 $worker1host =~ s/^\s+|\s+$//g; 358 $worker1port =~ s/^\s+|\s+$//g; 359 $worker2host =~ s/^\s+|\s+$//g; 360 $worker2port =~ s/^\s+|\s+$//g; 361 362 push(@workerPorts, $worker1port); 363 push(@workerPorts, $worker2port); 364 push(@workerHosts, $worker1host); 365 push(@workerHosts, $worker2host); 366 367 my $worker1hostReplaced = $worker1host; 368 my $worker2hostReplaced = $worker2host; 369 370 $worker1hostReplaced =~ s/\./\\\./g; 371 $worker2hostReplaced =~ s/\./\\\./g; 372 373 print $out "s/\\b$worker1hostReplaced\\b/<host>/g\n"; 374 print $out "s/\\b$worker2hostReplaced\\b/<host>/g\n"; 375} 376else 377{ 378 for (my $workerIndex = 1; $workerIndex <= $workerCount; $workerIndex++) { 379 my $workerPort = $masterPort + $workerIndex; 380 push(@workerPorts, $workerPort); 381 push(@workerHosts, "localhost"); 382 } 383} 384 385my $followerCoordPort = 9070; 386my @followerWorkerPorts = (); 387for (my $workerIndex = 1; $workerIndex <= $workerCount; $workerIndex++) { 388 my $workerPort = $followerCoordPort + $workerIndex; 389 push(@followerWorkerPorts, $workerPort); 390} 391 392my @pgOptions = (); 393 394# Postgres options set for the tests 395push(@pgOptions, "listen_addresses='${host}'"); 396push(@pgOptions, "fsync=off"); 397if (! $vanillatest) 398{ 399 push(@pgOptions, "extra_float_digits=0"); 400} 401 402my $sharedPreloadLibraries = "citus"; 403 404# check if pg_stat_statements extension is installed 405# if it is add it to shared preload libraries 406my $sharedir = `$pgConfig --sharedir`; 407chomp $sharedir; 408my $pg_stat_statements_control = catfile($sharedir, "extension", "pg_stat_statements.control"); 409if (-e $pg_stat_statements_control) 410{ 411 $sharedPreloadLibraries .= ',pg_stat_statements'; 412} 413 414# check if hll extension is installed 415# if it is add it to shared preload libraries 416my $hll_control = catfile($sharedir, "extension", "hll.control"); 417if (-e $hll_control) 418{ 419 $sharedPreloadLibraries .= ',hll'; 420} 421push(@pgOptions, "shared_preload_libraries='${sharedPreloadLibraries}'"); 422 423if ($vanillatest) { 424 # use the default used in vanilla tests 425 push(@pgOptions, "max_parallel_workers_per_gather=2"); 426}else { 427 # Avoid parallelism to stabilize explain plans 428 push(@pgOptions, "max_parallel_workers_per_gather=0"); 429} 430 431# Help with debugging 432push(@pgOptions, "log_error_verbosity = 'verbose'"); 433 434# Allow CREATE SUBSCRIPTION to work 435push(@pgOptions, "wal_level='logical'"); 436 437# Faster logical replication status update so tests with logical replication 438# run faster 439push(@pgOptions, "wal_receiver_status_interval=1"); 440 441# Faster logical replication apply worker launch so tests with logical 442# replication run faster. This is used in ApplyLauncherMain in 443# src/backend/replication/logical/launcher.c. 444push(@pgOptions, "wal_retrieve_retry_interval=1000"); 445 446# disable compute_query_id so that we don't get Query Identifiers 447# in explain outputs 448if ($majorversion >= "14") { 449 push(@pgOptions, "compute_query_id=off"); 450} 451 452# Citus options set for the tests 453push(@pgOptions, "citus.shard_count=4"); 454push(@pgOptions, "citus.max_adaptive_executor_pool_size=4"); 455push(@pgOptions, "citus.shard_max_size=1500kB"); 456push(@pgOptions, "citus.defer_shard_delete_interval=-1"); 457push(@pgOptions, "citus.repartition_join_bucket_count_per_node=2"); 458push(@pgOptions, "citus.sort_returning='on'"); 459push(@pgOptions, "citus.shard_replication_factor=2"); 460push(@pgOptions, "citus.node_connection_timeout=${connectionTimeout}"); 461push(@pgOptions, "citus.explain_analyze_sort_method='taskId'"); 462push(@pgOptions, "citus.enable_manual_changes_to_shards=on"); 463 464# we disable slow start by default to encourage parallelism within tests 465push(@pgOptions, "citus.executor_slow_start_interval=0ms"); 466 467if ($useMitmproxy) 468{ 469 # make tests reproducible by never trying to negotiate ssl 470 push(@pgOptions, "citus.node_conninfo='sslmode=disable'"); 471 # The commands that we intercept are based on the the text based protocol. 472 push(@pgOptions, "citus.enable_binary_protocol='false'"); 473} 474elsif ($followercluster) 475{ 476 # follower clusters don't work well when automatically generating certificates as the 477 # followers do not execute the extension creation sql scripts that trigger the creation 478 # of certificates 479 push(@pgOptions, "citus.node_conninfo='sslmode=prefer'"); 480} 481 482if ($useMitmproxy) 483{ 484 if (! -e $TMP_CHECKDIR) 485 { 486 make_path($TMP_CHECKDIR) or die "could not create $TMP_CHECKDIR directory"; 487 } 488 my $absoluteFifoPath = abs_path($mitmFifoPath); 489 die 'abs_path returned empty string' unless ($absoluteFifoPath ne ""); 490 push(@pgOptions, "citus.mitmfifo='$absoluteFifoPath'"); 491} 492 493if ($followercluster) 494{ 495 push(@pgOptions, "max_wal_senders=10"); 496 push(@pgOptions, "hot_standby=on"); 497 push(@pgOptions, "wal_level='replica'"); 498} 499 500 501# disable automatic distributed deadlock detection during the isolation testing 502# to make sure that we always get consistent test outputs. If we don't manually 503# (i.e., calling a UDF) detect the deadlocks, some sessions that do not participate 504# in the deadlock may interleave with the deadlock detection, which results in non- 505# consistent test outputs. 506# since we have CREATE/DROP distributed tables very frequently, we also set 507# shard_count to 4 to speed up the tests. 508if($isolationtester) 509{ 510 push(@pgOptions, "citus.worker_min_messages='warning'"); 511 push(@pgOptions, "citus.log_distributed_deadlock_detection=on"); 512 push(@pgOptions, "citus.distributed_deadlock_detection_factor=-1"); 513 push(@pgOptions, "citus.shard_count=4"); 514 push(@pgOptions, "citus.metadata_sync_interval=1000"); 515 push(@pgOptions, "citus.metadata_sync_retry_interval=100"); 516 push(@pgOptions, "client_min_messages='warning'"); # pg12 introduced notice showing during isolation tests 517 push(@pgOptions, "citus.running_under_isolation_test=true"); 518 519} 520 521# Add externally added options last, so they overwrite the default ones above 522for my $option (@userPgOptions) 523{ 524 push(@pgOptions, $option); 525} 526 527# define functions as signature->definition 528%functions = (); 529if (!$conninfo) 530{ 531 %functions = ('fake_fdw_handler()', 'fdw_handler AS \'citus\' LANGUAGE C STRICT;'); 532} 533else 534{ 535 # when running the tests on a cluster these will be created with run_command_on_workers 536 # so extra single quotes are needed 537 %functions = ('fake_fdw_handler()', 'fdw_handler AS \'\'citus\'\' LANGUAGE C STRICT;'); 538} 539 540#define fdws as name->handler name 541%fdws = ('fake_fdw', 'fake_fdw_handler'); 542 543#define server_name->fdw 544%fdwServers = ('fake_fdw_server', 'fake_fdw'); 545 546# Cleanup leftovers and prepare directories for the run 547if (-e catfile($TMP_CHECKDIR, $TMP_BINDIR)) 548{ 549 remove_tree(catfile($TMP_CHECKDIR, $TMP_BINDIR)) or die "Could not remove $TMP_BINDIR directory"; 550} 551 552if (-e catfile($TMP_CHECKDIR, $MASTERDIR)) 553{ 554 remove_tree(catfile($TMP_CHECKDIR, $MASTERDIR)) or die "Could not remove $MASTERDIR directory"; 555} 556 557for my $port (@workerPorts) 558{ 559 if (-e catfile($TMP_CHECKDIR, "worker.$port")) 560 { 561 remove_tree(catfile($TMP_CHECKDIR, "worker.$port")) or die "Could not remove worker directory"; 562 } 563} 564 565if (-e catfile($TMP_CHECKDIR, $MASTER_FOLLOWERDIR)) 566{ 567 remove_tree(catfile($TMP_CHECKDIR, $MASTER_FOLLOWERDIR)) or die "Could not remove $MASTER_FOLLOWERDIR directory"; 568} 569 570for my $port (@followerWorkerPorts) 571{ 572 if (-e catfile($TMP_CHECKDIR, "follower.$port")) 573 { 574 remove_tree(catfile($TMP_CHECKDIR, "follower.$port")) or die "Could not remove worker directory"; 575 } 576} 577 578# Prepare directory in which 'psql' has some helpful variables for locating the workers 579make_path(catfile($TMP_CHECKDIR, $TMP_BINDIR)) or die "Could not create $TMP_BINDIR directory $!\n"; 580 581my $psql_name = "psql"; 582if ($usingWindows) 583{ 584 $psql_name = "psql.cmd"; 585} 586 587sysopen my $fh, catfile($TMP_CHECKDIR, $TMP_BINDIR, $psql_name), O_CREAT|O_TRUNC|O_RDWR, 0700 588 or die "Could not create psql wrapper"; 589if ($usingWindows) 590{ 591 print $fh "\@echo off\n"; 592} 593print $fh catfile($bindir, "psql")." "; 594print $fh "--variable=master_port=$masterPort "; 595print $fh "--variable=worker_2_proxy_port=$mitmPort "; 596print $fh "--variable=follower_master_port=$followerCoordPort "; 597print $fh "--variable=default_user=$user "; 598print $fh "--variable=SHOW_CONTEXT=always "; 599for my $workeroff (0 .. $#workerPorts) 600{ 601 my $port = $workerPorts[$workeroff]; 602 print $fh "--variable=worker_".($workeroff+1)."_port=$port "; 603} 604for my $workeroff (0 .. $#workerHosts) 605{ 606 my $host = $workerHosts[$workeroff]; 607 print $fh "--variable=worker_".($workeroff+1)."_host=\"$host\" "; 608} 609print $fh "--variable=master_host=\"$host\" "; 610print $fh "--variable=public_worker_1_host=\"$publicWorker1Host\" "; 611print $fh "--variable=public_worker_2_host=\"$publicWorker2Host\" "; 612for my $workeroff (0 .. $#followerWorkerPorts) 613{ 614 my $port = $followerWorkerPorts[$workeroff]; 615 print $fh "--variable=follower_worker_".($workeroff+1)."_port=$port "; 616} 617 618if ($usingWindows) 619{ 620 print $fh "--variable=dev_null=\"/nul\" "; 621 print $fh "--variable=temp_dir=\"%TEMP%\" "; 622 print $fh "--variable=psql=\"".catfile($bindir, "psql")."\" "; 623} 624else 625{ 626 print $fh "--variable=dev_null=\"/dev/null\" "; 627 print $fh "--variable=temp_dir=\"/tmp/\" "; 628 print $fh "--variable=psql=\"psql\" "; 629} 630 631 632if ($usingWindows) 633{ 634 print $fh "%*\n"; # pass on the commandline arguments 635} 636else 637{ 638 print $fh "\"\$@\"\n"; # pass on the commandline arguments 639} 640close $fh; 641 642 643if (!$conninfo) 644{ 645 make_path(catfile($TMP_CHECKDIR, $MASTERDIR, 'log')) or die "Could not create $MASTERDIR directory"; 646 for my $port (@workerPorts) 647 { 648 make_path(catfile($TMP_CHECKDIR, "worker.$port", "log")) 649 or die "Could not create worker directory"; 650 } 651 652 if ($followercluster) 653 { 654 make_path(catfile($TMP_CHECKDIR, $MASTER_FOLLOWERDIR, 'log')) or die "Could not create $MASTER_FOLLOWERDIR directory"; 655 for my $port (@followerWorkerPorts) 656 { 657 make_path(catfile($TMP_CHECKDIR, "follower.$port", "log")) 658 or die "Could not create worker directory"; 659 } 660 } 661 662 # Create new data directories, copy workers for speed 663 # --allow-group-access is used to ensure we set permissions on private keys 664 # correctly 665 system(catfile("$bindir", "initdb"), ("--nosync", "--allow-group-access", "-U", $user, "--encoding", "UTF8", catfile($TMP_CHECKDIR, $MASTERDIR, "data"))) == 0 666 or die "Could not create $MASTERDIR data directory"; 667 668 if ($usingWindows) 669 { 670 for my $port (@workerPorts) 671 { 672 system(catfile("$bindir", "initdb"), ("--nosync", "--allow-group-access", "-U", $user, "--encoding", "UTF8", catfile($TMP_CHECKDIR, "worker.$port", "data"))) == 0 673 or die "Could not create worker data directory"; 674 } 675 } 676 else 677 { 678 for my $port (@workerPorts) 679 { 680 system("cp", ("-a", catfile($TMP_CHECKDIR, $MASTERDIR, "data"), catfile($TMP_CHECKDIR, "worker.$port", "data"))) == 0 681 or die "Could not create worker data directory"; 682 } 683 } 684} 685 686 687# Routine to shutdown servers at failure/exit 688sub ShutdownServers() 689{ 690 if (!$conninfo && $serversAreShutdown eq "FALSE") 691 { 692 system(catfile("$bindir", "pg_ctl"), 693 (@pg_ctl_args, 'stop', '-w', '-D', catfile($TMP_CHECKDIR, $MASTERDIR, 'data'))) == 0 694 or warn "Could not shutdown worker server"; 695 696 for my $port (@workerPorts) 697 { 698 system(catfile("$bindir", "pg_ctl"), 699 (@pg_ctl_args, 'stop', '-w', '-D', catfile($TMP_CHECKDIR, "worker.$port", "data"))) == 0 700 or warn "Could not shutdown worker server"; 701 } 702 703 if ($followercluster) 704 { 705 system(catfile("$bindir", "pg_ctl"), 706 (@pg_ctl_args, 'stop', '-w', '-D', catfile($TMP_CHECKDIR, $MASTER_FOLLOWERDIR, 'data'))) == 0 707 or warn "Could not shutdown worker server"; 708 709 for my $port (@followerWorkerPorts) 710 { 711 system(catfile("$bindir", "pg_ctl"), 712 (@pg_ctl_args, 'stop', '-w', '-D', catfile($TMP_CHECKDIR, "follower.$port", "data"))) == 0 713 or warn "Could not shutdown worker server"; 714 } 715 } 716 if ($mitmPid != 0) 717 { 718 # '-' means signal the process group, 2 is SIGINT 719 kill(-2, $mitmPid) or warn "could not interrupt mitmdump"; 720 } 721 $serversAreShutdown = "TRUE"; 722 } 723} 724 725# setup the signal handler before we fork 726$SIG{CHLD} = sub { 727 # If, for some reason, mitmproxy dies before we do, we should also die! 728 while ((my $waitpid = waitpid(-1, WNOHANG)) > 0) { 729 if ($mitmPid != 0 && $mitmPid == $waitpid) { 730 die "aborting tests because mitmdump failed unexpectedly"; 731 } 732 } 733}; 734 735if ($useMitmproxy) 736{ 737 if (! -e $mitmFifoPath) 738 { 739 mkfifo($mitmFifoPath, 0777) or die "could not create fifo"; 740 } 741 742 if (! -p $mitmFifoPath) 743 { 744 die "a file already exists at $mitmFifoPath, delete it before trying again"; 745 } 746 747 system("lsof -i :$mitmPort"); 748 if (! $?) { 749 die "cannot start mitmproxy because a process already exists on port $mitmPort"; 750 } 751 752 if ($Config{osname} eq "linux") 753 { 754 system("netstat --tcp -n | grep $mitmPort"); 755 } 756 else 757 { 758 system("netstat -p tcp -n | grep $mitmPort"); 759 } 760 761 my $childPid = fork(); 762 763 die("Failed to fork\n") 764 unless (defined $childPid); 765 766 die("No child process\n") 767 if ($childPid < 0); 768 769 $mitmPid = $childPid; 770 771 if ($mitmPid eq 0) { 772 print("forked, about to exec mitmdump\n"); 773 setpgrp(0,0); # we're about to spawn both a shell and a mitmdump, kill them as a group 774 exec("mitmdump --rawtcp -p $mitmPort --mode reverse:localhost:57638 -s $regressdir/mitmscripts/fluent.py --set fifo=$mitmFifoPath --set flow_detail=0 --set termlog_verbosity=warn >proxy.output 2>&1"); 775 die 'could not start mitmdump'; 776 } 777} 778 779# Set signals to shutdown servers 780$SIG{INT} = \&ShutdownServers; 781$SIG{QUIT} = \&ShutdownServers; 782$SIG{TERM} = \&ShutdownServers; 783$SIG{__DIE__} = \&ShutdownServers; 784 785# Shutdown servers on exit only if help option is not used 786END 787{ 788 if ($? != 1) 789 { 790 ShutdownServers(); 791 } 792 793 # At the end of a run, replace redirected binary with original again 794 if ($valgrind) 795 { 796 revert_replace_postgres(); 797 } 798} 799 800# want to use valgrind, replace binary before starting server 801if ($valgrind) 802{ 803 replace_postgres(); 804} 805 806 807# Signal that servers should be shutdown 808$serversAreShutdown = "FALSE"; 809 810# enable synchronous replication if needed 811if ($followercluster) 812{ 813 $synchronousReplication = "-c synchronous_standby_names='FIRST 1 (*)' -c synchronous_commit=remote_apply"; 814} 815 816# Start servers 817if (!$conninfo) 818{ 819 write_settings_to_postgres_conf(\@pgOptions, catfile($TMP_CHECKDIR, $MASTERDIR, "data/postgresql.conf")); 820 if(system(catfile("$bindir", "pg_ctl"), 821 (@pg_ctl_args, 'start', '-w', 822 '-o', " -c port=$masterPort $synchronousReplication", 823 '-D', catfile($TMP_CHECKDIR, $MASTERDIR, 'data'), '-l', catfile($TMP_CHECKDIR, $MASTERDIR, 'log', 'postmaster.log'))) != 0) 824 { 825 system("tail", ("-n20", catfile($TMP_CHECKDIR, $MASTERDIR, "log", "postmaster.log"))); 826 die "Could not start master server"; 827 } 828 829 for my $port (@workerPorts) 830 { 831 write_settings_to_postgres_conf(\@pgOptions, catfile($TMP_CHECKDIR, "worker.$port", "data/postgresql.conf")); 832 if(system(catfile("$bindir", "pg_ctl"), 833 (@pg_ctl_args, 'start', '-w', 834 '-o', " -c port=$port $synchronousReplication", 835 '-D', catfile($TMP_CHECKDIR, "worker.$port", "data"), 836 '-l', catfile($TMP_CHECKDIR, "worker.$port", "log", "postmaster.log"))) != 0) 837 { 838 system("tail", ("-n20", catfile($TMP_CHECKDIR, "worker.$port", "log", "postmaster.log"))); 839 die "Could not start worker server"; 840 } 841 } 842} 843 844# Setup the follower nodes 845if ($followercluster) 846{ 847 system(catfile("$bindir", "pg_basebackup"), 848 ("-D", catfile($TMP_CHECKDIR, $MASTER_FOLLOWERDIR, "data"), "--host=$host", "--port=$masterPort", 849 "--username=$user", "-R", "-X", "stream", "--no-sync")) == 0 850 or die 'could not take basebackup'; 851 852 for my $offset (0 .. $#workerPorts) 853 { 854 my $workerPort = $workerPorts[$offset]; 855 my $followerPort = $followerWorkerPorts[$offset]; 856 system(catfile("$bindir", "pg_basebackup"), 857 ("-D", catfile($TMP_CHECKDIR, "follower.$followerPort", "data"), "--host=$host", "--port=$workerPort", 858 "--username=$user", "-R", "-X", "stream")) == 0 859 or die "Could not take basebackup"; 860 } 861 862 write_settings_to_postgres_conf(\@pgOptions, catfile($TMP_CHECKDIR, $MASTER_FOLLOWERDIR, "data/postgresql.conf")); 863 if(system(catfile("$bindir", "pg_ctl"), 864 (@pg_ctl_args, 'start', '-w', 865 '-o', " -c port=$followerCoordPort", 866 '-D', catfile($TMP_CHECKDIR, $MASTER_FOLLOWERDIR, 'data'), '-l', catfile($TMP_CHECKDIR, $MASTER_FOLLOWERDIR, 'log', 'postmaster.log'))) != 0) 867 { 868 system("tail", ("-n20", catfile($TMP_CHECKDIR, $MASTER_FOLLOWERDIR, "log", "postmaster.log"))); 869 die "Could not start master follower server"; 870 } 871 872 for my $port (@followerWorkerPorts) 873 { 874 write_settings_to_postgres_conf(\@pgOptions, catfile($TMP_CHECKDIR, "follower.$port", "data/postgresql.conf")); 875 if(system(catfile("$bindir", "pg_ctl"), 876 (@pg_ctl_args, 'start', '-w', 877 '-o', " -c port=$port", 878 '-D', catfile($TMP_CHECKDIR, "follower.$port", "data"), 879 '-l', catfile($TMP_CHECKDIR, "follower.$port", "log", "postmaster.log"))) != 0) 880 { 881 system("tail", ("-n20", catfile($TMP_CHECKDIR, "follower.$port", "log", "postmaster.log"))); 882 die "Could not start follower server"; 883 } 884 } 885} 886 887### 888# Create database, extensions, types, functions and fdws on the workers, 889# pg_regress won't know to create them for us. 890### 891if (!$conninfo) 892{ 893 for my $port (@workerPorts) 894 { 895 system(catfile($bindir, "psql"), 896 ('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "postgres", 897 '-c', "CREATE DATABASE regression;")) == 0 898 or die "Could not create regression database on worker"; 899 900 for my $extension (@extensions) 901 { 902 system(catfile($bindir, "psql"), 903 ('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "regression", 904 '-c', "CREATE EXTENSION IF NOT EXISTS $extension;")) == 0 905 or die "Could not create extension on worker"; 906 } 907 908 foreach my $function (keys %functions) 909 { 910 system(catfile($bindir, "psql"), 911 ('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "regression", 912 '-c', "CREATE FUNCTION $function RETURNS $functions{$function};")) == 0 913 or die "Could not create FUNCTION $function on worker"; 914 } 915 916 foreach my $fdw (keys %fdws) 917 { 918 system(catfile($bindir, "psql"), 919 ('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "regression", 920 '-c', "CREATE FOREIGN DATA WRAPPER $fdw HANDLER $fdws{$fdw};")) == 0 921 or die "Could not create foreign data wrapper $fdw on worker"; 922 } 923 924 foreach my $fdwServer (keys %fdwServers) 925 { 926 system(catfile($bindir, "psql"), 927 ('-X', '-h', $host, '-p', $port, '-U', $user, "-d", "regression", 928 '-c', "CREATE SERVER $fdwServer FOREIGN DATA WRAPPER $fdwServers{$fdwServer};")) == 0 929 or die "Could not create server $fdwServer on worker"; 930 } 931 } 932} 933else 934{ 935 for my $extension (@extensions) 936 { 937 system(catfile($bindir, "psql"), 938 ('-X', '-h', $host, '-p', $masterPort, '-U', $user, "-d", $dbname, 939 '-c', "SELECT run_command_on_workers('CREATE EXTENSION IF NOT EXISTS $extension;');")) == 0 940 or die "Could not create extension on worker"; 941 } 942 foreach my $function (keys %functions) 943 { 944 system(catfile($bindir, "psql"), 945 ('-X', '-h', $host, '-p', $masterPort, '-U', $user, "-d", $dbname, 946 '-c', "SELECT run_command_on_workers('CREATE FUNCTION $function RETURNS $functions{$function};');")) == 0 947 or die "Could not create FUNCTION $function on worker"; 948 } 949 950 foreach my $fdw (keys %fdws) 951 { 952 system(catfile($bindir, "psql"), 953 ('-X', '-h', $host, '-p', $masterPort, '-U', $user, "-d", $dbname, 954 '-c', "SELECT run_command_on_workers('CREATE FOREIGN DATA WRAPPER $fdw HANDLER $fdws{$fdw};');")) == 0 955 or die "Could not create foreign data wrapper $fdw on worker"; 956 } 957 958 foreach my $fdwServer (keys %fdwServers) 959 { 960 system(catfile($bindir, "psql"), 961 ('-X', '-h', $host, '-p', $masterPort, '-U', $user, "-d", $dbname, 962 '-c', "SELECT run_command_on_workers('CREATE SERVER $fdwServer FOREIGN DATA WRAPPER $fdwServers{$fdwServer};');")) == 0 963 or die "Could not create server $fdwServer on worker"; 964 } 965} 966 967# Prepare pg_regress arguments 968my @arguments = ( 969 "--host", $host, 970 '--port', $masterPort, 971 '--user', $user, 972 '--bindir', catfile($TMP_CHECKDIR, $TMP_BINDIR) 973); 974 975# Add load extension parameters to the argument list 976for my $extension (@extensions) 977{ 978 push(@arguments, "--load-extension=$extension"); 979} 980 981# Append remaining ARGV arguments to pg_regress arguments 982push(@arguments, @ARGV); 983 984my $startTime = time(); 985 986my $exitcode = 0; 987 988# Finally run the tests 989if ($vanillatest) 990{ 991 $ENV{PGHOST} = $host; 992 $ENV{PGPORT} = $masterPort; 993 $ENV{PGUSER} = $user; 994 $ENV{VANILLATEST} = "1"; 995 996 if (-f "$vanillaSchedule") 997 { 998 rmdir "./testtablespace"; 999 mkdir "./testtablespace"; 1000 1001 my $pgregressdir=catfile(dirname("$pgxsdir"), "regress"); 1002 $exitcode = system("$plainRegress", ("--inputdir", $pgregressdir), 1003 ("--schedule", catfile("$pgregressdir", "parallel_schedule"))) 1004 } 1005 else 1006 { 1007 $exitcode = system("make", ("-C", catfile("$postgresBuilddir", "src", "test", "regress"), "installcheck-parallel")) 1008 } 1009} 1010elsif ($isolationtester) 1011{ 1012 push(@arguments, "--dbname=regression"); 1013 $exitcode = system("$isolationRegress", @arguments) 1014} 1015else 1016{ 1017 if ($conninfo) 1018 { 1019 push(@arguments, "--dbname=$dbname"); 1020 push(@arguments, "--use-existing"); 1021 } 1022 $exitcode = system("$plainRegress", @arguments); 1023} 1024 1025system ("copy_modified"); 1026my $endTime = time(); 1027 1028if ($exitcode == 0) { 1029 print "Finished in ". ($endTime - $startTime)." seconds. \n"; 1030 exit 0; 1031} 1032else { 1033 die "Failed in ". ($endTime - $startTime)." seconds. \n"; 1034 1035} 1036 1037