1#################################################################################################################################### 2# JobTest.pm - Run a test job and monitor progress 3#################################################################################################################################### 4package pgBackRestTest::Common::JobTest; 5 6#################################################################################################################################### 7# Perl includes 8#################################################################################################################################### 9use strict; 10use warnings FATAL => qw(all); 11use Carp qw(confess); 12use English '-no_match_vars'; 13 14use Cwd qw(abs_path); 15use Exporter qw(import); 16 our @EXPORT = qw(); 17use File::Basename qw(dirname basename); 18use POSIX qw(ceil); 19use Time::HiRes qw(gettimeofday usleep); 20 21use pgBackRestDoc::Common::Exception; 22use pgBackRestDoc::Common::Log; 23use pgBackRestDoc::Common::String; 24use pgBackRestDoc::ProjectInfo; 25 26use pgBackRestTest::Common::BuildTest; 27use pgBackRestTest::Common::ContainerTest; 28use pgBackRestTest::Common::CoverageTest; 29use pgBackRestTest::Common::DbVersion; 30use pgBackRestTest::Common::DefineTest; 31use pgBackRestTest::Common::ExecuteTest; 32use pgBackRestTest::Common::ListTest; 33use pgBackRestTest::Common::RunTest; 34use pgBackRestTest::Common::VmTest; 35 36#################################################################################################################################### 37# new 38#################################################################################################################################### 39sub new 40{ 41 my $class = shift; # Class name 42 43 # Create the class hash 44 my $self = {}; 45 bless $self, $class; 46 47 # Assign function parameters, defaults, and log debug info 48 ( 49 my $strOperation, 50 $self->{oStorageTest}, 51 $self->{strBackRestBase}, 52 $self->{strTestPath}, 53 $self->{oTest}, 54 $self->{bDryRun}, 55 $self->{bVmOut}, 56 $self->{iVmIdx}, 57 $self->{iVmMax}, 58 $self->{strMakeCmd}, 59 $self->{iTestIdx}, 60 $self->{iTestMax}, 61 $self->{strLogLevel}, 62 $self->{strLogLevelTest}, 63 $self->{strLogLevelTestFile}, 64 $self->{bLogTimestamp}, 65 $self->{bLogForce}, 66 $self->{bShowOutputAsync}, 67 $self->{bNoCleanup}, 68 $self->{iRetry}, 69 $self->{bValgrindUnit}, 70 $self->{bCoverageUnit}, 71 $self->{bCoverageSummary}, 72 $self->{bOptimize}, 73 $self->{bBackTrace}, 74 $self->{bProfile}, 75 $self->{iScale}, 76 $self->{strTimeZone}, 77 $self->{bDebug}, 78 $self->{bDebugTestTrace}, 79 $self->{iBuildMax}, 80 ) = 81 logDebugParam 82 ( 83 __PACKAGE__ . '->new', \@_, 84 {name => 'oStorageTest'}, 85 {name => 'strBackRestBase'}, 86 {name => 'strTestPath'}, 87 {name => 'oTest'}, 88 {name => 'bDryRun'}, 89 {name => 'bVmOut'}, 90 {name => 'iVmIdx'}, 91 {name => 'iVmMax'}, 92 {name => 'strMakeCmd'}, 93 {name => 'iTestIdx'}, 94 {name => 'iTestMax'}, 95 {name => 'strLogLevel'}, 96 {name => 'strLogLevelTest'}, 97 {name => 'strLogLevelTestFile'}, 98 {name => 'bLogTimestamp'}, 99 {name => 'bLogForce'}, 100 {name => 'bShowOutputAsync'}, 101 {name => 'bNoCleanup'}, 102 {name => 'iRetry'}, 103 {name => 'bValgrindUnit'}, 104 {name => 'bCoverageUnit'}, 105 {name => 'bCoverageSummary'}, 106 {name => 'bOptimize'}, 107 {name => 'bBackTrace'}, 108 {name => 'bProfile'}, 109 {name => 'iScale'}, 110 {name => 'strTimeZone', required => false}, 111 {name => 'bDebug'}, 112 {name => 'bDebugTestTrace'}, 113 {name => 'iBuildMax'}, 114 ); 115 116 # Set try to 0 117 $self->{iTry} = 0; 118 119 # Setup the path where gcc coverage will be performed 120 $self->{strGCovPath} = "$self->{strTestPath}/gcov-$self->{oTest}->{&TEST_VM}-$self->{iVmIdx}"; 121 $self->{strDataPath} = "$self->{strTestPath}/data-$self->{iVmIdx}"; 122 $self->{strRepoPath} = "$self->{strTestPath}/repo"; 123 124 # Return from function and log return values if any 125 return logDebugReturn 126 ( 127 $strOperation, 128 {name => 'self', value => $self, trace => true} 129 ); 130} 131 132#################################################################################################################################### 133# run 134#################################################################################################################################### 135sub run 136{ 137 my $self = shift; 138 139 # Assign function parameters, defaults, and log debug info 140 (my $strOperation) = logDebugParam (__PACKAGE__ . '->run', \@_,); 141 142 # Start the test timer 143 my $fTestStartTime = gettimeofday(); 144 145 # Was the job run? 146 my $bRun = false; 147 148 # Should the job be run? 149 $self->{iTry}++; 150 151 if ($self->{iTry} <= ($self->{iRetry} + 1)) 152 { 153 if ($self->{iTry} != 1 && $self->{iTry} == ($self->{iRetry} + 1)) 154 { 155 $self->{strLogLevel} = lc(DEBUG); 156 } 157 158 my $strTest = sprintf( 159 'P%0' . length($self->{iVmMax}) . 'd-T%0' . length($self->{iTestMax}) . 'd/%0' . 160 length($self->{iTestMax}) . "d - ", $self->{iVmIdx} + 1, $self->{iTestIdx} + 1, $self->{iTestMax}) . 161 'vm=' . $self->{oTest}->{&TEST_VM} . 162 ', module=' . $self->{oTest}->{&TEST_MODULE} . 163 ', test=' . $self->{oTest}->{&TEST_NAME} . 164 (defined($self->{oTest}->{&TEST_RUN}) ? ', run=' . join(',', sort(@{$self->{oTest}->{&TEST_RUN}})) : '') . 165 (defined($self->{oTest}->{&TEST_DB}) ? ', pg-version=' . $self->{oTest}->{&TEST_DB} : '') . 166 ($self->{iTry} > 1 ? ' (retry ' . ($self->{iTry} - 1) . ')' : ''); 167 168 my $strImage = 'test-' . $self->{iVmIdx}; 169 my $strDbVersion = (defined($self->{oTest}->{&TEST_DB}) ? $self->{oTest}->{&TEST_DB} : PG_VERSION_94); 170 $strDbVersion =~ s/\.//; 171 172 &log($self->{bDryRun} && !$self->{bVmOut} || $self->{bShowOutputAsync} ? INFO : DETAIL, "${strTest}" . 173 (!($self->{bDryRun} || !$self->{bVmOut}) || $self->{bShowOutputAsync} ? "\n" : '')); 174 175 my $strVmTestPath = '/home/' . TEST_USER . "/test/${strImage}"; 176 my $strHostTestPath = "$self->{strTestPath}/${strImage}"; 177 178 # Don't create the container if this is a dry run unless output from the VM is required. Output can be requested 179 # to get more information about the specific tests that will be run. 180 if (!$self->{bDryRun} || $self->{bVmOut}) 181 { 182 # Create host test directory 183 $self->{oStorageTest}->pathCreate($strHostTestPath, {strMode => '0770'}); 184 185 # Create gcov directory 186 my $bGCovExists = true; 187 188 if ($self->{oTest}->{&TEST_C} && !$self->{oStorageTest}->pathExists($self->{strGCovPath})) 189 { 190 $self->{oStorageTest}->pathCreate($self->{strGCovPath}, {strMode => '0770'}); 191 $bGCovExists = false; 192 } 193 194 # Create data directory 195 if ($self->{oTest}->{&TEST_C} && !$self->{oStorageTest}->pathExists($self->{strDataPath})) 196 { 197 $self->{oStorageTest}->pathCreate($self->{strDataPath}, {strMode => '0770'}); 198 } 199 200 if ($self->{oTest}->{&TEST_CONTAINER}) 201 { 202 if ($self->{oTest}->{&TEST_VM} ne VM_NONE) 203 { 204 my $strBinPath = $self->{strTestPath} . '/bin/' . $self->{oTest}->{&TEST_VM} . '/' . PROJECT_EXE; 205 206 executeTest( 207 'docker run -itd -h ' . $self->{oTest}->{&TEST_VM} . "-test --name=${strImage}" . 208 " -v ${strHostTestPath}:${strVmTestPath}" . 209 ($self->{oTest}->{&TEST_C} ? " -v $self->{strGCovPath}:$self->{strGCovPath}" : '') . 210 ($self->{oTest}->{&TEST_C} ? " -v $self->{strDataPath}:$self->{strDataPath}" : '') . 211 " -v $self->{strBackRestBase}:$self->{strBackRestBase}" . 212 " -v $self->{strRepoPath}:$self->{strRepoPath}:ro" . 213 ($self->{oTest}->{&TEST_BIN_REQ} ? " -v ${strBinPath}:${strBinPath}:ro" : '') . 214 ' ' . containerRepo() . ':' . $self->{oTest}->{&TEST_VM} . '-test', 215 {bSuppressStdErr => true}); 216 } 217 } 218 } 219 220 # Create run parameters 221 my $strCommandRunParam = ''; 222 223 foreach my $iRunIdx (@{$self->{oTest}->{&TEST_RUN}}) 224 { 225 $strCommandRunParam .= ' --run=' . $iRunIdx; 226 } 227 228 if (!$self->{bDryRun} || $self->{bVmOut}) 229 { 230 my $strCommand = undef; # Command to run test 231 232 # If testing C code 233 if ($self->{oTest}->{&TEST_C}) 234 { 235 my $strRepoCopyPath = $self->{strTestPath} . '/repo'; # Path to repo copy 236 my $strRepoCopySrcPath = $strRepoCopyPath . '/src'; # Path to repo copy src 237 my $strRepoCopyTestSrcPath = $strRepoCopyPath . '/test/src'; # Path to repo copy test src 238 my $strShimSrcPath = $self->{strGCovPath} . '/src'; # Path to shim src 239 my $strShimTestSrcPath = $self->{strGCovPath} . '/test/src'; # Path to shim test src 240 241 my $bCleanAll = false; # Do all object files need to be cleaned? 242 my $bConfigure = false; # Does configure need to be run? 243 244 # If the build.processing file exists then wipe the path to start clean 245 # ------------------------------------------------------------------------------------------------------------------ 246 my $strBuildProcessingFile = $self->{strGCovPath} . "/build.processing"; 247 248 # If the file exists then processing terminated before test.bin was run in the last test and the path might be in a 249 # bad state. 250 if ($self->{oStorageTest}->exists($strBuildProcessingFile)) 251 { 252 executeTest("find $self->{strGCovPath} -mindepth 1 -print0 | xargs -0 rm -rf"); 253 } 254 255 # Write build.processing to track processing of this test 256 $self->{oStorageTest}->put($strBuildProcessingFile); 257 258 # Create Makefile.in 259 # ------------------------------------------------------------------------------------------------------------------ 260 my $strMakefileIn = 261 "CC_CONFIG = \@CC\@\n" . 262 "CFLAGS_CONFIG = \@CFLAGS\@\n" . 263 "CPPFLAGS_CONFIG = \@CPPFLAGS\@\n" . 264 "LDFLAGS_CONFIG = \@LDFLAGS\@\n" . 265 "LIBS_CONFIG = \@LIBS\@ \@LIBS_BUILD\@\n"; 266 267 # If Makefile.in has changed then configure needs to be run and all files cleaned 268 if (buildPutDiffers($self->{oStorageTest}, $self->{strGCovPath} . "/Makefile.in", $strMakefileIn)) 269 { 270 $bConfigure = true; 271 $bCleanAll = true; 272 } 273 274 # Create Makefile.param 275 # ------------------------------------------------------------------------------------------------------------------ 276 # Disable debug/coverage for performance and profile tests 277 my $bPerformance = $self->{oTest}->{&TEST_TYPE} eq TESTDEF_PERFORMANCE; 278 279 if ($bPerformance || $self->{bProfile}) 280 { 281 $self->{bDebug} = false; 282 $self->{bDebugTestTrace} = false; 283 $self->{bCoverageUnit} = false; 284 } 285 286 # Is coverage being tested? 287 my $bCoverage = vmCoverageC($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit}; 288 289 # Generate Makefile.param 290 my $strMakefileParam = 291 "CFLAGS =" . 292 " \\\n\t-Werror -Wfatal-errors -g" . 293 ($self->{bProfile} ? " \\\n\t-pg" : '') . 294 (vmArchBits($self->{oTest}->{&TEST_VM}) == 32 ? " \\\n\t-D_FILE_OFFSET_BITS=64" : '') . 295 ($self->{bDebug} ? '' : " \\\n\t-DNDEBUG") . 296 ($self->{oTest}->{&TEST_VM} eq VM_CO7 ? " \\\n\t-DDEBUG_EXEC_TIME" : '') . 297 ($bCoverage ? " \\\n\t-DDEBUG_COVERAGE" : '') . 298 ($self->{bDebugTestTrace} && $self->{bDebug} ? " \\\n\t-DDEBUG_TEST_TRACE" : '') . 299 (vmWithBackTrace($self->{oTest}->{&TEST_VM}) && $self->{bBackTrace} ? " \\\n\t-DWITH_BACKTRACE" : '') . 300 ($self->{oTest}->{&TEST_CDEF} ? " \\\n\t$self->{oTest}->{&TEST_CDEF}" : '') . 301 "\n" . 302 "\n" . 303 "CFLAGS_TEST =" . 304 " \\\n\t" . (($self->{bOptimize} && ($self->{bProfile} || $bPerformance)) ? '-O2' : '-O0') . 305 " \\\n\t-DDEBUG_MEM" . 306 (!$self->{bDebugTestTrace} && $self->{bDebug} ? " \\\n\t-DDEBUG_TEST_TRACE" : '') . 307 ($bCoverage ? " \\\n\t-fprofile-arcs -ftest-coverage" : '') . 308 ($self->{oTest}->{&TEST_VM} eq VM_NONE ? '' : " \\\n\t-DTEST_CONTAINER_REQUIRED") . 309 ($self->{oTest}->{&TEST_CTESTDEF} ? " \\\n\t$self->{oTest}->{&TEST_CTESTDEF}" : '') . 310 "\n" . 311 "\n" . 312 "CFLAGS_HARNESS =" . 313 " \\\n\t" . ($self->{bOptimize} ? '-O2' : '-O0') . 314 ($self->{oTest}->{&TEST_CTESTDEF} ? " \\\n\t$self->{oTest}->{&TEST_CTESTDEF}" : '') . 315 "\n" . 316 "\n" . 317 "CFLAGS_CORE =" . 318 " \\\n\t" . ($self->{bOptimize} ? '-O2' : '-O0') . 319 "\n" . 320 "\n" . 321 "LDFLAGS =" . 322 ($self->{bProfile} ? " \\\n\t-pg" : '') . 323 "\n" . 324 "\n" . 325 "LIBS =" . 326 ($bCoverage ? " \\\n\t-lgcov" : '') . 327 (vmWithBackTrace($self->{oTest}->{&TEST_VM}) && $self->{bBackTrace} ? " \\\n\t-lbacktrace" : '') . 328 "\n" . 329 "\n" . 330 "INCLUDE =" . 331 " \\\n\t-I\"${strShimSrcPath}\"" . 332 " \\\n\t-I\"${strShimTestSrcPath}\"" . 333 " \\\n\t-I\"${strRepoCopySrcPath}\"" . 334 " \\\n\t-I\"${strRepoCopyTestSrcPath}\"" . 335 "\n" . 336 "\n" . 337 "vpath \%.c ${strShimSrcPath}:${strShimTestSrcPath}:${strRepoCopySrcPath}:${strRepoCopyTestSrcPath}\n"; 338 339 # If Makefile.param has changed then clean all files 340 if (buildPutDiffers($self->{oStorageTest}, $self->{strGCovPath} . "/Makefile.param", $strMakefileParam)) 341 { 342 $bCleanAll = true; 343 } 344 345 # Generate list of harness files 346 # ------------------------------------------------------------------------------------------------------------------ 347 my $hTest = (testDefModuleTest($self->{oTest}->{&TEST_MODULE}, $self->{oTest}->{&TEST_NAME})); 348 my $strRepoCopyTestSrcHarnessPath = $strRepoCopyTestSrcPath . '/common'; 349 350 # C modules included in harness files that should not be added to the make list 351 my $rhHarnessCModule = {}; 352 353 # List of harness files to include in make 354 my @stryHarnessFile = ('common/harnessTest'); 355 356 foreach my $rhHarness (@{$hTest->{&TESTDEF_HARNESS}}) 357 { 358 my $bFound = false; 359 my $strFile = "common/harness" . ucfirst($rhHarness->{&TESTDEF_HARNESS_NAME}); 360 361 # Include harness file if present 362 my $strHarnessSrcFile = "${strRepoCopyTestSrcPath}/${strFile}.c"; 363 364 if ($self->{oStorageTest}->exists($strHarnessSrcFile)) 365 { 366 $bFound = true; 367 368 if (!defined($hTest->{&TESTDEF_HARNESS_SHIM_DEF}{$rhHarness->{&TESTDEF_HARNESS_NAME}})) 369 { 370 push(@stryHarnessFile, $strFile); 371 } 372 373 # Install shim 374 my $rhShim = $rhHarness->{&TESTDEF_HARNESS_SHIM}; 375 376 if (defined($rhShim)) 377 { 378 my $strHarnessSrc = ${$self->{oStorageTest}->get($strHarnessSrcFile)}; 379 380 # Error if there is no placeholder for the shimmed modules 381 if ($strHarnessSrc !~ /\{\[SHIM\_MODULE\]\}/) 382 { 383 confess &log(ERROR, "{[SHIM_MODULE]} tag not found in '${strFile}' harness with shims"); 384 } 385 386 # Build list of shimmed C modules 387 my $strShimModuleList = undef; 388 389 foreach my $strShimModule (sort(keys(%{$rhShim}))) 390 { 391 # If there are shimmed elements the C module will need to be updated and saved to the test path 392 if (defined($rhShim->{$strShimModule})) 393 { 394 my $strShimModuleSrc = ${$self->{oStorageTest}->get( 395 "${strRepoCopySrcPath}/${strShimModule}.c")}; 396 my @stryShimModuleSrcRenamed; 397 my $strFunctionDeclaration = undef; 398 my $strFunctionShim = undef; 399 400 foreach my $strLine (split("\n", $strShimModuleSrc)) 401 { 402 # If shimmed function declaration construction is in progress 403 if (defined($strFunctionShim)) 404 { 405 # When the beginning of the function block is found, output both the constructed 406 # declaration and the renamed implementation. 407 if ($strLine =~ /^{/) 408 { 409 push(@stryShimModuleSrcRenamed, trim($strFunctionDeclaration) . ";"); 410 push(@stryShimModuleSrcRenamed, $strFunctionShim); 411 push(@stryShimModuleSrcRenamed, $strLine); 412 413 $strFunctionShim = undef; 414 } 415 # Else keep constructing the declaration and implementation 416 else 417 { 418 $strFunctionDeclaration .= "${strLine}\n"; 419 $strFunctionShim .= "${strLine}\n"; 420 } 421 } 422 # Else search for shimmed functions 423 else 424 { 425 # Rename shimmed functions 426 my $bFound = false; 427 428 foreach my $strFunction (@{$rhShim->{$strShimModule}{&TESTDEF_HARNESS_SHIM_FUNCTION}}) 429 { 430 # If the function to shim is static then we need to create a declaration with the 431 # original name so references to the original name in the C module will compile. 432 # This is not necessary for extern'd functions since they should already have a 433 # declaration in the header file. 434 if ($strLine =~ /^${strFunction}\(/ && $stryShimModuleSrcRenamed[-1] =~ /^static /) 435 { 436 my $strLineLast = pop(@stryShimModuleSrcRenamed); 437 438 $strFunctionDeclaration = "${strLineLast} ${strLine}\n"; 439 440 $strLine =~ s/^${strFunction}\(/${strFunction}_SHIMMED\(/; 441 $strFunctionShim = "${strLineLast}\n${strLine}\n"; 442 443 $bFound = true; 444 last; 445 } 446 } 447 448 # If the function was not found then just append the line 449 if (!$bFound) 450 { 451 push(@stryShimModuleSrcRenamed, $strLine); 452 } 453 } 454 } 455 456 buildPutDiffers( 457 $self->{oStorageTest}, "${strShimSrcPath}/${strShimModule}.c", 458 join("\n", @stryShimModuleSrcRenamed)); 459 } 460 461 # Build list to include in the harness 462 if (defined($strShimModuleList)) 463 { 464 $strShimModuleList .= "\n"; 465 } 466 467 $rhHarnessCModule->{$strShimModule} = true; 468 $strShimModuleList .= "#include \"${strShimModule}.c\""; 469 } 470 471 # Replace modules and save 472 $strHarnessSrc =~ s/\{\[SHIM\_MODULE\]\}/${strShimModuleList}/g; 473 buildPutDiffers($self->{oStorageTest}, "${strShimTestSrcPath}/${strFile}.c", $strHarnessSrc); 474 } 475 } 476 477 # Include files in the harness directory if present 478 for my $strFileSub ( 479 $self->{oStorageTest}->list("${strRepoCopyTestSrcPath}/${strFile}", 480 {bIgnoreMissing => true, strExpression => '\.c$'})) 481 { 482 push(@stryHarnessFile, "${strFile}/" . substr($strFileSub, 0, length($strFileSub) - 2)); 483 $bFound = true; 484 } 485 486 # Error when no harness files were found 487 if (!$bFound) 488 { 489 confess &log(ERROR, "no files found for harness '$rhHarness->{&TESTDEF_HARNESS_NAME}'"); 490 } 491 } 492 493 # Generate list of core files (files to be tested/included in this unit will be excluded) 494 # ------------------------------------------------------------------------------------------------------------------ 495 my $hTestCoverage = $hTest->{&TESTDEF_COVERAGE}; 496 497 my @stryCoreFile; 498 499 foreach my $strFile (@{$hTest->{&TESTDEF_CORE}}) 500 { 501 # Skip all files except .c files (including .auto.c and .vendor.c) 502 next if $strFile !~ /(?<!\.auto)$/ || $strFile !~ /(?<!\.vendor)$/; 503 504 # Skip if no C file exists 505 next if !$self->{oStorageTest}->exists("${strRepoCopySrcPath}/${strFile}.c"); 506 507 # Skip if the C file is included in the harness 508 next if defined($rhHarnessCModule->{$strFile}); 509 510 if (!defined($hTestCoverage->{$strFile}) && !grep(/^$strFile$/, @{$hTest->{&TESTDEF_INCLUDE}})) 511 { 512 push(@stryCoreFile, $strFile); 513 } 514 } 515 516 # Create Makefile 517 # ------------------------------------------------------------------------------------------------------------------ 518 my $strMakefile = 519 "include Makefile.config\n" . 520 "include Makefile.param\n" . 521 "\n" . 522 "SRCS = test.c \\\n" . 523 "\t" . join('.c ', @stryHarnessFile) . ".c \\\n" . 524 (@stryCoreFile > 0 ? "\t" . join('.c ', @stryCoreFile) . ".c\n" : ''). 525 "\n" . 526 ".build/test.o: CFLAGS += \$(CFLAGS_TEST)\n" . 527 "\n" . 528 ".build/" . join(".o: CFLAGS += \$(CFLAGS_HARNESS)\n.build/", @stryHarnessFile) . 529 ".o: CFLAGS += \$(CFLAGS_HARNESS)\n" . 530 "\n" . 531 ".build/" . join(".o: CFLAGS += \$(CFLAGS_CORE)\n.build/", @stryCoreFile) . 532 ".o: CFLAGS += \$(CFLAGS_CORE)\n" . 533 "\n" . 534 ".build/\%.o : \%.c\n" . 535 " \@if test ! -d \$(\@D); then mkdir -p \$(\@D); fi\n" . 536 " \$(CC_CONFIG) \$(INCLUDE) \$(CFLAGS_CONFIG) \$(CPPFLAGS_CONFIG) \$(CFLAGS)" . 537 " -c -o \$\@ \$< -MMD -MP -MF .build/\$*.dep\n" . 538 "\n" . 539 "OBJS = \$(patsubst \%.c,.build/\%.o,\$(SRCS))\n" . 540 "\n" . 541 "test: \$(OBJS)\n" . 542 " \$(CC_CONFIG) -o test.bin \$(OBJS) \$(LDFLAGS_CONFIG) \$(LDFLAGS) \$(LIBS_CONFIG) \$(LIBS)\n" . 543 "\n" . 544 "rwildcard = \$(wildcard \$1\$2) \$(foreach d,\$(wildcard \$1*),\$(call rwildcard,\$d/,\$2))\n" . 545 "DEP_FILES = \$(call rwildcard,.build,*.dep)\n" . 546 "include \$(DEP_FILES)\n"; 547 548 buildPutDiffers($self->{oStorageTest}, $self->{strGCovPath} . "/Makefile", $strMakefile); 549 550 # Create test.c 551 # ------------------------------------------------------------------------------------------------------------------ 552 # Generate list of C files to include for testing 553 my $strTestDepend = ''; 554 my $strTestFile = 555 "module/$self->{oTest}->{&TEST_MODULE}/" . testRunName($self->{oTest}->{&TEST_NAME}, false) . 'Test.c'; 556 my $strCInclude; 557 558 foreach my $strFile (sort(keys(%{$hTestCoverage}), @{$hTest->{&TESTDEF_INCLUDE}})) 559 { 560 # Don't include the test file as it is already included below 561 next if $strFile =~ /Test$/; 562 563 # Don't include auto files as they are included in their companion C files 564 next if $strFile =~ /auto$/; 565 566 # Don't include vendor files as they are included in regular C files 567 next if $strFile =~ /vendor$/; 568 569 # Skip if the C file is included in the harness 570 next if defined($rhHarnessCModule->{$strFile}); 571 572 # Include the C file if it exists 573 my $strCIncludeFile = "${strFile}.c"; 574 575 # If the C file does not exist use the header file instead 576 if (!$self->{oStorageTest}->exists("${strRepoCopySrcPath}/${strCIncludeFile}")) 577 { 578 # Error if code was expected 579 if ($hTestCoverage->{$strFile} ne TESTDEF_COVERAGE_NOCODE) 580 { 581 confess &log(ERROR, "unable to find source file '${strRepoCopySrcPath}/${strCIncludeFile}'"); 582 } 583 584 $strCIncludeFile = "${strFile}.h"; 585 } 586 587 $strCInclude .= (defined($strCInclude) ? "\n" : '') . "#include \"${strCIncludeFile}\""; 588 $strTestDepend .= " ${strCIncludeFile}"; 589 } 590 591 # Add harnesses with shims that are first defined in this module 592 foreach my $strHarness (sort(keys(%{$hTest->{&TESTDEF_HARNESS_SHIM_DEF}}))) 593 { 594 $strCInclude .= 595 (defined($strCInclude) ? "\n" : '') . "#include \"common/harness" . ucfirst($strHarness) . ".c\""; 596 } 597 598 # Update C test file with test module 599 my $strTestC = ${$self->{oStorageTest}->get("${strRepoCopyTestSrcPath}/test.c")}; 600 601 if (defined($strCInclude)) 602 { 603 $strTestC =~ s/\{\[C\_INCLUDE\]\}/$strCInclude/g; 604 } 605 else 606 { 607 $strTestC =~ s/\{\[C\_INCLUDE\]\}//g; 608 } 609 610 $strTestC =~ s/\{\[C\_TEST\_INCLUDE\]\}/\#include \"$strTestFile\"/g; 611 $strTestDepend .= " ${strTestFile}"; 612 613 # Determine where the project exe is located 614 my $strProjectExePath = "$self->{strTestPath}/bin/$self->{oTest}->{&TEST_VM}/" . PROJECT_EXE; 615 616 # Is this test running in a container? 617 my $strContainer = $self->{oTest}->{&TEST_VM} eq VM_NONE ? 'false' : 'true'; 618 619 # What test path should be passed to C? Containers always have their test path at ~/test but when running with 620 # vm=none it should be in a subdirectory of the current directory. 621 my $strTestPathC = $self->{oTest}->{&TEST_VM} eq VM_NONE ? $strHostTestPath : $strVmTestPath; 622 623 # Set globals 624 $strTestC =~ s/\{\[C\_TEST\_CONTAINER\]\}/$strContainer/g; 625 $strTestC =~ s/\{\[C\_TEST\_PROJECT\_EXE\]\}/$strProjectExePath/g; 626 $strTestC =~ s/\{\[C\_TEST\_PATH\]\}/$strTestPathC/g; 627 $strTestC =~ s/\{\[C\_TEST\_USER\]\}/${\TEST_USER}/g; 628 $strTestC =~ s/\{\[C\_TEST\_USER\_ID\]\}/${\TEST_USER_ID}/g; 629 $strTestC =~ s/\{\[C\_TEST\_GROUP\]\}/${\TEST_GROUP}/g; 630 $strTestC =~ s/\{\[C\_TEST\_GROUP\_ID\]\}/${\TEST_GROUP_ID}/g; 631 $strTestC =~ s/\{\[C\_TEST\_PGB\_PATH\]\}/$strRepoCopyPath/g; 632 $strTestC =~ s/\{\[C\_HRN\_PATH\]\}/$self->{strDataPath}/g; 633 $strTestC =~ s/\{\[C\_TEST\_IDX\]\}/$self->{iVmIdx}/g; 634 $strTestC =~ s/\{\[C\_HRN\_PATH\_REPO\]\}/$self->{strBackRestBase}/g; 635 $strTestC =~ s/\{\[C\_TEST\_SCALE\]\}/$self->{iScale}/g; 636 637 my $strLogTimestampC = $self->{bLogTimestamp} ? 'true' : 'false'; 638 $strTestC =~ s/\{\[C\_TEST\_TIMING\]\}/$strLogTimestampC/g; 639 640 # Set timezone 641 if (defined($self->{strTimeZone})) 642 { 643 $strTestC =~ s/\{\[C\_TEST\_TZ\]\}/setenv\("TZ", "$self->{strTimeZone}", true\);/g; 644 } 645 else 646 { 647 $strTestC =~ s/\{\[C\_TEST\_TZ\]\}/\/\/ No timezone specified/g; 648 } 649 650 # Set default log level 651 my $strLogLevelTestC = "logLevel" . ucfirst($self->{strLogLevelTest}); 652 $strTestC =~ s/\{\[C\_LOG\_LEVEL\_TEST\]\}/$strLogLevelTestC/g; 653 654 # Initialize tests 655 my $strTestInit; 656 657 for (my $iTestIdx = 1; $iTestIdx <= $hTest->{&TESTDEF_TOTAL}; $iTestIdx++) 658 { 659 my $bSelected = false; 660 661 if (!defined($self->{oTest}->{&TEST_RUN}) || @{$self->{oTest}->{&TEST_RUN}} == 0 || 662 grep(/^$iTestIdx$/, @{$self->{oTest}->{&TEST_RUN}})) 663 { 664 $bSelected = true; 665 } 666 667 $strTestInit .= 668 (defined($strTestInit) ? "\n " : '') . 669 sprintf("hrnAdd(%3d, %8s);" , $iTestIdx, ($bSelected ? 'true' : 'false')); 670 } 671 672 $strTestC =~ s/\{\[C\_TEST\_LIST\]\}/$strTestInit/g; 673 674 # Save test.c and make sure it gets a new timestamp 675 # ------------------------------------------------------------------------------------------------------------------ 676 my $strTestCFile = "$self->{strGCovPath}/test.c"; 677 678 if (buildPutDiffers($self->{oStorageTest}, "$self->{strGCovPath}/test.c", $strTestC)) 679 { 680 # Get timestamp for test.bin if it existss 681 my $oTestBinInfo = $self->{oStorageTest}->info("$self->{strGCovPath}/test.bin", {bIgnoreMissing => true}); 682 my $iTestBinOriginalTime = defined($oTestBinInfo) ? $oTestBinInfo->mtime : 0; 683 684 # Get timestamp for test.c 685 my $iTestCNewTime = $self->{oStorageTest}->info($strTestCFile)->mtime; 686 687 # Is the timestamp for test.c newer than test.bin? 688 while ($iTestCNewTime <= $iTestBinOriginalTime) 689 { 690 # If not then sleep until the next second 691 my $iTimeToSleep = ($iTestBinOriginalTime + 1) - gettimeofday(); 692 693 if ($iTimeToSleep > 0) 694 { 695 usleep($iTimeToSleep * 1000000); 696 } 697 698 # Save the file again 699 $self->{oStorageTest}->put($self->{oStorageTest}->openWrite($strTestCFile), $strTestC); 700 $iTestCNewTime = $self->{oStorageTest}->info($strTestCFile)->mtime; 701 } 702 } 703 704 # Create command 705 # ------------------------------------------------------------------------------------------------------------------ 706 # Build filename for valgrind suppressions 707 my $strValgrindSuppress = $self->{strRepoPath} . '/test/src/valgrind.suppress.' . $self->{oTest}->{&TEST_VM}; 708 709 $strCommand = 710 ($self->{oTest}->{&TEST_VM} ne VM_NONE ? "docker exec -i -u ${\TEST_USER} ${strImage} bash -l -c '" : '') . 711 " \\\n" . 712 "cd $self->{strGCovPath} && \\\n" . 713 # Clean build 714 ($bCleanAll ? "rm -rf .build && \\\n" : '') . 715 # Remove coverage data 716 (!$bCleanAll && $bCoverage ? "rm -rf .build/test.gcda && \\\n" : '') . 717 # Configure when required 718 ($bConfigure ? 719 "mv Makefile Makefile.tmp && ${strRepoCopySrcPath}/configure -q --enable-test" . 720 " && mv Makefile Makefile.config && mv Makefile.tmp Makefile && \\\n" : 721 '') . 722 $self->{strMakeCmd} . " -j $self->{iBuildMax} -s 2>&1 && \\\n" . 723 "rm ${strBuildProcessingFile} && \\\n" . 724 # Test with valgrind when requested 725 ($self->{bValgrindUnit} && $self->{oTest}->{&TEST_TYPE} ne TESTDEF_PERFORMANCE ? 726 'valgrind -q --gen-suppressions=all' . 727 ($self->{oStorageTest}->exists($strValgrindSuppress) ? " --suppressions=${strValgrindSuppress}" : '') . 728 " --leak-check=full --leak-resolution=high --error-exitcode=25" . ' ' : '') . 729 "./test.bin 2>&1" . 730 ($self->{oTest}->{&TEST_VM} ne VM_NONE ? "'" : ''); 731 } 732 else 733 { 734 $strCommand = 735 ($self->{oTest}->{&TEST_CONTAINER} ? 'docker exec -i -u ' . TEST_USER . " ${strImage} " : '') . 736 abs_path($0) . 737 " --test-path=${strVmTestPath}" . 738 " --vm=$self->{oTest}->{&TEST_VM}" . 739 " --vm-id=$self->{iVmIdx}" . 740 " --module=" . $self->{oTest}->{&TEST_MODULE} . 741 ' --test=' . $self->{oTest}->{&TEST_NAME} . 742 $strCommandRunParam . 743 (defined($self->{oTest}->{&TEST_DB}) ? ' --pg-version=' . $self->{oTest}->{&TEST_DB} : '') . 744 ($self->{strLogLevel} ne lc(INFO) ? " --log-level=$self->{strLogLevel}" : '') . 745 ($self->{strLogLevelTestFile} ne lc(TRACE) ? " --log-level-test-file=$self->{strLogLevelTestFile}" : '') . 746 ($self->{bLogTimestamp} ? '' : ' --no-log-timestamp') . 747 ' --pgsql-bin=' . $self->{oTest}->{&TEST_PGSQL_BIN} . 748 ($self->{strTimeZone} ? " --tz='$self->{strTimeZone}'" : '') . 749 ($self->{bLogForce} ? ' --log-force' : '') . 750 ($self->{bDryRun} ? ' --dry-run' : '') . 751 ($self->{bDryRun} ? ' --vm-out' : '') . 752 ($self->{bNoCleanup} ? " --no-cleanup" : ''); 753 } 754 755 my $oExec = new pgBackRestTest::Common::ExecuteTest( 756 $strCommand, {bSuppressError => true, bShowOutputAsync => $self->{bShowOutputAsync}}); 757 758 $oExec->begin(); 759 760 $self->{oProcess} = 761 { 762 exec => $oExec, 763 test => $strTest, 764 start_time => $fTestStartTime, 765 }; 766 767 $bRun = true; 768 } 769 } 770 771 # Return from function and log return values if any 772 return logDebugReturn 773 ( 774 $strOperation, 775 {name => 'bRun', value => $bRun, trace => true} 776 ); 777} 778 779#################################################################################################################################### 780# end 781#################################################################################################################################### 782sub end 783{ 784 my $self = shift; 785 786 # Assign function parameters, defaults, and log debug info 787 (my $strOperation) = logDebugParam (__PACKAGE__ . '->run', \@_,); 788 789 # Is the job done? 790 my $bDone = false; 791 my $bFail = false; 792 793 my $oExecDone = $self->{oProcess}{exec}; 794 my $strTestDone = $self->{oProcess}{test}; 795 my $iTestDoneIdx = $self->{oProcess}{idx}; 796 797 my $iExitStatus = $oExecDone->end($self->{iVmMax} == 1); 798 799 if (defined($iExitStatus)) 800 { 801 my $strImage = 'test-' . $self->{iVmIdx}; 802 803 if ($self->{bShowOutputAsync}) 804 { 805 syswrite(*STDOUT, "\n"); 806 } 807 808 # If C code generate profile info 809 if ($iExitStatus == 0 && $self->{oTest}->{&TEST_C} && $self->{bProfile}) 810 { 811 executeTest( 812 ($self->{oTest}->{&TEST_VM} ne VM_NONE ? 'docker exec -i -u ' . TEST_USER . " ${strImage} " : '') . 813 "gprof $self->{strGCovPath}/test.bin $self->{strGCovPath}/gmon.out > $self->{strGCovPath}/gprof.txt"); 814 815 $self->{oStorageTest}->pathCreate( 816 "$self->{strBackRestBase}/test/result/profile", {strMode => '0750', bIgnoreExists => true, bCreateParent => true}); 817 $self->{oStorageTest}->copy( 818 "$self->{strGCovPath}/gprof.txt", "$self->{strBackRestBase}/test/result/profile/gprof.txt"); 819 } 820 821 # If C code generate coverage info 822 if ($iExitStatus == 0 && $self->{oTest}->{&TEST_C} && vmCoverageC($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit}) 823 { 824 coverageExtract( 825 $self->{oStorageTest}, $self->{oTest}->{&TEST_MODULE}, $self->{oTest}->{&TEST_NAME}, 826 $self->{oTest}->{&TEST_VM} ne VM_NONE, $self->{bCoverageSummary}, 827 $self->{oTest}->{&TEST_VM} eq VM_NONE ? undef : $strImage, $self->{strTestPath}, "$self->{strTestPath}/temp", 828 $self->{strGCovPath}, $self->{strBackRestBase} . '/test/result'); 829 } 830 831 # Record elapsed time 832 my $fTestElapsedTime = ceil((gettimeofday() - $self->{oProcess}{start_time}) * 100) / 100; 833 834 # Output error 835 if ($iExitStatus != 0) 836 { 837 # Get stdout 838 my $strOutput = trim($oExecDone->{strOutLog}) ? "STDOUT:\n" . trim($oExecDone->{strOutLog}) : ''; 839 840 # Get stderr 841 if (trim($oExecDone->{strSuppressedErrorLog}) ne '') 842 { 843 if ($strOutput ne '') 844 { 845 $strOutput .= "\n"; 846 } 847 848 $strOutput .= "STDERR:\n" . trim($oExecDone->{strSuppressedErrorLog}); 849 } 850 851 # If no stdout or stderr output something rather than a blank line 852 if ($strOutput eq '') 853 { 854 $strOutput = 'NO OUTPUT ON STDOUT OR STDERR'; 855 } 856 857 &log(ERROR, "${strTestDone} (err${iExitStatus}" . ($self->{bLogTimestamp} ? "-${fTestElapsedTime}s)" : '') . 858 (defined($oExecDone->{strOutLog}) && !$self->{bShowOutputAsync} ? ":\n\n${strOutput}\n" : ''), undef, undef, 4); 859 860 $bFail = true; 861 } 862 # Output success 863 else 864 { 865 &log(INFO, "${strTestDone}" . ($self->{bLogTimestamp} ? " (${fTestElapsedTime}s)" : ''). 866 ($self->{bVmOut} && !$self->{bShowOutputAsync} ? 867 ":\n\n" . trim($oExecDone->{strOutLog}) . "\n" : ''), undef, undef, 4); 868 } 869 870 if (!$self->{bNoCleanup}) 871 { 872 my $strHostTestPath = "$self->{strTestPath}/${strImage}"; 873 874 if ($self->{oTest}->{&TEST_VM} ne VM_NONE) 875 { 876 containerRemove("test-$self->{iVmIdx}"); 877 } 878 879 executeTest("chmod -R 700 ${strHostTestPath}/* 2>&1;rm -rf ${strHostTestPath}"); 880 } 881 882 $bDone = true; 883 } 884 885 # Return from function and log return values if any 886 return logDebugReturn 887 ( 888 $strOperation, 889 {name => 'bDone', value => $bDone, trace => true}, 890 {name => 'bFail', value => $bFail, trace => true} 891 ); 892} 893 8941; 895