1#! /bin/sh 2 3# $Id: check.sh 568071 2018-07-30 14:34:02Z ivanov $ 4# Author: Denis Vakatov, NCBI 5# 6########################################################################### 7# 8# Auxilary script -- to be called by "./Makefile.check" 9# 10# Command line: 11# check.sh <signature> [reporting args] 12# 13# Reporting mode and addresses' args: 14# file:[full:]/abs_fname 15# file:[full:]rel_fname 16# mail:[full:]addr1,addr2,... 17# post:[full:]url 18# stat:addr1,addr2,... 19# watch:addr1,addr2,... 20# debug 21# sendonly 22# 23# Checks on $NCBI_CHECK_SPEED_LEVEL environment variable during test stage. 24# It defines maximum number of directories to run checks simultaneously. 25# 0 means no MT check, just a regualar check. 26# 27########################################################################### 28 29 30#### CONFIG 31 32# Allow to run checks in parallel 33use_mt_checks=true 34# Maximum number of directories to run checks simultaneously 35mt_max_dirs=3 36# Sleep timeout between checks on finished tasks (dirs) 37mt_sleeptime=30 38 39if [ -n "$NCBI_CHECK_SPEED_LEVEL" ]; then 40 if [ $NCBI_CHECK_SPEED_LEVEL -le 1 ]; then 41 use_mt_checks=false 42 else 43 mt_max_dirs=$NCBI_CHECK_SPEED_LEVEL 44 fi 45fi 46 47# The limit on the sending email size in Kbytes 48mail_limit=199 49 50 51#### ERROR STREAM 52 53err_log="/tmp/check.sh.err.$$" 54#exec 2>$err_log 55trap "rm -f $err_log" 1 2 15 56 57 58#### INCLUDE COMMON.SH 59 60script_name=`basename $0` 61script_dir=`dirname $0` 62script_dir=`(cd "${script_dir}" ; pwd)` 63. ${script_dir}/../../scripts/common/common.sh 64 65 66#### MISC 67 68script_args="$*" 69 70summary_res="check.sh.log" 71error_res="check.sh.out_err" 72build_info="build_info" 73 74if test -x /usr/sbin/sendmail; then 75 sendmail="/usr/sbin/sendmail -oi" 76else 77 sendmail="/usr/lib/sendmail -oi" 78fi 79 80 81#### FUNCTIONS 82 83 84# MIME headers intended to keep Outlook Exchange from mangling the body. 85# Perhaps we should hardcode UTF-8 rather than ISO 8859-1, but it's a 86# moot point since we'll generally just encounter ASCII. 87 88DoMime() 89{ 90 cat <<EOF 91MIME-Version: 1.0 92Content-Type: text/plain; charset="iso-8859-1" 93Content-Transfer-Encoding: binary 94EOF 95} 96 97 98# Error reporting 99 100Error() 101{ 102 cat <<EOF 1>&2 103 104----------------------------------------------------------------------- 105 106$script_name $script_args 107 108ERROR: $1 109EOF 110 111 cat $err_log 112 113 kill $$ 114 exit 1 115} 116 117 118 119#### SPECIAL ARGUMENTS (DEBUG, SENDONLY) 120 121debug="no" 122need_check="yes" 123 124for arg in "$@" ; do 125 case "$arg" in 126 debug ) 127 set -xv 128 debug="yes" 129 run_script="/bin/sh -xv" 130 ;; 131 sendonly ) 132 need_check="no" 133 if test ! -f $summary_res -o ! -f $error_res ; then 134 Error "Missing check result files" 135 fi 136 ;; 137 esac 138done 139 140 141#### ARGS 142 143signature="$1" 144shift 145 146# If "$need_check" = "no" that files with check results already 147# prepared and standing in the current directory. 148# Only post results in this case. 149 150if test "$need_check" = "yes" ; then 151 # Default check 152 test $# -ge 3 || Error "Wrong number of args: $#" 153 make="$1" 154 builddir="$2" 155 action="$3" 156 # build_info can be located in the current directory or 2 levels upper, 157 # depends on platform and arguments 158 test ! -f $build_info && build_info="$builddir/../../build_info" 159 160 shift 161 shift 162 shift 163 164 # Use passed $builddir for build_info only, and detect real directory 165 # to run tests in case of fake root. 166 builddir=`(cd ..; pwd)` 167 168 test -d "$builddir" && cd "$builddir" || \ 169 Error "Cannot change directory to: $builddir" 170 171 # Check if we have not a library-only build 172 173 appornull=`grep 'APP_OR_NULL' $builddir/Makefile.mk` 174 if [ -n "$appornull" ]; then 175 # No executables - no checks 176 grep 'APP_OR_NULL = null' $builddir/Makefile.mk >/dev/null && exit 0 177 fi 178 179 180 # Process action 181 182 case "$action" in 183 all ) 184 # continue 185 ;; 186 clean | purge ) 187 test -x ./check.sh && $run_script ./check.sh clean 188 exit 0 189 ;; 190 * ) 191 Error "Invalid action: $action" 192 ;; 193 esac 194 195 #### RUN CHECKS 196 197 export MAKEFLAGS 198 MAKEFLAGS= 199 200 scope='r' 201 if grep '^PROJECTS_ =.*[^ ]' Makefile.meta >/dev/null ; then 202 scope='p' 203 fi 204 205 # --- Multi thread tests 206 if $use_mt_checks; then 207 208 # Prepare global check list 209 210 CHECK_RUN_LIST="$builddir/$script_name.list" 211 export CHECK_RUN_LIST 212 rm $CHECK_RUN_LIST >/dev/null 213 "$make" check_add_${scope} RUN_CHECK=N || Error "MAKE CHECK_ADD_${scope} failed" 214 test -f $CHECK_RUN_LIST || Error "Error preparing check list" 215 216 # Get list of directories to check 217 218 list=`cat "$CHECK_RUN_LIST" | tr -d ' '` 219 dirs='' 220 for row in $list; do 221 dir=`echo "$row" | sed -e 's|/.*$||'` 222 dirs="$dirs $dir" 223 done 224 dirs=`echo $dirs | tr ' ' '\n' | uniq` 225 226 # Generate check.sh for parallel checks execution. 227 # It should look as any other check.sh, and can be 228 # used later to concat results. 229 230cat > check.sh <<EOF 231#! /bin/sh 232 233builddir="$builddir" 234script="\$builddir/check.sh" 235 236res_journal="\$script.journal" 237res_log="\$script.log" 238res_list="\$script.list" 239res_concat="\$script.out" 240res_concat_err="\$script.out_err" 241 242mt_max_dirs=$mt_max_dirs 243mt_sleeptime=$mt_sleeptime 244 245dirs="$dirs" 246 247 248#////////////////////////////////////////////////////////////////////////// 249 250 251# Printout USAGE info and exit 252 253Usage() { 254 cat <<EOF_usage 255 256USAGE: check.sh {run | clean | concat | concat_err} 257 258 run Run the tests. Create output file ("*.test_out") for each test, 259 plus journal and log files. 260 clean Remove all files created during the last "run" and this script 261 itself. 262 concat Concatenate all files created during the last "run" into one big 263 file "\$res_log". 264 concat_err Like previous. But into the file "\$res_concat_err" 265 will be added outputs of failed tests only. 266 report_err Report failed tests directly to developers (NCBI/NIH only). 267 268ERROR: \$1 269 270EOF_usage 271 exit 1 272} 273 274if test \$# -ne 1; then 275 Usage "Invalid number of arguments." 276fi 277 278 279### What to do (cmd-line arg) 280 281method="\$1" 282 283case "\$method" in 284#---------------------------------------------------------- 285 run ) 286 # See below 287 ;; 288#---------------------------------------------------------- 289 clean ) 290 for dir in \$dirs; do 291 \$builddir/\$dir/check.sh clean 292 done 293 rm -f \$res_journal \$res_log \$res_list \$res_concat \$res_concat_err > /dev/null 294 rm -f \$script > /dev/null 295 exit 0 296 ;; 297#---------------------------------------------------------- 298 concat ) 299 rm -f "\$res_concat" 300 ( 301 for dir in \$dirs; do 302 cat \$builddir/\$dir/check.sh.log 303 done 304 for dir in \$dirs; do 305 files=\`cat \$builddir/\$dir/check.sh.journal | sed -e 's/ /%gj_s4%/g'\` 306 for f in \$files; do 307 f=\`echo "\$f" | sed -e 's/%gj_s4%/ /g'\` 308 echo 309 echo 310 cat \$f 311 done 312 done 313 ) >> \$res_concat 314 exit 0 315 ;; 316#---------------------------------------------------------- 317 concat_err ) 318 rm -f "\$res_concat_err" 319 ( 320 for dir in \$dirs; do 321 cat \$builddir/\$dir/check.sh.log | egrep 'ERR \[|TO -' 322 done 323 for dir in \$dirs; do 324 files=\`cat \$builddir/\$dir/check.sh.journal | sed -e 's/ /%gj_s4%/g'\` 325 for f in \$files; do 326 f=\`echo "\$f" | sed -e 's/%gj_s4%/ /g'\` 327 code=\`cat \$f | grep -c '@@@ EXIT CODE:'\` 328 test \$code -ne 0 || continue 329 code=\`cat \$f | grep -c '@@@ EXIT CODE: 0'\` 330 if [ \$code -ne 1 ]; then 331 echo 332 echo 333 cat \$f 334 fi 335 done 336 done 337 ) >> \$res_concat_err 338 exit 0 339 ;; 340#---------------------------------------------------------- 341 report_err ) 342 # This method works inside NCBI only 343 test "\$NCBI_CHECK_MAILTO_AUTHORS." = 'Y.' || exit 0; 344 for dir in \$dirs; do 345 \$builddir/\$dir/check.sh report_err 346 done 347 return 0 348 ;; 349#---------------------------------------------------------- 350 * ) 351 Usage "Invalid method name." 352 ;; 353esac 354 355 356#////////////////////////////////////////////////////////////////////////// 357 358 359Error() 360{ 361 echo "ERROR: \$1" 362 echo 363 kill \$\$ 364 exit 1 365} 366 367tasks_pids='' 368 369Cleanup() 370{ 371 touch \$builddir/check.failed 372 for p in \$tasks_pids; do 373 kill -9 \$p 2>/dev/null 374 done 375# killall -9 -q -e check.sh 2>/dev/null 376 exit 1 377} 378 379 380 381#////////////////////////////////////////////////////////////////////////// 382 383# --- Run 384 385trap "Cleanup" 1 2 15 386rm \$builddir/check.failed \$builddir/check.success > /dev/null 2>&1 387rm \$res_journal \$res_log \$res_concat \$res_concat_err > /dev/null 2>&1 388 389# Task queue and number of tasks in it 390tasks='' 391tasks_count=0 392 393timestamp="date +'%Y-%m-%d %H:%M'" 394 395 396# Run checks in directories in parallel (no more than \$mt_max_dirs simultaneously) 397 398for dir in \$dirs; do 399 # Run checks in \$dir 400 rm \$builddir/\$dir/check.success \$builddir/\$dir/check.failed >/dev/null 2>&1 401 echo [\`eval \$timestamp\`] Checking \'\$dir\' 402 cd \$builddir/\$dir || Error "Cannot change directory to: \$dir" 403 ./check.sh run >/dev/null 2>&1 & 404 tasks_pids="\$tasks_pids \$!" 405 406 # Add it into the task list 407 tasks="\$tasks \$dir " 408 tasks_count=\`expr \$tasks_count + 1\` 409 if [ \$tasks_count -lt \$mt_max_dirs ]; then 410 # If have space in queue, run another task, otherwise wait (below) for empty slot 411 continue 412 fi 413 414 # Wait any running task to finish 415 while true; do 416 # Checks on finished tasks 417 finished_task='' 418 for t in \$tasks; do 419 if [ -f "\$builddir/\$t/check.success" -o \\ 420 -f "\$builddir/\$t/check.failed" ]; then 421 finished_task=\$t 422 break 423 fi 424 done 425 if [ -n "\$finished_task" ]; then 426 echo [\`eval \$timestamp\`] Checking \'\$finished_task\' -- finished 427 # Remove task from queue 428 tasks=\`echo "\$tasks" | sed -e "s/ \$finished_task //"\` 429 tasks_count=\`expr \$tasks_count - 1\` 430 # Ready to process next task (dir) 431 break; 432 else 433 # All slots busy -- waiting before check again 434 sleep \$mt_sleeptime 435 fi 436 done 437done 438 439 440# Wait unfinished tasks 441wait 442tasks_pids='' 443 444 445# Collect check results and logs 446 447failed=false 448 449for dir in \$dirs; do 450 if [ -f "\$builddir/\$dir/check.failed" ]; then 451 failed=true 452 fi 453 cat \$builddir/\$dir/check.sh.journal >> \$res_journal 454 cat \$builddir/\$dir/check.sh.log >> \$res_log 455done 456 457echo 458cat \$res_log 459echo 460 461if \$failed; then 462 touch \$builddir/check.failed 463 exit 1 464fi 465 466touch \$builddir/check.success 467exit 0 468 469EOF 470 471 # Set execute mode to script 472 chmod a+x check.sh 473 474 $ Generate check script for every directory in the list 475 for dir in $dirs; do 476 cd $builddir/$dir || Error "Cannot change directory to: $dir" 477 "$make" check_${scope} RUN_CHECK=N || Error "$dir: MAKE CHECK_${scope} failed" 478 done 479 480 481 # --- Single thread tests 482 else 483 "$make" check_${scope} RUN_CHECK=N || Error "MAKE CHECK_${scope} failed" 484 fi 485 486 # Run checks 487 cd "$builddir" 488 $run_script ./check.sh run 489fi 490 491 492 493#### POST RESULTS 494 495# Parse the destination location list 496for dest in "$@" ; do 497 type=`echo "$dest" | sed 's%\([^:][^:]*\):.*%\1%'` 498 loc=`echo "$dest" | sed 's%.*:\([^:][^:]*\)$%\1%'` 499 500 full="" 501 echo "$dest" | grep ':full:' >/dev/null && full="yes" 502 503 case "$type" in 504 file ) 505 echo "$loc" | grep '^/' >/dev/null || loc="$builddir/$loc" 506 loc=`echo "$loc" | sed 's%{}%'${signature}'%g'` 507 if test -n "$full" ; then 508 file_list_full="$file_list_full $loc" 509 else 510 file_list="$file_list $loc" 511 fi 512 ;; 513 mail ) 514 if test -n "$full" ; then 515 mail_list_full="$mail_list_full $loc" 516 else 517 mail_list="$mail_list $loc" 518 fi 519 ;; 520 stat ) 521 stat_list="$stat_list $loc" 522 ;; 523 watch ) 524 loc=`echo "$loc" | sed 's/,/ /g'` 525 watch_list="$watch_list $loc" 526 ;; 527 debug | sendonly ) 528 ;; 529 * ) 530 err_list="$err_list BAD_TYPE:\"$dest\"" 531 ;; 532 esac 533done 534 535 536# Compose "full" results archive, if necessary 537if test "$need_check" = "yes" ; then 538 $run_script ./check.sh concat_err 539fi 540 541# Post results to the specified locations 542if test -n "$file_list_full" ; then 543 for loc in $file_list_full ; do 544 cp -p $error_res $loc || err_list="$err_list COPY_ERR:\"$loc\"" 545 done 546fi 547 548# Report check results to authors (Unix only) 549if test "$need_check" = "yes" ; then 550 if test "$NCBI_CHECK_MAILTO_AUTHORS." = 'Y.' ; then 551 $run_script ./check.sh report_err 552 fi 553fi 554 555if test -n "$file_list" ; then 556 for loc in $file_list ; do 557 cp -p $summary_res $loc || err_list="$err_list COPY_ERR:\"$loc\"" 558 done 559fi 560 561n_ok=`grep '^OK -- ' "$summary_res" | wc -l | sed 's/ //g'` 562n_err=`grep '^ERR \[[0-9][0-9]*\] -- ' "$summary_res" | wc -l | sed 's/ //g'` 563n_abs=`grep '^ABS -- ' "$summary_res" | wc -l | sed 's/ //g'` 564 565subject="${signature} OK:$n_ok ERR:$n_err ABS:$n_abs" 566tmp_src="/tmp/check.sh.$$.src" 567tmp_dst="/tmp/check.sh.$$.dst" 568 569if test -n "$mail_list_full" ; then 570 { 571 cat $summary_res 572 echo ; echo '%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%' ; echo 573 cat $error_res 574 } > $tmp_src 575 COMMON_LimitTextFileSize $tmp_src $tmp_dst $mail_limit 576 for loc in $mail_list_full ; do 577 mailto=`echo "$loc" | sed 's/,/ /g'` 578 { 579 echo "To: $mailto" 580 echo "Subject: [C++ CHECK] $subject" 581 DoMime 582 echo 583 echo "$subject" 584 echo 585 test -f $build_info && cat $build_info 586 echo 587 cat $tmp_dst 588 } | $sendmail $mailto || err_list="$err_list MAIL_ERR:\"$loc\"" 589 done 590fi 591 592if test -n "$mail_list" -a -s "$error_res" ; then 593 COMMON_LimitTextFileSize $error_res $tmp_dst $mail_limit 594 for loc in $mail_list ; do 595 mailto=`echo "$loc" | sed 's/,/ /g'` 596 { 597 echo "To: $mailto" 598 echo "Subject: [C++ ERRORS] $subject" 599 DoMime 600 echo 601 echo "$subject" 602 echo 603 test -f $build_info && cat $build_info 604 echo 605 cat $tmp_dst 606 } | $sendmail $mailto || err_list="$err_list MAIL_ERR:\"$loc\"" 607 done 608fi 609 610# Post check statistics 611if test -n "$stat_list" ; then 612 COMMON_LimitTextFileSize $summary_res $tmp_dst $mail_limit 613 for loc in $stat_list ; do 614 mailto=`echo "$loc" | sed 's/,/ /g'` 615 { 616 echo "To: $mailto" 617 echo "Subject: [`date '+%Y-%m-%d %H:%M'`] $subject" 618 DoMime 619 echo 620 cat $tmp_dst 621 } | $sendmail $mailto || err_list="$err_list STAT_ERR:\"$loc\"" 622 done 623fi 624 625# Post errors to watchers 626if test -n "$watch_list" ; then 627 if test -n "$err_list" -o "$debug" = "yes" ; then 628 { 629 echo "To: $watch_list" 630 echo "Subject: [C++ WATCH] $signature" 631 DoMime 632 echo 633 echo "$err_list" 634 echo 635 test -f $build_info && cat $build_info 636 echo 637 echo "========================================" 638 COMMON_LimitTextFileSize $err_log $tmp_dst $mail_limit 639 cat $tmp_dst 640 } | $sendmail $watch_list 641 fi 642fi 643 644# Cleanup 645rm -f $tmp_src $tmp_dst $err_log >/dev/null 2>&1 646 647exit 0 648