1# Copyright (c) 2019, Parallax Software, Inc. 2# 3# This program is free software: you can redistribute it and/or modify 4# it under the terms of the GNU General Public License as published by 5# the Free Software Foundation, either version 3 of the License, or 6# (at your option) any later version. 7# 8# This program is distributed in the hope that it will be useful, 9# but WITHOUT ANY WARRANTY; without even the implied warranty of 10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11# GNU General Public License for more details. 12# 13# You should have received a copy of the GNU General Public License 14# along with this program. If not, see <https://www.gnu.org/licenses/>. 15 16# This is a generic regression test script used to compare application 17# output to a known good "ok" file. 18# 19# Use the "regression" command to run the regressions. 20# 21# regression -help | [-valgrind] test1 [test2...] 22# 23# where test is "all" or the name of a test group defined in regression_vars.tcl 24# Wildcards can be used in test names if the name is enclosed in ""s to suppress 25# shell globbing. For example, 26# 27# regression "init_floorplan*" 28# 29# will run all tests with names that begin with "init_floorplan". 30# Each test name is printed before it runs. Once it finishes pass, 31# fail, *NO OK FILE* or *SEG FAULT* is printed after the test name. 32# 33# The results of each test are in the file test/results/<test>.log 34# The diffs for all tests are in test/results/diffs. 35# A list of failed tests is in test/results/failures. 36# To save a log file as the correct output use the save_ok command. 37# 38# save_ok failures | test1 [test2...] 39# 40# This copies test/results/test.log to test/test.ok 41# Using the test name 'failures' copies the ok files for all failed tests. 42# This is a quick way to update the failing test ok files after examining 43# the differences. 44# 45# You should NOT need to modify this script. 46# Customization unique to an application is in "regression_vars.tcl". 47# In this case the application is OpenROAD, so nothing should need to be changed 48# in "regression_vars.tcl". 49# 50# Customize the scripts "regresssion" and "save_ok" to source this file 51# and a file that defines the test scripts, "regresion_tests.tcl". 52# Each test is a tcl command file. 53 54set openroad_test_dir [file join $openroad_dir "test"] 55 56source [file join $openroad_test_dir "regression_vars.tcl"] 57source [file join $openroad_test_dir "flow_metrics.tcl"] 58 59proc regression_main {} { 60 global argv 61 exit [regression_body $argv] 62} 63 64proc regression_body { cmd_argv } { 65 setup 66 parse_args $cmd_argv 67 run_tests 68 show_summary 69 return [found_errors] 70} 71 72proc setup {} { 73 global result_dir diff_file failure_file errors 74 global use_valgrind valgrind_shared_lib_failure 75 76 set use_valgrind 0 77 78 if { !([file exists $result_dir] && [file isdirectory $result_dir]) } { 79 file mkdir $result_dir 80 } 81 file delete $diff_file 82 file delete $failure_file 83 84 set errors(error) 0 85 set errors(memory) 0 86 set errors(leak) 0 87 set errors(fail) 0 88 set errors(no_cmd) 0 89 set errors(no_ok) 0 90 set valgrind_shared_lib_failure 0 91} 92 93proc parse_args { cmd_argv } { 94 global app_options tests test_groups cmd_paths 95 global use_valgrind 96 global result_dir tests 97 98 while { $cmd_argv != {} } { 99 set arg [lindex $cmd_argv 0] 100 if { $arg == "help" || $arg == "-help" } { 101 puts {Usage: regression [-help] [-threads threads] [-valgrind] tests...} 102 puts " -threads max|integer - number of threads to use" 103 puts " -valgrind - run valgrind (linux memory checker)" 104 puts " Wildcarding for test names is supported (enclose in \"'s)" 105 puts " Tests are: all, fast, med, slow, or a test group or test name" 106 puts "" 107 puts " If 'limit coredumpsize unlimited' corefiles are saved in $result_dir/test.core" 108 exit 109 } elseif { $arg == "-threads" } { 110 set threads [lindex $cmd_argv 1] 111 if { !([string is integer $threads] || $threads == "max") } { 112 puts "Error: -threads arg $threads is not an integer or max." 113 exit 0 114 } 115 lappend app_options "-threads" 116 lappend app_options $threads 117 set cmd_argv [lrange $cmd_argv 2 end] 118 } elseif { $arg == "-valgrind" } { 119 set use_valgrind 1 120 set cmd_argv [lrange $cmd_argv 1 end] 121 } else { 122 break 123 } 124 } 125 if { $cmd_argv == {} } { 126 # Default is to run all tests. 127 set tests [group_tests all] 128 } else { 129 set tests [expand_tests $cmd_argv] 130 } 131} 132 133proc expand_tests { tests_arg } { 134 global test_groups 135 136 set tests {} 137 foreach arg $tests_arg { 138 if { [info exists test_groups($arg)] } { 139 set tests [concat $tests $test_groups($arg)] 140 } elseif { [string first "*" $arg] != -1 \ 141 || [string first "?" $arg] != -1 } { 142 # Find wildcard matches. 143 foreach test [group_tests "all"] { 144 if [string match $arg $test] { 145 lappend tests $test 146 } 147 } 148 } elseif { [lsearch [group_tests "all"] $arg] != -1 } { 149 lappend tests $arg 150 } else { 151 puts "Error: test $arg not found." 152 } 153 } 154 return $tests 155} 156 157proc run_tests {} { 158 global tests errors app_path 159 160 foreach test $tests { 161 run_test $test 162 } 163 # Macos debug info generated by valgrind. 164 file delete -force "$app_path.dSYM" 165} 166 167proc run_test { test } { 168 global result_dir diff_file errors diff_options 169 170 set cmd_file [test_cmd_file $test] 171 if [file exists $cmd_file] { 172 set ok_file [test_ok_file $test] 173 set log_file [test_log_file $test] 174 foreach file [glob -nocomplain [file join $result_dir $test.*]] { 175 file delete -force $file 176 } 177 puts -nonewline $test 178 flush stdout 179 set test_errors [run_test_app $test $cmd_file $log_file] 180 if { [lindex $test_errors 0] == "ERROR" } { 181 puts " *ERROR* [lrange $test_errors 1 end]" 182 append_failure $test 183 incr errors(error) 184 185 # For some reason seg faults aren't echoed in the log - add them. 186 if [file exists $log_file] { 187 set log_ch [open $log_file "a"] 188 puts $log_ch "$test_errors" 189 close $log_ch 190 } 191 192 # Report partial log diff anyway. 193 if [file exists $ok_file] { 194 catch [concat exec diff $diff_options $ok_file $log_file \ 195 >> $diff_file] 196 } 197 } else { 198 set error_msg "" 199 if { [lsearch $test_errors "MEMORY"] != -1 } { 200 append error_msg " *MEMORY*" 201 append_failure $test 202 incr errors(memory) 203 } 204 if { [lsearch $test_errors "LEAK"] != -1 } { 205 append error_msg " *LEAK*" 206 append_failure $test 207 incr errors(leak) 208 } 209 210 switch [test_pass_criteria $test] { 211 compare_logfile { 212 if { [file exists $ok_file] } { 213 # Filter dos '/r's from log file. 214 set tmp_file [file join $result_dir $test.tmp] 215 exec tr -d "\r" < $log_file > $tmp_file 216 file rename -force $tmp_file $log_file 217 if [catch [concat exec diff $diff_options $ok_file $log_file \ 218 >> $diff_file]] { 219 puts " *FAIL*$error_msg" 220 append_failure $test 221 incr errors(fail) 222 } else { 223 puts " pass$error_msg" 224 } 225 } else { 226 puts " *NO OK FILE*$error_msg" 227 append_failure $test 228 incr errors(no_ok) 229 } 230 } 231 pass_fail { 232 set error_msg [find_log_pass_fail $log_file] 233 if { $error_msg != "pass" } { 234 puts " *FAIL* $error_msg" 235 append_failure $test 236 incr errors(fail) 237 } else { 238 puts " pass" 239 } 240 } 241 check_metrics { 242 set error_msg [check_test_metrics $test] 243 if { $error_msg != "pass" } { 244 puts " *FAIL* $error_msg" 245 append_failure $test 246 incr errors(fail) 247 } else { 248 puts " pass" 249 } 250 } 251 } 252 } 253 } else { 254 puts "$test *NO CMD FILE*" 255 incr errors(no_cmd) 256 } 257} 258 259proc find_log_pass_fail { log_file } { 260 if { [file exists $log_file] } { 261 set stream [open $log_file r] 262 set last_line "" 263 while { [gets $stream line] >= 0 } { 264 set last_line $line 265 } 266 close $stream 267 if { [string match "pass*" $last_line] } { 268 return "pass" 269 } else { 270 return $last_line 271 } 272 } 273 return "fail - reason not found" 274} 275 276proc append_failure { test } { 277 global failure_file 278 set fail_ch [open $failure_file "a"] 279 puts $fail_ch $test 280 close $fail_ch 281} 282 283# Return error. 284proc run_test_app { test cmd_file log_file } { 285 global app_path errorCode use_valgrind 286 if { $use_valgrind } { 287 return [run_test_valgrind $test $cmd_file $log_file] 288 } else { 289 return [run_test_plain $test $cmd_file $log_file] 290 } 291} 292 293proc run_test_plain { test cmd_file log_file } { 294 global app_path app_options result_dir errorCode 295 296 if { ![file exists $app_path] } { 297 return "ERROR $app_path not found." 298 } elseif { ![file executable $app_path] } { 299 return "ERROR $app_path is not executable." 300 } else { 301 set save_dir [pwd] 302 cd [file dirname $cmd_file] 303 if { [catch [concat exec $app_path $app_options -metrics [test_metrics_result_file $test]\ 304 [file tail $cmd_file] >& $log_file]] } { 305 cd $save_dir 306 set signal [lindex $errorCode 2] 307 set error [lindex $errorCode 3] 308 # Errors strings are not consistent across platforms but signal 309 # names are. 310 if { $signal == "SIGSEGV" } { 311 # Save corefiles to regression results directory. 312 set pid [lindex $errorCode 1] 313 set sys_corefile [test_sys_core_file $test $pid] 314 if { [file exists $sys_corefile] } { 315 file copy $sys_corefile [test_core_file $test] 316 } 317 } 318 cleanse_logfile $test $log_file 319 return "ERROR $error" 320 } 321 cd $save_dir 322 cleanse_logfile $test $log_file 323 return "" 324 } 325} 326 327proc run_test_valgrind { test cmd_file log_file } { 328 global app_path app_options valgrind_options result_dir errorCode 329 330 set vg_cmd_file [test_valgrind_cmd_file $test] 331 set vg_stream [open $vg_cmd_file "w"] 332 puts $vg_stream "cd [file dirname $cmd_file]" 333 puts $vg_stream "source [file tail $cmd_file]" 334 puts $vg_stream "ord::delete_all_memory" 335 close $vg_stream 336 337 set cmd [concat exec valgrind $valgrind_options \ 338 $app_path $app_options $vg_cmd_file >& $log_file] 339 set error_msg "" 340 if { [catch $cmd] } { 341 set error_msg "ERROR [lindex $errorCode 3]" 342 } 343 file delete $vg_cmd_file 344 cleanse_logfile $test $log_file 345 lappend error_msg [cleanse_valgrind_logfile $test $log_file] 346 return $error_msg 347} 348 349# Error messages can be found in "valgrind/memcheck/mc_errcontext.c". 350# 351# "Conditional jump or move depends on uninitialised value(s)" 352# "%s contains unaddressable byte(s)" 353# "%s contains uninitialised or unaddressable byte(s)" 354# "Use of uninitialised value of size %d" 355# "Invalid read of size %d" 356# "Syscall param %s contains uninitialised or unaddressable byte(s)" 357# "Unaddressable byte(s) found during client check request" 358# "Uninitialised or unaddressable byte(s) found during client check request" 359# "Invalid free() / delete / delete[]" 360# "Mismatched free() / delete / delete []" 361set valgrind_mem_regexp "(depends on uninitialised value)|(contains unaddressable)|(contains uninitialised)|(Use of uninitialised value)|(Invalid read)|(Unaddressable byte)|(Uninitialised or unaddressable)|(Invalid free)|(Mismatched free)" 362 363# "%d bytes in %d blocks are definitely lost in loss record %d of %d" 364# "%d bytes in %d blocks are possibly lost in loss record %d of %d" 365#set valgrind_leak_regexp "blocks are (possibly|definitely) lost" 366set valgrind_leak_regexp "blocks are definitely lost" 367 368# Valgrind fails on executables using shared libraries. 369set valgrind_shared_lib_failure_regexp "No malloc'd blocks -- no leaks are possible" 370 371# Scan the log file to separate valgrind notifications and check for 372# valgrind errors. 373proc cleanse_valgrind_logfile { test log_file } { 374 global valgrind_mem_regexp valgrind_leak_regexp 375 global valgrind_shared_lib_failure_regexp 376 global valgrind_shared_lib_failure 377 378 set tmp_file [test_tmp_file $test] 379 set valgrind_log_file [test_valgrind_file $test] 380 file copy -force $log_file $tmp_file 381 set tmp [open $tmp_file "r"] 382 set log [open $log_file "w"] 383 set valgrind [open $valgrind_log_file "w"] 384 set leaks 0 385 set mem_errors 0 386 gets $tmp line 387 while { ![eof $tmp] } { 388 if {[regexp "^==" $line]} { 389 puts $valgrind $line 390 if {[regexp $valgrind_leak_regexp $line]} { 391 set leaks 1 392 } 393 if {[regexp $valgrind_mem_regexp $line]} { 394 set mem_errors 1 395 } 396 if {[regexp $valgrind_shared_lib_failure_regexp $line]} { 397 set valgrind_shared_lib_failure 1 398 } 399 } elseif {[regexp {^--[0-9]+} $line]} { 400 # Valgrind notification line. 401 } else { 402 puts $log $line 403 } 404 gets $tmp line 405 } 406 close $log 407 close $tmp 408 close $valgrind 409 file delete $tmp_file 410 411 set errors {} 412 if { $mem_errors } { 413 lappend errors "MEMORY" 414 } 415 if { $leaks } { 416 lappend errors "LEAK" 417 } 418 return $errors 419} 420 421################################################################ 422 423proc show_summary {} { 424 global errors tests diff_file result_dir valgrind_shared_lib_failure 425 global app_path app 426 427 puts "------------------------------------------------------" 428 set test_count [llength $tests] 429 if { [found_errors] } { 430 if { $errors(error) != 0 } { 431 puts "Errored $errors(error)/$test_count" 432 } 433 if { $errors(fail) != 0 } { 434 puts "Failed $errors(fail)/$test_count" 435 } 436 if { $errors(leak) != 0 } { 437 puts "Memory leaks in $errors(leak)/$test_count" 438 } 439 if { $errors(memory) != 0 } { 440 puts "Memory corruption in $errors(memory)/$test_count" 441 } 442 if { $errors(no_ok) != 0 } { 443 puts "No ok file for $errors(no_ok)/$test_count" 444 } 445 if { $errors(no_cmd) != 0 } { 446 puts "No cmd tcl file for $errors(no_cmd)/$test_count" 447 } 448 if { $errors(fail) != 0 } { 449 puts "See $diff_file for differences" 450 } 451 } else { 452 puts "Passed $test_count" 453 } 454 if { $valgrind_shared_lib_failure } { 455 puts "WARNING: valgrind failed because the executable is not statically linked." 456 } 457 puts "See $result_dir for log files" 458} 459 460proc found_errors {} { 461 global errors 462 463 return [expr $errors(error) != 0 || $errors(fail) != 0 \ 464 || $errors(no_cmd) != 0 || $errors(no_ok) != 0 \ 465 || $errors(memory) != 0 || $errors(leak) != 0 ] 466} 467 468################################################################ 469 470proc save_ok_main {} { 471 global argv 472 if { $argv == "help" || $argv == "-help" } { 473 puts {Usage: save_ok [failures] test1 [test2]...} 474 } else { 475 if { $argv == "failures" } { 476 set tests [failed_tests] 477 } else { 478 set tests $argv 479 } 480 foreach test $tests { 481 if { [lsearch [group_tests "all"] $test] == -1 } { 482 puts "Error: test $test not found." 483 } else { 484 save_ok $test 485 } 486 } 487 } 488} 489 490proc failed_tests {} { 491 global failure_file 492 493 set failures {} 494 if { [file exists $failure_file] } { 495 set fail_ch [open $failure_file "r"] 496 while { ![eof $fail_ch] } { 497 set test [gets $fail_ch] 498 if { $test != "" } { 499 lappend failures $test 500 } 501 } 502 close $fail_ch 503 } 504 return $failures 505} 506 507proc save_ok { test } { 508 set ok_file [test_ok_file $test] 509 set log_file [test_log_file $test] 510 if { ! [file exists $log_file] } { 511 puts "Error: log file $log_file not found." 512 } else { 513 file copy -force $log_file $ok_file 514 } 515} 516 517################################################################ 518 519proc save_defok_main {} { 520 global argv 521 if { $argv == "help" || $argv == "-help" } { 522 puts {Usage: save_defok [failures] test1 [test2]...} 523 } else { 524 if { $argv == "failures" } { 525 set tests [failed_tests] 526 } else { 527 set tests $argv 528 } 529 foreach test $tests { 530 if { [lsearch [group_tests "all"] $test] == -1 } { 531 puts "Error: test $test not found." 532 } else { 533 save_defok $test 534 } 535 } 536 } 537} 538 539proc save_defok { test } { 540 set defok_file [test_defok_file $test] 541 set def_file [test_def_result_file $test] 542 if { [file exists $def_file] } { 543 file copy -force $def_file $defok_file 544 } 545} 546 547################################################################ 548 549proc test_cmd_dir { test } { 550 global cmd_dirs 551 552 if {[info exists cmd_dirs($test)]} { 553 return $cmd_dirs($test) 554 } else { 555 return "" 556 } 557} 558 559proc test_cmd_file { test } { 560 return [file join [test_cmd_dir $test] "$test.tcl"] 561} 562 563proc test_ok_file { test } { 564 global test_dir 565 return [file join $test_dir "$test.ok"] 566} 567 568proc test_defok_file { test } { 569 global test_dir 570 return [file join $test_dir "$test.defok"] 571} 572 573proc test_log_file { test } { 574 global result_dir 575 return [file join $result_dir "$test.log"] 576} 577 578proc test_def_result_file { test } { 579 global result_dir 580 return [file join $result_dir "$test.def"] 581} 582 583proc test_tmp_file { test } { 584 global result_dir 585 return [file join $result_dir $test.tmp] 586} 587 588proc test_valgrind_cmd_file { test } { 589 global result_dir 590 return [file join $result_dir $test.vg_cmd] 591} 592 593proc test_valgrind_file { test } { 594 global result_dir 595 return [file join $result_dir $test.valgrind] 596} 597 598proc test_core_file { test } { 599 global result_dir 600 return [file join $result_dir $test.core] 601} 602 603proc test_sys_core_file { test pid } { 604 global cmd_dirs 605 606 # macos 607 # return [file join "/cores" "core.$pid"] 608 609 # Suse 610 return [file join [test_cmd_dir $test] "core"] 611} 612 613proc test_pass_criteria { test } { 614 global test_pass_criteria 615 616 return $test_pass_criteria($test) 617} 618 619################################################################ 620 621# Local Variables: 622# mode:tcl 623# End: 624