1# Copyright (c) 2019, Parallax Software, Inc.
2#
3# This program is free software: you can redistribute it and/or modify
4# it under the terms of the GNU General Public License as published by
5# the Free Software Foundation, either version 3 of the License, or
6# (at your option) any later version.
7#
8# This program is distributed in the hope that it will be useful,
9# but WITHOUT ANY WARRANTY; without even the implied warranty of
10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11# GNU General Public License for more details.
12#
13# You should have received a copy of the GNU General Public License
14# along with this program.  If not, see <https://www.gnu.org/licenses/>.
15
16# This is a generic regression test script used to compare application
17# output to a known good "ok" file.
18#
19# Use the "regression" command to run the regressions.
20#
21#  regression -help | [-valgrind] test1 [test2...]
22#
23# where test is "all" or the name of a test group defined in regression_vars.tcl
24# Wildcards can be used in test names if the name is enclosed in ""s to suppress
25# shell globbing. For example,
26#
27#  regression "init_floorplan*"
28#
29# will run all tests with names that begin with "init_floorplan".
30# Each test name is printed before it runs. Once it finishes pass,
31# fail, *NO OK FILE* or *SEG FAULT* is printed after the test name.
32#
33# The results of each test are in the file test/results/<test>.log
34# The diffs for all tests are in test/results/diffs.
35# A list of failed tests is in test/results/failures.
36# To save a log file as the correct output use the save_ok command.
37#
38#  save_ok failures | test1 [test2...]
39#
40# This copies test/results/test.log to test/test.ok
41# Using the test name 'failures' copies the ok files for all failed tests.
42# This is a quick way to update the failing test ok files after examining
43# the differences.
44#
45# You should NOT need to modify this script.
46# Customization unique to an application is in "regression_vars.tcl".
47# In this case the application is OpenROAD, so nothing should need to be changed
48# in "regression_vars.tcl".
49#
50# Customize the scripts "regresssion" and "save_ok" to source this file
51# and a file that defines the test scripts, "regresion_tests.tcl".
52# Each test is a tcl command file.
53
54set openroad_test_dir [file join $openroad_dir "test"]
55
56source [file join $openroad_test_dir "regression_vars.tcl"]
57source [file join $openroad_test_dir "flow_metrics.tcl"]
58
59proc regression_main {} {
60  global argv
61  exit [regression_body $argv]
62}
63
64proc regression_body { cmd_argv } {
65  setup
66  parse_args $cmd_argv
67  run_tests
68  show_summary
69  return [found_errors]
70}
71
72proc setup {} {
73  global result_dir diff_file failure_file errors
74  global use_valgrind valgrind_shared_lib_failure
75
76  set use_valgrind 0
77
78  if { !([file exists $result_dir] && [file isdirectory $result_dir]) } {
79    file mkdir $result_dir
80  }
81  file delete $diff_file
82  file delete $failure_file
83
84  set errors(error) 0
85  set errors(memory) 0
86  set errors(leak) 0
87  set errors(fail) 0
88  set errors(no_cmd) 0
89  set errors(no_ok) 0
90  set valgrind_shared_lib_failure 0
91}
92
93proc parse_args { cmd_argv } {
94  global app_options tests test_groups cmd_paths
95  global use_valgrind
96  global result_dir tests
97
98  while { $cmd_argv != {} } {
99    set arg [lindex $cmd_argv 0]
100    if { $arg == "help" || $arg == "-help" } {
101      puts {Usage: regression [-help] [-threads threads] [-valgrind] tests...}
102      puts "  -threads max|integer - number of threads to use"
103      puts "  -valgrind - run valgrind (linux memory checker)"
104      puts "  Wildcarding for test names is supported (enclose in \"'s)"
105      puts "  Tests are: all, fast, med, slow, or a test group or test name"
106      puts ""
107      puts "  If 'limit coredumpsize unlimited' corefiles are saved in $result_dir/test.core"
108      exit
109    } elseif { $arg == "-threads" } {
110      set threads [lindex $cmd_argv 1]
111      if { !([string is integer $threads] || $threads == "max") } {
112	puts "Error: -threads arg $threads is not an integer or max."
113	exit 0
114      }
115      lappend app_options "-threads"
116      lappend app_options $threads
117      set cmd_argv [lrange $cmd_argv 2 end]
118    } elseif { $arg == "-valgrind" } {
119      set use_valgrind 1
120      set cmd_argv [lrange $cmd_argv 1 end]
121    } else {
122      break
123    }
124  }
125  if { $cmd_argv == {} } {
126    # Default is to run all tests.
127    set tests [group_tests all]
128  } else {
129    set tests [expand_tests $cmd_argv]
130  }
131}
132
133proc expand_tests { tests_arg } {
134  global test_groups
135
136  set tests {}
137  foreach arg $tests_arg {
138    if { [info exists test_groups($arg)] } {
139      set tests [concat $tests $test_groups($arg)]
140    } elseif { [string first "*" $arg] != -1 \
141	       || [string first "?" $arg] != -1 } {
142      # Find wildcard matches.
143      foreach test [group_tests "all"] {
144	if [string match $arg $test] {
145	  lappend tests $test
146	}
147      }
148    } elseif { [lsearch [group_tests "all"] $arg] != -1 } {
149      lappend tests $arg
150    } else {
151      puts "Error: test $arg not found."
152    }
153  }
154  return $tests
155}
156
157proc run_tests {} {
158  global tests errors app_path
159
160  foreach test $tests {
161    run_test $test
162  }
163  # Macos debug info generated by valgrind.
164  file delete -force "$app_path.dSYM"
165}
166
167proc run_test { test } {
168  global result_dir diff_file errors diff_options
169
170  set cmd_file [test_cmd_file $test]
171  if [file exists $cmd_file] {
172    set ok_file [test_ok_file $test]
173    set log_file [test_log_file $test]
174    foreach file [glob -nocomplain [file join $result_dir $test.*]] {
175      file delete -force $file
176    }
177    puts -nonewline $test
178    flush stdout
179    set test_errors [run_test_app $test $cmd_file $log_file]
180    if { [lindex $test_errors 0] == "ERROR" } {
181      puts " *ERROR* [lrange $test_errors 1 end]"
182      append_failure $test
183      incr errors(error)
184
185      # For some reason seg faults aren't echoed in the log - add them.
186      if [file exists $log_file] {
187	set log_ch [open $log_file "a"]
188	puts $log_ch "$test_errors"
189	close $log_ch
190      }
191
192      # Report partial log diff anyway.
193      if [file exists $ok_file] {
194	catch [concat exec diff $diff_options $ok_file $log_file \
195		 >> $diff_file]
196      }
197    } else {
198      set error_msg ""
199      if { [lsearch $test_errors "MEMORY"] != -1 } {
200	append error_msg " *MEMORY*"
201	append_failure $test
202	incr errors(memory)
203      }
204      if { [lsearch $test_errors "LEAK"] != -1 } {
205	append error_msg " *LEAK*"
206	append_failure $test
207	incr errors(leak)
208      }
209
210      switch [test_pass_criteria $test] {
211        compare_logfile {
212          if { [file exists $ok_file] } {
213            # Filter dos '/r's from log file.
214            set tmp_file [file join $result_dir $test.tmp]
215            exec tr -d "\r" < $log_file > $tmp_file
216            file rename -force $tmp_file $log_file
217            if [catch [concat exec diff $diff_options $ok_file $log_file \
218                         >> $diff_file]] {
219              puts " *FAIL*$error_msg"
220              append_failure $test
221              incr errors(fail)
222            } else {
223              puts " pass$error_msg"
224            }
225          } else {
226            puts " *NO OK FILE*$error_msg"
227            append_failure $test
228            incr errors(no_ok)
229          }
230        }
231        pass_fail {
232          set error_msg [find_log_pass_fail $log_file]
233          if { $error_msg != "pass" } {
234            puts " *FAIL* $error_msg"
235            append_failure $test
236            incr errors(fail)
237          } else {
238            puts " pass"
239          }
240        }
241        check_metrics {
242          set error_msg [check_test_metrics $test]
243          if { $error_msg != "pass" } {
244            puts " *FAIL* $error_msg"
245            append_failure $test
246            incr errors(fail)
247          } else {
248            puts " pass"
249          }
250        }
251      }
252    }
253  } else {
254    puts "$test *NO CMD FILE*"
255    incr errors(no_cmd)
256  }
257}
258
259proc find_log_pass_fail { log_file } {
260  if { [file exists $log_file] } {
261    set stream [open $log_file r]
262    set last_line ""
263    while { [gets $stream line] >= 0 } {
264      set last_line $line
265    }
266    close $stream
267    if { [string match "pass*" $last_line] } {
268      return "pass"
269    } else {
270      return $last_line
271    }
272  }
273  return "fail - reason not found"
274}
275
276proc append_failure { test } {
277  global failure_file
278  set fail_ch [open $failure_file "a"]
279  puts $fail_ch $test
280  close $fail_ch
281}
282
283# Return error.
284proc run_test_app { test cmd_file log_file } {
285  global app_path errorCode use_valgrind
286  if { $use_valgrind } {
287    return [run_test_valgrind $test $cmd_file $log_file]
288  } else {
289    return [run_test_plain $test $cmd_file $log_file]
290  }
291}
292
293proc run_test_plain { test cmd_file log_file } {
294  global app_path app_options result_dir errorCode
295
296  if { ![file exists $app_path] } {
297    return "ERROR $app_path not found."
298  } elseif { ![file executable $app_path] } {
299    return "ERROR $app_path is not executable."
300  } else {
301    set save_dir [pwd]
302    cd [file dirname $cmd_file]
303    if { [catch [concat exec $app_path $app_options -metrics [test_metrics_result_file $test]\
304                   [file tail $cmd_file] >& $log_file]] } {
305      cd $save_dir
306      set signal [lindex $errorCode 2]
307      set error [lindex $errorCode 3]
308      # Errors strings are not consistent across platforms but signal
309      # names are.
310      if { $signal == "SIGSEGV" } {
311        # Save corefiles to regression results directory.
312        set pid [lindex $errorCode 1]
313        set sys_corefile [test_sys_core_file $test $pid]
314        if { [file exists $sys_corefile] } {
315          file copy $sys_corefile [test_core_file $test]
316        }
317      }
318      cleanse_logfile $test $log_file
319      return "ERROR $error"
320    }
321    cd $save_dir
322    cleanse_logfile $test $log_file
323    return ""
324  }
325}
326
327proc run_test_valgrind { test cmd_file log_file } {
328  global app_path app_options valgrind_options result_dir errorCode
329
330  set vg_cmd_file [test_valgrind_cmd_file $test]
331  set vg_stream [open $vg_cmd_file "w"]
332  puts $vg_stream "cd [file dirname $cmd_file]"
333  puts $vg_stream "source [file tail $cmd_file]"
334  puts $vg_stream "ord::delete_all_memory"
335  close $vg_stream
336
337  set cmd [concat exec valgrind $valgrind_options \
338             $app_path $app_options $vg_cmd_file >& $log_file]
339  set error_msg ""
340  if { [catch $cmd] } {
341    set error_msg "ERROR [lindex $errorCode 3]"
342  }
343  file delete $vg_cmd_file
344  cleanse_logfile $test $log_file
345  lappend error_msg [cleanse_valgrind_logfile $test $log_file]
346  return $error_msg
347}
348
349# Error messages can be found in "valgrind/memcheck/mc_errcontext.c".
350#
351# "Conditional jump or move depends on uninitialised value(s)"
352# "%s contains unaddressable byte(s)"
353# "%s contains uninitialised or unaddressable byte(s)"
354# "Use of uninitialised value of size %d"
355# "Invalid read of size %d"
356# "Syscall param %s contains uninitialised or unaddressable byte(s)"
357# "Unaddressable byte(s) found during client check request"
358# "Uninitialised or unaddressable byte(s) found during client check request"
359# "Invalid free() / delete / delete[]"
360# "Mismatched free() / delete / delete []"
361set valgrind_mem_regexp "(depends on uninitialised value)|(contains unaddressable)|(contains uninitialised)|(Use of uninitialised value)|(Invalid read)|(Unaddressable byte)|(Uninitialised or unaddressable)|(Invalid free)|(Mismatched free)"
362
363# "%d bytes in %d blocks are definitely lost in loss record %d of %d"
364# "%d bytes in %d blocks are possibly lost in loss record %d of %d"
365#set valgrind_leak_regexp "blocks are (possibly|definitely) lost"
366set valgrind_leak_regexp "blocks are definitely lost"
367
368# Valgrind fails on executables using shared libraries.
369set valgrind_shared_lib_failure_regexp "No malloc'd blocks -- no leaks are possible"
370
371# Scan the log file to separate valgrind notifications and check for
372# valgrind errors.
373proc cleanse_valgrind_logfile { test log_file } {
374  global valgrind_mem_regexp valgrind_leak_regexp
375  global valgrind_shared_lib_failure_regexp
376  global valgrind_shared_lib_failure
377
378  set tmp_file [test_tmp_file $test]
379  set valgrind_log_file [test_valgrind_file $test]
380  file copy -force $log_file $tmp_file
381  set tmp [open $tmp_file "r"]
382  set log [open $log_file "w"]
383  set valgrind [open $valgrind_log_file "w"]
384  set leaks 0
385  set mem_errors 0
386  gets $tmp line
387  while { ![eof $tmp] } {
388    if {[regexp "^==" $line]} {
389      puts $valgrind $line
390      if {[regexp $valgrind_leak_regexp $line]} {
391        set leaks 1
392      }
393      if {[regexp $valgrind_mem_regexp $line]} {
394        set mem_errors 1
395      }
396      if {[regexp $valgrind_shared_lib_failure_regexp $line]} {
397        set valgrind_shared_lib_failure 1
398      }
399    } elseif {[regexp {^--[0-9]+} $line]} {
400      # Valgrind notification line.
401    } else {
402      puts $log $line
403    }
404    gets $tmp line
405  }
406  close $log
407  close $tmp
408  close $valgrind
409  file delete $tmp_file
410
411  set errors {}
412  if { $mem_errors } {
413    lappend errors "MEMORY"
414  }
415  if { $leaks } {
416    lappend errors "LEAK"
417  }
418  return $errors
419}
420
421################################################################
422
423proc show_summary {} {
424  global errors tests diff_file result_dir valgrind_shared_lib_failure
425  global app_path app
426
427  puts "------------------------------------------------------"
428  set test_count [llength $tests]
429  if { [found_errors] } {
430    if { $errors(error) != 0 } {
431      puts "Errored $errors(error)/$test_count"
432    }
433    if { $errors(fail) != 0 } {
434      puts "Failed $errors(fail)/$test_count"
435    }
436    if { $errors(leak) != 0 } {
437      puts "Memory leaks in $errors(leak)/$test_count"
438    }
439    if { $errors(memory) != 0 } {
440      puts "Memory corruption in $errors(memory)/$test_count"
441    }
442    if { $errors(no_ok) != 0 } {
443      puts "No ok file for $errors(no_ok)/$test_count"
444    }
445    if { $errors(no_cmd) != 0 } {
446      puts "No cmd tcl file for $errors(no_cmd)/$test_count"
447    }
448    if { $errors(fail) != 0 } {
449      puts "See $diff_file for differences"
450    }
451  } else {
452    puts "Passed $test_count"
453  }
454  if { $valgrind_shared_lib_failure } {
455    puts "WARNING: valgrind failed because the executable is not statically linked."
456  }
457  puts "See $result_dir for log files"
458}
459
460proc found_errors {} {
461  global errors
462
463  return [expr $errors(error) != 0 || $errors(fail) != 0 \
464            || $errors(no_cmd) != 0 || $errors(no_ok) != 0 \
465            || $errors(memory) != 0 || $errors(leak) != 0 ]
466}
467
468################################################################
469
470proc save_ok_main {} {
471  global argv
472  if { $argv == "help" || $argv == "-help" } {
473    puts {Usage: save_ok [failures] test1 [test2]...}
474  } else {
475    if { $argv == "failures" } {
476      set tests [failed_tests]
477    } else {
478      set tests $argv
479    }
480    foreach test $tests {
481      if { [lsearch [group_tests "all"] $test] == -1 } {
482        puts "Error: test $test not found."
483      } else {
484        save_ok $test
485      }
486    }
487  }
488}
489
490proc failed_tests {} {
491  global failure_file
492
493  set failures {}
494  if { [file exists $failure_file] } {
495    set fail_ch [open $failure_file "r"]
496    while { ![eof $fail_ch] } {
497      set test [gets $fail_ch]
498      if { $test != "" } {
499        lappend failures $test
500      }
501    }
502    close $fail_ch
503  }
504  return $failures
505}
506
507proc save_ok { test } {
508  set ok_file [test_ok_file $test]
509  set log_file [test_log_file $test]
510  if { ! [file exists $log_file] } {
511    puts "Error: log file $log_file not found."
512  } else {
513    file copy -force $log_file $ok_file
514  }
515}
516
517################################################################
518
519proc save_defok_main {} {
520  global argv
521  if { $argv == "help" || $argv == "-help" } {
522    puts {Usage: save_defok [failures] test1 [test2]...}
523  } else {
524    if { $argv == "failures" } {
525      set tests [failed_tests]
526    } else {
527      set tests $argv
528    }
529    foreach test $tests {
530      if { [lsearch [group_tests "all"] $test] == -1 } {
531        puts "Error: test $test not found."
532      } else {
533        save_defok $test
534      }
535    }
536  }
537}
538
539proc save_defok { test } {
540  set defok_file [test_defok_file $test]
541  set def_file [test_def_result_file $test]
542  if { [file exists $def_file] } {
543    file copy -force $def_file $defok_file
544  }
545}
546
547################################################################
548
549proc test_cmd_dir { test } {
550  global cmd_dirs
551
552  if {[info exists cmd_dirs($test)]} {
553    return $cmd_dirs($test)
554  } else {
555    return ""
556  }
557}
558
559proc test_cmd_file { test } {
560  return [file join [test_cmd_dir $test] "$test.tcl"]
561}
562
563proc test_ok_file { test } {
564  global test_dir
565  return [file join $test_dir "$test.ok"]
566}
567
568proc test_defok_file { test } {
569  global test_dir
570  return [file join $test_dir "$test.defok"]
571}
572
573proc test_log_file { test } {
574  global result_dir
575  return [file join $result_dir "$test.log"]
576}
577
578proc test_def_result_file { test } {
579  global result_dir
580  return [file join $result_dir "$test.def"]
581}
582
583proc test_tmp_file { test } {
584  global result_dir
585  return [file join $result_dir $test.tmp]
586}
587
588proc test_valgrind_cmd_file { test } {
589  global result_dir
590  return [file join $result_dir $test.vg_cmd]
591}
592
593proc test_valgrind_file { test } {
594  global result_dir
595  return [file join $result_dir $test.valgrind]
596}
597
598proc test_core_file { test } {
599  global result_dir
600  return [file join $result_dir $test.core]
601}
602
603proc test_sys_core_file { test pid } {
604  global cmd_dirs
605
606  # macos
607  # return [file join "/cores" "core.$pid"]
608
609  # Suse
610  return [file join [test_cmd_dir $test] "core"]
611}
612
613proc test_pass_criteria { test } {
614  global test_pass_criteria
615
616  return $test_pass_criteria($test)
617}
618
619################################################################
620
621# Local Variables:
622# mode:tcl
623# End:
624