1#!/usr/bin/env python
2
3# Copied from:
4# https://github.com/lammps/lammps-testing/blob/ce8df751e1e39fd933991fa7d258336cb45cf32c/lammps_testing/regression.py
5
6#   LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator
7#   http://lammps.sandia.gov, Sandia National Laboratories
8#   Steve Plimpton, sjplimp@sandia.gov
9
10#   Copyright (2003) Sandia Corporation.  Under the terms of Contract
11#   DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
12#   certain rights in this software.  This software is distributed under
13#   the GNU General Public License.
14
15#   See the README file in the top-level LAMMPS directory.
16
17
18#regression.py
19
20#Tool for numerical comparisions of benchmark log files
21#Created by Stan Moore (SNL), email: stamoor at sandia.gov
22#based on benchmark.py created by Reese Jones (SNL)
23#Requires log.py from Pizza.py toolkit, http://pizza.sandia.gov/
24
25
26#SYNTAX: regression.py <descriptor> <LAMMPS_args> <test_dirs> <options>
27#
28#    descriptor = any string without spaces, appended to log files, allows multiple tests in the same directory
29#    LAMMPS_args = string to launch the benchmark calculation
30#      the path to the executable must be an absolute path
31#      e.g. ~/lammps/src/lmp_g++ or "mpirun -np 4 ~/lammps/src/lmp_g++ -v x 10"
32#    test_dirs = list of one or more dirs to recursively search for scripts
33#      scripts = any in.* file
34#    options = one or more keyword/value pairs
35#      wildcards are expanded by Python, not the shell, so it may be necessary to escape * as \*
36#
37#"Gold standard" logfiles are automatically generated if they don't exist
38
39
40#EXAMPLES:
41
42#python regression.py mpi_16 "mpiexec -np 16 ~/lammps/src/lmp_mpi" ~/regression/examples -only colloid 2>&1 |tee test_mpi_16.out
43#python regression.py kk_gpu_2 "mpiexec -np 2 .lmp_kokkos_cuda_openmpi -k on g 2 -sf kk -pk kokkos comm/forward device comm/exchange device newton on neigh half" ~/lammps/examples -error_norm L1 -relative_error True -tolerance 0.05 -min-same-rows 2 -exclude hugoniostat nb3b colloid indent snap peri dreiding ASPHERE pour streitz srd min balance USER/cg-cmm/sds-monolayer USER/gauss-diel USER/awpmd/H USER/fep USER/misc/basal USER/lb USER/eff USER/drude USER/qtb/alpha_quartz_qbmsst USER/tally ELASTIC ellipse body dipole comb kim rigid VISCOSITY micelle USER/gle USER/qtb/methane_qbmsst USER/qtb/methane_qtb USER/cg-cmm/peg-verlet USER/pimd/para-h2 reax voronoi USER/qtb/alpha_quartz_qtb MC meam shear USER/dpd 2>&1 |tee test_kk_gpu_2_half.out
44
45
46#OPTIONS:
47#      -exclude <subdir1 subdir2* ...>
48#        do not run tests from these sub-dirs or their children
49#        default = none
50#      -only <subdir1 subdir2* ...>
51#        only run tests from these sub-dirs or their children
52#        default = none
53#      -customonly <file1 file2* ...>
54#        only run tests from sub-dirs that contain these files
55#        default = none
56#      -custom <file_prefix>
57#        read options from this file_prefix plus test name in each sub-dir, if it exists
58#        valid options are: launch, descriptors, tolerance, error_norm, relative_error
59#        the number of launches and descriptors must match
60#        lines in file have syntax like:
61#          descriptors = ["1","4","8"]
62#          error_norm = "L1"
63#        default = "options"
64#      -error_norm <"L1" or "L2" or "max">
65#        metric for comparing a column of output to gold standard answer
66#        these are vector norms, treating the column as a vector
67#        default = "max"
68#      -relative_error <"True" or "False">
69#        treat tolerance as a relative error or not
70#        default = "False"
71#      -tolerance <float>
72#        column difference > tolerance = fail, else success
73#        default = 1.0e-7
74#      -logread <dir module>
75#        path for where to find the log-file reading Python module
76#        default = . log (log.py in this dir or somewhere Python can find it)
77#
78
79usage = """
80  regression.py: numerical comparisions of logs and corresponding benchmarks
81  Syntax: regression.py <descriptor> <LAMMPS_args> <test_dirs> <options>
82    descriptor = any string without spaces, appended to log files
83    LAMMPS_args = string to launch the benchmark calculation
84      the path to the executable must be an absolute path
85      e.g. ~/lammps/src/lmp_g++ or "mpirun -np 4 ~/lammps/src/lmp_g++ -v x 10"
86    test_dirs = list of one or more dirs to recursively search for scripts
87      scripts = any in.* file
88    options = one or more keyword/value pairs
89      wildcards are expanded by Python, not the shell, so it may be necessary to escape * as \*
90      -exclude <subdir1 subdir2* ...>
91        do not run tests from these sub-dirs or their children
92        default = none
93      -only <subdir1 subdir2* ...>
94        only run tests from these sub-dirs or their children
95        default = none
96      -customonly <file1 file2* ...>
97        only run tests from sub-dirs that contain these files
98        default = none
99      -customtest in.test
100        only run a single test with this "in" file
101      -custom <file_prefix>
102        read options from this file_prefix plus test name in each sub-dir, if it exists
103        valid options are: launch, descriptors, tolerance, error_norm, relative_error
104        the number of launches and descriptors must match
105        lines in file have syntax like:
106          descriptors = ["1","4","8"]
107          error_norm = "L1"
108        default = "options"
109      -error_norm <"L1" or "L2" or "max">
110        metric for comparing a column of output to gold standard answer
111        these are vector norms, treating the column as a vector
112        default = "max"
113      -relative_error <"True" or "False">
114        treat tolerance as a relative error or not
115        default = "False"
116      -tolerance <float>
117        column difference > tolerance = fail, else success
118        default = 1.0e-7
119      -logread <dir module>
120        path for where to find the log-file reading Python module
121        default = . log (log.py in this dir or somewhere Python can find it)
122"""
123
124import sys
125import os
126import math
127import re
128from operator import itemgetter
129from glob import glob
130import time
131
132import shutil
133import platform
134
135#====================================================
136### global variables
137#====================================================
138nrows = 0
139not_same_rows = set()
140auto_rebless_flag = False
141min_same_rows = -1
142custom_file = "options"
143default_error_norm = "max"
144default_relative_error = False
145default_tolerance = 1.e-7
146logread = [".","log"]
147
148#====================================================
149### constants
150#====================================================
151fail_pattern = re.compile("FAIL");
152warn_pattern = re.compile("WARNING");
153
154#====================================================
155### date
156#====================================================
157def date():
158 return time.asctime()
159
160#====================================================
161### timer
162#====================================================
163def start():
164  global dt
165  dt = -(time.time())
166def stop():
167  global dt
168  dt += (time.time())
169  return dt
170
171#====================================================
172### run a regression test
173#====================================================
174def run_test(test,lmps,descriptor):
175  global not_same_rows
176  global nrows
177  msg = ""
178  input = "in."+test
179  log = "log."+descriptor+"."+test
180  stdout = "stdout."+descriptor+"."+test
181  new_flag = False
182
183  # print test header
184  print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
185  print "dir =",os.getcwd()
186  print "test =",test
187  sys.stdout.flush()
188  if (custom_flag):
189    print "descriptor =",descriptor
190    print "norm =",error_norm
191    print "tolerance =",tolerance
192    print "relative error =",relative_error
193    sys.stdout.flush()
194
195  # check if gold standard exists, if not create it
196  system_name = platform.system()
197  gold_standard = glob("log.*"+"."+descriptor+"."+test)
198  if (len(gold_standard) > 0):
199    ref = (gold_standard)[0];
200    print "gold standard =",ref
201    sys.stdout.flush()
202  else:
203    new_flag = True
204    msg += add_test(test,lmps,descriptor)
205    gold_standard = glob("log.*"+"."+descriptor+"."+test)
206    if not (len(gold_standard) > 0):
207      raise Exception("No logfile found")
208    ref = (gold_standard)[0];
209
210  # compare current run to gold standard
211  msg += "==== comparing "+log+" with "+ref+" ====\n"
212  # remove old log and stdout files
213  if (os.path.isfile(log)): os.remove(log)
214  if (os.path.isfile(stdout)): os.remove(stdout)
215  # run the test
216  os.system(lmps+" -in "+input+" -log "+log+" >& "+stdout);
217  # check if a log file was generated
218  if (not os.path.isfile(log)) :
219    msg += "!!! no "+log+"\n";
220    msg += "!!! test "+test+" FAILED\n"
221    return msg
222  # extract data
223  [passing,cdict,cdata,emsg] = extract_data(log,stdout);
224  [passing,bdict,bdata,emsg] = extract_data(ref,stdout);
225  msg += emsg
226  fail = False
227  if (not passing) : fail = True
228  if (fail) :
229    msg += "!!! test "+test+" FAILED\n"
230    if (new_flag):
231      os.remove(ref)
232    return msg
233  # compare columns
234  not_same_rows.clear()
235  cols = range(len(bdict))
236  if (len(cdata) != len(bdata)):
237    msg += "!!! data size "+str(len(cdata))+" does not match data "+str(len(bdata))+" in "+ref+"\n";
238    msg += "!!! test "+test+" FAILED\n"
239    return msg
240  i = 0
241  for name in bdict:
242    [passing,cmsg] = compare(name,cdata[cols[i]],bdata[cols[i]]);
243    i += 1
244    msg += cmsg
245    if (not passing) : fail = True
246
247  # print out results
248  nsame_rows = nrows - len(not_same_rows)
249  if (not fail):
250    if (nrows == nsame_rows):
251      msg += "\nAll rows are identical\n"
252    else:
253      msg += "\nWARNING: Only "+str(nsame_rows)+" out of "+str(nrows)+" rows are identical\n"
254    if (auto_rebless_flag):
255      dmy = time.strftime("%d%b%y")
256      hms = time.strftime("%H:%M:%S")
257      shutil.copyfile(ref,"old_"+ref+"_"+dmy+"_"+hms)
258      shutil.copyfile(log,ref)
259      msg += "WARNING: Gold standard for test "+test+" has been auto-reblessed\n"
260  if (fail) :
261    msg += "!!! test "+test+" FAILED\n"
262  else :
263    msg += "*** test "+test+" passed\n"
264  return msg
265
266#====================================================
267### add a regression test
268#====================================================
269def add_test(test,lmps,descriptor):
270  input = "in."+test;
271  log = "log."+descriptor+"."+test
272  stdout = "stdout."+descriptor+"."+test
273  msg = "==== generating gold standard for test "+test+" ====\n"
274  if (os.path.isfile(log)): os.remove(log)
275  if (os.path.isfile(stdout)): os.remove(stdout)
276  os.system(lmps+" -in "+input+" -log "+log+" >& "+stdout);
277  if (not os.path.isfile(log)) :
278    msg += "!!! no "+log+"\n";
279    msg += "!!! test "+test+" FAILED\n"
280    return msg
281  dmy = time.strftime("%d%b%y")
282  system_name = platform.system()
283  shutil.copyfile(log,"log.archive."+dmy+"."+descriptor+"."+test)
284  return msg
285
286#====================================================
287### extract data from log file
288#====================================================
289def extract_data(file,stdout):
290  msg = ""
291  dictionary = [];
292  data = []
293  read = False
294
295  msg = error_check(file,stdout)
296  if (msg != ""):
297    return [False,dictionary,data,msg]
298
299  try:
300    if logreader.__name__ == "log": lg= logreader(file)
301    elif logreader.__name__ == "olog": lg= logreader(file,"Step")
302    else: raise Exception("Unknown log reader")
303  except:
304    msg += "Invalid logfile found\n"
305    return [False,dictionary,data,msg]
306
307  if (len(lg.names) <= 0):
308    msg += "Invalid logfile found\n"
309    return [False,dictionary,data,msg]
310
311  for name in lg.names:
312    if (name == "CPU"): continue
313    dictionary.append(name)
314    data.append(lg.get(name))
315  return [True,dictionary,data,msg]
316
317#====================================================
318### check log and stdout file for errors
319#====================================================
320def error_check(file,stdout):
321  msg = ""
322  text_file = open(file, "r")
323  lines = text_file.readlines()
324  text_file.close()
325  num_lines = int(len(lines))
326  # check for errors
327  for i in xrange(num_lines):
328    if "ERROR" in lines[i] or "exited on signal" in lines[i]:
329      msg += lines[i]
330
331  text_file = open(stdout, "r")
332  lines = text_file.readlines()
333  text_file.close()
334  num_lines = int(len(lines))
335  # check for errors
336  for i in xrange(num_lines):
337    if "ERROR" in lines[i] or "exited on signal" in lines[i]:
338      msg += lines[i]
339
340  return msg
341
342#====================================================
343### compare columns of current run and gold standard
344#====================================================
345def compare(name,colA,colB):
346  msg = ""
347  err1 = 0.
348  err2 = 0.
349  errmax = 0.
350  norm1 = 0.
351  norm2 = 0.
352  normmax = 0.
353  nsame_rows = 0
354  global nrows
355  global not_same_rows
356  n = len(colB)
357  nrows = n
358  if (len(colA) != len(colB)):
359    msg = "Cannot compare columns\n"
360    return [False,msg];
361  for i in range(n):
362    vA = float(colA[i])
363    vB = float(colB[i])
364    norm1 += abs(vB)
365    norm2 += vB*vB
366    normmax = max(normmax,abs(vB))
367    dv = vA-vB
368    if (abs(dv) > tolerance):
369      not_same_rows.add(i)
370    else:
371      nsame_rows += 1
372    err1 += abs(dv)
373    err2 += dv*dv
374    errmax = max(errmax,abs(dv))
375  norm1 /= n
376  norm2 = math.sqrt(norm2/n)
377  err1 /= n
378  err2 = math.sqrt(err2/n)
379
380  if error_norm == "L1":
381    err = err1
382    norm = norm1
383  elif error_norm == "L2":
384    err = err2
385    norm = norm2
386  elif error_norm == "max":
387    err = errmax
388    norm = normmax
389  else:
390    raise Exception("Invalid error norm")
391
392  if (relative_error and norm > tolerance):
393    err /= norm
394    norm = 1.0
395
396  if (norm > tolerance) :
397    msg = "{0:7s}  error {1:4} wrt norm {2:7}\n".format(name,err,norm)
398  else :
399    msg = "{0:7s}           error {1:4}\n"               .format(name,err)
400  pass_flag = False
401  if min_same_rows >= 0:
402    pass_flag = err < tolerance or nsame_rows >= min_same_rows or nsame_rows == nrows;
403  else:
404    pass_flag = err < tolerance
405  return [pass_flag,msg];
406
407#====================================================
408### run and time tests
409#====================================================
410def execute(test):
411  global tolerance,error_norm,relative_error,custom_flag
412  global launch,descriptors
413  msg = ""
414  os.chdir(test[0])
415  tolerance = default_tolerance
416  error_norm = default_error_norm
417  relative_error = default_relative_error
418  launch = [default_lmps]
419  descriptors = [default_descriptor]
420  options = custom_file+"."+test[1]
421  custom_flag = False
422  if os.path.isfile(os.path.join(test[0],options)):
423    custom_flag = True
424    exec(open(options).read(),globals())
425  for i,lmp_args in enumerate(launch):
426    start()
427    msg += run_test(test[1],lmp_args,descriptors[i])
428    elapsed_time = stop()
429    msg += "elapsed time = "+str(elapsed_time)+" s for test "+test[1]+"\n"
430    msg += "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
431  os.chdir(home)
432  return msg
433
434#====================================================
435### parse inputs
436#====================================================
437def init() :
438  global default_descriptor, ntests, default_lmps, home
439  global auto_rebless_flag, min_same_rows, custom_file
440  global default_error_norm, default_relative_error, default_tolerance
441  global logread
442
443  # parse input arguments
444  if (len(sys.argv) < 4):
445    print usage
446    sys.exit(1)
447  default_descriptor = sys.argv[1]
448  default_lmps = sys.argv[2]
449  top_dir = os.path.abspath(sys.argv[3])
450  dirs = [name for name in os.listdir(top_dir) if os.path.isdir(os.path.join(top_dir, name))]
451  tests = []
452  exclude_dirs = []
453  only_dirs = []
454  only_files = []
455  only_tests = []
456  os.chdir(top_dir)
457  home = os.getcwd()
458  cnt = 4
459  keywords = ["-auto-rebless","-min-same-rows","-exclude","-only",
460              "-customonly","-customtest","-custom","-error_norm",
461              "-relative_error","-tolerance","-logread"]
462  while (cnt < len(sys.argv)):
463    option = sys.argv[cnt]
464    if ("-auto-rebless" == option):
465      flag = sys.argv[cnt+1]
466      if (flag.lower() == "true"):
467        auto_rebless_flag = True
468      elif (flag.lower() == "false"):
469        auto_rebless_flag = False
470      else:
471        raise Exception("Invalid optional arguements for regression.py")
472      cnt += 2
473    elif ("-min-same-rows" == option):
474      min_same_rows = int(sys.argv[cnt+1])
475      cnt += 2
476    elif ("-exclude" == option):
477      cnt += 1
478      while sys.argv[cnt] not in keywords:
479        names = [name for name in glob(os.path.join(top_dir, sys.argv[cnt])) if os.path.isdir(name)]
480        if len(names) == 0:
481          raise Exception("Directory " + sys.argv[cnt] + " not found")
482        for name in names:
483          exclude_dirs.append(name)
484        cnt += 1
485        if cnt == len(sys.argv): break
486    elif ("-only" == option):
487      cnt += 1
488      while sys.argv[cnt] not in keywords:
489        names = [name for name in glob(os.path.join(top_dir, sys.argv[cnt])) if os.path.isdir(name)]
490        if len(names) == 0:
491          raise Exception("Directory " + sys.argv[cnt] + " not found")
492        for name in names:
493          only_dirs.append(name)
494        cnt += 1
495        if cnt == len(sys.argv): break
496    elif ("-customonly" == option):
497      cnt += 1
498      while sys.argv[cnt] not in keywords:
499        only_files.append(sys.argv[cnt])
500        cnt += 1
501        if cnt == len(sys.argv): break
502    elif ("-customtest" == option):
503      cnt += 1
504      only_tests.append(sys.argv[cnt])
505      cnt += 1
506      if cnt == len(sys.argv): break
507    elif ("-custom" == option):
508      custom_file = sys.argv[cnt+1]
509      cnt += 2
510    elif ("-error_norm" == option):
511      default_error_norm = sys.argv[cnt+1]
512      cnt += 2
513    elif ("-relative_error" == option):
514      flag = sys.argv[cnt+1]
515      if (flag == "True"):
516        default_relative_error = True
517      elif (flag == "False"):
518        default_relative_error = False
519      else:
520        raise Exception("Invalid optional arguements for regression.py")
521      cnt += 2
522    elif ("-tolerance" == option):
523      default_tolerance = float(sys.argv[cnt+1])
524      cnt += 2
525    elif ("-logread" == option):
526      logread = [os.path.abspath(sys.argv[cnt+1]),sys.argv[cnt+2]]
527      cnt += 3
528    else:
529      raise Exception("Invalid optional arguements for regression.py")
530
531  if len(only_tests) == 0:
532    # recursively loop through directories to get tests
533    for x in os.walk(top_dir):
534      dir = x[0];
535      if ".svn" in dir: continue
536
537      # exclude these directories
538      if len(exclude_dirs) > 0:
539        skip = False
540        for i in exclude_dirs:
541          for y in os.walk(i):
542            if y[0] == dir: skip = True
543        if skip: continue
544
545      # only include these directories
546      if len(only_dirs) > 0:
547        skip = True
548        for i in only_dirs:
549          for y in os.walk(i):
550            if y[0] == dir: skip = False
551        if skip: continue
552
553      # only include directories with these files
554      if len(only_files) > 0:
555        skip = True
556        for name in only_files:
557          for path in glob(os.path.join(dir,name)):
558            if os.path.isfile(path):
559              skip = False
560        if skip: continue
561
562      os.chdir(dir);
563      for path in glob("./in.*"):
564        test = path[5:];
565        tests.append([dir,test])
566      os.chdir(home)
567    ntests = len(tests)
568    #tests.sort()
569  else:
570    ntests = 1
571    os.chdir(top_dir)
572    path = glob("./"+only_tests[0])[0]
573    test = os.path.basename(path)[3:]
574    dir = os.path.dirname(top_dir+"/"+path)
575    tests.append([dir,test])
576    os.chdir(home)
577
578  # print header
579  print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
580  print "start: ",date()
581  print "ntests:",ntests
582  print "default descriptor:",default_descriptor
583  print "default norm:",default_error_norm
584  print "default tolerance:",default_tolerance
585  print "default relative error:",default_relative_error
586  print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
587  print
588  print "subdirs =",dirs
589  print
590  sys.stdout.flush()
591
592  return tests
593
594#====================================================
595### main
596#====================================================
597if __name__ == '__main__':
598  tests = init()
599
600  if logread[0] != ".": sys.path.append(logread[0])
601  strcmd = "from %s import %s as logreader" % (logread[1],logread[1])
602  exec strcmd
603
604  nfails = 0
605  fail_list = []
606  nwarnings = 0
607  warn_list = []
608
609  # run the tests
610
611  for test in tests:
612    msg = execute(test)
613    if (fail_pattern.search(msg)) :
614      nfails += 1
615      fail_list.append(test)
616    elif (warn_pattern.search(msg)) :
617      nwarnings += 1
618      warn_list.append(test)
619    print msg
620    sys.stdout.flush()
621
622  # print out results
623
624  print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
625  print "end:",date()
626  if (nfails == 0):
627    print ntests,"tests passed"
628    print "*** no failures ***"
629  else:
630    print "!!!",nfails,"of",ntests,"tests failed"
631    for test in fail_list:
632      print test
633  if (nwarnings > 0):
634    print "\n!!! Warnings were generated in the following tests"
635    for test in warn_list:
636      print test
637  print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
638  sys.stdout.flush()
639