1#!/usr/local/bin/python3.8
2"""
3Parse the test file and return a dictionary.
4
5Quick usage::
6
7  lib/petsc/bin/maint/testparse.py -t src/ksp/ksp/tutorials/ex1.c
8
9From the command line, it prints out the dictionary.
10This is meant to be used by other scripts, but it is
11useful to debug individual files.
12
13Example language
14----------------
15
16/*TEST
17   build:
18     requires: moab
19   # This is equivalent to test:
20   testset:
21      args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full
22
23   testset:
24      suffix: 2
25      nsize: 2
26      args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full
27
28   testset:
29      suffix: 2
30      nsize: 2
31      args: -pc_type mg -ksp_type fgmres -da_refine 2 -ksp_monitor_short -mg_levels_ksp_monitor_short -mg_levels_ksp_norm_type unpreconditioned -ksp_view -pc_mg_type full
32      test:
33
34TEST*/
35
36"""
37from __future__ import print_function
38
39import os, re, glob, types
40import sys
41import logging
42from gmakegen import *
43sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
44
45import inspect
46thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
47maintdir=os.path.join(os.path.join(os.path.dirname(thisscriptdir),'bin'),'maint')
48sys.path.insert(0,maintdir)
49
50# These are special keys describing build
51buildkeys="requires TODO SKIP depends".split()
52
53acceptedkeys="test nsize requires command suffix diff_args args filter filter_output localrunfiles comments TODO SKIP output_file timeoutfactor".split()
54appendlist="args diff_args requires comments".split()
55
56import re
57
58def getDefaultOutputFileRoot(testname):
59  """
60  Given testname, give DefaultRoot and DefaultOutputFilename
61  e.g., runex1 gives ex1_1, output/ex1_1.out
62  """
63  defroot=(re.sub("run","",testname) if testname.startswith("run") else testname)
64  if not "_" in defroot: defroot=defroot+"_1"
65  return defroot
66
67def _stripIndent(block,srcfile,entireBlock=False,fileNums=[]):
68  """
69  Go through and remove a level of indentation
70  Also strip of trailing whitespace
71  """
72  # The first entry should be test: but it might be indented.
73  ext = getlangext(srcfile)
74  stripstr=" "
75  if len(fileNums)>0: lineNum=fileNums[0]-1
76  for lline in block.split("\n"):
77    if len(fileNums)>0: lineNum+=1
78    line=lline[1:] if lline.startswith("!") else lline
79    if not line.strip(): continue
80    if line.strip().startswith('#'): continue
81    if entireBlock:
82      var=line.split(":")[0].strip()
83      if not var in ['test','testset','build']:
84        raise Exception("Formatting error: Cannot find test in file: "+srcfile+" at line: "+str(lineNum)+"\n")
85    nspace=len(line)-len(line.lstrip(stripstr))
86    newline=line[nspace:]
87    break
88
89  # Strip off any indentation for the whole string and any trailing
90  # whitespace for convenience
91  newTestStr="\n"
92  if len(fileNums)>0: lineNum=fileNums[0]-1
93  firstPass=True
94  for lline in block.split("\n"):
95    if len(fileNums)>0: lineNum+=1
96    line=lline[1:] if lline.startswith("!") else lline
97    if not line.strip(): continue
98    if line.strip().startswith('#'):
99      newTestStr+=line+'\n'
100    else:
101      newline=line[nspace:]
102      newTestStr+=newline.rstrip()+"\n"
103    # Do some basic indentation checks
104    if entireBlock:
105      # Don't need to check comment lines
106      if line.strip().startswith('#'): continue
107      if not newline.startswith(" "):
108        var=newline.split(":")[0].strip()
109        if not var in ['test','testset','build']:
110          err="Formatting error in file "+srcfile+" at line: " +line+"\n"
111          if len(fileNums)>0:
112            raise Exception(err+"Check indentation at line number: "+str(lineNum))
113          else:
114            raise Exception(err)
115      else:
116        var=line.split(":")[0].strip()
117        if var in ['test','testset','build']:
118          subnspace=len(line)-len(line.lstrip(stripstr))
119          if firstPass:
120            firstsubnspace=subnspace
121            firstPass=False
122          else:
123            if firstsubnspace!=subnspace:
124              err="Formatting subtest error in file "+srcfile+" at line: " +line+"\n"
125              if len(fileNums)>0:
126                raise Exception(err+"Check indentation at line number: "+str(lineNum))
127              else:
128                raise Exception(err)
129
130  # Allow line continuation character '\'
131  return newTestStr.replace('\\\n', ' ')
132
133def parseLoopArgs(varset):
134  """
135  Given:   String containing loop variables
136  Return: tuple containing separate/shared and string of loop vars
137  """
138  keynm=varset.split("{{")[0].strip().lstrip('-')
139  if not keynm.strip(): keynm='nsize'
140  lvars=varset.split('{{')[1].split('}')[0]
141  suffx=varset.split('{{')[1].split('}')[1]
142  ftype='separate' if suffx.startswith('separate') else 'shared'
143  return keynm,lvars,ftype
144
145def _getLoopVars(testDict):
146  """
147  Given: dictionary that may have
148  Return:  Variables that cause a test split
149  """
150  vals=None
151  loopVars={}
152  loopVars['separate']=[]
153  loopVars['shared']=[]
154  # Check nsize
155  if 'nsize' in testDict:
156    varset=testDict['nsize']
157    if '{{' in varset:
158      keynm,lvars,ftype=parseLoopArgs(varset)
159      if ftype=='separate': loopVars['separate'].append(keynm)
160
161  # Now check args
162  if 'args' not in testDict: return loopVars
163  for varset in re.split('(^|\W)-(?=[a-zA-Z])',testDict['args']):
164    if not varset.strip(): continue
165    if '{{' in varset:
166      # Assuming only one for loop per var specification
167      keynm,lvars,ftype=parseLoopArgs(varset)
168      loopVars[ftype].append(keynm)
169
170  return loopVars
171
172def _getNewArgs(args,separate=True):
173  """
174  Given: String that has args that might have loops in them
175  Return:  All of the arguments/values that do not have
176             for 'separate output' in for loops
177             unless separate=False
178  """
179  newargs=''
180  if not args.strip(): return args
181  for varset in re.split('(^|\W)-(?=[a-zA-Z])',args):
182    if not varset.strip(): continue
183    if '{{' in varset:
184      if separate:
185         if 'separate' in varset: continue
186      else:
187         if 'separate' not in varset: continue
188
189    newargs+="-"+varset.strip()+" "
190
191  return newargs
192
193def _getVarVals(findvar,testDict):
194  """
195  Given: variable that is either nsize or in args
196  Return:  Values to loop over and the other arguments
197    Note that we keep the other arguments even if they have
198    for loops to enable stepping through all of the for lops
199  """
200  save_vals=None
201  if findvar=='nsize':
202    varset=testDict[findvar]
203    keynm,save_vals,ftype=parseLoopArgs('nsize '+varset)
204  else:
205    varlist=[]
206    for varset in re.split('-(?=[a-zA-Z])',testDict['args']):
207      if not varset.strip(): continue
208      if '{{' not in varset: continue
209      keyvar,vals,ftype=parseLoopArgs(varset)
210      if keyvar==findvar:
211        save_vals=vals
212
213  if not save_vals: raise Exception("Could not find separate_testvar: "+findvar)
214  return save_vals
215
216def genTestsSeparateTestvars(intests,indicts,final=False):
217  """
218  Given: testname, sdict with 'separate_testvars
219  Return: testnames,sdicts: List of generated tests
220    The tricky part here is the {{ ... }separate output}
221    that can be used multiple times
222  """
223  testnames=[]; sdicts=[]
224  for i in range(len(intests)):
225    testname=intests[i]; sdict=indicts[i]; i+=1
226    loopVars=_getLoopVars(sdict)
227    if len(loopVars['shared'])>0 and not final:
228      # Need to remove shared loop vars and push down to subtests
229      if 'subtests' in sdict:
230        for varset in re.split('(^|\W)-(?=[a-zA-Z])',sdict['args']):
231          if '{{' in varset:
232              for stest in sdict['subtests']:
233                if 'args' in sdict[stest]:
234                  sdict[stest]['args']+=' -'+varset
235                else:
236                  sdict[stest]['args']="-"+varset
237        sdict['args']=_getNewArgs(sdict['args'],separate=False)
238    if len(loopVars['separate'])>0:
239      sep_dicts=[sdict.copy()]
240      if 'args' in sep_dicts[0]:
241        sep_dicts[0]['args']=_getNewArgs(sdict['args'])
242      sep_testnames=[testname]
243      for kvar in loopVars['separate']:
244        kvals=_getVarVals(kvar,sdict)
245
246        # Have to do loop over previous var/val combos as well
247        # and accumulate as we go
248        val_testnames=[]; val_dicts=[]
249        for val in kvals.split():
250          gensuffix="_"+kvar+"-"+val.replace(',','__')
251          for kvaltestnm in sep_testnames:
252            val_testnames.append(kvaltestnm+gensuffix)
253          for kv in sep_dicts:
254            kvardict=kv.copy()
255            # If the last var then we have the final version
256            if 'suffix' in sdict:
257              kvardict['suffix']+=gensuffix
258            else:
259              kvardict['suffix']=gensuffix
260            if kvar=='nsize':
261              kvardict[kvar]=val
262            else:
263              kvardict['args']+="-"+kvar+" "+val+" "
264            val_dicts.append(kvardict)
265        sep_testnames=val_testnames
266        sep_dicts=val_dicts
267      testnames+=sep_testnames
268      sdicts+=sep_dicts
269    else:
270      # These are plain vanilla tests (no subtests, no loops) that
271      # do not have a suffix.  This makes the targets match up with
272      # the output file (testname_1.out)
273      if final:
274          if '_' not in testname: testname+='_1'
275      testnames.append(testname)
276      sdicts.append(sdict)
277  return testnames,sdicts
278
279def genTestsSubtestSuffix(testnames,sdicts):
280  """
281  Given: testname, sdict with separate_testvars
282  Return: testnames,sdicts: List of generated tests
283  """
284  tnms=[]; sdcts=[]
285  for i in range(len(testnames)):
286    testname=testnames[i]
287    rmsubtests=[]; keepSubtests=False
288    if 'subtests' in sdicts[i]:
289      for stest in sdicts[i]["subtests"]:
290        if 'suffix' in sdicts[i][stest]:
291          rmsubtests.append(stest)
292          gensuffix="_"+sdicts[i][stest]['suffix']
293          newtestnm=testname+gensuffix
294          tnms.append(newtestnm)
295          newsdict=sdicts[i].copy()
296          del newsdict['subtests']
297          # Have to hand update
298          # Append
299          for kup in appendlist:
300            if kup in sdicts[i][stest]:
301              if kup in sdicts[i]:
302                newsdict[kup]=sdicts[i][kup]+" "+sdicts[i][stest][kup]
303              else:
304                newsdict[kup]=sdicts[i][stest][kup]
305          # Promote
306          for kup in acceptedkeys:
307            if kup in appendlist: continue
308            if kup in sdicts[i][stest]:
309              newsdict[kup]=sdicts[i][stest][kup]
310          # Cleanup
311          for st in sdicts[i]["subtests"]: del newsdict[st]
312          sdcts.append(newsdict)
313        else:
314          keepSubtests=True
315    else:
316      tnms.append(testnames[i])
317      sdcts.append(sdicts[i])
318    # If a subtest without a suffix exists, then save it
319    if keepSubtests:
320      tnms.append(testnames[i])
321      newsdict=sdicts[i].copy()
322      # Prune the tests to prepare for keeping
323      for rmtest in rmsubtests:
324        newsdict['subtests'].remove(rmtest)
325        del newsdict[rmtest]
326      sdcts.append(newsdict)
327    i+=1
328  return tnms,sdcts
329
330def splitTests(testname,sdict):
331  """
332  Given: testname and dictionary generated from the YAML-like definition
333  Return: list of names and dictionaries corresponding to each test
334          given that the YAML-like language allows for multiple tests
335  """
336
337  # Order: Parent sep_tv, subtests suffix, subtests sep_tv
338  testnames,sdicts=genTestsSeparateTestvars([testname],[sdict])
339  testnames,sdicts=genTestsSubtestSuffix(testnames,sdicts)
340  testnames,sdicts=genTestsSeparateTestvars(testnames,sdicts,final=True)
341
342  # Because I am altering the list, I do this in passes.  Inelegant
343
344  return testnames, sdicts
345
346
347def testSplit(striptest):
348  """
349  Split up a test into lines, but use a shell parser to detect when newlines are within quotation marks
350  and keep those together
351  """
352  import shlex
353
354  sl = shlex.shlex()
355  sl.whitespace_split = True # only split at whitespace
356  sl.commenters = ''
357  sl.push_source(striptest)
358  last_pos = sl.instream.tell()
359  try:
360    last_token = sl.read_token()
361  except ValueError:
362    print(striptest)
363    raise ValueError
364  last_line = ''
365  while last_token != '':
366    new_pos = sl.instream.tell()
367    block = striptest[last_pos:new_pos]
368    token_start = block.find(last_token)
369    leading = block[0:token_start]
370    trailing = block[(token_start + len(last_token)):]
371    leading_split = leading.split('\n')
372    if len(leading_split) > 1:
373      yield last_line
374      last_line = ''
375    last_line += leading_split[-1]
376    last_line += last_token
377    trailing_split = trailing.split('\n')
378    last_line += trailing_split[0]
379    if len(trailing_split) > 1:
380      yield last_line
381      last_line = ''
382    last_pos = new_pos
383    try:
384      last_token = sl.read_token()
385    except ValueError:
386      print(striptest)
387      raise ValueError
388  yield last_line
389
390
391def parseTest(testStr,srcfile,verbosity):
392  """
393  This parses an individual test
394  Our YAML-like language is hierarchial so should use a state machine in the general case,
395  but in practice we only support two levels of test:
396  """
397  basename=os.path.basename(srcfile)
398  # Handle the new at the begininng
399  bn=re.sub("new_","",basename)
400  # This is the default
401  testname="run"+getlangsplit(bn)
402
403  # Tests that have default everything (so empty effectively)
404  if len(testStr)==0:
405      if '_' not in testname: testname+='_1'
406      return [testname], [{}]
407
408  striptest=_stripIndent(testStr,srcfile)
409
410  # go through and parse
411  subtestnum=0
412  subdict={}
413  comments=[]
414  indentlevel=0
415  for ln in testSplit(striptest):
416    line=ln.split('#')[0].rstrip()
417    if verbosity>2: print(line)
418    comment=("" if len(ln.split("#"))>0 else " ".join(ln.split("#")[1:]).strip())
419    if comment: comments.append(comment)
420    if not line.strip(): continue
421    lsplit=line.split(':')
422    if len(lsplit)==0: raise Exception("Missing : in line: "+line)
423    indentcount=lsplit[0].count(" ")
424    var=lsplit[0].strip()
425    val=line[line.find(':')+1:].strip()
426    if not var in acceptedkeys: raise Exception("Not a defined key: "+var+" from:  "+line)
427    # Start by seeing if we are in a subtest
428    if line.startswith(" "):
429      if var in subdict[subtestname]:
430        subdict[subtestname][var]+=" "+val
431      else:
432        subdict[subtestname][var]=val
433      if not indentlevel: indentlevel=indentcount
434      #if indentlevel!=indentcount: print("Error in indentation:", ln)
435    # Determine subtest name and make dict
436    elif var=="test":
437      subtestname="test"+str(subtestnum)
438      subdict[subtestname]={}
439      if "subtests" not in subdict: subdict["subtests"]=[]
440      subdict["subtests"].append(subtestname)
441      subtestnum=subtestnum+1
442    # The rest are easy
443    else:
444      # For convenience, it is sometimes convenient to list twice
445      if var in subdict:
446        if var in appendlist:
447          subdict[var]+=" "+val
448        else:
449          raise Exception(var+" entered twice: "+line)
450      else:
451        subdict[var]=val
452      if var=="suffix":
453        if len(val)>0:
454          testname+="_"+val
455
456  if len(comments): subdict['comments']="\n".join(comments).lstrip("\n")
457
458  # A test block can create multiple tests.  This does that logic
459  testnames,subdicts=splitTests(testname,subdict)
460  return testnames,subdicts
461
462def parseTests(testStr,srcfile,fileNums,verbosity):
463  """
464  Parse the YAML-like string describing tests and return
465  a dictionary with the info in the form of:
466    testDict[test][subtest]
467  """
468
469  testDict={}
470
471  # The first entry should be test: but it might be indented.
472  newTestStr=_stripIndent(testStr,srcfile,entireBlock=True,fileNums=fileNums)
473  if verbosity>2: print(srcfile)
474
475  ## Check and see if we have build requirements
476  addToRunRequirements=None
477  if "\nbuild:" in newTestStr:
478    testDict['build']={}
479    # The file info is already here and need to append
480    Part1=newTestStr.split("build:")[1]
481    fileInfo=re.split("\ntest(?:set)?:",newTestStr)[0]
482    for bkey in buildkeys:
483      if bkey+":" in fileInfo:
484        testDict['build'][bkey]=fileInfo.split(bkey+":")[1].split("\n")[0].strip()
485        #if verbosity>1: bkey+": "+testDict['build'][bkey]
486      # If a runtime requires are put into build, push them down to all run tests
487      # At this point, we are working with strings and not lists
488      if 'requires' in testDict['build']:
489         addToRunRequirements=testDict['build']['requires']
490         # People put datafilespath into build, but it needs to be a runtime
491         if 'datafilespath' in testDict['build']['requires']:
492             newreqs=re.sub('datafilespath','',testDict['build']['requires'])
493             testDict['build']['requires']=newreqs.strip()
494
495
496  # Now go through each test.  First elem in split is blank
497  for test in re.split("\ntest(?:set)?:",newTestStr)[1:]:
498    testnames,subdicts=parseTest(test,srcfile,verbosity)
499    for i in range(len(testnames)):
500      if testnames[i] in testDict:
501        raise RuntimeError("Multiple test names specified: "+testnames[i]+" in file: "+srcfile)
502      # Add in build requirements that need to be moved
503      if addToRunRequirements:
504          if 'requires' in subdicts[i]:
505              subdicts[i]['requires']+=' '+addToRunRequirements
506          else:
507              subdicts[i]['requires']=addToRunRequirements
508      testDict[testnames[i]]=subdicts[i]
509
510  return testDict
511
512def parseTestFile(srcfile,verbosity):
513  """
514  Parse single example files and return dictionary of the form:
515    testDict[srcfile][test][subtest]
516  """
517  debug=False
518  basename=os.path.basename(srcfile)
519  if basename=='makefile': return {}
520
521  curdir=os.path.realpath(os.path.curdir)
522  basedir=os.path.dirname(os.path.realpath(srcfile))
523  os.chdir(basedir)
524
525  testDict={}
526  sh=open(basename,"r"); fileStr=sh.read(); sh.close()
527
528  ## Start with doing the tests
529  #
530  fsplit=fileStr.split("/*TEST\n")[1:]
531  fstart=len(fileStr.split("/*TEST\n")[0].split("\n"))+1
532  # Allow for multiple "/*TEST" blocks even though it really should be
533  # one
534  srcTests=[]
535  for t in fsplit: srcTests.append(t.split("TEST*/")[0])
536  testString=" ".join(srcTests)
537  flen=len(testString.split("\n"))
538  fend=fstart+flen-1
539  fileNums=range(fstart,fend)
540  testDict[basename]=parseTests(testString,srcfile,fileNums,verbosity)
541  # Massage dictionary for build requirements
542  if 'build' in testDict[basename]:
543    testDict[basename].update(testDict[basename]['build'])
544    del testDict[basename]['build']
545
546
547  os.chdir(curdir)
548  return testDict
549
550def parseTestDir(directory,verbosity):
551  """
552  Parse single example files and return dictionary of the form:
553    testDict[srcfile][test][subtest]
554  """
555  curdir=os.path.realpath(os.path.curdir)
556  basedir=os.path.realpath(directory)
557  os.chdir(basedir)
558
559  tDict={}
560  for test_file in sorted(glob.glob("new_ex*.*")):
561    tDict.update(parseTestFile(test_file,verbosity))
562
563  os.chdir(curdir)
564  return tDict
565
566def printExParseDict(rDict):
567  """
568  This is useful for debugging
569  """
570  indent="   "
571  for sfile in rDict:
572    print(sfile)
573    sortkeys=list(rDict[sfile].keys())
574    sortkeys.sort()
575    for runex in sortkeys:
576      if runex == 'requires':
577        print(indent+runex+':'+str(rDict[sfile][runex]))
578        continue
579      print(indent+runex)
580      if type(rDict[sfile][runex])==bytes:
581        print(indent*2+rDict[sfile][runex])
582      else:
583        for var in rDict[sfile][runex]:
584          if var.startswith("test"): continue
585          print(indent*2+var+": "+str(rDict[sfile][runex][var]))
586        if 'subtests' in rDict[sfile][runex]:
587          for var in rDict[sfile][runex]['subtests']:
588            print(indent*2+var)
589            for var2 in rDict[sfile][runex][var]:
590              print(indent*3+var2+": "+str(rDict[sfile][runex][var][var2]))
591      print("\n")
592  return
593
594def main(directory='',test_file='',verbosity=0):
595
596    if directory:
597      tDict=parseTestDir(directory,verbosity)
598    else:
599      tDict=parseTestFile(test_file,verbosity)
600    if verbosity>0: printExParseDict(tDict)
601
602    return
603
604if __name__ == '__main__':
605    import optparse
606    parser = optparse.OptionParser()
607    parser.add_option('-d', '--directory', dest='directory',
608                      default="", help='Directory containing files to parse')
609    parser.add_option('-t', '--test_file', dest='test_file',
610                      default="", help='Test file, e.g., ex1.c, to parse')
611    parser.add_option('-v', '--verbosity', dest='verbosity',
612                      help='Verbosity of output by level: 1, 2, or 3', default=0)
613    opts, extra_args = parser.parse_args()
614
615    if extra_args:
616        import sys
617        sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args))
618        exit(1)
619    if not opts.test_file and not opts.directory:
620      print("test file or directory is required")
621      parser.print_usage()
622      sys.exit()
623
624    # Need verbosity to be an integer
625    try:
626      verbosity=int(opts.verbosity)
627    except:
628      raise Exception("Error: Verbosity must be integer")
629
630    main(directory=opts.directory,test_file=opts.test_file,verbosity=verbosity)
631