1#!/usr/local/bin/python3.8 2 3from __future__ import print_function 4import pickle 5import os,shutil, string, re 6import sys 7import logging, time 8import types 9sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 10from collections import defaultdict 11from gmakegen import * 12 13import inspect 14thisscriptdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 15sys.path.insert(0,thisscriptdir) 16import testparse 17import example_template 18 19 20""" 21 22There are 2 modes of running tests: Normal builds and run from prefix of 23install. They affect where to find things: 24 25 26Case 1. Normal builds: 27 28 +---------------------+----------------------------------+ 29 | PETSC_DIR | <git dir> | 30 +---------------------+----------------------------------+ 31 | PETSC_ARCH | arch-foo | 32 +---------------------+----------------------------------+ 33 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 34 +---------------------+----------------------------------+ 35 | PETSC_EXAMPLESDIR | PETSC_DIR/src | 36 +---------------------+----------------------------------+ 37 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 38 +---------------------+----------------------------------+ 39 | PETSC_GMAKEFILETEST | PETSC_DIR/gmakefile.test | 40 +---------------------+----------------------------------+ 41 | PETSC_GMAKEGENTEST | PETSC_DIR/config/gmakegentest.py | 42 +---------------------+----------------------------------+ 43 44 45Case 2. From install dir: 46 47 +---------------------+-------------------------------------------------------+ 48 | PETSC_DIR | <prefix dir> | 49 +---------------------+-------------------------------------------------------+ 50 | PETSC_ARCH | '' | 51 +---------------------+-------------------------------------------------------+ 52 | PETSC_LIBDIR | PETSC_DIR/PETSC_ARCH/lib | 53 +---------------------+-------------------------------------------------------+ 54 | PETSC_EXAMPLESDIR | PETSC_DIR/share/petsc/examples/src | 55 +---------------------+-------------------------------------------------------+ 56 | PETSC_TESTDIR | PETSC_DIR/PETSC_ARCH/tests | 57 +---------------------+-------------------------------------------------------+ 58 | PETSC_GMAKEFILETEST | PETSC_DIR/share/petsc/examples/gmakefile.test | 59 +---------------------+-------------------------------------------------------+ 60 | PETSC_GMAKEGENTEST | PETSC_DIR/share/petsc/examples/config/gmakegentest.py | 61 +---------------------+-------------------------------------------------------+ 62 63""" 64 65def install_files(source, destdir): 66 """Install file or directory 'source' to 'destdir'. Does not preserve 67 mode (permissions). 68 """ 69 if not os.path.isdir(destdir): 70 os.makedirs(destdir) 71 if os.path.isdir(source): 72 for name in os.listdir(source): 73 install_files(os.path.join(source, name), os.path.join(destdir, os.path.basename(source))) 74 else: 75 shutil.copyfile(source, os.path.join(destdir, os.path.basename(source))) 76 77def nameSpace(srcfile,srcdir): 78 """ 79 Because the scripts have a non-unique naming, the pretty-printing 80 needs to convey the srcdir and srcfile. There are two ways of doing this. 81 """ 82 if srcfile.startswith('run'): srcfile=re.sub('^run','',srcfile) 83 prefix=srcdir.replace("/","_")+"-" 84 nameString=prefix+srcfile 85 return nameString 86 87class generateExamples(Petsc): 88 """ 89 gmakegen.py has basic structure for finding the files, writing out 90 the dependencies, etc. 91 """ 92 def __init__(self,petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, pkg_name=None, pkg_pkgs=None, testdir='tests', verbose=False, single_ex=False, srcdir=None, check=False): 93 super(generateExamples, self).__init__(petsc_dir=petsc_dir, petsc_arch=petsc_arch, pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, verbose=verbose) 94 95 self.single_ex=single_ex 96 self.srcdir=srcdir 97 self.check_output=check 98 99 # Set locations to handle movement 100 self.inInstallDir=self.getInInstallDir(thisscriptdir) 101 102 if self.inInstallDir: 103 # Case 2 discussed above 104 # set PETSC_ARCH to install directory to allow script to work in both 105 dirlist=thisscriptdir.split(os.path.sep) 106 installdir=os.path.sep.join(dirlist[0:len(dirlist)-4]) 107 self.arch_dir=installdir 108 if self.srcdir is None: 109 self.srcdir=os.path.join(os.path.dirname(thisscriptdir),'src') 110 else: 111 if petsc_arch == '': 112 raise RuntimeError('PETSC_ARCH must be set when running from build directory') 113 # Case 1 discussed above 114 self.arch_dir=os.path.join(self.petsc_dir,self.petsc_arch) 115 if self.srcdir is None: 116 self.srcdir=os.path.join(self.petsc_dir,'src') 117 118 self.testroot_dir=os.path.abspath(testdir) 119 120 self.verbose=verbose 121 # Whether to write out a useful debugging 122 self.summarize=True if verbose else False 123 124 # For help in setting the requirements 125 self.precision_types="single double __float128 int32".split() 126 self.integer_types="int32 int64 long32 long64".split() 127 self.languages="fortran cuda hip sycl cxx cpp".split() # Always requires C so do not list 128 129 # Things that are not test 130 self.buildkeys=testparse.buildkeys 131 132 # Adding a dictionary for storing sources, objects, and tests 133 # to make building the dependency tree easier 134 self.sources={} 135 self.objects={} 136 self.tests={} 137 for pkg in self.pkg_pkgs: 138 self.sources[pkg]={} 139 self.objects[pkg]=[] 140 self.tests[pkg]={} 141 for lang in LANGS: 142 self.sources[pkg][lang]={} 143 self.sources[pkg][lang]['srcs']=[] 144 self.tests[pkg][lang]={} 145 146 if not os.path.isdir(self.testroot_dir): os.makedirs(self.testroot_dir) 147 148 self.indent=" " 149 if self.verbose: print('Finishing the constructor') 150 return 151 152 def srcrelpath(self,rdir): 153 """ 154 Get relative path to source directory 155 """ 156 return os.path.relpath(rdir,self.srcdir) 157 158 def getInInstallDir(self,thisscriptdir): 159 """ 160 When petsc is installed then this file in installed in: 161 <PREFIX>/share/petsc/examples/config/gmakegentest.py 162 otherwise the path is: 163 <PETSC_DIR>/config/gmakegentest.py 164 We use this difference to determine if we are in installdir 165 """ 166 dirlist=thisscriptdir.split(os.path.sep) 167 if len(dirlist)>4: 168 lastfour=os.path.sep.join(dirlist[len(dirlist)-4:]) 169 if lastfour==os.path.join('share','petsc','examples','config'): 170 return True 171 else: 172 return False 173 else: 174 return False 175 176 def getLanguage(self,srcfile): 177 """ 178 Based on the source, determine associated language as found in gmakegen.LANGS 179 Can we just return srcext[1:\] now? 180 """ 181 langReq=None 182 srcext = getlangext(srcfile) 183 if srcext in ".F90".split(): langReq="F90" 184 if srcext in ".F".split(): langReq="F" 185 if srcext in ".cxx".split(): langReq="cxx" 186 if srcext in ".kokkos.cxx".split(): langReq="kokkos_cxx" 187 if srcext in ".cpp".split(): langReq="cpp" 188 if srcext == ".cu": langReq="cu" 189 if srcext == ".c": langReq="c" 190 #if not langReq: print("ERROR: ", srcext, srcfile) 191 return langReq 192 193 def _getAltList(self,output_file,srcdir): 194 ''' Calculate AltList based on output file-- see 195 src/snes/tutorials/output/ex22*.out 196 ''' 197 altlist=[output_file] 198 basefile = getlangsplit(output_file) 199 for i in range(1,9): 200 altroot=basefile+"_alt" 201 if i > 1: altroot=altroot+"_"+str(i) 202 af=altroot+".out" 203 srcaf=os.path.join(srcdir,af) 204 fullaf=os.path.join(self.petsc_dir,srcaf) 205 if os.path.isfile(fullaf): altlist.append(srcaf) 206 207 return altlist 208 209 210 def _getLoopVars(self,inDict,testname, isSubtest=False): 211 """ 212 Given: 'args: -bs {{1 2 3 4 5}} -pc_type {{cholesky sor}} -ksp_monitor' 213 Return: 214 inDict['args']: -ksp_monitor 215 inDict['subargs']: -bs ${bs} -pc_type ${pc_type} 216 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 217 loopVars['subargs']['bs']=[["bs"],["1 2 3 4 5"]] 218 loopVars['subargs']['pc_type']=[["pc_type"],["cholesky sor"]] 219 subst should be passed in instead of inDict 220 """ 221 loopVars={}; newargs=[] 222 lsuffix='+' 223 argregex = re.compile(' (?=-[a-zA-Z])') 224 from testparse import parseLoopArgs 225 for key in inDict: 226 if key in ('SKIP', 'regexes'): 227 continue 228 akey=('subargs' if key=='args' else key) # what to assign 229 if akey not in inDict: inDict[akey]='' 230 if akey == 'nsize' and not inDict['nsize'].startswith('{{'): 231 # Always generate a loop over nsize, even if there is only one value 232 inDict['nsize'] = '{{' + inDict['nsize'] + '}}' 233 keystr = str(inDict[key]) 234 varlist = [] 235 for varset in argregex.split(keystr): 236 if not varset.strip(): continue 237 if '{{' in varset: 238 keyvar,lvars,ftype=parseLoopArgs(varset) 239 if akey not in loopVars: loopVars[akey]={} 240 varlist.append(keyvar) 241 loopVars[akey][keyvar]=[keyvar,lvars] 242 if akey=='nsize': 243 if len(lvars.split()) > 1: 244 lsuffix += akey +'-${i' + keyvar + '}' 245 else: 246 inDict[akey] += ' -'+keyvar+' ${i' + keyvar + '}' 247 lsuffix+=keyvar+'-${i' + keyvar + '}_' 248 else: 249 if key=='args': 250 newargs.append(varset.strip()) 251 if varlist: 252 loopVars[akey]['varlist']=varlist 253 254 # For subtests, args are always substituted in (not top level) 255 if isSubtest: 256 inDict['subargs'] += " "+" ".join(newargs) 257 inDict['args']='' 258 if 'label_suffix' in inDict: 259 inDict['label_suffix']+=lsuffix.rstrip('+').rstrip('_') 260 else: 261 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 262 else: 263 if loopVars: 264 inDict['args'] = ' '.join(newargs) 265 inDict['label_suffix']=lsuffix.rstrip('+').rstrip('_') 266 return loopVars 267 268 def getArgLabel(self,testDict): 269 """ 270 In all of the arguments in the test dictionary, create a simple 271 string for searching within the makefile system. For simplicity in 272 search, remove "-", for strings, etc. 273 Also, concatenate the arg commands 274 For now, ignore nsize -- seems hard to search for anyway 275 """ 276 # Collect all of the args associated with a test 277 argStr=("" if 'args' not in testDict else testDict['args']) 278 if 'subtests' in testDict: 279 for stest in testDict["subtests"]: 280 sd=testDict[stest] 281 argStr=argStr+("" if 'args' not in sd else sd['args']) 282 283 # Now go through and cleanup 284 argStr=re.sub('{{(.*?)}}',"",argStr) 285 argStr=re.sub('-'," ",argStr) 286 for digit in string.digits: argStr=re.sub(digit," ",argStr) 287 argStr=re.sub("\.","",argStr) 288 argStr=re.sub(",","",argStr) 289 argStr=re.sub('\+',' ',argStr) 290 argStr=re.sub(' +',' ',argStr) # Remove repeated white space 291 return argStr.strip() 292 293 def addToSources(self,exfile,rpath,srcDict): 294 """ 295 Put into data structure that allows easy generation of makefile 296 """ 297 pkg=rpath.split(os.path.sep)[0] 298 relpfile=os.path.join(rpath,exfile) 299 lang=self.getLanguage(exfile) 300 if not lang: return 301 if pkg not in self.sources: return 302 self.sources[pkg][lang]['srcs'].append(relpfile) 303 self.sources[pkg][lang][relpfile] = [] 304 if 'depends' in srcDict: 305 depSrcList=srcDict['depends'].split() 306 for depSrc in depSrcList: 307 depObj = getlangsplit(depSrc)+'.o' 308 self.sources[pkg][lang][relpfile].append(os.path.join(rpath,depObj)) 309 310 # In gmakefile, ${TESTDIR} var specifies the object compilation 311 testsdir=rpath+"/" 312 objfile="${TESTDIR}/"+testsdir+getlangsplit(exfile)+'.o' 313 self.objects[pkg].append(objfile) 314 return 315 316 def addToTests(self,test,rpath,exfile,execname,testDict): 317 """ 318 Put into data structure that allows easy generation of makefile 319 Organized by languages to allow testing of languages 320 """ 321 pkg=rpath.split("/")[0] 322 nmtest=os.path.join(rpath,test) 323 lang=self.getLanguage(exfile) 324 if not lang: return 325 if pkg not in self.tests: return 326 self.tests[pkg][lang][nmtest]={} 327 self.tests[pkg][lang][nmtest]['exfile']=os.path.join(rpath,exfile) 328 self.tests[pkg][lang][nmtest]['exec']=execname 329 self.tests[pkg][lang][nmtest]['argLabel']=self.getArgLabel(testDict) 330 return 331 332 def getExecname(self,exfile,rpath): 333 """ 334 Generate bash script using template found next to this file. 335 This file is read in at constructor time to avoid file I/O 336 """ 337 if self.single_ex: 338 execname=rpath.split("/")[1]+"-ex" 339 else: 340 execname=getlangsplit(exfile) 341 return execname 342 343 def getSubstVars(self,testDict,rpath,testname): 344 """ 345 Create a dictionary with all of the variables that get substituted 346 into the template commands found in example_template.py 347 """ 348 subst={} 349 350 # Handle defaults of testparse.acceptedkeys (e.g., ignores subtests) 351 if 'nsize' not in testDict: testDict['nsize'] = '1' 352 if 'timeoutfactor' not in testDict: testDict['timeoutfactor']="1" 353 for ak in testparse.acceptedkeys: 354 if ak=='test': continue 355 subst[ak]=(testDict[ak] if ak in testDict else '') 356 357 # Now do other variables 358 subst['execname']=testDict['execname'] 359 subst['error']='' 360 if 'filter' in testDict: 361 if testDict['filter'].startswith("Error:"): 362 subst['error']="Error" 363 subst['filter']=testDict['filter'].lstrip("Error:") 364 else: 365 subst['filter']=testDict['filter'] 366 367 # Others 368 subst['subargs']='' # Default. For variables override 369 subst['srcdir']=os.path.join(self.srcdir, rpath) 370 subst['label_suffix']='' 371 subst['comments']="\n#".join(subst['comments'].split("\n")) 372 if subst['comments']: subst['comments']="#"+subst['comments'] 373 subst['exec']="../"+subst['execname'] 374 subst['testroot']=self.testroot_dir 375 subst['testname']=testname 376 dp = self.conf.get('DATAFILESPATH','') 377 subst['datafilespath_line'] = 'DATAFILESPATH=${DATAFILESPATH:-"'+dp+'"}' 378 379 # This is used to label some matrices 380 subst['petsc_index_size']=str(self.conf['PETSC_INDEX_SIZE']) 381 subst['petsc_scalar_size']=str(self.conf['PETSC_SCALAR_SIZE']) 382 383 #Conf vars 384 if self.petsc_arch.find('valgrind')>=0: 385 subst['mpiexec']='petsc_mpiexec_valgrind ' + self.conf['MPIEXEC'] 386 else: 387 subst['mpiexec']=self.conf['MPIEXEC'] 388 subst['pkg_name']=self.pkg_name 389 subst['pkg_dir']=self.pkg_dir 390 subst['pkg_arch']=self.petsc_arch 391 subst['CONFIG_DIR']=thisscriptdir 392 subst['PETSC_BINDIR']=os.path.join(self.petsc_dir,'lib','petsc','bin') 393 subst['diff']=self.conf['DIFF'] 394 subst['rm']=self.conf['RM'] 395 subst['grep']=self.conf['GREP'] 396 subst['petsc_lib_dir']=self.conf['PETSC_LIB_DIR'] 397 subst['wpetsc_dir']=self.conf['wPETSC_DIR'] 398 399 # Output file is special because of subtests override 400 defroot = testparse.getDefaultOutputFileRoot(testname) 401 if 'output_file' not in testDict: 402 subst['output_file']="output/"+defroot+".out" 403 subst['redirect_file']=defroot+".tmp" 404 subst['label']=nameSpace(defroot,self.srcrelpath(subst['srcdir'])) 405 406 # Add in the full path here. 407 subst['output_file']=os.path.join(subst['srcdir'],subst['output_file']) 408 409 subst['regexes']={} 410 for subkey in subst: 411 if subkey=='regexes': continue 412 if not isinstance(subst[subkey],str): continue 413 patt="@"+subkey.upper()+"@" 414 subst['regexes'][subkey]=re.compile(patt) 415 416 return subst 417 418 def _substVars(self,subst,origStr): 419 """ 420 Substitute variables 421 """ 422 Str=origStr 423 for subkey in subst: 424 if subkey=='regexes': continue 425 if not isinstance(subst[subkey],str): continue 426 if subkey.upper() not in Str: continue 427 Str=subst['regexes'][subkey].sub(lambda x: subst[subkey],Str) 428 return Str 429 430 def getCmds(self,subst,i, debug=False): 431 """ 432 Generate bash script using template found next to this file. 433 This file is read in at constructor time to avoid file I/O 434 """ 435 nindnt=i # the start and has to be consistent with below 436 cmdindnt=self.indent*nindnt 437 cmdLines="" 438 439 # MPI is the default -- but we have a few odd commands 440 if not subst['command']: 441 cmd=cmdindnt+self._substVars(subst,example_template.mpitest) 442 else: 443 cmd=cmdindnt+self._substVars(subst,example_template.commandtest) 444 cmdLines+=cmd+"\n"+cmdindnt+"res=$?\n\n" 445 446 cmdLines+=cmdindnt+'if test $res = 0; then\n' 447 diffindnt=self.indent*(nindnt+1) 448 449 # Do some checks on existence of output_file and alt files 450 if not os.path.isfile(os.path.join(self.petsc_dir,subst['output_file'])): 451 if not subst['TODO']: 452 print("Warning: "+subst['output_file']+" not found.") 453 altlist=self._getAltList(subst['output_file'], subst['srcdir']) 454 455 # altlist always has output_file 456 if len(altlist)==1: 457 cmd=diffindnt+self._substVars(subst,example_template.difftest) 458 else: 459 if debug: print("Found alt files: ",altlist) 460 # Have to do it by hand a bit because of variable number of alt files 461 rf=subst['redirect_file'] 462 cmd=diffindnt+example_template.difftest.split('@')[0] 463 for i in range(len(altlist)): 464 af=altlist[i] 465 cmd+=af+' '+rf 466 if i!=len(altlist)-1: 467 cmd+=' > diff-${testname}-'+str(i)+'.out 2> diff-${testname}-'+str(i)+'.out' 468 cmd+=' || ${diff_exe} ' 469 else: 470 cmd+='" diff-${testname}.out diff-${testname}.out diff-${label}' 471 cmd+=subst['label_suffix']+' ""' # Quotes are painful 472 cmdLines+=cmd+"\n" 473 cmdLines+=cmdindnt+'else\n' 474 cmdLines+=diffindnt+'petsc_report_tapoutput "" ${label} "SKIP Command failed so no diff"\n' 475 cmdLines+=cmdindnt+'fi\n' 476 return cmdLines 477 478 def _writeTodoSkip(self,fh,tors,reasons,footer): 479 """ 480 Write out the TODO and SKIP lines in the file 481 The TODO or SKIP variable, tors, should be lower case 482 """ 483 TORS=tors.upper() 484 template=eval("example_template."+tors+"line") 485 tsStr=re.sub("@"+TORS+"COMMENT@",', '.join(reasons),template) 486 tab = '' 487 if reasons: 488 fh.write('if ! $force; then\n') 489 tab = tab + ' ' 490 if reasons == ["Requires DATAFILESPATH"]: 491 # The only reason not to run is DATAFILESPATH, which we check at run-time 492 fh.write(tab + 'if test -z "${DATAFILESPATH}"; then\n') 493 tab = tab + ' ' 494 if reasons: 495 fh.write(tab+tsStr+"\n" + tab + "total=1; "+tors+"=1\n") 496 fh.write(tab+footer+"\n") 497 fh.write(tab+"exit\n") 498 if reasons == ["Requires DATAFILESPATH"]: 499 fh.write(' fi\n') 500 if reasons: 501 fh.write('fi\n') 502 fh.write('\n\n') 503 return 504 505 def getLoopVarsHead(self,loopVars,i,usedVars={}): 506 """ 507 Generate a nicely indented string with the format loops 508 Here is what the data structure looks like 509 loopVars['subargs']['varlist']=['bs' 'pc_type'] # Don't worry about OrderedDict 510 loopVars['subargs']['bs']=["i","1 2 3 4 5"] 511 loopVars['subargs']['pc_type']=["j","cholesky sor"] 512 """ 513 outstr=''; indnt=self.indent 514 515 for key in loopVars: 516 for var in loopVars[key]['varlist']: 517 varval=loopVars[key][var] 518 outstr += "{0}_in=${{{0}:-{1}}}\n".format(*varval) 519 outstr += "\n\n" 520 521 for key in loopVars: 522 for var in loopVars[key]['varlist']: 523 varval=loopVars[key][var] 524 outstr += indnt * i + "for i{0} in ${{{0}_in}}; do\n".format(*varval) 525 i = i + 1 526 return (outstr,i) 527 528 def getLoopVarsFoot(self,loopVars,i): 529 outstr=''; indnt=self.indent 530 for key in loopVars: 531 for var in loopVars[key]['varlist']: 532 i = i - 1 533 outstr += indnt * i + "done\n" 534 return (outstr,i) 535 536 def genRunScript(self,testname,root,isRun,srcDict): 537 """ 538 Generate bash script using template found next to this file. 539 This file is read in at constructor time to avoid file I/O 540 """ 541 # runscript_dir directory has to be consistent with gmakefile 542 testDict=srcDict[testname] 543 rpath=self.srcrelpath(root) 544 runscript_dir=os.path.join(self.testroot_dir,rpath) 545 if not os.path.isdir(runscript_dir): os.makedirs(runscript_dir) 546 with open(os.path.join(runscript_dir,testname+".sh"),"w") as fh: 547 548 # Get variables to go into shell scripts. last time testDict used 549 subst=self.getSubstVars(testDict,rpath,testname) 550 loopVars = self._getLoopVars(subst,testname) # Alters subst as well 551 if 'subtests' in testDict: 552 # The subtests inherit inDict, so we don't need top-level loops. 553 loopVars = {} 554 555 #Handle runfiles 556 for lfile in subst.get('localrunfiles','').split(): 557 install_files(os.path.join(root, lfile), 558 os.path.join(runscript_dir, os.path.dirname(lfile))) 559 # Check subtests for local runfiles 560 for stest in subst.get("subtests",[]): 561 for lfile in testDict[stest].get('localrunfiles','').split(): 562 install_files(os.path.join(root, lfile), 563 os.path.join(runscript_dir, os.path.dirname(lfile))) 564 565 # Now substitute the key variables into the header and footer 566 header=self._substVars(subst,example_template.header) 567 # The header is done twice to enable @...@ in header 568 header=self._substVars(subst,header) 569 footer=re.sub('@TESTROOT@',subst['testroot'],example_template.footer) 570 571 # Start writing the file 572 fh.write(header+"\n") 573 574 # If there is a TODO or a SKIP then we do it before writing out the 575 # rest of the command (which is useful for working on the test) 576 # SKIP and TODO can be for the source file or for the runs 577 self._writeTodoSkip(fh,'todo',[s for s in [srcDict.get('TODO',''), testDict.get('TODO','')] if s],footer) 578 self._writeTodoSkip(fh,'skip',srcDict.get('SKIP',[]) + testDict.get('SKIP',[]),footer) 579 580 j=0 # for indentation 581 582 if loopVars: 583 (loopHead,j) = self.getLoopVarsHead(loopVars,j) 584 if (loopHead): fh.write(loopHead+"\n") 585 586 # Subtests are special 587 allLoopVars=list(loopVars.keys()) 588 if 'subtests' in testDict: 589 substP=subst # Subtests can inherit args but be careful 590 k=0 # for label suffixes 591 for stest in testDict["subtests"]: 592 subst=substP.copy() 593 subst.update(testDict[stest]) 594 subst['label_suffix']='+'+string.ascii_letters[k]; k+=1 595 sLoopVars = self._getLoopVars(subst,testname,isSubtest=True) 596 if sLoopVars: 597 (sLoopHead,j) = self.getLoopVarsHead(sLoopVars,j,allLoopVars) 598 allLoopVars+=list(sLoopVars.keys()) 599 fh.write(sLoopHead+"\n") 600 fh.write(self.getCmds(subst,j)+"\n") 601 if sLoopVars: 602 (sLoopFoot,j) = self.getLoopVarsFoot(sLoopVars,j) 603 fh.write(sLoopFoot+"\n") 604 else: 605 fh.write(self.getCmds(subst,j)+"\n") 606 607 if loopVars: 608 (loopFoot,j) = self.getLoopVarsFoot(loopVars,j) 609 fh.write(loopFoot+"\n") 610 611 fh.write(footer+"\n") 612 613 os.chmod(os.path.join(runscript_dir,testname+".sh"),0o755) 614 #if '10_9' in testname: sys.exit() 615 return 616 617 def genScriptsAndInfo(self,exfile,root,srcDict): 618 """ 619 Generate scripts from the source file, determine if built, etc. 620 For every test in the exfile with info in the srcDict: 621 1. Determine if it needs to be run for this arch 622 2. Generate the script 623 3. Generate the data needed to write out the makefile in a 624 convenient way 625 All tests are *always* run, but some may be SKIP'd per the TAP standard 626 """ 627 debug=False 628 rpath=self.srcrelpath(root) 629 execname=self.getExecname(exfile,rpath) 630 isBuilt=self._isBuilt(exfile,srcDict) 631 for test in srcDict: 632 if test in self.buildkeys: continue 633 if debug: print(nameSpace(exfile,root), test) 634 srcDict[test]['execname']=execname # Convenience in generating scripts 635 isRun=self._isRun(srcDict[test]) 636 self.genRunScript(test,root,isRun,srcDict) 637 srcDict[test]['isrun']=isRun 638 self.addToTests(test,rpath,exfile,execname,srcDict[test]) 639 640 # This adds to datastructure for building deps 641 if isBuilt: self.addToSources(exfile,rpath,srcDict) 642 return 643 644 def _isBuilt(self,exfile,srcDict): 645 """ 646 Determine if this file should be built. 647 """ 648 # Get the language based on file extension 649 srcDict['SKIP'] = [] 650 lang=self.getLanguage(exfile) 651 if (lang=="F" or lang=="F90"): 652 if not self.have_fortran: 653 srcDict["SKIP"].append("Fortran required for this test") 654 elif lang=="F90" and 'PETSC_USING_F90FREEFORM' not in self.conf: 655 srcDict["SKIP"].append("Fortran f90freeform required for this test") 656 if lang=="cu" and 'PETSC_HAVE_CUDA' not in self.conf: 657 srcDict["SKIP"].append("CUDA required for this test") 658 if lang=="hip" and 'PETSC_HAVE_HIP' not in self.conf: 659 srcDict["SKIP"].append("HIP required for this test") 660 if lang=="sycl" and 'PETSC_HAVE_SYCL' not in self.conf: 661 srcDict["SKIP"].append("SYCL required for this test") 662 if lang=="kokkos_cxx" and 'PETSC_HAVE_KOKKOS' not in self.conf: 663 srcDict["SKIP"].append("KOKKOS required for this test") 664 if lang=="cxx" and 'PETSC_HAVE_CXX' not in self.conf: 665 srcDict["SKIP"].append("C++ required for this test") 666 if lang=="cpp" and 'PETSC_HAVE_CXX' not in self.conf: 667 srcDict["SKIP"].append("C++ required for this test") 668 669 # Deprecated source files 670 if srcDict.get("TODO"): 671 return False 672 673 # isRun can work with srcDict to handle the requires 674 if "requires" in srcDict: 675 if srcDict["requires"]: 676 return self._isRun(srcDict) 677 678 return srcDict['SKIP'] == [] 679 680 681 def _isRun(self,testDict, debug=False): 682 """ 683 Based on the requirements listed in the src file and the petscconf.h 684 info, determine whether this test should be run or not. 685 """ 686 indent=" " 687 688 if 'SKIP' not in testDict: 689 testDict['SKIP'] = [] 690 # MPI requirements 691 if 'MPI_IS_MPIUNI' in self.conf: 692 if testDict.get('nsize', '1') != '1': 693 testDict['SKIP'].append("Parallel test with serial build") 694 695 # The requirements for the test are the sum of all the run subtests 696 if 'subtests' in testDict: 697 if 'requires' not in testDict: testDict['requires']="" 698 for stest in testDict['subtests']: 699 if 'requires' in testDict[stest]: 700 testDict['requires']+=" "+testDict[stest]['requires'] 701 if testDict[stest].get('nsize', '1') != '1': 702 testDict['SKIP'].append("Parallel test with serial build") 703 break 704 705 # Now go through all requirements 706 if 'requires' in testDict: 707 for requirement in testDict['requires'].split(): 708 requirement=requirement.strip() 709 if not requirement: continue 710 if debug: print(indent+"Requirement: ", requirement) 711 isNull=False 712 if requirement.startswith("!"): 713 requirement=requirement[1:]; isNull=True 714 # Precision requirement for reals 715 if requirement in self.precision_types: 716 if self.conf['PETSC_PRECISION']==requirement: 717 if isNull: 718 testDict['SKIP'].append("not "+requirement+" required") 719 continue 720 continue # Success 721 elif not isNull: 722 testDict['SKIP'].append(requirement+" required") 723 continue 724 # Precision requirement for ints 725 if requirement in self.integer_types: 726 if requirement=="int32": 727 if self.conf['PETSC_SIZEOF_INT']==4: 728 if isNull: 729 testDict['SKIP'].append("not int32 required") 730 continue 731 continue # Success 732 elif not isNull: 733 testDict['SKIP'].append("int32 required") 734 continue 735 if requirement=="int64": 736 if self.conf['PETSC_SIZEOF_INT']==8: 737 if isNull: 738 testDict['SKIP'].append("NOT int64 required") 739 continue 740 continue # Success 741 elif not isNull: 742 testDict['SKIP'].append("int64 required") 743 continue 744 if requirement.startswith("long"): 745 reqsize = int(requirement[4:])//8 746 longsize = int(self.conf['PETSC_SIZEOF_LONG'].strip()) 747 if longsize==reqsize: 748 if isNull: 749 testDict['SKIP'].append("not %s required" % requirement) 750 continue 751 continue # Success 752 elif not isNull: 753 testDict['SKIP'].append("%s required" % requirement) 754 continue 755 # Datafilespath 756 if requirement=="datafilespath" and not isNull: 757 testDict['SKIP'].append("Requires DATAFILESPATH") 758 continue 759 # Defines -- not sure I have comments matching 760 if "define(" in requirement.lower(): 761 reqdef=requirement.split("(")[1].split(")")[0] 762 if reqdef in self.conf: 763 if isNull: 764 testDict['SKIP'].append("Null requirement not met: "+requirement) 765 continue 766 continue # Success 767 elif not isNull: 768 testDict['SKIP'].append("Required: "+requirement) 769 continue 770 771 # Rest should be packages that we can just get from conf 772 if requirement in ["complex","debug"]: 773 petscconfvar="PETSC_USE_"+requirement.upper() 774 pkgconfvar=self.pkg_name.upper()+"_USE_"+requirement.upper() 775 else: 776 petscconfvar="PETSC_HAVE_"+requirement.upper() 777 pkgconfvar=self.pkg_name.upper()+'_HAVE_'+requirement.upper() 778 petsccv = self.conf.get(petscconfvar) 779 pkgcv = self.conf.get(pkgconfvar) 780 781 if petsccv or pkgcv: 782 if isNull: 783 if petsccv: 784 testDict['SKIP'].append("Not "+petscconfvar+" requirement not met") 785 continue 786 else: 787 testDict['SKIP'].append("Not "+pkgconfvar+" requirement not met") 788 continue 789 continue # Success 790 elif not isNull: 791 if not petsccv and not pkgcv: 792 if debug: print("requirement not found: ", requirement) 793 if self.pkg_name == 'petsc': 794 testDict['SKIP'].append(petscconfvar+" requirement not met") 795 else: 796 testDict['SKIP'].append(petscconfvar+" or "+pkgconfvar+" requirement not met") 797 continue 798 return testDict['SKIP'] == [] 799 800 def checkOutput(self,exfile,root,srcDict): 801 """ 802 Check and make sure the output files are in the output directory 803 """ 804 debug=False 805 rpath=self.srcrelpath(root) 806 for test in srcDict: 807 if test in self.buildkeys: continue 808 if debug: print(rpath, exfile, test) 809 if 'output_file' in srcDict[test]: 810 output_file=srcDict[test]['output_file'] 811 else: 812 defroot = testparse.getDefaultOutputFileRoot(test) 813 if 'TODO' in srcDict[test]: continue 814 output_file="output/"+defroot+".out" 815 816 fullout=os.path.join(root,output_file) 817 if debug: print("---> ",fullout) 818 if not os.path.exists(fullout): 819 self.missing_files.append(fullout) 820 821 return 822 823 def genPetscTests_summarize(self,dataDict): 824 """ 825 Required method to state what happened 826 """ 827 if not self.summarize: return 828 indent=" " 829 fhname=os.path.join(self.testroot_dir,'GenPetscTests_summarize.txt') 830 with open(fhname, "w") as fh: 831 for root in dataDict: 832 relroot=self.srcrelpath(root) 833 pkg=relroot.split("/")[1] 834 fh.write(relroot+"\n") 835 allSrcs=[] 836 for lang in LANGS: allSrcs+=self.sources[pkg][lang]['srcs'] 837 for exfile in dataDict[root]: 838 # Basic information 839 rfile=os.path.join(relroot,exfile) 840 builtStatus=(" Is built" if rfile in allSrcs else " Is NOT built") 841 fh.write(indent+exfile+indent*4+builtStatus+"\n") 842 for test in dataDict[root][exfile]: 843 if test in self.buildkeys: continue 844 line=indent*2+test 845 fh.write(line+"\n") 846 # Looks nice to have the keys in order 847 #for key in dataDict[root][exfile][test]: 848 for key in "isrun abstracted nsize args requires script".split(): 849 if key not in dataDict[root][exfile][test]: continue 850 line=indent*3+key+": "+str(dataDict[root][exfile][test][key]) 851 fh.write(line+"\n") 852 fh.write("\n") 853 fh.write("\n") 854 fh.write("\n") 855 return 856 857 def genPetscTests(self,root,dirs,files,dataDict): 858 """ 859 Go through and parse the source files in the directory to generate 860 the examples based on the metadata contained in the source files 861 """ 862 debug=False 863 # Use examplesAnalyze to get what the makefles think are sources 864 #self.examplesAnalyze(root,dirs,files,anlzDict) 865 866 dataDict[root]={} 867 868 for exfile in files: 869 #TST: Until we replace files, still leaving the orginals as is 870 #if not exfile.startswith("new_"+"ex"): continue 871 #if not exfile.startswith("ex"): continue 872 873 # Ignore emacs and other temporary files 874 if exfile.startswith("."): continue 875 if exfile.startswith("#"): continue 876 if exfile.endswith("~"): continue 877 # Only parse source files 878 ext=getlangext(exfile).lstrip('.').replace('.','_') 879 if ext not in LANGS: continue 880 881 # Convenience 882 fullex=os.path.join(root,exfile) 883 if self.verbose: print(' --> '+fullex) 884 dataDict[root].update(testparse.parseTestFile(fullex,0)) 885 if exfile in dataDict[root]: 886 if not self.check_output: 887 self.genScriptsAndInfo(exfile,root,dataDict[root][exfile]) 888 else: 889 self.checkOutput(exfile,root,dataDict[root][exfile]) 890 891 return 892 893 def walktree(self,top): 894 """ 895 Walk a directory tree, starting from 'top' 896 """ 897 if self.check_output: 898 print("Checking for missing output files") 899 self.missing_files=[] 900 901 # Goal of action is to fill this dictionary 902 dataDict={} 903 for root, dirs, files in os.walk(top, topdown=True): 904 dirs.sort() 905 files.sort() 906 if "/tests" not in root and "/tutorials" not in root: continue 907 if "dSYM" in root: continue 908 if "tutorials"+os.sep+"build" in root: continue 909 if os.path.basename(root.rstrip("/")) == 'output': continue 910 if self.verbose: print(root) 911 self.genPetscTests(root,dirs,files,dataDict) 912 913 # If checking output, report results 914 if self.check_output: 915 if self.missing_files: 916 for file in set(self.missing_files): # set uniqifies 917 print(file) 918 sys.exit(1) 919 920 # Now summarize this dictionary 921 if self.verbose: self.genPetscTests_summarize(dataDict) 922 return dataDict 923 924 def gen_gnumake(self, fd): 925 """ 926 Overwrite of the method in the base PETSc class 927 """ 928 def write(stem, srcs): 929 for lang in LANGS: 930 if srcs[lang]['srcs']: 931 fd.write('%(stem)s.%(lang)s := %(srcs)s\n' % dict(stem=stem, lang=lang.replace('_','.'), srcs=' '.join(srcs[lang]['srcs']))) 932 for pkg in self.pkg_pkgs: 933 srcs = self.gen_pkg(pkg) 934 write('testsrcs-' + pkg, srcs) 935 # Handle dependencies 936 for lang in LANGS: 937 for exfile in srcs[lang]['srcs']: 938 if exfile in srcs[lang]: 939 ex='$(TESTDIR)/'+getlangsplit(exfile) 940 exfo=ex+'.o' 941 deps = [os.path.join('$(TESTDIR)', dep) for dep in srcs[lang][exfile]] 942 if deps: 943 # The executable literally depends on the object file because it is linked 944 fd.write(ex +": " + " ".join(deps) +'\n') 945 # The object file containing 'main' does not normally depend on other object 946 # files, but it does when it includes their modules. This dependency is 947 # overly blunt and could be reduced to only depend on object files for 948 # modules that are used, like "*f90aux.o". 949 fd.write(exfo +": " + " ".join(deps) +'\n') 950 951 return self.gendeps 952 953 def gen_pkg(self, pkg): 954 """ 955 Overwrite of the method in the base PETSc class 956 """ 957 return self.sources[pkg] 958 959 def write_gnumake(self, dataDict, output=None): 960 """ 961 Write out something similar to files from gmakegen.py 962 963 Test depends on script which also depends on source 964 file, but since I don't have a good way generating 965 acting on a single file (oops) just depend on 966 executable which in turn will depend on src file 967 """ 968 # Different options for how to set up the targets 969 compileExecsFirst=False 970 971 # Open file 972 with open(output, 'w') as fd: 973 # Write out the sources 974 gendeps = self.gen_gnumake(fd) 975 976 # Write out the tests and execname targets 977 fd.write("\n#Tests and executables\n") # Delimiter 978 979 for pkg in self.pkg_pkgs: 980 # These grab the ones that are built 981 for lang in LANGS: 982 testdeps=[] 983 for ftest in self.tests[pkg][lang]: 984 test=os.path.basename(ftest) 985 basedir=os.path.dirname(ftest) 986 testdeps.append(nameSpace(test,basedir)) 987 fd.write("test-"+pkg+"."+lang.replace('_','.')+" := "+' '.join(testdeps)+"\n") 988 fd.write('test-%s.%s : $(test-%s.%s)\n' % (pkg, lang.replace('_','.'), pkg, lang.replace('_','.'))) 989 990 # test targets 991 for ftest in self.tests[pkg][lang]: 992 test=os.path.basename(ftest) 993 basedir=os.path.dirname(ftest) 994 testdir="${TESTDIR}/"+basedir+"/" 995 nmtest=nameSpace(test,basedir) 996 rundir=os.path.join(testdir,test) 997 script=test+".sh" 998 999 # Deps 1000 exfile=self.tests[pkg][lang][ftest]['exfile'] 1001 fullex=os.path.join(self.srcdir,exfile) 1002 localexec=self.tests[pkg][lang][ftest]['exec'] 1003 execname=os.path.join(testdir,localexec) 1004 fullscript=os.path.join(testdir,script) 1005 tmpfile=os.path.join(testdir,test,test+".tmp") 1006 1007 # *.counts depends on the script and either executable (will 1008 # be run) or the example source file (SKIP or TODO) 1009 fd.write('%s.counts : %s %s' 1010 % (os.path.join('$(TESTDIR)/counts', nmtest), 1011 fullscript, 1012 execname if exfile in self.sources[pkg][lang]['srcs'] else fullex) 1013 ) 1014 if exfile in self.sources[pkg][lang]: 1015 for dep in self.sources[pkg][lang][exfile]: 1016 fd.write(' %s' % os.path.join('$(TESTDIR)',dep)) 1017 fd.write('\n') 1018 1019 # Now write the args: 1020 fd.write(nmtest+"_ARGS := '"+self.tests[pkg][lang][ftest]['argLabel']+"'\n") 1021 1022 return 1023 1024 def write_db(self, dataDict, testdir): 1025 """ 1026 Write out the dataDict into a pickle file 1027 """ 1028 with open(os.path.join(testdir,'datatest.pkl'), 'wb') as fd: 1029 pickle.dump(dataDict,fd) 1030 return 1031 1032def main(petsc_dir=None, petsc_arch=None, pkg_dir=None, pkg_arch=None, 1033 pkg_name=None, pkg_pkgs=None, verbose=False, single_ex=False, 1034 srcdir=None, testdir=None, check=False): 1035 # Allow petsc_arch to have both petsc_dir and petsc_arch for convenience 1036 testdir=os.path.normpath(testdir) 1037 if petsc_arch: 1038 petsc_arch=petsc_arch.rstrip(os.path.sep) 1039 if len(petsc_arch.split(os.path.sep))>1: 1040 petsc_dir,petsc_arch=os.path.split(petsc_arch) 1041 output = os.path.join(testdir, 'testfiles') 1042 1043 pEx=generateExamples(petsc_dir=petsc_dir, petsc_arch=petsc_arch, 1044 pkg_dir=pkg_dir, pkg_arch=pkg_arch, pkg_name=pkg_name, pkg_pkgs=pkg_pkgs, 1045 verbose=verbose, single_ex=single_ex, srcdir=srcdir, 1046 testdir=testdir,check=check) 1047 dataDict=pEx.walktree(os.path.join(pEx.srcdir)) 1048 if not pEx.check_output: 1049 pEx.write_gnumake(dataDict, output) 1050 pEx.write_db(dataDict, testdir) 1051 1052if __name__ == '__main__': 1053 import optparse 1054 parser = optparse.OptionParser() 1055 parser.add_option('--verbose', help='Show mismatches between makefiles and the filesystem', action='store_true', default=False) 1056 parser.add_option('--petsc-dir', help='Set PETSC_DIR different from environment', default=os.environ.get('PETSC_DIR')) 1057 parser.add_option('--petsc-arch', help='Set PETSC_ARCH different from environment', default=os.environ.get('PETSC_ARCH')) 1058 parser.add_option('--srcdir', help='Set location of sources different from PETSC_DIR/src', default=None) 1059 parser.add_option('-s', '--single_executable', dest='single_executable', action="store_false", help='Whether there should be single executable per src subdir. Default is false') 1060 parser.add_option('-t', '--testdir', dest='testdir', help='Test directory [$PETSC_ARCH/tests]') 1061 parser.add_option('-c', '--check-output', dest='check_output', action="store_true", 1062 help='Check whether output files are in output director') 1063 parser.add_option('--pkg-dir', help='Set the directory of the package (different from PETSc) you want to generate the makefile rules for', default=None) 1064 parser.add_option('--pkg-name', help='Set the name of the package you want to generate the makefile rules for', default=None) 1065 parser.add_option('--pkg-arch', help='Set the package arch name you want to generate the makefile rules for', default=None) 1066 parser.add_option('--pkg-pkgs', help='Set the package folders (comma separated list, different from the usual sys,vec,mat etc) you want to generate the makefile rules for', default=None) 1067 1068 opts, extra_args = parser.parse_args() 1069 if extra_args: 1070 import sys 1071 sys.stderr.write('Unknown arguments: %s\n' % ' '.join(extra_args)) 1072 exit(1) 1073 if opts.testdir is None: 1074 opts.testdir = os.path.join(opts.petsc_arch, 'tests') 1075 1076 main(petsc_dir=opts.petsc_dir, petsc_arch=opts.petsc_arch, 1077 pkg_dir=opts.pkg_dir,pkg_arch=opts.pkg_arch,pkg_name=opts.pkg_name,pkg_pkgs=opts.pkg_pkgs, 1078 verbose=opts.verbose, 1079 single_ex=opts.single_executable, srcdir=opts.srcdir, 1080 testdir=opts.testdir, check=opts.check_output) 1081