1# -*- coding: utf-8 -*- 2 3# ============================================================================== 4# COPYRIGHT (C) 1991 - 2015 EDF R&D WWW.CODE-ASTER.ORG 5# THIS PROGRAM IS FREE SOFTWARE; YOU CAN REDISTRIBUTE IT AND/OR MODIFY 6# IT UNDER THE TERMS OF THE GNU GENERAL PUBLIC LICENSE AS PUBLISHED BY 7# THE FREE SOFTWARE FOUNDATION; EITHER VERSION 2 OF THE LICENSE, OR 8# (AT YOUR OPTION) ANY LATER VERSION. 9# 10# THIS PROGRAM IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, BUT 11# WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED WARRANTY OF 12# MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. SEE THE GNU 13# GENERAL PUBLIC LICENSE FOR MORE DETAILS. 14# 15# YOU SHOULD HAVE RECEIVED A COPY OF THE GNU GENERAL PUBLIC LICENSE 16# ALONG WITH THIS PROGRAM; IF NOT, WRITE TO EDF R&D CODE_ASTER, 17# 1 AVENUE DU GENERAL DE GAULLE, 92141 CLAMART CEDEX, FRANCE. 18# ============================================================================== 19 20""" 21This modules gives functions to start up Code_Aster executions. 22""" 23 24 25import os 26import os.path as osp 27import re 28import stat 29from glob import glob 30from math import log10 31from warnings import warn 32 33from asrun.installation import confdir 34from asrun.common.i18n import _ 35from asrun.mystring import convert, ufmt, file_cleanCR, to_unicode 36from asrun.build import glob_unigest 37from asrun.runner import Runner 38from asrun.profil import AsterProfil, ExportEntry 39from asrun.common_func import get_tmpname 40from asrun.common.utils import getpara, check_joker, hms2s, make_writable 41from asrun.common.sysutils import on_64bits 42 43 44def dict_typ_test(test): 45 """supported file types + ['com?', '[0-9]*'] 46 """ 47 return { 48 'comm' : ('fort.1' , 1), 49 'mail' : ('fort.20', 20), 50 'mmed' : ('fort.20', 20), 51 'med' : ('fort.20', 20), 52 'datg' : ('fort.16', 16), 53 'mgib' : ('fort.19', 19), 54 'msh' : ('fort.19', 19), 55 'msup' : ('fort.19', 19), 56 'para' : ('%s.para' % test, 0), 57 'ensi' : ('DONNEES_ENSIGHT', 0), 58 'repe' : ('REPE_IN', 0), 59 } 60 61def dict_typ_result(): 62 """supported file types : resu, rmed 63 """ 64 return { 65 'mess' : ('fort.6', 6), 66 'resu' : ('fort.8', 8), 67 'dat' : ('fort.29', 29), 68 'rmed' : ('fort.80', 80), 69 'pos' : ('fort.37', 37), 70 'base' : ('unused', 0), 71 'bhdf' : ('unused', 0), 72 } 73 74 75 76def execute(reptrav, multiple=False, with_dbg=False, only_env=False, 77 follow_output=True, fpara=None, facmtps=1., 78 runner=None, **kargs): 79 """Run a Code_Aster execution in 'reptrav'. 80 Arguments : 81 multiple : False if only one execution is run (so stop if it fails), 82 True if several executions are run (don't stop when error occurs) 83 with_dbg : start debugger or not, 84 fpara : deprecated, 85 follow_output : print output to follow the execution, 86 kargs give "run, conf, prof, build" instances + exec name 87 Return a tuple (diag, tcpu, tsys, ttot, validbase). 88 """ 89 # 1. ----- initializations 90 run = kargs['run'] 91 conf = kargs['conf'] 92 prof = kargs['prof'] 93 build = kargs['build'] 94 exetmp = kargs['exe'] 95 ctest = prof['parent'][0] == "astout" 96 waf_inst = build.support('waf') 97 waf_nosupv = build.support('nosuperv') 98 waf_noresu = build.support('noresu') 99 waf_orb = build.support('orbinitref') 100 use_numthreads = build.support('use_numthreads') 101 run.DBG("version supports: waf ({0}), nosuperv ({1}), " 102 "orbinitref ({2}), numthreads ({3})".format(waf_inst, waf_nosupv, 103 waf_orb, use_numthreads)) 104 if not waf_inst: 105 exetmp = osp.join('.', osp.basename(exetmp)) 106 tcpu = 0. 107 tsys = 0. 108 ttot = 0. 109 validbase = True 110 if runner is None: 111 runner = Runner() 112 runner.set_rep_trav(reptrav) 113 114 interact = ('interact' in prof.args or 115 prof.args.get('args', '').find('-interact') > -1) 116 hide_command = ("hide-command" in prof.args or 117 prof.args.get('args', '').find('--hide-command') > -1) 118 119 os.chdir(reptrav) 120 121 # 2. ----- list of command files 122 list_comm = glob('fort.1.*') 123 list_comm.sort() 124 if osp.exists('fort.1'): 125 list_comm.insert(0, 'fort.1') 126 if waf_nosupv: 127 for fcomm in list_comm: 128 add_import_commands(fcomm) 129 130 # 3. ----- arguments list 131 drep = { 'REPOUT' : 'rep_outils', 'REPMAT' : 'rep_mat', 'REPDEX' : 'rep_dex' } 132 cmd = [] 133 if waf_nosupv: 134 if interact: 135 cmd.append('-i') 136 cmd.append('fort.1') 137 else: 138 if waf_inst: 139 cmd.append(osp.join(conf['SRCPY'][0], conf['ARGPYT'][0])) 140 else: 141 cmd.append(osp.join(conf['REPPY'][0], conf['ARGPYT'][0])) 142 cmd.extend(conf['ARGEXE']) 143 # warning: using --commandes will turn off backward compatibility 144 cmd.append('-commandes fort.1') 145 # cmd.append(_fmtoption('command', 'fort.1')) 146 147 # remove deprecated options 148 long_opts_rm = ['rep', 'mem', 'mxmemdy', 'memory_stat', 'memjeveux_stat', 149 'type_alloc', 'taille', 'partition', 150 'origine', 'eficas_path'] 151 # for version < 12.6/13.2 that does not support --ORBInitRef=, ignore it 152 if not waf_orb: 153 long_opts_rm.append('ORBInitRef') 154 cmd_memtps = {} 155 for k, v in list(prof.args.items()): 156 if k == 'args': 157 cmd.append(prof.args[k]) 158 elif k in long_opts_rm: 159 warn("this command line option is deprecated : --%s" % k, 160 DeprecationWarning, stacklevel=3) 161 elif k in ('memjeveux', 'tpmax'): 162 cmd_memtps[k] = v 163 elif v.strip() == '' and k in list(drep.values()): 164 run.Mess(_('empty value not allowed for "%s"') % k, '<A>_INVALID_PARAMETER') 165 else: 166 cmd.append(_fmtoption(k, v)) 167 168 # add arguments to find the process (for as_actu/as_del) 169 if not 'astout' in prof['actions'] and not 'distribution' in prof['actions']: 170 cmd.append(_fmtoption('num_job', run['num_job'])) 171 cmd.append(_fmtoption('mode', prof['mode'][0])) 172 173 # arguments which can be set in file 'config.txt' 174 for kconf, karg in list(drep.items()): 175 if conf[kconf][0] != '' and not karg in list(prof.args.keys()): 176 cmd.append(_fmtoption(karg, conf[kconf][0])) 177 178 ncpus = prof['ncpus'][0] 179 try: 180 ncpus = max(1, int(ncpus)) 181 except ValueError: 182 ncpus = '' 183 if use_numthreads: 184 if ncpus == '': 185 ncpus = max([run[prof['mode'][0] + '_nbpmax'] // 2, 1]) 186 cmd.append(_fmtoption('numthreads', ncpus)) 187 elif ncpus == '': 188 ncpus = '1' 189 190 # 4. ----- add parameters from prof 191 if on_64bits(): 192 facW = 8 193 else: 194 facW = 4 195 tps = 0 196 memj = 0 197 nbp = 0 198 try: 199 tps = int(float(prof.args['tpmax'])) 200 except KeyError: 201 run.Mess(_('tpmax not provided in profile'), '<E>_INCORRECT_PARA') 202 except ValueError: 203 run.Mess(_('incorrect value for tpmax (%s) in profile') % \ 204 prof.args['tpmax'], '<E>_INCORRECT_PARA') 205 try: 206 memj = float(prof.args['memjeveux']) 207 except KeyError: 208 run.Mess(_('memjeveux not provided in profile'), '<E>_INCORRECT_PARA') 209 except ValueError: 210 run.Mess(_('incorrect value for memjeveux (%s) in profile') % \ 211 prof.args['memjeveux'], '<E>_INCORRECT_PARA') 212 213 try: 214 nbp = int(ncpus) 215 except ValueError: 216 run.Mess(_('incorrect value for ncpus (%s) in profile') % \ 217 prof['ncpus'][0], '<E>_INCORRECT_PARA') 218 219 # 4.1. check for memory, time and procs limits 220 run.Mess(_('Parameters : memory %d MB - time limit %d s') % (memj*facW, tps)) 221 check_limits(run, prof['mode'][0], tps, memj*facW, nbp, runner.nbnode(), runner.nbcpu()) 222 # check for previous errors (parameters) 223 if not multiple: 224 run.CheckOK() 225 elif run.GetGrav(run.diag) > run.GetGrav('<A>'): 226 run.Mess(_('error in parameters : %s') % run.diag) 227 return run.diag, tcpu, tsys, ttot, validbase 228 229 # 5. ----- only environment, print command lines to execute 230 if only_env: 231 run.Mess(ufmt(_('Code_Aster environment prepared in %s'), reptrav), 'OK') 232 run.Mess(_('To start execution copy/paste following lines in a ksh/bash shell :')) 233 run.Mess(' cd %s' % reptrav, 'SILENT') 234 run.Mess(' . %s' % osp.join(confdir, 'profile.sh'), 'SILENT') 235 tmp_profile = "profile_tmp.sh" 236 with open(tmp_profile, 'w') as f: 237 f.write(""" 238export PYTHONPATH=$PYTHONPATH:. 239export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:. 240""") 241 # set per version environment 242 for f in conf.get_with_absolute_path('ENV_SH') + [tmp_profile]: 243 run.Mess(' . %s' % f, 'SILENT') 244 245 cmd.insert(0, exetmp) 246 # add memjeveux and tpmax 247 cmd.extend([_fmtoption(k, v) for k, v in list(cmd_memtps.items())]) 248 cmdline = ' '.join(cmd) 249 250 # get pdb.py path 251 pdbpy_cmd = "import os, sys ; " \ 252 "pdbpy = os.path.join(sys.prefix, 'lib', 'python' + sys.version[:3], 'pdb.py')" 253 d = {} 254 exec(pdbpy_cmd, d) 255 pdbpy = d['pdbpy'] 256 257 if runner.really(): #XXX and if not ? perhaps because of exit_code 258 cmdline = runner.get_exec_command(cmdline, 259 add_tee=False, 260 env=conf.get_with_absolute_path('ENV_SH')) 261 262 # print command lines 263 k = 0 264 for fcomm in list_comm: 265 k += 1 266 run.Mess(_("Command line %d :") % k) 267 run.Mess("cp %s fort.1" % fcomm, 'SILENT') 268 run.Mess(cmdline, 'SILENT') 269 # how to start the Python debugger 270 if not runner.really(): 271 run.Mess(_('To start execution in the Python debugger you could type :'), 'SILENT') 272 pdb_cmd = cmdline.replace(exetmp, 273 '%s %s' % (exetmp, ' '.join(pdbpy.splitlines()))) 274 run.Mess("cp %s fort.1" % fcomm, 'SILENT') 275 run.Mess(pdb_cmd, 'SILENT') 276 277 diag = 'OK' 278 279 # 6. ----- really execute 280 else: 281 # 6.1. content of reptrav 282 if not ctest: 283 run.Mess(ufmt(_('Content of %s before execution'), reptrav), 'TITLE') 284 out = run.Shell(cmd='ls -la')[1] 285 print(out) 286 287 if len(list_comm) == 0: 288 run.Mess(_('no .comm file found'), '<E>_NO_COMM_FILE') 289 # check for previous errors (copy datas) 290 if not multiple: 291 run.CheckOK() 292 elif run.GetGrav(run.diag) > run.GetGrav('<A>'): 293 run.Mess(_('error occurs during preparing data : %s') % run.diag) 294 return run.diag, tcpu, tsys, ttot, validbase 295 296 # 6.2. complete command line 297 cmd.append('--suivi_batch') 298 add_tee = False 299 if with_dbg: 300 # how to run the debugger 301 cmd_dbg = run.config.get('cmd_dbg', '') 302 if not cmd_dbg: 303 run.Mess(_('command line to run the debugger not defined'), 304 '<F>_DEBUG_ERROR') 305 if cmd_dbg.find('gdb') > -1: 306 ldbg = ['break main', ] 307 else: 308 ldbg = ['stop in main', ] 309 # add memjeveux and tpmax 310 update_cmd_memtps(cmd_memtps) 311 cmd.extend([_fmtoption(k, v) for k, v in list(cmd_memtps.items())]) 312 pos_memtps = -1 313 314 cmd_args = ' '.join(cmd) 315 ldbg.append('run ' + cmd_args) 316 cmd = getdbgcmd(cmd_dbg, exetmp, '', ldbg, cmd_args) 317 else: 318 add_tee = True 319 cmd.insert(0, exetmp) 320 # position where insert memjeveux+tpmax 321 pos_memtps = len(cmd) 322 # keep for compatibility with version < 13.1 323 os.environ['OMP_NUM_THREADS'] = str(ncpus) 324 # unlimit coredump size 325 try: 326 corefilesize = int(prof['corefilesize'][0]) * 1024*1024 327 except ValueError: 328 corefilesize = 'unlimited' 329 run.SetLimit('core', corefilesize) 330 331 # 6.3. if multiple .comm files, keep previous bases 332 if len(list_comm) > 1: 333 run.Mess(_('%d command files') % len(list_comm)) 334 validbase = False 335 BASE_PREC = osp.join(reptrav, 'BASE_PREC') 336 run.MkDir(BASE_PREC) 337 338 # 6.4. for each .comm file 339 diag = '?' 340 diag_ok = None 341 k = 0 342 for fcomm in list_comm: 343 k += 1 344 os.chdir(runner.reptrav()) 345 run.Copy('fort.1', fcomm) 346 347 # start execution 348 tit = _('Code_Aster run') 349 run.timer.Start(tit) 350 351 # add memjeveux and tpmax at the right position 352 cmd_i = cmd[:] 353 update_cmd_memtps(cmd_memtps) 354 if pos_memtps > -1: 355 for key, value in list(cmd_memtps.items()): 356 cmd_i.insert(pos_memtps, _fmtoption(key, value)) 357 358 if True or not ctest: 359 run.Mess(tit, 'TITLE') 360 run.Mess(_('Command line %d :') % k) 361 if not run['verbose']: 362 run.Mess(' '.join(cmd_i)) 363 if waf_nosupv and not hide_command: 364 dash = "# " + "-" * 90 365 with open('fort.1', 'rb') as f: 366 content =[_("Content of the file to execute"), 367 dash, 368 to_unicode(f.read()), 369 dash] 370 run.Mess(os.linesep.join(content)) 371 372 cmd_exec = runner.get_exec_command(' '.join(cmd_i), 373 add_tee=add_tee, 374 env=conf.get_with_absolute_path('ENV_SH')) 375 376 # go 377 iret, exec_output = run.Shell(cmd_exec, follow_output=follow_output, interact=interact) 378 if iret != 0: 379 cats = ['fort.6'] 380 if not waf_noresu: 381 cats.extend(['fort.8', 'fort.9']) 382 for f in cats: 383 run.FileCat(text="""\n <I>_EXIT_CODE = %s""" % iret, dest=f) 384 if not follow_output and not ctest: 385 print(exec_output) 386 387 # mpirun does not include cpu/sys time of childrens, add it in timer 388 runner.add_to_timer(exec_output, tit) 389 390 run.timer.Stop(tit) 391 if k < len(list_comm): 392 for b in glob('vola.*')+glob('loca.*'): 393 run.Delete(b) 394 395 if len(list_comm) > 1: 396 ldiag = build.getDiag(cas_test=ctest) 397 diag_k = ldiag[0] 398 tcpu += ldiag[1] 399 tsys += ldiag[2] 400 ttot += ldiag[3] 401 run.FileCat('fort.6', 'fort_bis.6') 402 run.Delete('fort.6') 403 if not waf_noresu: 404 run.FileCat('fort.8', 'fort_bis.8') 405 run.Delete('fort.8') 406 run.FileCat('fort.9', 'fort_bis.9') 407 run.Delete('fort.9') 408 if re.search('<[ESF]{1}>', diag_k): 409 # switch <F> to <E> if multiple .comm 410 if diag_k.find('<F>') > -1: 411 diag_k = diag_k.replace('<F>', '<E>') 412 # ...and try to restore previous bases 413 run.Mess(ufmt(_('restore bases from %s'), BASE_PREC)) 414 lbas = glob(osp.join(BASE_PREC, 'glob.*')) + \ 415 glob(osp.join(BASE_PREC, 'bhdf.*')) + \ 416 glob(osp.join(BASE_PREC, 'pick.*')) 417 if len(lbas) > 0: 418 run.Copy(os.getcwd(), niverr='INFO', verbose=follow_output, *lbas) 419 else: 420 run.Mess(_('no glob/bhdf base to restore'), '<A>_ALARM') 421 run.Mess(_('execution aborted (comm file #%d)') % k, diag_k) 422 diag = diag_k 423 break 424 else: 425 # save bases in BASE_PREC if next execution fails 426 validbase = True 427 if k < len(list_comm): 428 if not ctest: 429 run.Mess(ufmt(_('save bases into %s'), BASE_PREC)) 430 lbas = glob('glob.*') + \ 431 glob('bhdf.*') + \ 432 glob('pick.*') 433 run.Copy(BASE_PREC, niverr='INFO', verbose=follow_output, *lbas) 434 run.Mess(_('execution ended (comm file #%d)') % k, diag_k) 435 # at least one is ok/alarm ? keep the "worse good" status! 436 if run.GetGrav(diag_k) in (0, 1): 437 diag_ok = diag_ok or 'OK' 438 if run.GetGrav(diag_ok) < run.GetGrav(diag_k): 439 diag_ok = diag_k 440 # the worst diagnostic 441 if run.GetGrav(diag) < run.GetGrav(diag_k): 442 diag = diag_k 443 444 # 6.5. global diagnostic 445 if len(list_comm) > 1: 446 run.Rename('fort_bis.6', 'fort.6') 447 run.Rename('fort_bis.8', 'fort.8') 448 run.Rename('fort_bis.9', 'fort.9') 449 else: 450 diag, tcpu, tsys, ttot = build.getDiag(cas_test=ctest)[:4] 451 validbase = run.GetGrav(diag) <= run.GetGrav('<S>') 452 if ctest and run.GetGrav(diag) < 0: 453 diag = '<F>_' + diag 454 if ctest and diag == 'NO_TEST_RESU' and diag_ok: 455 diag = diag_ok 456 run.ReinitDiag(diag) 457 # expected diagnostic ? 458 if prof['expected_diag'][0]: 459 expect = prof['expected_diag'][0] 460 if run.GetGrav(diag) >= run.GetGrav('<E>'): 461 diag = '<F>_ERROR' 462 if run.GetGrav(diag) == run.GetGrav(expect): 463 run.Mess(_('Diagnostic is as expected.')) 464 diag = 'OK' 465 else: 466 run.Mess(_("Diagnostic is not as expected (got '%s').") % diag) 467 diag = 'NOOK_TEST_RESU' 468 run.ReinitDiag(diag) 469 run.Mess(_('Code_Aster run ended, diagnostic : %s') % diag) 470 471 # 6.6. post-mortem analysis of the core file 472 if not with_dbg: 473 cmd_dbg = run.config.get('cmd_post', '') 474 lcor = glob('core*') 475 if cmd_dbg and lcor: 476 run.Mess(_('Code_Aster run created a coredump'), 477 '<E>_CORE_FILE') 478 if not multiple: 479 # take the first one if several core files 480 core = lcor[0] 481 run.Mess(ufmt(_('core file name : %s'), core)) 482 cmd = getdbgcmd(cmd_dbg, exetmp, core, 483 ('where', 'quit'), '') 484 tit = _('Coredump analysis') 485 run.Mess(tit, 'TITLE') 486 run.timer.Start(tit) 487 iret, output = run.Shell(' '.join(cmd), 488 alt_comment='coredump analysis...', verbose=True) 489 if iret == 0 and not ctest: 490 print(output) 491 run.timer.Stop(tit) 492 493 if not ctest: 494 # 6.7. content of reptrav 495 run.Mess(ufmt(_('Content of %s after execution'), os.getcwd()), 'TITLE') 496 out = run.Shell(cmd='ls -la . REPE_OUT')[1] 497 print(out) 498 499 # 6.8. print some informations 500 run.Mess(_('Size of bases'), 'TITLE') 501 lf = glob('vola.*') 502 lf.extend(glob('loca.*')) 503 lf.extend(glob('glob.*')) 504 lf.extend(glob('bhdf.*')) 505 lf.extend(glob('pick.*')) 506 for f in lf: 507 run.Mess(_('size of %s : %12d bytes') % (f, os.stat(f).st_size)) 508 509 return diag, tcpu, tsys, ttot, validbase 510 511 512def _fmtoption(key, value=None): 513 """Format an option""" 514 key = key.lstrip('-') 515 if value is None or (type(value) is str and not value.strip()): 516 fmt = '--{0}'.format(key) 517 else: 518 fmt = '--{0}={1}'.format(key, value) 519 return fmt 520 521def copyfiles(run, mode, prof, copybase=True, alarm=True): 522 """Copy datas from profile into `pwd` (if mode=='DATA') 523 or results from `pwd` into destination given by profile (if mode=='RESU'). 524 Aster bases are copied only if copybase is True. 525 Raise only <E> if an error occurs run CheckOK() after. 526 """ 527 if mode == 'DATA': 528 l_dico = prof.data 529 icomm = 0 530 ncomm = len(prof.Get('D', 'comm')) 531 for df in l_dico: 532 icomm = copyfileD(run, df, icomm, ncomm) 533 else: 534 l_dico = prof.resu 535 for df in l_dico: 536 copyfileR(run, df, copybase, alarm) 537 538 539def copyfileD(run, df, icomm, ncomm): 540 """Copy datas from `df` into current directory. 541 Raise only <E> if an error occurs run CheckOK() after. 542 """ 543 dest = None 544 # 1. ----- if logical unit is set : fort.* 545 if df['ul'] != 0 or df['type'] in ('nom',): 546 dest = 'fort.%d' % df['ul'] 547 if df['type'] == 'nom': 548 dest = osp.basename(df['path']) 549 # exception for multiple command files (adding _N) 550 if df['ul'] == 1: 551 icomm += 1 552 format = '%%0%dd' % (int(log10(max(1, ncomm))) + 1) 553 dest = dest + '.' + format % icomm 554 # warning if file already exists 555 if run.Exists(dest): 556 run.Mess(ufmt(_("'%s' overwrites '%s'"), df['path'], dest), '<A>_COPY_DATA') 557 if df['compr']: 558 dest = dest + '.gz' 559 560 # 2. ----- bases and directories (ul=0) 561 else: 562 # base 563 if df['type'] in ('base', 'bhdf'): 564 dest = osp.basename(df['path']) 565 # ensi 566 elif df['type'] == 'ensi': 567 dest = 'DONNEES_ENSIGHT' 568 # repe 569 elif df['type'] == 'repe': 570 dest = 'REPE_IN' 571 572 if dest is not None: 573 # 3. --- copy 574 kret = run.Copy(dest, df['path'], niverr='<E>_COPY_ERROR', verbose=True) 575 576 # 4. --- decompression 577 if kret == 0 and df['compr']: 578 kret, dest = run.Gunzip(dest, niverr='<E>_DECOMPRESSION', verbose=True) 579 580 # 5. --- move the bases in main directory 581 if df['type'] in ('base', 'bhdf'): 582 for f in glob(osp.join(dest, '*')): 583 run.Rename(f, osp.basename(f)) 584 585 # force the file to be writable 586 make_writable(dest) 587 588 # clean text files if necessary 589 if df['ul'] != 0 and run.IsTextFileWithCR(dest): 590 file_cleanCR(dest) 591 print(ufmt(' ` ' + _('line terminators have been removed from %s'), dest)) 592 return icomm 593 594 595def copyfileR(run, df, copybase=True, alarm=True): 596 """Copy results from current directory into destination given by `df`. 597 Aster bases are copied only if copybase is True. 598 Raise only <E> if an error occurs run CheckOK() after. 599 """ 600 # list of files 601 lf = [] 602 isdir = False 603 604 # 1. ----- files 605 if df['ul'] != 0 or df['type'] in ('nom', ): 606 # if logical unit is set : fort.* 607 if df['ul'] != 0: 608 lf.append('fort.%d' % df['ul']) 609 elif df['type'] == 'nom': 610 lf.append(osp.basename(df['path'])) 611 612 # 2. ----- bases and directories (ul=0) 613 else: 614 isdir = True 615 # base 616 if df['type'] == 'base' and copybase: 617 lf.extend(glob('glob.*')) 618 lf.extend(glob('pick.*')) 619 # bhdf 620 elif df['type'] == 'bhdf' and copybase: 621 lbas = glob('bhdf.*') 622 if len(lbas) == 0: 623 if alarm: 624 run.Mess(_("No 'bhdf' found, saving 'glob' instead"), '<A>_COPY_BASE') 625 lbas = glob('glob.*') 626 lf.extend(lbas) 627 lf.extend(glob('pick.*')) 628 # repe 629 elif df['type'] == 'repe': 630 rep = 'REPE_OUT' 631 lfrep = glob(osp.join(rep, '*')) 632 if len(lfrep) == 0 and alarm: 633 run.Mess(ufmt(_("%s directory is empty !"), rep), '<A>_COPY_REPE') 634 lf.extend(lfrep) 635 636 # 3. ----- compression 637 kret = 0 638 if df['compr']: 639 lfnam = lf[:] 640 lf = [] 641 for fnam in lfnam: 642 kret, f = run.Gzip(fnam, niverr='<E>_COMPRES_ERROR', verbose=True) 643 if kret == 0: 644 lf.append(f) 645 else: 646 lf.append(fnam) 647 run.Mess(_("Warning: The uncompressed file will be returned " 648 "without changing the target filename\n(eventually " 649 "ending with '.gz' even if it is not compressed; " 650 "you may have to rename it before use)."), 651 '<A>_COPYFILE') 652 653 # 4. ----- copy 654 if len(lf) > 0: 655 # create destination 656 if isdir: 657 kret = run.MkDir(df['path'], '<E>_MKDIR_ERROR') 658 else: 659 if len(lf) > 1: 660 run.Mess(ufmt(_("""Only the first one of [%s] is copied."""), ', '.join(lf)), 661 '<A>_COPYFILE') 662 lf = [lf[0],] 663 kret = run.MkDir(osp.dirname(df['path']), '<E>_MKDIR_ERROR') 664 # copy 665 if kret == 0: 666 lfc = lf[:] 667 for fname in lfc: 668 if not osp.exists(fname): 669 if alarm: 670 run.Mess(ufmt("no such file or directory: %s", fname), '<A>_COPYFILE') 671 lf.remove(fname) 672 if len(lf) > 0: 673 kret = run.Copy(df['path'], niverr='<E>_COPY_ERROR', verbose=True, *lf) 674 # save base if failure 675 if kret != 0: 676 rescue = get_tmpname(run, basename='save_results') 677 run.Mess(ufmt(_("Saving results in a temporary directory (%s)."), rescue), 678 '<A>_COPY_RESULTS', store=True) 679 kret = run.MkDir(rescue, chmod=0o700) 680 kret = run.Copy(rescue, niverr='<E>_COPY_ERROR', 681 verbose=True, *lf) 682 683 684def build_test_export(run, conf, REPREF, reptest, test, resutest=None, 685 with_default=True, d_unig=None): 686 """Return a profile for a testcase. 687 """ 688 lrep = [osp.join(REPREF, dt) for dt in conf['SRCTEST']] 689 if reptest: 690 lrep.extend(reptest) 691 for rep in lrep: 692 if run.IsRemote(rep): 693 run.Mess(ufmt(_('reptest (%s) must be on exec host'), rep), 694 '<F>_INVALID_DIR') 695 lrep = [run.PathOnly(rep) for rep in lrep] 696 lrm = [] 697 if d_unig: 698 d_unig = glob_unigest(d_unig, REPREF) 699 lrm = set([osp.basename(f) for f in d_unig['test']]) 700 701 export = _existing_file(test + '.export', lrep, last=True) 702 # new testcase with .export 703 if export: 704 prof = AsterProfil(run=run) 705 if with_default: 706 prof.add_default_parameters() 707 pexp = AsterProfil(export, run) 708 pexp.set_param_limits() 709 for entry in pexp.get_data(): 710 if osp.basename(entry.path) in lrm: 711 run.Mess(ufmt(_('deleting %s (matches unigest)'), 712 osp.basename(entry.path))) 713 pexp.remove(entry) 714 found = _existing_file(entry.path, lrep, last=True) 715 if found is None: 716 run.Mess(ufmt(_('file not found : %s'), entry.path), 717 '<E>_FILE_NOT_FOUND') 718 pexp.remove(entry) 719 else: 720 entry.path = found 721 pexp._compatibility() 722 prof.update(pexp) 723 else: 724 # old version using .para 725 lall = [] 726 for r in lrep: 727 f = osp.join(r, '%s.*' % test) 728 lall.extend(glob(f)) 729 lf = [] 730 for f in lall: 731 if osp.basename(f) in lrm: 732 run.Mess(ufmt(_('deleting %s (matches unigest)'), osp.basename(f))) 733 else: 734 lf.append(f) 735 if not lf: 736 run.Mess(ufmt(_('no such file : %s.*'), test), 737 '<E>_FILE_NOT_FOUND') 738 prof = build_export_from_files(run, lf, test, with_default=with_default) 739 if resutest: 740 ftyp = { 'resu' : 8, 'mess' : 6, 'code' : 15 } 741 for typ, ul in list(ftyp.items()): 742 new = ExportEntry(osp.join(resutest, '%s.%s' % (test, typ)), 743 type=typ, ul=ul, 744 result=True) 745 prof.add(new) 746 return prof 747 748 749def build_export_from_files(run, lf, root="", with_default=True, with_results=False): 750 """Build an export file from a list of files. 751 """ 752 prof = AsterProfil(run=run) 753 if with_default: 754 prof.add_default_parameters() 755 ddat = build_dict_file(run, prof, dict_typ_test(root), lf, ['com?', '[0-9]*']) 756 dres = {} 757 if with_results: 758 dres = build_dict_file(run, prof, dict_typ_result(), lf) 759 760 for dicf, dr in ((ddat, 'D'), (dres, 'R')): 761 lcom_i = [] 762 for f, dico in list(dicf.items()): 763 if dico['type'] != 'comm' or osp.splitext(f)[-1] == '.comm': 764 prof.Set(dr, dico) 765 else: 766 lcom_i.append([f, dico]) 767 lcom_i.sort() 768 for f, dico in lcom_i: 769 prof.Set(dr, dico) 770 return prof 771 772 773def build_dict_file(run, prof, dtyp, lf, opt_suffix=[]): 774 """Build the dictionnary of files 'lf' matching the types defined 775 in 'dtyp'.""" 776 dvu = {} 777 for typ0 in list(dtyp.keys()) + opt_suffix: 778 if typ0 == 'comm': # always in com? 779 continue 780 for f in lf: 781 typ = typ0 782 if not check_joker(f, "."+typ): 783 continue 784 base = osp.basename(f) 785 if dvu.get(base): 786 run.DBG("'%s' overwrites by '%s'" % (base, f)) 787 if typ == 'para': 788 iret, dico, msg = getpara(f, run['plate-forme']) 789 if iret != 0: 790 run.DBG("ERROR getpara :", msg) 791 prof.add_param_from_dict(dico) 792 continue 793 if typ[:3] == 'com': # comm ou com? 794 typ = 'comm' 795 ul = 1 796 elif dtyp.get(typ) != None: 797 ul = dtyp[typ][1] 798 else: 799 typ = 'libr' 800 try: 801 ul = int(f.split('.')[-1]) 802 except ValueError: 803 ul = 0 804 dvu[base] = { 'type' : typ, 805 'isrep' : False, 806 'path' : f, 807 'ul' : ul, 808 'compr' : False } 809 return dvu 810 811def add_all_results(prof, dest, jobname, dtyp=None): 812 """Add all known results files to 'prof'.""" 813 if dtyp is None: 814 dtyp = dict_typ_result 815 prof['copy_result_alarm'] = 'no' 816 for typ, value in list(dtyp().items()): 817 if typ in ('base', 'bhdf'): 818 continue 819 path = osp.join(dest, "%s.%s" % (jobname, typ)) 820 ul = value[1] 821 prof.Set('R', 822 { 'path' : path, 'ul' : ul, 'type' : typ, 823 'isrep' : False, 'compr' : False}) 824 825def getdbgcmd(cmd_dbg, exe, core, lcmd, args): 826 """Return the command line (as list) to run 'lcmd' in the debugger. 827 @E : executable 828 @C : coredump filename 829 @D : filename of debugger commands 830 @d : string of debugger commands 831 """ 832 # debugger commands string 833 cstr = ' ; '.join(lcmd) 834 ftmp = 'dbg_cmdfile' 835 # fill debugger commands file 836 f = open(ftmp, 'w') 837 f.write(os.linesep.join(lcmd) + os.linesep) 838 f.close() 839 # replace @ codes 840 cmd = cmd_dbg.replace('@E', exe) 841 cmd = cmd.replace('@C', core) 842 cmd = cmd.replace('@D', ftmp) 843 cmd = cmd.replace('@d', cstr) 844 cmd = cmd.replace('@a', args) 845 return [cmd] 846 847 848def check_limits(run, mode, tps, mem, nbp, mpi_nbnoeud, mpi_nbcpu): 849 """Return True if args are less than limits defined in 850 configuration file, False else. 851 """ 852 # time 853 try: 854 tpsmax = hms2s(run[mode+'_tpsmax']) 855 except ValueError: 856 run.Mess(_('Incorrect value (%s) for %s') % \ 857 (str(run[mode+'_tpsmax']), mode+'_tpsmax'), '<F>_CONFIG_ERROR') 858 if tps > tpsmax: 859 run.Mess(_("""Requested time (%s s) is higher than the limit (%s s)""") % \ 860 (tps, tpsmax), '<E>_INCORRECT_PARA') 861 # memory 862 try: 863 limit = int(run[mode+'_memmax']) 864 except ValueError: 865 run.Mess(_('Incorrect value (%s) for %s') % \ 866 (str(run[mode+'_memmax']), mode+'_memmax'), '<F>_CONFIG_ERROR') 867 if mem > limit: 868 run.Mess(_("""Requested memory (%s MB) is higher than the limit (%s MB)""")%\ 869 (mem, limit), '<E>_INCORRECT_PARA') 870 # ncpus 871 try: 872 limit = int(run[mode+'_nbpmax']) 873 except ValueError: 874 run.Mess(_('Incorrect value (%s) for %s') % \ 875 (str(run[mode+'_nbpmax']), mode+'_nbpmax'), '<F>_CONFIG_ERROR') 876 if nbp > limit: 877 run.Mess(_("""Requested number of processors (%s) is higher than the limit (%s)""") %\ 878 (nbp, limit), '<E>_INCORRECT_PARA') 879 # mpi nbcpu 880 para = mode + '_mpi_nbpmax' 881 try: 882 limit = int(run.get(para, 1)) 883 except ValueError: 884 run.Mess(_('Incorrect value (%s) for %s') % \ 885 (str(run.get(para, 1)), para), '<F>_CONFIG_ERROR') 886 if mpi_nbcpu > limit: 887 run.Mess(_("""Requested number of MPI processors (%s) is higher than the limit (%s)""") %\ 888 (mpi_nbcpu, limit), '<E>_INCORRECT_PARA') 889 890 891def update_cmd_memtps(dico): 892 """Met à jour (ou non) les infos du dictionnaire contenant memjeveux et tpmax. 893 On traite uniquement : info_cpu (produit par E_JDC.py) 894 """ 895 # dictionnaire des infos lues 896 d = {} 897 try: 898 with open('info_cpu', 'r') as f: 899 content = f.read().splitlines() 900 l_str = content[0].split() 901 l_fl = [float(s) for s in l_str] 902 d['cpu_total'], d['cpu_total_user'], d['cpu_total_syst'], d['cpu_restant'] = l_fl 903 except: 904 pass 905 # new values 906 dnew = {} 907 # tpmax 908 if d.get('cpu_restant') is not None: 909 dnew['tpmax'] = int(d['cpu_restant']) 910 dico.update(dnew) 911 912 913def _existing_file(fname, l_paths, last=False): 914 """Return the first (or `last` if True) existing file in `l_paths`. 915 Return None if `fname` is not found.""" 916 l_paths = l_paths[:] 917 if last: 918 l_paths.reverse() 919 path = None 920 for dirn in l_paths: 921 if osp.exists(osp.join(dirn, fname)): 922 path = osp.join(dirn, fname) 923 break 924 return path 925 926 927def add_import_commands(filename): 928 """Add import of code_aster commands if not present. 929 930 Arguments: 931 filename (str): Path of the comm file to check. 932 """ 933 with open(filename, 'r') as fobj: 934 txt = fobj.read() 935 936 re_done = re.compile(r"^from +code_aster\.Commands", re.M) 937 if re_done.search(txt): 938 return 939 940 re_init = re.compile("^(?P<init>(DEBUT|POURSUITE))", re.M) 941 if re_init.search(txt): 942 starter = r"\g<init>" 943 else: 944 starter = "code_aster.init()\n" 945 946 re_replacement = \ 947r""" 948# temporarly added for compatibility with code_aster legacy 949from math import * 950 951import code_aster 952from code_aster.Commands import * 953 954{starter}""" 955 956 txt = re_init.sub(re_replacement.format(starter=starter), txt) 957 txt = convert(txt, encoding="utf-8") 958 re_coding = re.compile(r'^#( *(?:|\-\*\- *|en)coding.*)' + '\n', re.M) 959 if not re_coding.search(txt): 960 txt = "# coding=utf-8\n" + txt 961 with open(filename, 'w') as fobj: 962 fobj.write(txt) 963