1################################################################## 2## (c) Copyright 2015- by Jaron T. Krogel ## 3################################################################## 4 5 6#====================================================================# 7# simulation.py # 8# Provides base classes for simulation objects, including input # 9# and analysis. The Simulation base class enables a large amount # 10# of the functionality of Nexus, including workflow construction # 11# and monitoring, in tandem with the ProjectManager class. # 12# # 13# Content summary: # 14# SimulationInput # 15# Abstract base class for simulation input. # 16# # 17# SimulationAnalyzer # 18# Abstract base class for simulation data analysis. # 19# # 20# Simulation # 21# Major Nexus class representing a simulation prior to, during, # 22# and after execution. Checks dependencies between simulations # 23# connected in workflows, manages input file writing, # 24# participates in job submission, checks for successful # 25# simulation completion, and analyzes output data. Saves state # 26# image of simulation progress regularly. Contains # 27# SimulationInput, SimulationAnalyzer, and Job objects (also # 28# optionally contains a PhysicalSystem object). Derived # 29# classes tailor specific functions such as passing dependency # 30# data and checking simulation state to a target simulation # 31# code. Derived classes include Qmcpack, Pwscf, Vasp, Gamess, # 32# Convert4qmc, Pw2qmcpack, SimulationBundle, and # 33# TemplateSimulation. # 34# # 35# NullSimulationInput # 36# Simulation input class intended for codes that do not use an # 37# input file. # 38# # 39# NullSimulationAnalyzer # 40# Simulation input class intended for codes that do not produce # 41# or need to analyze output data. # 42# # 43# SimulationInputTemplate # 44# Supports template input files. A template input file is a # 45# standard text input file provided by the user that optionally # 46# has specially marked keywords. Using find and replace # 47# operations, Nexus can produce variations on the template # 48# input file (e.g. to scan over a parameter). In this way # 49# Nexus can drive codes that do not have specialized classes # 50# derived from Simulation, SimulationInput, or # 51# SimulationAnalyzer. # 52# # 53# SimulationInputMultiTemplate # 54# Supports templated input files for codes that take many # 55# different files as input (VASP is an example of this). The # 56# multi-template is essentially a collection of individual # 57# template input files. # 58# # 59# input_template # 60# User-facing function to create SimulationInputTemplate's. # 61# # 62# multi_input_template # 63# User-facing function to create SimulationInputMultiTemplate's.# 64# # 65#====================================================================# 66 67 68import os 69import sys 70import shutil 71import string 72from subprocess import Popen,PIPE 73from developer import unavailable,ci 74from generic import obj 75from periodic_table import is_element 76from physical_system import PhysicalSystem 77from machines import Job 78from pseudopotential import ppset 79from nexus_base import NexusCore,nexus_core 80 81 82class SimulationInput(NexusCore): 83 def is_valid(self): 84 self.not_implemented() 85 #end def is_valid 86 87 def read_file_text(self,filepath): 88 if not os.path.exists(filepath): 89 self.error('file does not exist: '+filepath) 90 #end if 91 fobj = open(filepath,'r') 92 text = fobj.read() 93 fobj.close() 94 return text 95 #end def read_file_text 96 97 def write_file_text(self,filepath,text): 98 fobj = open(filepath,'w') 99 fobj.write(text) 100 fobj.flush() 101 fobj.close() 102 #end def write_file_text 103 104 def read(self,filepath): 105 tokens = filepath.split(None,1) 106 if len(tokens)>1: 107 text = filepath 108 self.read_text(text) 109 else: 110 text = self.read_file_text(filepath) 111 self.read_text(text,filepath) 112 #end if 113 #end def read 114 115 def write(self,filepath=None): 116 text = self.write_text(filepath) 117 if filepath!=None: 118 self.write_file_text(filepath,text) 119 #end if 120 return text 121 #end def write 122 123 def return_structure(self): 124 return self.return_system(structure_only=True) 125 #end def return_structure 126 127 def read_text(self,text,filepath=None): 128 self.not_implemented() 129 #end def read_text 130 131 def write_text(self,filepath=None): 132 self.not_implemented() 133 #end def write_text 134 135 def incorporate_system(self,system): 136 #take information from a physical system object and fill in input file 137 self.not_implemented() 138 #end def incorporate_system 139 140 def return_system(self,structure_only=False): 141 #create a physical system object from input file information 142 self.not_implemented() 143 #end def return_system 144#end class SimulationInput 145 146 147 148 149class SimulationAnalyzer(NexusCore): 150 def __init__(self,sim): 151 self.not_implemented() 152 #end def __init__ 153 154 def analyze(self): 155 self.not_implemented() 156 #end def analyze 157#end class SimulationAnalyzer 158 159 160 161 162class SimulationEmulator(NexusCore): 163 def run(self): 164 self.not_implemented() 165 #end def run 166#end class SimulationEmulator 167 168 169 170class SimulationImage(NexusCore): 171 save_only_fields = set([ 172 # user block (temporary) of (sim+) subcascade 173 'block', 174 'block_subcascade', 175 # local/remote/results directories 176 'locdir', 177 'remdir', 178 'resdir', 179 # image directories 180 'imlocdir', 181 'imremdir', 182 'imresdir', 183 ]) 184 185 load_fields = set([ 186 # important sim variables 187 'identifier', 188 'path', 189 'process_id', 190 # properties of the executable 191 'app_name', 192 'app_props', 193 # names of in/out/err files 194 'infile', 195 'outfile', 196 'errfile', 197 # directory and image file names for sim/input/analyzer 198 'image_dir', 199 'sim_image', 200 'input_image', 201 'analyzer_image', 202 # files copied in/out before/after run 203 'files', 204 'outputs', 205 # simulation status flags 206 'setup', 207 'sent_files', 208 'submitted', 209 'finished', 210 'failed', 211 'got_output', 212 'analyzed', 213 # cascade status flag 214 'subcascade_finished', 215 ]) 216 217 save_fields = load_fields | save_only_fields 218 219 def __init__(self): 220 None 221 #end def __init__ 222 223 def save_image(self,sim,imagefile): 224 self.clear() 225 self.transfer_from(sim,SimulationImage.save_fields) 226 self.save(imagefile) 227 self.clear() 228 #end def save_image 229 230 def load_image(self,sim,imagefile): 231 self.clear() 232 self.load(imagefile) 233 self.transfer_to(sim,SimulationImage.load_fields) 234 self.clear() 235 #end def load_image 236 237#end class SimulationImage 238 239 240 241class Simulation(NexusCore): 242 input_type = SimulationInput 243 analyzer_type = SimulationAnalyzer 244 generic_identifier = 'sim' 245 infile_extension = '.in' 246 outfile_extension = '.out' 247 errfile_extension = '.err' 248 application = 'simapp' 249 application_properties = set(['serial']) 250 application_results = set() 251 allow_overlapping_files = False 252 allowed_inputs = set(['identifier','path','infile','outfile','errfile','imagefile', 253 'input','job','files','dependencies','analysis_request', 254 'block','block_subcascade','app_name','app_props','system', 255 'skip_submit','force_write','simlabel','fake_sim', 256 'restartable','force_restart']) 257 sim_imagefile = 'sim.p' 258 input_imagefile = 'input.p' 259 analyzer_imagefile = 'analyzer.p' 260 image_directory = 'sim' 261 supports_restarts = False 262 renew_app_command = False 263 264 is_bundle = False 265 266 sim_count = 0 267 creating_fake_sims = False 268 269 sim_directories = dict() 270 all_sims = [] 271 272 273 @classmethod 274 def clear_all_sims(cls): 275 cls.sim_directories.clear() 276 cls.all_sims = [] 277 cls.sim_count = 0 278 #end def clear_all_sims 279 280 @classmethod 281 def code_name(cls): 282 return cls.generic_identifier 283 #end def code_name 284 285 # test needed 286 @classmethod 287 def separate_inputs(cls,kwargs,overlapping_kw=-1,copy_pseudos=True,sim_kw=None): 288 if overlapping_kw==-1: 289 overlapping_kw = set(['system']) 290 elif overlapping_kw is None: 291 overlapping_kw = set() 292 #end if 293 if sim_kw is None: 294 sim_kw = set() 295 else: 296 sim_kw = set(sim_kw) 297 #end if 298 kw = set(kwargs.keys()) 299 sim_kw = kw & (Simulation.allowed_inputs | sim_kw) 300 inp_kw = (kw - sim_kw) | (kw & overlapping_kw) 301 sim_args = obj() 302 inp_args = obj() 303 sim_args.transfer_from(kwargs,sim_kw) 304 inp_args.transfer_from(kwargs,inp_kw) 305 if 'system' in inp_args: 306 system = inp_args.system 307 if not isinstance(system,PhysicalSystem): 308 extra='' 309 if not isinstance(extra,obj): 310 extra = '\nwith value: {0}'.format(system) 311 #end if 312 cls.class_error('invalid input for variable "system"\nsystem object must be of type PhysicalSystem\nyou provided type: {0}'.format(system.__class__.__name__)+extra) 313 #end if 314 #end if 315 if 'pseudos' in inp_args and inp_args.pseudos!=None: 316 pseudos = inp_args.pseudos 317 # support ppset labels 318 if isinstance(pseudos,str): 319 code = cls.code_name() 320 if not ppset.supports_code(code): 321 cls.class_error('ppset labeled pseudopotential groups are not supported for code "{0}"'.format(code)) 322 #end if 323 if 'system' not in inp_args: 324 cls.class_error('system must be provided when using a ppset label') 325 #end if 326 system = inp_args.system 327 pseudos = ppset.get(pseudos,code,system) 328 if 'pseudos' in sim_args: 329 sim_args.pseudos = pseudos 330 #end if 331 inp_args.pseudos = pseudos 332 #end if 333 if copy_pseudos: 334 if 'files' in sim_args: 335 sim_args.files = list(sim_args.files) 336 else: 337 sim_args.files = list() 338 #end if 339 sim_args.files.extend(list(pseudos)) 340 #end if 341 if 'system' in inp_args: 342 system = inp_args.system 343 species_labels,species = system.structure.species(symbol=True) 344 pseudopotentials = nexus_core.pseudopotentials 345 for ppfile in pseudos: 346 if not ppfile in pseudopotentials: 347 cls.class_error('pseudopotential file {0} cannot be found'.format(ppfile)) 348 #end if 349 pp = pseudopotentials[ppfile] 350 if not pp.element_label in species_labels and not pp.element in species: 351 cls.class_error('the element {0} for pseudopotential file {1} is not in the physical system provided'.format(pp.element,ppfile)) 352 #end if 353 #end for 354 #end if 355 #end if 356 # this is already done in Simulation.__init__() 357 #if 'system' in inp_args and isinstance(inp_args.system,PhysicalSystem): 358 # inp_args.system = inp_args.system.copy() 359 ##end if 360 return sim_args,inp_args 361 #end def separate_inputs 362 363 364 def __init__(self,**kwargs): 365 #user specified variables 366 self.path = '' #directory where sim will be run 367 self.job = None #Job object for machine 368 self.dependencies = obj() #Simulation results on which sim serially depends 369 self.restartable = False #if True, job can be automatically restarted as deemed appropriate 370 self.force_restart = False #force a restart of the run 371 372 #variables determined by self 373 self.identifier = self.generic_identifier 374 self.simid = Simulation.sim_count 375 self.simlabel = None 376 Simulation.sim_count+=1 377 self.files = set() 378 self.app_name = self.application 379 self.app_props = list(self.application_properties) 380 self.sim_image = self.sim_imagefile 381 self.input_image = self.input_imagefile 382 self.analyzer_image = self.analyzer_imagefile 383 self.image_dir = self.image_directory 384 self.input = self.input_type() 385 self.system = None 386 self.dependents = obj() 387 self.created_directories = False 388 self.got_dependencies = False 389 self.setup = False 390 self.sent_files = False 391 self.submitted = False 392 self.finished = False 393 self.failed = False 394 self.got_output = False 395 self.analyzed = False 396 self.subcascade_finished = False 397 self.dependency_ids = set() 398 self.wait_ids = set() 399 self.block = False 400 self.block_subcascade = False 401 self.skip_submit = nexus_core.skip_submit 402 self.force_write = False 403 self.loaded = False 404 self.ordered_dependencies = [] 405 self.process_id = None 406 self.infile = None 407 self.outfile = None 408 self.errfile = None 409 self.bundleable = True 410 self.bundled = False 411 self.bundler = None 412 self.fake_sim = Simulation.creating_fake_sims 413 414 #variables determined by derived classes 415 self.outputs = None #object representing output data 416 # accessed by dependents when calling get_dependencies 417 418 self.set(**kwargs) 419 self.pre_init() 420 self.set_directories() 421 self.set_files() 422 self.propagate_identifier() 423 if len(kwargs)>0: 424 self.init_job() 425 #end if 426 self.post_init() 427 428 Simulation.all_sims.append(self) 429 #end def __init__ 430 431 432 def fake(self): 433 return self.fake_sim 434 #end def fake 435 436 437 def init_job(self): 438 if self.job is None: 439 self.error('job not provided. Input field job must be set to a Job object.') 440 elif not isinstance(self.job,Job): 441 self.error('Input field job must be set to a Job object\nyou provided an object of type: {0}\nwith value: {1}'.format(self.job.__class__.__name__,self.job)) 442 #end if 443 self.job = self.job.copy() 444 self.init_job_extra() 445 self.job.initialize(self) 446 #end def init_job 447 448 449 def init_job_extra(self): 450 None 451 #end def init_job_extra 452 453 454 def set_app_name(self,app_name): 455 self.app_name = app_name 456 #end def set_app_name 457 458 459 def set(self,**kw): 460 cls = self.__class__ 461 if 'dependencies' in kw: 462 self.depends(*kw['dependencies']) 463 del kw['dependencies'] 464 #end if 465 kwset = set(kw.keys()) 466 invalid = kwset - self.allowed_inputs 467 if len(invalid)>0: 468 self.error('received invalid inputs\ninvalid inputs: {0}\nallowed inputs are: {1}'.format(sorted(invalid),sorted(self.allowed_inputs))) 469 #end if 470 allowed = kwset & self.allowed_inputs 471 for name in allowed: 472 self[name] = kw[name] 473 #end for 474 if 'path' in allowed: 475 p = self.path 476 if not isinstance(p,str): 477 self.error('path must be a string, you provided {0} (type {1})'.format(p,p.__class__.__name__)) 478 #end if 479 if p.startswith('./'): 480 p = p[2:] 481 #end if 482 ld = nexus_core.local_directory 483 if p.startswith(ld): 484 p = p.split(ld)[1].lstrip('/') 485 #end if 486 self.path = p 487 #end if 488 if 'files' in allowed: 489 self.files = set(self.files) 490 #end if 491 if not isinstance(self.input,(self.input_type,GenericSimulationInput)): 492 self.error('input must be of type {0}\nreceived {1}\nplease provide input appropriate to {2}'.format(self.input_type.__name__,self.input.__class__.__name__,self.__class__.__name__)) 493 #end if 494 if isinstance(self.system,PhysicalSystem): 495 self.system = self.system.copy() 496 consistent,msg = self.system.check_consistent(exit=False,message=True) 497 if not consistent: 498 locdir = os.path.join(nexus_core.local_directory,nexus_core.runs,self.path) 499 self.error('user provided physical system is not internally consistent\nsimulation identifier: {0}\nlocal directory: {1}\nmore details on the user error are given below\n\n{2}'.format(self.identifier,locdir,msg)) 500 #end if 501 elif self.system!=None: 502 self.error('system must be a PhysicalSystem object\nyou provided an object of type: {0}'.format(self.system.__class__.__name__)) 503 #end if 504 if self.restartable or self.force_restart: 505 if not cls.supports_restarts: 506 self.warn('restarts are not supported by {0}, request ignored'.format(cls.__name__)) 507 #end if 508 #end if 509 #end def set 510 511 512 def set_directories(self): 513 self.locdir = os.path.join(nexus_core.local_directory,nexus_core.runs,self.path) 514 self.remdir = os.path.join(nexus_core.remote_directory,nexus_core.runs,self.path) 515 self.resdir = os.path.join(nexus_core.local_directory,nexus_core.results,nexus_core.runs,self.path) 516 517 if not self.fake(): 518 #print ' creating sim {0} in {1}'.format(self.simid,self.locdir) 519 520 if not self.locdir in self.sim_directories: 521 self.sim_directories[self.locdir] = set([self.identifier]) 522 else: 523 idset = self.sim_directories[self.locdir] 524 if not self.identifier in idset: 525 idset.add(self.identifier) 526 else: 527 self.error('multiple simulations in a single directory have the same identifier\nplease assign unique identifiers to each simulation\nsimulation directory: {0}\nrepeated identifier: {1}\nother identifiers: {2}\nbetween the directory shown and the identifiers listed, it should be clear which simulations are involved\nmost likely, you described two simulations with identifier {3}'.format(self.locdir,self.identifier,sorted(idset),self.identifier)) 528 #end if 529 #end if 530 #end if 531 532 self.image_dir = self.image_dir+'_'+self.identifier 533 self.imlocdir = os.path.join(self.locdir,self.image_dir) 534 self.imremdir = os.path.join(self.remdir,self.image_dir) 535 self.imresdir = os.path.join(self.resdir,self.image_dir) 536 #end def set_directories 537 538 539 def set_files(self): 540 if self.infile is None: 541 self.infile = self.identifier + self.infile_extension 542 #end if 543 if self.outfile is None: 544 self.outfile = self.identifier + self.outfile_extension 545 #end if 546 if self.errfile is None: 547 self.errfile = self.identifier + self.errfile_extension 548 #end if 549 #end def set_files 550 551 552 def reset_indicators(self): 553 #this is needed to support restarts 554 self.got_dependencies = False 555 self.setup = False 556 self.sent_files = False 557 self.submitted = False 558 self.finished = False 559 self.failed = False 560 self.got_output = False 561 self.analyzed = False 562 #end def reset_indicators 563 564 565 def completed(self): 566 completed = self.setup 567 completed &= self.sent_files 568 completed &= self.submitted 569 completed &= self.finished 570 completed &= self.got_output 571 completed &= self.analyzed 572 completed &= not self.failed 573 return completed 574 #end def completed 575 576 577 def active(self): 578 deps_completed = True 579 for dep in self.dependencies: 580 deps_completed &= dep.sim.completed() 581 #end for 582 active = deps_completed and not self.completed() 583 return active 584 #end def active 585 586 587 def ready(self): 588 ready = self.active() 589 ready &= not self.submitted 590 ready &= not self.finished 591 ready &= not self.got_output 592 ready &= not self.analyzed 593 ready &= not self.failed 594 return ready 595 #end def ready 596 597 598 def check_result(self,result_name,sim): 599 self.not_implemented() 600 #end def check_result 601 602 def get_result(self,result_name,sim): 603 self.not_implemented() 604 #end def get_result 605 606 def incorporate_result(self,result_name,result,sim): 607 self.not_implemented() 608 #end def incorporate_result 609 610 def app_command(self): 611 self.not_implemented() 612 #end def app_command 613 614 def check_sim_status(self): 615 self.not_implemented() 616 #end def check_sim_status 617 618 def get_output_files(self): # returns list of output files to save 619 self.not_implemented() 620 #end def get_output_files 621 622 623 def propagate_identifier(self): 624 None 625 #end def propagate_identifier 626 627 def pre_init(self): 628 None 629 #end def pre_init 630 631 def post_init(self): 632 None 633 #end def post_init 634 635 def pre_create_directories(self): 636 None 637 #end def pre_create_directories 638 639 def write_prep(self): 640 None 641 #end def write_prep 642 643 def pre_write_inputs(self,save_image): 644 None 645 #end def pre_write_inputs 646 647 def pre_send_files(self,enter): 648 None 649 #end def pre_send_files 650 651 def post_submit(self): 652 None 653 #end def post_submit 654 655 def pre_check_status(self): 656 None 657 #end def pre_check_status 658 659 def post_analyze(self,analyzer): 660 None 661 #end def post_analyze 662 663 664 def condense_name(self,name): 665 return name.strip().lower().replace('-','_').replace(' ','_') 666 #end def condense_name 667 668 669 def has_generic_input(self): 670 return isinstance(self.input,GenericSimulationInput) 671 #end def has_generic_input 672 673 def outfile_text(self): 674 return self._file_text('outfile') 675 #end def outfile_text 676 677 def errfile_text(self): 678 return self._file_text('errfile') 679 #end def errfile_text 680 681 def _file_text(self,filename): 682 filepath = os.path.join(self.locdir,self[filename]) 683 fobj = open(filepath,'r') 684 text = fobj.read() 685 fobj.close() 686 return text 687 #end def _file_text 688 689 690 def _create_dir(self,dir): 691 if not os.path.exists(dir): 692 os.makedirs(dir) 693 elif os.path.isfile(dir): 694 self.error('cannot create directory {0}\na file exists at this location'.format(dir)) 695 #end if 696 #end def _create_dir 697 698 def create_directories(self): 699 self.pre_create_directories() 700 self._create_dir(self.locdir) 701 self._create_dir(self.imlocdir) 702 self.created_directories = True 703 #end def create_directories 704 705 706 def depends(self,*dependencies): 707 if len(dependencies)==0: 708 return 709 #end if 710 if isinstance(dependencies[0],Simulation): 711 dependencies = [dependencies] 712 #end if 713 for d in dependencies: 714 sim = d[0] 715 if not isinstance(sim,Simulation): 716 self.error('first element in a dependency tuple must be a Simulation object\nyou provided a '+sim.__class__.__name__) 717 #end if 718 dep = obj() 719 dep.sim = sim 720 rn = [] 721 unrecognized_names = False 722 app_results = sim.application_results | set(['other']) 723 for name in d[1:]: 724 result_name = self.condense_name(name) 725 if result_name in app_results: 726 rn.append(result_name) 727 else: 728 unrecognized_names = True 729 self.error(name+' is not known to be a result of '+sim.__class__.__name__,exit=False) 730 #end if 731 #end for 732 if unrecognized_names: 733 self.error('unrecognized dependencies specified for simulation '+self.identifier) 734 #end if 735 dep.result_names = rn 736 dep.results = obj() 737 if not sim.simid in self.dependencies: 738 self.ordered_dependencies.append(dep) 739 self.dependencies[sim.simid]=dep 740 sim.dependents[self.simid]=self 741 self.dependency_ids.add(sim.simid) 742 self.wait_ids.add(sim.simid) 743 else: 744 self.dependencies[sim.simid].result_names.extend(dep.result_names) 745 #end if 746 #end for 747 #end def depends 748 749 750 def undo_depends(self,sim): 751 i=0 752 for dep in self.ordered_dependencies: 753 if dep.sim.simid==sim.simid: 754 break 755 #end if 756 i+=1 757 #end for 758 self.ordered_dependencies.pop(i) 759 del self.dependencies[sim.simid] 760 del sim.dependents[self.simid] 761 self.dependency_ids.remove(sim.simid) 762 if sim.simid in self.wait_ids: 763 self.wait_ids.remove(sim.simid) 764 #end if 765 #end def undo_depends 766 767 768 # remove? 769 def acquire_dependents(self,sim): 770 # acquire the dependents from the other simulation 771 dsims = obj(sim.dependents) 772 for dsim in dsims: 773 dep = dsim.dependencies[sim.simid] 774 dsim.depends(self,*dep.result_names) 775 #end for 776 # eliminate the other simulation 777 # this renders it void (fake) and removes all dependency relationships 778 sim.eliminate() 779 #end def acquire_dependents 780 781 782 # remove? 783 def eliminate(self): 784 # reverse relationship of dependents (downstream) 785 dsims = obj(self.dependents) 786 for dsim in dsims: 787 dsim.undo_depends(self) 788 #end for 789 # reverse relationship of dependencies (upstream) 790 deps = obj(self.dependencies) 791 for dep in deps: 792 self.undo_depends(dep.sim) 793 #end for 794 # mark sim to be ignored in all future interactions 795 self.fake_sim = True 796 #end def eliminate 797 798 799 def check_dependencies(self,result): 800 dep_satisfied = result.dependencies_satisfied 801 for dep in self.dependencies: 802 sim = dep.sim 803 for result_name in dep.result_names: 804 if result_name!='other': 805 if sim.has_generic_input(): 806 calculating_result = False 807 cls = self.__class__ 808 self.warn('a simulation result cannot be inferred from generic formatted or template input\nplease use {0} instead of {1}\nsee error below for information identifying this simulation instance'.format(cls.input_type.__class__.__name__,sim.input.__class__.__name__)) 809 else: 810 calculating_result = sim.check_result(result_name,self) 811 #end if 812 if not calculating_result: 813 self.error('simulation {0} id {1} is not calculating result {2}\nrequired by simulation {3} id {4}\n{5} {6} directory: {7}\n{8} {9} directory: {10}'.format(sim.identifier,sim.simid,result_name,self.identifier,self.simid,sim.identifier,sim.simid,sim.locdir,self.identifier,self.simid,self.locdir),exit=False) 814 #end if 815 else: 816 calculating_result = True 817 #end if 818 dep_satisfied = dep_satisfied and calculating_result 819 #end for 820 #end for 821 result.dependencies_satisfied = dep_satisfied 822 #end def check_dependencies 823 824 825 def get_dependencies(self): 826 if nexus_core.generate_only or self.finished: 827 for dep in self.dependencies: 828 for result_name in dep.result_names: 829 dep.results[result_name] = result_name 830 #end for 831 #end for 832 else: 833 for dep in self.dependencies: 834 sim = dep.sim 835 for result_name in dep.result_names: 836 if result_name!='other': 837 if sim.has_generic_input(): 838 self.error('a simulation result cannot be inferred from generic formatted or template input\nplease use {0} instead of {1}\nsim id: {2}\ndirectory: {3}\nresult: {4}'.format(cls.input_type.__class__.__name__,sim.input.__class__.__name__,sim.id,sim.locdir,result_name)) 839 #end if 840 dep.results[result_name] = sim.get_result(result_name,sim) 841 else: 842 dep.results['other'] = obj() 843 #end if 844 #end for 845 #end for 846 if not self.got_dependencies: 847 for dep in self.ordered_dependencies: 848 sim = dep.sim 849 for result_name,result in dep.results.items(): 850 if result_name!='other': 851 if self.has_generic_input(): 852 self.error('a simulation result cannot be incorporated into generic formatted or template input\nplease use {0} instead of {1}\nsim id: {2}\ndirectory: {3}\nresult: {4}'.format(cls.input_type.__class__.__name__,self.input.__class__.__name__,self.id,self.locdir,result_name)) 853 #end if 854 self.incorporate_result(result_name,result,sim) 855 #end if 856 #end for 857 #end for 858 #end if 859 #end if 860 if self.renew_app_command: 861 self.job.renew_app_command(self) 862 #end if 863 self.got_dependencies = True 864 #end def get_dependencies 865 866 867 def downstream_simids(self,simids=None): 868 if simids is None: 869 simids = set() 870 #end if 871 for sim in self.dependents: 872 simids.add(sim.simid) 873 sim.downstream_simids(simids) 874 #end for 875 return simids 876 #end def downstream_simids 877 878 879 def copy_file(self,sourcefile,dest): 880 src = os.path.dirname(os.path.abspath(sourcefile)) 881 dst = os.path.abspath(dest) 882 if src!=dst: 883 shutil.copy2(sourcefile,dest) 884 #end if 885 #end def copy_file 886 887 888 def save_image(self,all=False): 889 imagefile = os.path.join(self.imlocdir,self.sim_image) 890 if os.path.exists(imagefile): 891 os.system('rm '+imagefile) 892 #end if 893 if not all: 894 sim_image = SimulationImage() 895 sim_image.save_image(self,imagefile) 896 else: 897 self.error('attempting to save full object!') 898 self.save(imagefile) 899 #end if 900 #end def save_image 901 902 903 def load_image(self,imagepath=None,all=False): 904 if imagepath==None: 905 imagepath=os.path.join(self.imlocdir,self.sim_image) 906 #end if 907 if not all: 908 sim_image = SimulationImage() 909 sim_image.load_image(self,imagepath) 910 else: 911 self.load(imagepath) 912 #end if 913 # update process id for backwards compatibility 914 if 'process_id' not in self: 915 self.process_id = self.job.system_id 916 #end if 917 #end def load_image 918 919 920 def load_analyzer_image(self,imagepath=None): 921 if imagepath==None: 922 imagepath = os.path.join(self.imresdir,self.analyzer_image) 923 #end if 924 analyzer = self.analyzer_type(self) 925 analyzer.load(imagepath) 926 return analyzer 927 #end def load_analyzer_image 928 929 930 def attempt_files(self): 931 return (self.infile,self.outfile,self.errfile) 932 #end def attempt_files 933 934 935 def save_attempt(self): 936 local = self.attempt_files() 937 filepaths = [] 938 for file in local: 939 filepath = os.path.join(self.locdir,file) 940 if os.path.exists(filepath): 941 filepaths.append(filepath) 942 #end if 943 #end for 944 if len(filepaths)>0: 945 prefix = self.identifier+'_attempt' 946 n=0 947 for dir in os.listdir(self.locdir): 948 if dir.startswith(prefix): 949 n=max(n,int(dir.replace(prefix,''))) 950 #end if 951 #end for 952 n+=1 953 attempt_dir = os.path.join(self.locdir,prefix+str(n)) 954 os.makedirs(attempt_dir) 955 for filepath in filepaths: 956 os.system('mv {0} {1}'.format(filepath,attempt_dir)) 957 #end for 958 #end if 959 #end def save_attempt 960 961 962 def idstr(self): 963 return ' '+str(self.simid)+' '+str(self.identifier) 964 #end def idstr 965 966 967 def write_inputs(self,save_image=True): 968 self.pre_write_inputs(save_image) 969 self.enter(self.locdir,False,self.simid) 970 self.log('writing input files'+self.idstr(),n=3) 971 self.write_prep() 972 if self.infile is not None: 973 infile = os.path.join(self.locdir,self.infile) 974 self.input.write(infile) 975 #end if 976 self.job.write(file=True) 977 self.setup = True 978 if save_image: 979 self.save_image() 980 self.input.save(os.path.join(self.imlocdir,self.input_image)) 981 #end if 982 #try to also write structure information 983 if self.system is not None: 984 filebase = os.path.join(self.locdir,self.identifier+'.struct') 985 try: 986 self.system.structure.write(filebase+'.xyz') 987 except: 988 None 989 #end try 990 try: 991 if self.system.structure.has_axes(): 992 self.system.structure.write(filebase+'.xsf') 993 #end if 994 except: 995 None 996 #end try 997 #end if 998 #end def write_inputs 999 1000 1001 def send_files(self,enter=True): 1002 self.pre_send_files(enter) 1003 if enter: 1004 self.enter(self.locdir,False,self.simid) 1005 #end if 1006 self.log('sending required files'+self.idstr(),n=3) 1007 if not os.path.exists(self.remdir): 1008 os.makedirs(self.remdir) 1009 #end if 1010 if not os.path.exists(self.imremdir): 1011 os.makedirs(self.imremdir) 1012 #end if 1013 if self.infile is not None: 1014 self.files.add(self.infile) 1015 #end if 1016 send_files = self.files 1017 file_locations = [self.locdir]+nexus_core.file_locations 1018 remote = self.remdir 1019 for file in send_files: 1020 found_file = False 1021 for location in file_locations: 1022 local = os.path.join(location,file) 1023 found_file = os.path.exists(local) 1024 if found_file: 1025 break 1026 #end if 1027 #end if 1028 if found_file: 1029 self.copy_file(local,remote) 1030 else: 1031 self.error('file {0} not found\nlocations checked: {1}'.format(file,file_locations)) 1032 #end if 1033 #end for 1034 self.sent_files = True 1035 self.save_image() 1036 send_imfiles=[self.sim_image,self.input_image] 1037 remote = self.imremdir 1038 for imfile in send_imfiles: 1039 local = os.path.join(self.imlocdir,imfile) 1040 if os.path.exists(local): 1041 self.copy_file(local,remote) 1042 #end if 1043 #end for 1044 #end def send_files 1045 1046 1047 def submit(self): 1048 if not self.submitted: 1049 if self.skip_submit and not self.bundled: 1050 self.block_dependents(block_self=True) 1051 return 1052 #end if 1053 self.log('submitting job'+self.idstr(),n=3) 1054 if not self.skip_submit: 1055 if not self.job.local: 1056 self.job.submit() 1057 else: 1058 self.execute() # execute local job immediately 1059 #end if 1060 #end if 1061 self.submitted = True 1062 if (self.job.batch_mode or not nexus_core.monitor) and not nexus_core.generate_only: 1063 self.save_image() 1064 #end if 1065 elif not self.finished: 1066 self.check_status() 1067 #end if 1068 self.post_submit() 1069 #end def submit 1070 1071 1072 def update_process_id(self): 1073 if self.process_id is None and self.job.system_id is not None: 1074 self.process_id = self.job.system_id 1075 self.save_image() 1076 #end if 1077 #end def update_process_id 1078 1079 1080 def check_status(self): 1081 self.pre_check_status() 1082 if nexus_core.generate_only: 1083 self.finished = self.job.finished 1084 elif self.job.finished: 1085 should_check = True 1086 if self.outfile is not None: 1087 outfile = os.path.join(self.locdir,self.outfile) 1088 should_check &= os.path.exists(outfile) 1089 #end if 1090 if self.errfile is not None: 1091 errfile = os.path.join(self.locdir,self.errfile) 1092 should_check &= os.path.exists(errfile) 1093 #end if 1094 if not self.finished and should_check: 1095 self.check_sim_status() 1096 #end if 1097 if self.failed: 1098 self.finished = True 1099 #end if 1100 #end if 1101 if self.finished: 1102 self.save_image() 1103 #end if 1104 #end def check_status 1105 1106 1107 def get_output(self): 1108 if not os.path.exists(self.resdir): 1109 os.makedirs(self.resdir) 1110 #end if 1111 if not os.path.exists(self.imresdir): 1112 os.makedirs(self.imresdir) 1113 #end if 1114 images = [self.sim_image,self.input_image] 1115 for image in images: 1116 remote_image = os.path.join(self.imremdir,image) 1117 if os.path.exists(remote_image): 1118 self.copy_file(remote_image,self.imresdir) 1119 #end if 1120 #end for 1121 results_image = os.path.join(self.imresdir,self.sim_image) 1122 if os.path.exists(results_image): 1123 self.load_image(results_image) 1124 #end if 1125 if self.finished: 1126 self.enter(self.locdir,False,self.simid) 1127 self.log('copying results'+self.idstr(),n=3) 1128 if not nexus_core.generate_only: 1129 output_files = self.get_output_files() 1130 if self.infile is not None: 1131 output_files.append(self.infile) 1132 #end if 1133 if self.outfile is not None: 1134 output_files.append(self.outfile) 1135 #end if 1136 if self.errfile is not None: 1137 output_files.append(self.errfile) 1138 #end if 1139 files_missing = [] 1140 for file in output_files: 1141 remfile = os.path.join(self.remdir,file) 1142 if os.path.exists(remfile): 1143 self.copy_file(remfile,self.resdir) 1144 else: 1145 files_missing.append(file) 1146 #end if 1147 #end for 1148 if len(files_missing)>0: 1149 self.log('warning: the following files were missing',n=4) 1150 for file in files_missing: 1151 self.log(file,n=5) 1152 #end for 1153 #end if 1154 #end if 1155 self.got_output = True 1156 self.save_image() 1157 #end if 1158 #end def get_output 1159 1160 1161 def analyze(self): 1162 if not os.path.exists(self.imresdir): 1163 os.makedirs(self.imresdir) 1164 #end if 1165 if self.finished: 1166 self.enter(self.locdir,False,self.simid) 1167 self.log('analyzing'+self.idstr(),n=3) 1168 if not nexus_core.generate_only: 1169 analyzer = self.analyzer_type(self) 1170 analyzer.analyze() 1171 self.post_analyze(analyzer) 1172 analyzer.save(os.path.join(self.imresdir,self.analyzer_image)) 1173 del analyzer 1174 #end if 1175 self.analyzed = True 1176 self.save_image() 1177 #end if 1178 #end def analyze 1179 1180 1181 def reset_wait_ids(self): 1182 self.wait_ids = set(self.dependency_ids) 1183 for sim in self.dependents: 1184 sim.reset_wait_ids() 1185 #end for 1186 #end def reset_wait_ids 1187 1188 1189 def check_subcascade(self): 1190 finished = self.finished or self.block 1191 if not self.block and not self.block_subcascade and not self.failed: 1192 for sim in self.dependents: 1193 finished &= sim.check_subcascade() 1194 #end for 1195 #end if 1196 self.subcascade_finished = finished 1197 return finished 1198 #end def check_subcascade 1199 1200 1201 def block_dependents(self,block_self=True): 1202 if block_self: 1203 self.block = True 1204 #end if 1205 self.block_subcascade = True 1206 for sim in self.dependents: 1207 sim.block_dependents() 1208 #end for 1209 #end def block_dependents 1210 1211 1212 def progress(self,dependency_id=None): 1213 if dependency_id is not None: 1214 self.wait_ids.remove(dependency_id) 1215 #end if 1216 if len(self.wait_ids)==0 and not self.block and not self.failed: 1217 modes = nexus_core.modes 1218 mode = nexus_core.mode 1219 progress = True 1220 if mode==modes.none: 1221 return 1222 elif mode==modes.setup: 1223 self.write_inputs() 1224 elif mode==modes.send_files: 1225 self.send_files() 1226 elif mode==modes.submit: 1227 self.submit() 1228 progress = self.finished 1229 elif mode==modes.get_output: 1230 self.get_output() 1231 progress = self.finished 1232 elif mode==modes.analyze: 1233 self.analyze() 1234 progress = self.finished 1235 elif mode==modes.stages: 1236 if not self.created_directories: 1237 self.create_directories() 1238 #end if 1239 if not self.got_dependencies: 1240 self.get_dependencies() 1241 #end if 1242 if not self.setup and 'setup' in nexus_core.stages: 1243 self.write_inputs() 1244 #end if 1245 if not self.sent_files and 'send_files' in nexus_core.stages: 1246 self.send_files() 1247 #end if 1248 if not self.finished and 'submit' in nexus_core.stages: 1249 self.submit() 1250 #end if 1251 if nexus_core.dependent_modes <= nexus_core.stages_set: 1252 progress_post = self.finished 1253 progress = self.finished and self.analyzed 1254 else: 1255 progress_post = progress 1256 #end if 1257 if progress_post: 1258 if not self.got_output and 'get_output' in nexus_core.stages: 1259 self.get_output() 1260 #end if 1261 if not self.analyzed and 'analyze' in nexus_core.stages: 1262 self.analyze() 1263 #end if 1264 #end if 1265 elif mode==modes.all: 1266 if not self.setup: 1267 self.write_inputs() 1268 self.send_files(False) 1269 #end if 1270 if not self.finished: 1271 self.submit() 1272 #end if 1273 if self.finished: 1274 if not self.got_output: 1275 self.get_output() 1276 #end if 1277 if not self.analyzed: 1278 self.analyze() 1279 #end if 1280 #end if 1281 progress = self.finished 1282 #end if 1283 if progress and not self.block_subcascade and not self.failed: 1284 for sim in self.dependents: 1285 if not sim.bundled: 1286 sim.progress(self.simid) 1287 #end if 1288 #end for 1289 #end if 1290 elif len(self.wait_ids)==0 and self.force_write: 1291 modes = nexus_core.modes 1292 mode = nexus_core.mode 1293 if mode==modes.stages: 1294 if not self.got_dependencies: 1295 self.get_dependencies() 1296 #end if 1297 if 'setup' in nexus_core.stages: 1298 self.write_inputs() 1299 #end if 1300 if not self.sent_files and 'send_files' in nexus_core.stages: 1301 self.send_files() 1302 #end if 1303 #end if 1304 #end if 1305 #end def progress 1306 1307 1308 def reconstruct_cascade(self): 1309 imagefile = os.path.join(self.imlocdir,self.sim_image) 1310 if os.path.exists(imagefile) and not self.loaded: 1311 self.load_image() 1312 # continue from interruption 1313 if self.submitted and not self.finished and self.process_id is not None: 1314 self.job.system_id = self.process_id # load process id of job 1315 self.job.reenter_queue() 1316 #end if 1317 self.loaded = True 1318 #end if 1319 for sim in self.dependents: 1320 sim.reconstruct_cascade() 1321 #end for 1322 return self 1323 #end def reconstruct_cascade 1324 1325 1326 def traverse_cascade(self,operation,*args,**kwargs): 1327 if 'dependency_id' in kwargs: 1328 self.wait_ids.remove(kwargs['dependency_id']) 1329 del kwargs['dependency_id'] 1330 #end if 1331 if len(self.wait_ids)==0: 1332 operation(self,*args,**kwargs) 1333 for sim in self.dependents: 1334 kwargs['dependency_id'] = self.simid 1335 sim.traverse_cascade(operation,*args,**kwargs) 1336 #end for 1337 #end if 1338 #end def traverse_cascade 1339 1340 1341 # used only in tests 1342 def traverse_full_cascade(self,operation,*args,**kwargs): 1343 operation(self,*args,**kwargs) 1344 for sim in self.dependents: 1345 sim.traverse_full_cascade(operation,*args,**kwargs) 1346 #end for 1347 #end def traverse_full_cascade 1348 1349 1350 def write_dependents(self,n=0,location=False,block_status=False): 1351 outs = [self.__class__.__name__,self.identifier,self.simid] 1352 if location: 1353 outs.append(self.locdir) 1354 #end if 1355 if block_status: 1356 if self.block: 1357 outs.append('blocked') 1358 else: 1359 outs.append('unblocked') 1360 #end if 1361 #end if 1362 outs.append(list(self.dependency_ids)) 1363 self.log(*outs,n=n) 1364 n+=1 1365 for sim in self.dependents: 1366 sim.write_dependents(n=n,location=location,block_status=block_status) 1367 #end for 1368 #end def write_dependents 1369 1370 1371 def execute(self,run_command=None): 1372 pad = self.enter(self.locdir) 1373 if run_command is None: 1374 job = self.job 1375 command = 'export OMP_NUM_THREADS='+str(job.threads)+'\n' 1376 if len(job.presub)>0: 1377 command += job.presub+'\n' 1378 #end if 1379 machine = job.get_machine() 1380 command += job.run_command(machine.app_launcher) 1381 if len(job.postsub)>0: 1382 command += job.postsub+'\n' 1383 #end if 1384 command = ('\n'+command).replace('\n','\n '+pad) 1385 run_command = command 1386 #end if 1387 if self.job is None: 1388 env = os.environ.copy() 1389 else: 1390 env = job.env 1391 #end if 1392 if nexus_core.generate_only: 1393 self.log(pad+'Would have executed: '+command) 1394 else: 1395 self.log(pad+'Executing: '+command) 1396 fout = open(self.outfile,'w') 1397 ferr = open(self.errfile,'w') 1398 out,err = Popen(command,env=env,stdout=fout,stderr=ferr,shell=True,close_fds=True).communicate() 1399 #end if 1400 self.leave() 1401 self.submitted = True 1402 if self.job is not None: 1403 job.status = job.states.finished 1404 self.job.finished = True 1405 #end if 1406 #end def execute 1407 1408 1409 def show_input(self,exit=True): 1410 print() 1411 print(80*'=') 1412 print('Input file for simulation "{}"\nDirectory: {}'.format(self.identifier,self.locdir)) 1413 print(80*'-') 1414 print(self.input.write()) 1415 print(80*'=') 1416 if exit: 1417 exit_call() 1418 #end if 1419 #end def show_input 1420#end class Simulation 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432class NullSimulationInput(SimulationInput): 1433 def is_valid(self): 1434 return True 1435 #end def is_valid 1436 1437 def read(self,filepath): 1438 None 1439 #end def read 1440 1441 def write(self,filepath=None): 1442 None 1443 #end def write 1444 1445 def read_text(self,text,filepath=None): 1446 None 1447 #end def read_text 1448 1449 def write_text(self,filepath=None): 1450 None 1451 #end def write_text 1452 1453 def incorporate_system(self,system): 1454 None 1455 #end def incorporate_system 1456 1457 def return_system(self): 1458 self.not_implemented() 1459 #end def return_system 1460#end class NullSimulationInput 1461 1462 1463 1464 1465class NullSimulationAnalyzer(SimulationAnalyzer): 1466 def __init__(self,sim): 1467 None 1468 #end def __init__ 1469 1470 def analyze(self): 1471 None 1472 #end def analyze 1473#end class NullSimulationAnalyzer 1474 1475 1476class GenericSimulationInput: # marker class for generic user input 1477 None 1478#end class GenericSimulationInput 1479 1480 1481class GenericSimulation(Simulation): 1482 def __init__(self,**kwargs): 1483 self.input_type = NullSimulationInput 1484 self.analyzer_type = NullSimulationAnalyzer 1485 if 'input_type' in kwargs: 1486 self.input_type = kwargs['input_type'] 1487 del kwargs['input_type'] 1488 #end if 1489 if 'analyzer_type' in kwargs: 1490 self.analyzer_type = kwargs['analyzer_type'] 1491 del kwargs['analyzer_type'] 1492 #end if 1493 if 'input' in kwargs: 1494 self.input_type = kwargs['input'].__class__ 1495 #end if 1496 if 'analyzer' in kwargs: 1497 self.analyzer_type = kwargs['analyzer'].__class__ 1498 #end if 1499 Simulation.__init__(self,**kwargs) 1500 #end def __init__ 1501 1502 def check_sim_status(self): 1503 self.finished = True 1504 #end def check_sim_status 1505 1506 def get_output_files(self): 1507 return [] 1508 #end def get_output_files 1509#end class GenericSimulation 1510 1511 1512 1513from string import Template 1514class SimulationInputTemplateDev(SimulationInput): 1515 def __init__(self,filepath=None,text=None): 1516 self.reset() 1517 if filepath is not None: 1518 self.read(filepath) 1519 elif text is not None: 1520 self.read_text(text) 1521 #end if 1522 #end def __init__ 1523 1524 def reset(self): 1525 self.template = None 1526 self.keywords = set() 1527 self.values = obj() 1528 self.allow_not_set = set() 1529 #end def reset 1530 1531 def clear(self): 1532 self.values.clear() 1533 #end def clear 1534 1535 def allow_no_assign(self,*keys): 1536 for k in keys: 1537 self.allow_not_set.add(k) 1538 #end for 1539 #end def allow_no_assign 1540 1541 def assign(self,**values): 1542 if self.template is None: 1543 self.error('cannot assign values prior to reading template') 1544 #end if 1545 invalid = set(values.keys()) - self.keywords - self.allow_not_set 1546 if len(invalid)>0: 1547 self.error('attempted to assign invalid keywords\ninvalid keywords: {0}\nvalid options are: {1}'.format(sorted(invalid),sorted(self.keywords))) 1548 #end if 1549 self.values.set(**values) 1550 #end def assign 1551 1552 def read_text(self,text,filepath=None): 1553 text = self.preprocess(text,filepath) # for derived class intervention 1554 try: 1555 template = Template(text) 1556 key_tuples = Template.pattern.findall(text) 1557 except Exception as e: 1558 self.error('exception encountered during read\nfile: {0}\nexception: {1}'.format(filepath,e)) 1559 #end try 1560 for ktup in key_tuples: 1561 if len(ktup[1])>0: # normal keyword, e.g. $key 1562 self.keywords.add(ktup[1]) 1563 elif len(ktup[2])>0: # braced keyword, e.g. ${key} 1564 self.keywords.add(ktup[2]) 1565 #end if 1566 #end for 1567 self.template = template 1568 #end def read_text 1569 1570 def write_text(self,filepath=None): 1571 kw_rem = self.keywords-set(self.values.keys()) 1572 if len(kw_rem)>0: 1573 self.error('not all keywords for this template have been assigned\nkeywords remaining: {0}'.format(sorted(kw_rem))) 1574 #end if 1575 try: 1576 text = self.template.substitute(**self.values) 1577 except Exception as e: 1578 self.error('exception encountered during write:\n'+str(e)) 1579 #end try 1580 return text 1581 #end def write_text 1582 1583 def preprocess(self,text,filepath): 1584 return text # derived classes can modify text prior to template creation 1585 #end def preprocess 1586#end class SimulationInputTemplateDev 1587 1588 1589 1590 1591class SimulationInputMultiTemplateDev(SimulationInput): 1592 def __init__(self,**file_templates): 1593 self.filenames = obj() 1594 if len(file_templates)>0: 1595 self.set_templates(**file_templates) 1596 #end if 1597 #end def __init__ 1598 1599 1600 def set_templates(self,**file_templates): 1601 for name,val in file_templates.items(): 1602 if isinstance(val,str): 1603 if ' ' in val: 1604 self.error('filename cannot have any spaces\nbad filename provided with keyword '+name) 1605 #end if 1606 self.filenames[name] = val 1607 elif isinstance(val,tuple) and len(val)==2: 1608 filename,template_path = val 1609 self[name] = SimulationInputTemplate(template_path) 1610 self.filenames[name] = filename 1611 else: 1612 self.error('keyword inputs must either be all filenames or all filename/filepath pairs') 1613 #end if 1614 #end for 1615 #end def set_templates 1616 1617 1618 def read(self,filepath): 1619 if len(self.filenames)==0: 1620 self.error('cannot perform read, filenames are not set') 1621 #end if 1622 base,filename = os.path.split(filepath) 1623 filenames = self.filenames 1624 self.clear() 1625 self.filenames = filenames 1626 templates = dict() 1627 for name,filename in filenames.items(): 1628 templates[name] = filename, os.path.join(base,filename) 1629 #end for 1630 self.set_templates(**templates) 1631 #end def read 1632 1633 1634 def write(self,filepath=None): 1635 if filepath is None: 1636 contents = obj() 1637 for name in self.filenames.keys(): 1638 contents[name] = self[name].write() 1639 #end for 1640 return contents 1641 else: 1642 base,filename = os.path.split(filepath) 1643 for name,filename in self.filenames.items(): 1644 self[name].write(os.path.join(base,filename)) 1645 #end for 1646 #end if 1647 #end def write 1648#end class SimulationInputMultiTemplateDev 1649 1650 1651 1652# these are for user access, *Dev are for development 1653class SimulationInputTemplate(SimulationInputTemplateDev,GenericSimulationInput): 1654 None 1655#end class SimulationInputTemplate 1656 1657class SimulationInputMultiTemplate(SimulationInputMultiTemplateDev,GenericSimulationInput): 1658 None 1659#end class SimulationInputMultiTemplate 1660 1661 1662 1663# developer functions 1664 1665def input_template_dev(*args,**kwargs): 1666 return SimulationInputTemplateDev(*args,**kwargs) 1667#end def input_template_dev 1668 1669 1670def multi_input_template_dev(*args,**kwargs): 1671 return SimulationInputMultiTemplateDev(*args,**kwargs) 1672#end def multi_input_template_dev 1673 1674 1675 1676 1677 1678 1679# user functions 1680 1681def input_template(*args,**kwargs): 1682 return SimulationInputTemplate(*args,**kwargs) 1683#end def input_template 1684 1685 1686def multi_input_template(*args,**kwargs): 1687 return SimulationInputMultiTemplate(*args,**kwargs) 1688#end def multi_input_template 1689 1690 1691def generate_template_input(*args,**kwargs): 1692 return SimulationInputTemplate(*args,**kwargs) 1693#end def generate_template_input 1694 1695 1696def generate_multi_template_input(*args,**kwargs): 1697 return SimulationInputMultiTemplate(*args,**kwargs) 1698#end def generate_multi_template_input 1699 1700 1701def generate_simulation(**kwargs): 1702 sim_type='generic' 1703 if 'sim_type' in kwargs: 1704 sim_type = kwargs['sim_type'] 1705 del kwargs['sim_type'] 1706 #end if 1707 if sim_type=='generic': 1708 return GenericSimulation(**kwargs) 1709 else: 1710 Simulation.class_error('sim_type {0} is unrecognized'.format(sim_type),'generate_simulation') 1711 #end if 1712#end def generate_simulation 1713 1714 1715 1716 1717# ability to graph simulation workflows 1718try: 1719 from pydot import Dot,Node,Edge 1720except: 1721 Dot,Node,Edge = unavailable('pydot','Dot','Node','Edge') 1722#end try 1723try: 1724 from matplotlib.image import imread 1725 from matplotlib.pyplot import imshow,show,xticks,yticks 1726except: 1727 imread = unavailable('matplotlib.image','imread') 1728 imshow,show,xticks,yticks = unavailable('matplotlib.pyplot','imshow','show','xticks','yticks') 1729#end try 1730import tempfile 1731exit_call = sys.exit 1732def graph_sims(sims=None,savefile=None,useid=False,exit=True,quants=True,display=True): 1733 if sims is None: 1734 sims = Simulation.all_sims 1735 #end if 1736 graph = Dot(graph_type='digraph',dpi=300) 1737 graph.set_label('simulation workflows') 1738 graph.set_labelloc('t') 1739 nodes = obj() 1740 for sim in sims: 1741 if 'fake_sim' in sim and sim.fake_sim: 1742 continue 1743 #end if 1744 if sim.simlabel is not None and not useid: 1745 nlabel = sim.simlabel+' '+str(sim.simid) 1746 else: 1747 nlabel = sim.identifier+' '+str(sim.simid) 1748 #end if 1749 nopts = obj() 1750 if 'block' in sim and sim.block: 1751 nopts.color = 'black' 1752 nopts.fontcolor = 'white' 1753 #end if 1754 node = obj( 1755 id = sim.simid, 1756 sim = sim, 1757 node = Node(nlabel,style='filled',shape='Mrecord',**nopts), 1758 edges = obj(), 1759 ) 1760 nodes[node.id] = node 1761 graph.add_node(node.node) 1762 #end for 1763 for node in nodes: 1764 for simid,dep in node.sim.dependencies.items(): 1765 other = nodes[simid].node 1766 if quants: 1767 for quantity in dep.result_names: 1768 edge = Edge(other,node.node,label=quantity,fontsize='10.0') 1769 graph.add_edge(edge) 1770 #end for 1771 else: 1772 edge = Edge(other,node.node) 1773 graph.add_edge(edge) 1774 #end if 1775 #end for 1776 #end for 1777 1778 if savefile is None: 1779 fout = tempfile.NamedTemporaryFile(suffix='.png') 1780 savefile = fout.name 1781 #savefile = './sims.png' 1782 #end if 1783 fmt = savefile.rsplit('.',1)[1] 1784 graph.write(savefile,format=fmt,prog='dot') 1785 1786 # display the image 1787 if fmt=='png' and display: 1788 imshow(imread(savefile)) 1789 xticks([]) 1790 yticks([]) 1791 show() 1792 #end if 1793 1794 if exit: 1795 exit_call() 1796 #end if 1797#end def graph_sims 1798 1799 1800