1 2import testing 3from testing import value_eq,object_eq 4from testing import FailedTest,failed 5from testing import divert_nexus_log,restore_nexus_log 6from testing import divert_nexus,restore_nexus 7 8import versions 9 10from generic import obj 11from simulation import Simulation,SimulationInput,SimulationAnalyzer 12 13 14testing.divert_nexus_errors() 15 16 17class SimulationInputForTests(SimulationInput): 18 def __init__(self,*args,**kwargs): 19 SimulationInput.__init__(self,*args,**kwargs) 20 21 self.result_data = obj() 22 #end def __init__ 23 24 def is_valid(self): 25 return True 26 #end def is_valid 27 28 def read(self,filepath): 29 None 30 #end def read 31 32 def write(self,filepath=None): 33 None 34 #end def write 35 36 def read_text(self,text,filepath=None): 37 None 38 #end def read_text 39 40 def write_text(self,filepath=None): 41 None 42 #end def write_text 43 44 def incorporate_system(self,system): 45 None 46 #end def incorporate_system 47 48 def return_system(self): 49 self.not_implemented() 50 #end def return_system 51#end class SimulationInputForTests 52 53 54class SimulationAnalyzerForTests(SimulationAnalyzer): 55 def __init__(self,sim): 56 self.analysis_performed = False 57 #end def __init__ 58 59 def analyze(self): 60 self.analysis_performed = True 61 #end def analyze 62#end class SimulationAnalyzerForTests 63 64 65class SimulationForTests(Simulation): 66 input_type = SimulationInputForTests 67 analyzer_type = SimulationAnalyzerForTests 68 69 application_results = set(['quant1','quant2','quant3']) 70 71 def check_sim_status(self): 72 self.finished = True 73 #end def check_sim_status 74 75 def get_output_files(self): 76 return [] 77 #end def get_output_files 78 79 def check_result(self,result_name,sim): 80 return result_name in self.application_results 81 #end def check_result 82 83 def get_result(self,result_name,sim): 84 result = obj() 85 result.name = result_name 86 result.simid = sim.simid 87 return result 88 #end def get_result 89 90 def incorporate_result(self,result_name,result,sim): 91 self.input.result_data[result.simid] = result.name 92 #end def incorporate_result 93#end class SimulationForTests 94 95 96 97 98get_sim_simulations = [] 99 100def get_sim(**kwargs): 101 from machines import job 102 from simulation import Simulation 103 104 test_job = job(machine='ws1',app_command='test.x') 105 106 n = len(get_sim_simulations) 107 108 sim = Simulation(identifier='sim'+str(n),job=test_job,**kwargs) 109 110 get_sim_simulations.append(sim) 111 112 return sim 113#end def get_sim 114 115 116get_test_sim_simulations = [] 117 118def get_test_sim(**kwargs): 119 from machines import job 120 121 test_job = kwargs.pop('job',None) 122 if test_job is None: 123 test_job = job(machine='ws128',serial=True,app_command='echo run') 124 #end if 125 126 n = len(get_test_sim_simulations) 127 128 test_sim = SimulationForTests( 129 identifier = 'test_sim'+str(n), 130 job = test_job, 131 **kwargs 132 ) 133 134 get_test_sim_simulations.append(test_sim) 135 136 return test_sim 137#end def get_test_sim 138 139 140 141n_test_workflows = 9 142 143 144def generate_network(): 145 from numpy.random import randint 146 147 nsims = 10 148 nconnections = 3 149 nheads = 3 150 151 sims = [] 152 153 for n in range(randint(nheads)+1): 154 sims.append([]) 155 #end for 156 157 for isim in range(nsims): 158 deps = [] 159 for idep in range(randint(nconnections)+1): 160 if len(sims)>0: 161 i = randint(len(sims)) 162 deps.append(i) 163 #end if 164 #end for 165 sims.append(list(sorted(set(deps)))) 166 #end for 167 168 sims_dict = {} 169 for i,d in enumerate(sims): 170 sims_dict[i] = d 171 #end for 172 173 return sims_dict 174#end def generate_network 175 176 177def get_test_workflow(index,**kwargs): 178 from generic import obj 179 180 def make_network(network,**kwargs): 181 sims = obj() 182 for i in range(len(network)): 183 s = get_test_sim(**kwargs) 184 for j in network[i]: 185 s.depends((sims['s'+str(j)],'other')) 186 #end for 187 sims['s'+str(i)] = s 188 #end for 189 return sims 190 #end def make_network 191 192 nsims_bef = len(get_test_sim_simulations) 193 194 sims = obj() 195 196 if index==0: 197 # single simulation 198 sims.s = get_test_sim(**kwargs) 199 elif index==1: 200 # linear simulation chain 201 sims.s1 = get_test_sim(**kwargs) 202 sims.s2 = get_test_sim(dependencies=(sims.s1,'other'),**kwargs) 203 sims.s3 = get_test_sim(dependencies=(sims.s2,'other'),**kwargs) 204 elif index==2: 205 # linear chain with split 206 sims.s1 = get_test_sim(**kwargs) 207 sims.s2 = get_test_sim(dependencies=(sims.s1,'other'),**kwargs) 208 sims.s3 = get_test_sim(dependencies=(sims.s2,'other'),**kwargs) 209 sims.s41 = get_test_sim(dependencies=(sims.s3,'other'),**kwargs) 210 sims.s51 = get_test_sim(dependencies=(sims.s41,'other'),**kwargs) 211 sims.s42 = get_test_sim(dependencies=(sims.s3,'other'),**kwargs) 212 sims.s52 = get_test_sim(dependencies=(sims.s42,'other'),**kwargs) 213 elif index==3: 214 # linear chains with join 215 sims.s11 = get_test_sim(**kwargs) 216 sims.s21 = get_test_sim(dependencies=(sims.s11,'other'),**kwargs) 217 sims.s12 = get_test_sim(**kwargs) 218 sims.s22 = get_test_sim(dependencies=(sims.s12,'other'),**kwargs) 219 sims.s3 = get_test_sim(dependencies=[(sims.s21,'other'),(sims.s22,'other')]) 220 sims.s4 = get_test_sim(dependencies=(sims.s3,'other'),**kwargs) 221 sims.s5 = get_test_sim(dependencies=(sims.s4,'other'),**kwargs) 222 elif index==4: 223 # net-like workflow 224 sims.s11 = get_test_sim(**kwargs) 225 sims.s12 = get_test_sim(**kwargs) 226 sims.s13 = get_test_sim(**kwargs) 227 228 sims.s21 = get_test_sim( 229 dependencies = [ 230 (sims.s11,'other'), 231 (sims.s12,'other'), 232 ], 233 **kwargs 234 ) 235 sims.s22 = get_test_sim( 236 dependencies = [ 237 (sims.s12,'other'), 238 (sims.s13,'other'), 239 ], 240 **kwargs 241 ) 242 243 sims.s31 = get_test_sim( 244 dependencies = [ 245 (sims.s21,'other'), 246 (sims.s22,'other'), 247 ], 248 **kwargs 249 ) 250 sims.s32 = get_test_sim( 251 dependencies = [ 252 (sims.s21,'other'), 253 (sims.s22,'other'), 254 ], 255 **kwargs 256 ) 257 sims.s33 = get_test_sim( 258 dependencies = [ 259 (sims.s21,'other'), 260 (sims.s22,'other'), 261 ], 262 **kwargs 263 ) 264 265 sims.s41 = get_test_sim( 266 dependencies = [ 267 (sims.s11,'other'), 268 (sims.s22,'other'), 269 (sims.s32,'other'), 270 ], 271 **kwargs 272 ) 273 elif index==5: 274 # random network 1 275 network = { 276 0 : [], 277 1 : [], 278 2 : [1], 279 3 : [2], 280 4 : [1, 2], 281 5 : [0, 4], 282 6 : [4], 283 7 : [1], 284 8 : [7], 285 9 : [5], 286 10 : [0, 3, 9], 287 11 : [4, 5, 7], 288 } 289 sims = make_network(network,**kwargs) 290 elif index==6: 291 # random network 2 292 network = { 293 0 : [], 294 1 : [0], 295 2 : [0], 296 3 : [1, 2], 297 4 : [1, 2, 3], 298 5 : [0, 1, 2], 299 6 : [1, 3], 300 7 : [2], 301 8 : [1, 3, 7], 302 9 : [1, 8], 303 10 : [2, 4], 304 } 305 sims = make_network(network,**kwargs) 306 elif index==7: 307 # random network 3 308 network = { 309 0 : [], 310 1 : [], 311 2 : [], 312 3 : [0, 1, 2], 313 4 : [2, 3], 314 5 : [1], 315 6 : [3, 5], 316 7 : [0], 317 8 : [3], 318 9 : [6], 319 10 : [0], 320 11 : [10], 321 12 : [8, 10, 11], 322 } 323 sims = make_network(network,**kwargs) 324 elif index==8: 325 # larger random network 326 network = { 327 0 : [], 328 1 : [], 329 2 : [], 330 3 : [0], 331 4 : [2], 332 5 : [0, 2], 333 6 : [2, 5], 334 7 : [1], 335 8 : [1, 5, 7], 336 9 : [0, 8], 337 10 : [5], 338 11 : [0, 10], 339 12 : [1], 340 13 : [6, 9, 11], 341 14 : [2, 4], 342 15 : [8, 9, 10, 12], 343 16 : [3], 344 17 : [7], 345 18 : [6, 13], 346 19 : [7, 11, 14], 347 20 : [8, 16, 17], 348 21 : [0, 8], 349 22 : [14], 350 23 : [2], 351 24 : [16], 352 25 : [13, 16, 22, 24], 353 26 : [6, 10], 354 27 : [12, 17, 24], 355 28 : [8], 356 29 : [10, 12, 23, 26], 357 30 : [1], 358 31 : [28], 359 32 : [20], 360 } 361 sims = make_network(network,**kwargs) 362 else: 363 failed('index exceeds available workflows') 364 #end if 365 366 nsims_aft = len(get_test_sim_simulations) 367 368 assert(len(sims)==nsims_aft-nsims_bef) 369 370 return sims 371#end def get_test_workflow 372 373 374 375 376def test_import(): 377 import simulation 378 from simulation import Simulation,SimulationInput,SimulationAnalyzer 379 from simulation import SimulationImage 380 from simulation import NullSimulationInput,NullSimulationAnalyzer 381 from simulation import GenericSimulation 382 from simulation import SimulationInputTemplate 383 from simulation import SimulationInputMultiTemplate 384 from simulation import input_template,multi_input_template 385 from simulation import generate_simulation 386#end def test_import 387 388 389 390def test_simulation_input(): 391 import os 392 from generic import NexusError 393 from simulation import SimulationInput 394 395 tpath = testing.setup_unit_test_output_directory('simulation','test_simulation_input') 396 397 # empty init 398 si = SimulationInput() 399 400 # write 401 infile = os.path.join(tpath,'sim_input.in') 402 wtext = 'simulation input' 403 si.write_file_text(infile,wtext) 404 assert(os.path.exists(infile)) 405 406 # read 407 rtext = si.read_file_text(infile) 408 assert(rtext==wtext) 409 410 # virtuals 411 virts = [ 412 si.is_valid, 413 si.return_structure, 414 (si.read_text,[None]), 415 si.write_text, 416 (si.incorporate_system,[None]), 417 si.return_system, 418 ] 419 for v in virts: 420 args = [] 421 if isinstance(v,tuple): 422 v,args = v 423 #end if 424 try: 425 v(*args) 426 raise FailedTest 427 except NexusError: 428 None 429 except FailedTest: 430 failed(str(v)) 431 except Exception as e: 432 failed(str(e)) 433 #end try 434 #end for 435#end def test_simulation_input 436 437 438 439def test_simulation_analyzer(): 440 import os 441 from generic import NexusError 442 from simulation import SimulationAnalyzer 443 444 # empty init 445 try: 446 SimulationAnalyzer() 447 raise FailedTest 448 except FailedTest: 449 failed() 450 except: 451 None 452 #end try 453 454 # virtuals 455 try: 456 SimulationAnalyzer(None) 457 raise FailedTest 458 except NexusError: 459 None 460 except FailedTest: 461 failed() 462 except Exception as e: 463 failed(str(e)) 464 #end try 465#end def test_simulation_analyzer 466 467 468 469def test_simulation_input_template(): 470 import os 471 from string import Template 472 from generic import obj,NexusError 473 from simulation import SimulationInput 474 from simulation import GenericSimulationInput 475 from simulation import SimulationInputTemplate 476 from simulation import input_template 477 478 tpath = testing.setup_unit_test_output_directory('simulation','test_simulation_input_template') 479 480 481 # empty init 482 si_empty = input_template() 483 assert(isinstance(si_empty,SimulationInput)) 484 assert(isinstance(si_empty,GenericSimulationInput)) 485 assert(isinstance(si_empty,SimulationInputTemplate)) 486 487 si_empty_ref = obj( 488 template = None, 489 keywords = set(), 490 values = obj(), 491 allow_not_set = set(), 492 ) 493 494 assert(len(si_empty)==4) 495 assert(object_eq(si_empty.to_obj(),si_empty_ref)) 496 497 498 # template reference data 499 template_text = ''' 500a = "$a" 501b = $b 502file1 = "$file.$ext1" 503file2 = "$file.$ext2" 504''' 505 506 template_filepath = os.path.join(tpath,'template_file.txt') 507 508 open(template_filepath,'w').write(template_text) 509 510 511 # read 512 si_read = input_template(template_filepath) 513 514 assert(isinstance(si_read.template,Template)) 515 assert(si_read.keywords==set(['a','b','ext1','ext2','file'])) 516 517 518 # assign 519 si_assign = input_template() 520 try: 521 si_assign.assign(b=1) 522 raise FailedTest 523 except NexusError: 524 None 525 except FailedTest: 526 failed() 527 except Exception as e: 528 failed(str(e)) 529 #end try 530 531 si_assign = input_template(template_filepath) 532 try: 533 si_assign.assign(c=1) 534 raise FailedTest 535 except NexusError: 536 None 537 except FailedTest: 538 failed() 539 except Exception as e: 540 failed(str(e)) 541 #end try 542 543 values = obj( 544 a = 'name', 545 b = 1, 546 file = 'my_file', 547 ext1 = 'txt', 548 ext2 = 'dat', 549 ) 550 551 si_assign.assign(**values) 552 553 assert(object_eq(si_assign.values,values)) 554 555 556 # write 557 def try_write(si): 558 try: 559 si.write() 560 raise FailedTest 561 except NexusError: 562 None 563 except FailedTest: 564 failed() 565 except Exception as e: 566 failed(str(e)) 567 #end try 568 #end def try_write 569 570 si_write = input_template() 571 try_write(si_write) 572 573 si_write = input_template(template_filepath) 574 try_write(si_write) 575 576 si_write.assign(b=1) 577 try_write(si_write) 578 579 580 text_ref = ''' 581a = "name" 582b = 1 583file1 = "my_file.txt" 584file2 = "my_file.dat" 585''' 586 587 si_write.assign(**values) 588 text = si_write.write() 589 assert(text==text_ref) 590 591 input_filepath = os.path.join(tpath,'input_file.txt') 592 si_write.write(input_filepath) 593 assert(open(input_filepath,'r').read()==text_ref) 594 595#end def test_simulation_input_template 596 597 598 599def test_simulation_input_multi_template(): 600 import os 601 from string import Template 602 from generic import obj,NexusError 603 from simulation import SimulationInput 604 from simulation import GenericSimulationInput 605 from simulation import SimulationInputMultiTemplate 606 from simulation import multi_input_template 607 608 tpath = testing.setup_unit_test_output_directory('simulation','test_simulation_input_multi_template') 609 610 # make template files 611 template1_filepath = os.path.join(tpath,'template1.txt') 612 template2_filepath = os.path.join(tpath,'template2.txt') 613 template3_filepath = os.path.join(tpath,'template3.txt') 614 615 open(template1_filepath,'w').write(''' 616name = "$name" 617a = $a 618''') 619 open(template2_filepath,'w').write(''' 620name = "$name" 621b = $b 622''') 623 open(template3_filepath,'w').write(''' 624name = "$name" 625c = $c 626''') 627 628 input1_filepath = os.path.join(tpath,'input_file1.txt') 629 input2_filepath = os.path.join(tpath,'input_file2.txt') 630 input3_filepath = os.path.join(tpath,'input_file3.txt') 631 632 633 # empty init 634 si_empty = multi_input_template() 635 636 assert(isinstance(si_empty,SimulationInput)) 637 assert(isinstance(si_empty,GenericSimulationInput)) 638 assert(isinstance(si_empty,SimulationInputMultiTemplate)) 639 640 si_empty_ref = obj( 641 filenames = obj(), 642 ) 643 644 assert(len(si_empty)==1) 645 assert(object_eq(si_empty.to_obj(),si_empty_ref)) 646 647 648 # filename init 649 filenames = obj( 650 input1 = 'input_file1.txt', 651 input2 = 'input_file2.txt', 652 input3 = 'input_file3.txt', 653 ) 654 655 si = multi_input_template(**filenames) 656 657 assert(len(si)==1) 658 assert(len(si.filenames)==3) 659 assert(object_eq(si.filenames,filenames)) 660 661 662 # init read 663 si_init = multi_input_template( 664 input1 = ('input_file1.txt',template1_filepath), 665 input2 = ('input_file2.txt',template2_filepath), 666 input3 = ('input_file3.txt',template3_filepath), 667 ) 668 si = si_init 669 assert(len(si)==4) 670 assert(len(si.filenames)==3) 671 assert(object_eq(si.filenames,filenames)) 672 keywords_ref = dict( 673 input1 = set(['a', 'name']), 674 input2 = set(['b', 'name']), 675 input3 = set(['c', 'name']), 676 ) 677 for name,keyword_set in keywords_ref.items(): 678 assert(name in si) 679 sit = si[name] 680 assert(sit.keywords==keyword_set) 681 assert(isinstance(sit.template,Template)) 682 assert(object_eq(sit.values,obj())) 683 assert(sit.allow_not_set==set()) 684 #end for 685 686 687 # write 688 write_ref = obj( 689 input1 = ''' 690name = "name1" 691a = 1 692''', 693 input2 = ''' 694name = "name2" 695b = 2 696''', 697 input3 = ''' 698name = "name3" 699c = 3 700''', 701 ) 702 703 si_write = multi_input_template( 704 input1 = ('input_file1.txt',template1_filepath), 705 input2 = ('input_file2.txt',template2_filepath), 706 input3 = ('input_file3.txt',template3_filepath), 707 ) 708 si_write.input1.assign( 709 name = 'name1', 710 a = 1, 711 ) 712 si_write.input2.assign( 713 name = 'name2', 714 b = 2, 715 ) 716 si_write.input3.assign( 717 name = 'name3', 718 c = 3, 719 ) 720 assert(object_eq(si_write.write(),write_ref)) 721 722 si_write.write(input1_filepath) 723 assert(os.path.exists(input1_filepath)) 724 assert(os.path.exists(input2_filepath)) 725 assert(os.path.exists(input3_filepath)) 726 727 728 # read 729 si_read = multi_input_template(**filenames) 730 731 si_read.read(input1_filepath) 732 733 si = si_read 734 assert(len(si)==4) 735 assert(len(si.filenames)==3) 736 assert(object_eq(si.filenames,filenames)) 737 for name,keyword_set in keywords_ref.items(): 738 assert(name in si) 739 sit = si[name] 740 assert(sit.keywords==set()) 741 assert(isinstance(sit.template,Template)) 742 assert(object_eq(sit.values,obj())) 743 assert(sit.allow_not_set==set()) 744 #end for 745 assert(object_eq(si_read.write(),write_ref)) 746 747#end def test_simulation_input_multi_template 748 749 750 751def test_code_name(): 752 from simulation import Simulation 753 754 cn = Simulation.code_name() 755 assert(isinstance(cn,str)) 756 assert(' ' not in cn) 757#end def test_code_name 758 759 760 761def test_init(): 762 from generic import obj 763 from machines import job,Job 764 from simulation import Simulation,SimulationInput 765 766 # empty init, tests set(), set_directories(), set_files() 767 se = Simulation() 768 769 se_ref = obj( 770 analyzed = False, 771 analyzer_image = 'analyzer.p', 772 app_name = 'simapp', 773 app_props = ['serial'], 774 block = False, 775 block_subcascade = False, 776 bundleable = True, 777 bundled = False, 778 bundler = None, 779 created_directories = False, 780 dependency_ids = set([]), 781 errfile = 'sim.err', 782 failed = False, 783 fake_sim = False, 784 files = set([]), 785 finished = False, 786 force_restart = False, 787 force_write = False, 788 got_dependencies = False, 789 got_output = False, 790 identifier = 'sim', 791 image_dir = 'sim_sim', 792 imlocdir = './runs/sim_sim', 793 imremdir = './runs/sim_sim', 794 imresdir = './runs/sim_sim', 795 infile = 'sim.in', 796 input_image = 'input.p', 797 locdir = './runs/', 798 job = None, 799 loaded = False, 800 ordered_dependencies = [], 801 outfile = 'sim.out', 802 outputs = None, 803 path = '', 804 process_id = None, 805 restartable = False, 806 remdir = './runs/', 807 resdir = './runs/', 808 sent_files = False, 809 setup = False, 810 sim_image = 'sim.p', 811 simlabel = None, 812 skip_submit = False, 813 subcascade_finished = False, 814 submitted = False, 815 system = None, 816 wait_ids = set([]), 817 dependencies = obj(), 818 dependents = obj(), 819 input = SimulationInput(), 820 ) 821 822 assert(object_eq(se.obj(list(se_ref.keys())),se_ref)) 823 assert(isinstance(se.simid,int)) 824 assert(se.simid>=0) 825 assert(se.simid<Simulation.sim_count) 826 827 Simulation.clear_all_sims() 828 assert(len(Simulation.all_sims)==0) 829 assert(len(Simulation.sim_directories)==0) 830 assert(Simulation.sim_count==0) 831 832 833 # make a test job 834 test_job = job(machine='ws1',app_command='test.x') 835 836 837 # minimal non-empty init, tests init_job() 838 sm = Simulation(job=test_job) 839 840 sm_ref = se_ref.copy() 841 del sm_ref.job 842 assert(object_eq(sm.obj(list(sm_ref.keys())),sm_ref)) 843 assert(isinstance(se.simid,int)) 844 assert(se.simid>=0) 845 assert(se.simid<Simulation.sim_count) 846 assert(isinstance(sm.job,Job)) 847 assert(id(sm.job)!=id(test_job)) 848 849 850 # initialization tests for set_directories() 851 # two sims in same directory w/ same identifier should fail 852 try: 853 s1 = Simulation( 854 identifier = 'same_identifier', 855 path = 'same_directory', 856 job = test_job, 857 ) 858 s2 = Simulation( 859 identifier = 'same_identifier', 860 path = 'same_directory', 861 job = test_job, 862 ) 863 raise FailedTest 864 except FailedTest: 865 failed() 866 except: 867 None 868 #end try 869 870 # two sims in same directory w/ different identifiers should be ok 871 s1 = Simulation( 872 identifier = 'identifier1', 873 path = 'same_directory', 874 job = test_job, 875 ) 876 s2 = Simulation( 877 identifier = 'identifier2', 878 path = 'same_directory', 879 job = test_job, 880 ) 881 882 # two sims in different directories w/ same identifier should be ok 883 s1 = Simulation( 884 identifier = 'same_identifier', 885 path = 'directory1', 886 job = test_job, 887 ) 888 s2 = Simulation( 889 identifier = 'same_identifier', 890 path = 'directory2', 891 job = test_job, 892 ) 893 894 Simulation.clear_all_sims() 895 896#end def test_init 897 898 899 900def test_virtuals(): 901 from generic import NexusError 902 from simulation import Simulation 903 904 s = Simulation() 905 906 virts = [ 907 (s.check_result,[None,None]), 908 (s.get_result,[None,None]), 909 (s.incorporate_result,[None,None,None]), 910 s.app_command, 911 s.check_sim_status, 912 s.get_output_files, 913 ] 914 for v in virts: 915 args = [] 916 if isinstance(v,tuple): 917 v,args = v 918 #end if 919 try: 920 v(*args) 921 raise FailedTest 922 except NexusError: 923 None 924 except FailedTest: 925 failed(str(v)) 926 except Exception as e: 927 failed(str(e)) 928 #end try 929 #end for 930 931 vacuous_virts = [ 932 s.propagate_identifier, 933 s.pre_init, 934 s.post_init, 935 s.pre_create_directories, 936 s.write_prep, 937 (s.pre_write_inputs,[None]), 938 (s.pre_send_files,[None]), 939 s.post_submit, 940 s.pre_check_status, 941 (s.post_analyze,[None]), 942 ] 943 for v in vacuous_virts: 944 args = [] 945 if isinstance(v,tuple): 946 v,args = v 947 #end if 948 v(*args) 949 #end for 950 951 Simulation.clear_all_sims() 952 953#end def test_virtuals 954 955 956 957def test_reset_indicators(): 958 from simulation import Simulation 959 960 indicators = ''' 961 got_dependencies 962 setup 963 sent_files 964 submitted 965 finished 966 failed 967 got_output 968 analyzed 969 '''.split() 970 971 s = Simulation() 972 973 for i in indicators: 974 s[i] = True 975 #end for 976 977 s.reset_indicators() 978 979 for i in indicators: 980 ind = s[i] 981 assert(isinstance(ind,bool)) 982 assert(not ind) 983 #end for 984 985 Simulation.clear_all_sims() 986#end def test_reset_indicators 987 988 989 990def test_indicator_checks(): 991 from machines import job 992 from simulation import Simulation 993 994 def complete(sim): 995 sim.setup = True 996 sim.sent_files = True 997 sim.submitted = True 998 sim.finished = True 999 sim.got_output = True 1000 sim.analyzed = True 1001 sim.failed = False 1002 #end def complete 1003 1004 # test completed() 1005 s = Simulation() 1006 assert(not s.completed()) 1007 complete(s) 1008 assert(s.completed()) 1009 s.reset_indicators() 1010 Simulation.clear_all_sims() 1011 1012 # test ready() and active() 1013 test_job = job(machine='ws1',app_command='test.x') 1014 1015 simdeps = [] 1016 for i in range(5): 1017 s = Simulation(identifier='id'+str(i),job=test_job) 1018 complete(s) 1019 simdeps.append((s,'other')) 1020 #end for 1021 1022 s = Simulation(job=test_job,dependencies=simdeps) 1023 assert(s.ready()) 1024 assert(s.active()) 1025 s.submitted = True 1026 assert(not s.ready()) 1027 assert(s.active()) 1028 1029 Simulation.clear_all_sims() 1030 1031#end def test_indicator_checks 1032 1033 1034 1035def test_create_directories(): 1036 import os 1037 from simulation import Simulation 1038 1039 tpath = testing.setup_unit_test_output_directory('simulation','test_create_directories',divert=True) 1040 1041 s = Simulation() 1042 1043 assert(not os.path.exists(s.locdir)) 1044 assert(not os.path.exists(s.imlocdir)) 1045 assert(not s.created_directories) 1046 1047 s.create_directories() 1048 1049 assert(os.path.exists(s.locdir)) 1050 assert(os.path.exists(s.imlocdir)) 1051 assert(s.created_directories) 1052 1053 restore_nexus() 1054 1055 Simulation.clear_all_sims() 1056#end def test_create_directories 1057 1058 1059 1060def test_file_text(): 1061 import os 1062 from simulation import Simulation 1063 1064 tpath = testing.setup_unit_test_output_directory('simulation','test_create_directories',divert=True) 1065 1066 s = Simulation() 1067 s.create_directories() 1068 1069 outfile = os.path.join(s.locdir,s.outfile) 1070 errfile = os.path.join(s.locdir,s.errfile) 1071 1072 out_text = 'output' 1073 err_text = 'error' 1074 1075 open(outfile,'w').write(out_text) 1076 open(errfile,'w').write(err_text) 1077 1078 assert(s.outfile_text()==out_text) 1079 assert(s.errfile_text()==err_text) 1080 1081 restore_nexus() 1082 1083 Simulation.clear_all_sims() 1084#end def test_file_text 1085 1086 1087 1088def check_dependency_objects(*sims,**kwargs): 1089 from generic import obj 1090 from simulation import Simulation 1091 empty = kwargs.get('empty',False) 1092 wait_ids = kwargs.get('wait_ids',True) 1093 if len(sims)==1 and isinstance(sims[0],list): 1094 sims = sims[0] 1095 #end if 1096 for sim in sims: 1097 if empty: 1098 assert(value_eq(sim.ordered_dependencies,[])) 1099 assert(isinstance(sim.dependencies,obj)) 1100 assert(len(sim.dependencies)==0) 1101 assert(sim.dependency_ids==set()) 1102 assert(sim.wait_ids==set()) 1103 else: 1104 # check dependencies object 1105 for simid,dep in sim.dependencies.items(): 1106 assert(isinstance(simid,int)) 1107 assert(isinstance(dep,obj)) 1108 assert('result_names' in dep) 1109 assert('results' in dep) 1110 assert('sim' in dep) 1111 assert(len(dep)==3) 1112 assert(isinstance(dep.sim,Simulation)) 1113 assert(simid==dep.sim.simid) 1114 assert(isinstance(dep.result_names,list)) 1115 for name in dep.result_names: 1116 assert(isinstance(name,str)) 1117 #end for 1118 assert(isinstance(dep.results,obj)) 1119 assert(len(dep.results)==0) 1120 #end for 1121 # check ordered_dependencies object 1122 for dep in sim.ordered_dependencies: 1123 dep2 = sim.dependencies[dep.sim.simid] 1124 assert(id(dep2)==id(dep)) 1125 #end for 1126 # check dependents object 1127 for dsimid,dsim in sim.dependents.items(): 1128 assert(isinstance(dsimid,int)) 1129 assert(isinstance(dsim,Simulation)) 1130 assert(dsimid==dsim.simid) 1131 assert(sim.simid in dsim.dependency_ids) 1132 assert(sim.simid in dsim.dependencies) 1133 assert(id(sim)==id(dsim.dependencies[sim.simid].sim)) 1134 found = False 1135 for s in dsim.ordered_dependencies: 1136 found |= id(s.sim)==id(sim) 1137 #end for 1138 assert(found) 1139 #end for 1140 # check dependency_ids 1141 for simid in sim.dependency_ids: 1142 assert(isinstance(simid,int)) 1143 assert(simid in sim.dependencies) 1144 #end for 1145 # check wait_ids 1146 if wait_ids: 1147 assert(sim.wait_ids==sim.dependency_ids) 1148 #end if 1149 #end if 1150 #end if 1151#end def check_dependency_objects 1152 1153 1154 1155def check_dependency(sim2,sim1,quants=['other'],only=False,objects=False): 1156 # sim2 depends on sim1 for all quantities 1157 if objects: 1158 check_dependency_objects(sim1) 1159 check_dependency_objects(sim2) 1160 #end if 1161 assert(sim2.simid in sim1.dependents) 1162 assert(id(sim1.dependents[sim2.simid])==id(sim2)) 1163 assert(sim1.simid in sim2.dependency_ids) 1164 assert(sim1.simid in sim2.dependencies) 1165 assert(id(sim2.dependencies[sim1.simid].sim)==id(sim1)) 1166 assert(set(sim2.dependencies[sim1.simid].result_names)==set(quants)) 1167 if only: 1168 assert(len(sim1.dependents)==1) 1169 assert(len(sim2.dependency_ids)==1) 1170 assert(len(sim2.dependencies)==1) 1171 assert(len(sim2.ordered_dependencies)==1) 1172 #end if 1173#end def check_dependency 1174 1175 1176def test_depends(): 1177 from generic import NexusError 1178 from simulation import Simulation 1179 1180 # single dependency, single quantity 1181 s1 = get_sim() 1182 s2 = get_sim() 1183 1184 check_dependency_objects(s1,empty=True) 1185 check_dependency_objects(s2,empty=True) 1186 1187 s2.depends(s1,'other') 1188 1189 check_dependency(s2,s1,objects=True,only=True) 1190 del s1,s2 1191 1192 s1 = get_sim() 1193 s2 = get_sim() 1194 s2.depends((s1,'other')) 1195 check_dependency(s2,s1,objects=True,only=True) 1196 del s1,s2 1197 1198 s1 = get_sim() 1199 s2 = get_sim( 1200 dependencies = (s1,'other'), 1201 ) 1202 check_dependency(s2,s1,objects=True,only=True) 1203 del s1,s2 1204 1205 s1 = get_sim() 1206 s2 = get_sim( 1207 dependencies = [(s1,'other')], 1208 ) 1209 check_dependency(s2,s1,objects=True,only=True) 1210 del s1,s2 1211 1212 1213 # single dependency, multiple quantities 1214 s1 = get_test_sim() 1215 s2 = get_test_sim() 1216 1217 quants = ['quant1','quant2','quant3'] 1218 1219 check_dependency_objects(s1,empty=True) 1220 check_dependency_objects(s2,empty=True) 1221 1222 s2.depends(s1,'quant1','quant2','quant3') 1223 1224 check_dependency(s2,s1,quants,objects=True,only=True) 1225 del s1,s2 1226 1227 s1 = get_test_sim() 1228 s2 = get_test_sim() 1229 s2.depends(s1,'quant1') 1230 s2.depends(s1,'quant2') 1231 s2.depends(s1,'quant3') 1232 check_dependency(s2,s1,quants,objects=True,only=True) 1233 del s1,s2 1234 1235 s1 = get_test_sim() 1236 s2 = get_test_sim() 1237 s2.depends(s1,'quant1','quant2') 1238 s2.depends(s1,'quant3') 1239 check_dependency(s2,s1,quants,objects=True,only=True) 1240 del s1,s2 1241 1242 s1 = get_test_sim() 1243 s2 = get_test_sim() 1244 s2.depends((s1,'quant1','quant2','quant3')) 1245 check_dependency(s2,s1,quants,objects=True,only=True) 1246 del s1,s2 1247 1248 s1 = get_test_sim() 1249 s2 = get_test_sim() 1250 s2.depends((s1,'quant1')) 1251 s2.depends((s1,'quant2')) 1252 s2.depends((s1,'quant3')) 1253 check_dependency(s2,s1,quants,objects=True,only=True) 1254 del s1,s2 1255 1256 s1 = get_test_sim() 1257 s2 = get_test_sim( 1258 dependencies = (s1,'quant1','quant2','quant3'), 1259 ) 1260 check_dependency(s2,s1,quants,objects=True,only=True) 1261 del s1,s2 1262 1263 s1 = get_test_sim() 1264 s2 = get_test_sim( 1265 dependencies = [(s1,'quant1','quant2','quant3')], 1266 ) 1267 check_dependency(s2,s1,quants,objects=True,only=True) 1268 del s1,s2 1269 1270 s1 = get_test_sim() 1271 s2 = get_test_sim( 1272 dependencies = [ 1273 (s1,'quant1'), 1274 (s1,'quant2'), 1275 (s1,'quant3'), 1276 ], 1277 ) 1278 check_dependency(s2,s1,quants,objects=True,only=True) 1279 del s1,s2 1280 1281 1282 # multiple dependencies 1283 s11 = get_test_sim() 1284 s12 = get_test_sim() 1285 s13 = get_test_sim() 1286 1287 s21 = get_test_sim( 1288 dependencies = [ 1289 (s11,'quant1'), 1290 (s12,'quant2'), 1291 ] 1292 ) 1293 s22 = get_test_sim( 1294 dependencies = [ 1295 (s12,'quant2'), 1296 (s13,'quant3'), 1297 ] 1298 ) 1299 1300 s31 = get_test_sim( 1301 dependencies = [ 1302 (s21,'quant1'), 1303 (s22,'quant2'), 1304 ] 1305 ) 1306 s32 = get_test_sim( 1307 dependencies = [ 1308 (s21,'quant1'), 1309 (s22,'quant2'), 1310 ] 1311 ) 1312 s33 = get_test_sim( 1313 dependencies = [ 1314 (s21,'quant1'), 1315 (s22,'quant2'), 1316 ] 1317 ) 1318 1319 s41 = get_test_sim( 1320 dependencies = [ 1321 (s11,'quant1'), 1322 (s22,'quant2'), 1323 (s32,'quant3'), 1324 ] 1325 ) 1326 1327 check_dependency_objects(s11,s12,s13,s21,s22,s31,s32,s33,s41) 1328 1329 check_dependency(s21,s11,['quant1']) 1330 check_dependency(s21,s12,['quant2']) 1331 1332 check_dependency(s22,s12,['quant2']) 1333 check_dependency(s22,s13,['quant3']) 1334 1335 check_dependency(s31,s21,['quant1']) 1336 check_dependency(s31,s22,['quant2']) 1337 1338 check_dependency(s32,s21,['quant1']) 1339 check_dependency(s32,s22,['quant2']) 1340 1341 check_dependency(s33,s21,['quant1']) 1342 check_dependency(s33,s22,['quant2']) 1343 1344 check_dependency(s41,s11,['quant1']) 1345 check_dependency(s41,s22,['quant2']) 1346 check_dependency(s41,s32,['quant3']) 1347 1348 del s11,s12,s13,s21,s22,s31,s32,s33,s41 1349 1350 1351 # fail when dependency does not exist 1352 try: 1353 s1 = get_sim() 1354 s2 = get_sim( 1355 dependencies = [(s1,'quant1')], 1356 ) 1357 raise FailedTest 1358 except NexusError: 1359 None 1360 except: 1361 failed() 1362 #end try 1363 1364 try: 1365 s1 = get_sim() 1366 s2 = get_sim( 1367 dependencies = [(s1,'other','quant2')], 1368 ) 1369 raise FailedTest 1370 except NexusError: 1371 None 1372 except: 1373 failed() 1374 #end try 1375 1376 try: 1377 s1 = get_test_sim() 1378 s2 = get_test_sim( 1379 dependencies = [(s1,'quant1','apple')], 1380 ) 1381 raise FailedTest 1382 except NexusError: 1383 None 1384 except: 1385 failed() 1386 #end try 1387 1388 Simulation.clear_all_sims() 1389#end def test_depends 1390 1391 1392 1393def test_undo_depends(): 1394 from simulation import Simulation 1395 1396 # single dependency, single quantity 1397 s1 = get_sim() 1398 s2 = get_sim() 1399 1400 check_dependency_objects(s1,empty=True) 1401 check_dependency_objects(s2,empty=True) 1402 1403 s2.depends(s1,'other') 1404 1405 check_dependency(s2,s1,objects=True,only=True) 1406 1407 s2.undo_depends(s1) 1408 1409 check_dependency_objects(s1,empty=True) 1410 check_dependency_objects(s2,empty=True) 1411 1412 Simulation.clear_all_sims() 1413#end def test_undo_depends 1414 1415 1416 1417def test_has_generic_input(): 1418 from simulation import Simulation 1419 from simulation import SimulationInput,GenericSimulationInput 1420 1421 s = get_sim() 1422 assert(not s.has_generic_input()) 1423 del s 1424 1425 class GenInput(SimulationInput,GenericSimulationInput): 1426 None 1427 #end class GenInput 1428 1429 s = get_sim( 1430 input = GenInput(), 1431 ) 1432 assert(s.has_generic_input()) 1433 del s 1434 1435 Simulation.clear_all_sims() 1436#end def test_has_generic_input 1437 1438 1439 1440def test_check_dependencies(): 1441 from generic import obj,NexusError 1442 from simulation import Simulation 1443 from simulation import SimulationInput,GenericSimulationInput 1444 1445 result = obj() 1446 result.dependencies_satisfied = True 1447 1448 s11 = get_test_sim() 1449 s12 = get_test_sim() 1450 s13 = get_test_sim() 1451 1452 s21 = get_test_sim( 1453 dependencies = [ 1454 (s11,'quant1'), 1455 (s12,'quant2'), 1456 ] 1457 ) 1458 s22 = get_test_sim( 1459 dependencies = [ 1460 (s12,'quant2'), 1461 (s13,'quant3'), 1462 ] 1463 ) 1464 1465 s31 = get_test_sim( 1466 dependencies = [ 1467 (s21,'quant1'), 1468 (s22,'quant2'), 1469 ] 1470 ) 1471 s32 = get_test_sim( 1472 dependencies = [ 1473 (s21,'quant1'), 1474 (s22,'quant2'), 1475 ] 1476 ) 1477 s33 = get_test_sim( 1478 dependencies = [ 1479 (s21,'quant1'), 1480 (s22,'quant2'), 1481 ] 1482 ) 1483 1484 s41 = get_test_sim( 1485 dependencies = [ 1486 (s11,'quant1'), 1487 (s22,'quant2'), 1488 (s32,'quant3'), 1489 ] 1490 ) 1491 1492 sims = [s11,s12,s13,s21,s22,s31,s32,s33,s41] 1493 for s in sims: 1494 s.check_dependencies(result) 1495 #end for 1496 assert(result.dependencies_satisfied) 1497 1498 1499 # non-existent dependency 1500 try: 1501 s = get_test_sim() 1502 s2 = get_test_sim(dependencies=((s,'nonexistent'))) 1503 raise FailedTest 1504 except NexusError: 1505 None 1506 except FailedTest: 1507 failed() 1508 except Exception as e: 1509 failed(str(e)) 1510 #end try 1511 1512 1513 # existent dependency but generic input 1514 divert_nexus_log() 1515 class GenInput(SimulationInput,GenericSimulationInput): 1516 None 1517 #end class GenInput 1518 1519 s = get_test_sim( 1520 input = GenInput(), 1521 ) 1522 1523 s2 = get_test_sim( 1524 input = GenInput(), 1525 dependencies = (s,'quant1') 1526 ) 1527 1528 result = obj(dependencies_satisfied=True) 1529 s.check_dependencies(result) 1530 assert(result.dependencies_satisfied) 1531 1532 try: 1533 s2.check_dependencies(result) 1534 except NexusError: 1535 None 1536 except FailedTest: 1537 failed() 1538 except Exception as e: 1539 failed(str(e)) 1540 #end try 1541 restore_nexus_log() 1542 1543 Simulation.clear_all_sims() 1544#end def test_check_dependencies 1545 1546 1547 1548def test_get_dependencies(): 1549 from generic import obj 1550 from simulation import Simulation 1551 1552 simdeps = obj() 1553 1554 deps = [] 1555 s11 = get_test_sim() 1556 simdeps[s11.simid] = deps 1557 1558 s12 = get_test_sim() 1559 simdeps[s12.simid] = deps 1560 1561 s13 = get_test_sim() 1562 simdeps[s13.simid] = deps 1563 1564 deps = [ 1565 (s11,'quant1'), 1566 (s12,'quant2'), 1567 ] 1568 s21 = get_test_sim(dependencies=deps) 1569 simdeps[s21.simid] = deps 1570 1571 deps = [ 1572 (s12,'quant2'), 1573 (s13,'quant3'), 1574 ] 1575 s22 = get_test_sim(dependencies=deps) 1576 simdeps[s22.simid] = deps 1577 1578 dependencies = [ 1579 (s21,'quant1'), 1580 (s22,'quant2'), 1581 ] 1582 s31 = get_test_sim(dependencies=deps) 1583 simdeps[s31.simid] = deps 1584 1585 deps = [ 1586 (s21,'quant1'), 1587 (s22,'quant2'), 1588 ] 1589 s32 = get_test_sim(dependencies=deps) 1590 simdeps[s32.simid] = deps 1591 1592 deps = [ 1593 (s21,'quant1'), 1594 (s22,'quant2'), 1595 ] 1596 s33 = get_test_sim(dependencies=deps) 1597 simdeps[s33.simid] = deps 1598 1599 deps = [ 1600 (s11,'quant1'), 1601 (s22,'quant2'), 1602 (s32,'quant3'), 1603 ] 1604 s41 = get_test_sim(dependencies=deps) 1605 simdeps[s41.simid] = deps 1606 1607 sims = [s11,s12,s13,s21,s22,s31,s32,s33,s41] 1608 assert(len(simdeps)==len(sims)) 1609 for s in sims: 1610 assert(not s.got_dependencies) 1611 assert(len(s.input.result_data)==0) 1612 s.get_dependencies() 1613 resdata = s.input.result_data 1614 deps = simdeps[s.simid] 1615 for sim,resname in deps: 1616 assert(sim.simid in resdata) 1617 assert(resdata[sim.simid]==resname) 1618 #end for 1619 #end for 1620 1621 Simulation.clear_all_sims() 1622#end def test_get_dependencies 1623 1624 1625 1626def test_downstream_simids(): 1627 from generic import obj 1628 from simulation import Simulation 1629 1630 s11 = get_test_sim() 1631 s12 = get_test_sim() 1632 s13 = get_test_sim() 1633 1634 s21 = get_test_sim( 1635 dependencies = [ 1636 (s11,'quant1'), 1637 (s12,'quant2'), 1638 ] 1639 ) 1640 s22 = get_test_sim( 1641 dependencies = [ 1642 (s12,'quant2'), 1643 (s13,'quant3'), 1644 ] 1645 ) 1646 1647 s31 = get_test_sim( 1648 dependencies = [ 1649 (s21,'quant1'), 1650 (s22,'quant2'), 1651 ] 1652 ) 1653 s32 = get_test_sim( 1654 dependencies = [ 1655 (s21,'quant1'), 1656 (s22,'quant2'), 1657 ] 1658 ) 1659 s33 = get_test_sim( 1660 dependencies = [ 1661 (s21,'quant1'), 1662 (s22,'quant2'), 1663 ] 1664 ) 1665 1666 s41 = get_test_sim( 1667 dependencies = [ 1668 (s11,'quant1'), 1669 (s22,'quant2'), 1670 (s32,'quant3'), 1671 ] 1672 ) 1673 1674 sims = obj( 1675 s11 = s11, 1676 s12 = s12, 1677 s13 = s13, 1678 s21 = s21, 1679 s22 = s22, 1680 s31 = s31, 1681 s32 = s32, 1682 s33 = s33, 1683 s41 = s41, 1684 ) 1685 1686 downstream_sims = obj( 1687 s11 = [s21,s31,s32,s33,s41], 1688 s12 = [s21,s22,s31,s32,s33,s41], 1689 s13 = [s22,s31,s32,s33,s41], 1690 s21 = [s31,s32,s33,s41], 1691 s22 = [s31,s32,s33,s41], 1692 s31 = [], 1693 s32 = [s41], 1694 s33 = [], 1695 s41 = [], 1696 ) 1697 1698 n = 0 1699 for sname in sorted(sims.keys()): 1700 s = sims[sname] 1701 ds_ids = s.downstream_simids() 1702 ds_ids_ref = set([sd.simid for sd in downstream_sims[sname]]) 1703 assert(ds_ids==ds_ids_ref) 1704 n+=1 1705 #end for 1706 assert(n==9) 1707 1708 Simulation.clear_all_sims() 1709#end def test_downstream_simids 1710 1711 1712 1713def test_copy_file(): 1714 import os 1715 from simulation import Simulation 1716 1717 tpath = testing.setup_unit_test_output_directory('simulation','test_copy_file') 1718 1719 opath = os.path.join(tpath,'other') 1720 if not os.path.exists(opath): 1721 os.makedirs(opath) 1722 #end if 1723 1724 file1 = os.path.join(tpath,'file.txt') 1725 file2 = os.path.join(opath,'file.txt') 1726 1727 open(file1,'w').write('text') 1728 assert(os.path.exists(file1)) 1729 1730 s = get_sim() 1731 1732 s.copy_file(file1,opath) 1733 1734 assert(os.path.exists(file2)) 1735 assert(open(file2,'r').read().strip()=='text') 1736 1737 Simulation.clear_all_sims() 1738#end def test_copy_file 1739 1740 1741 1742def test_save_load_image(): 1743 import os 1744 from generic import obj 1745 from simulation import Simulation,SimulationImage 1746 1747 tpath = testing.setup_unit_test_output_directory('simulation','test_save_load_image',divert=True) 1748 1749 nsave = 30 1750 nload = 22 1751 1752 assert(len(SimulationImage.save_fields)==nsave) 1753 assert(len(SimulationImage.load_fields)==nload) 1754 assert(len(SimulationImage.save_only_fields&SimulationImage.load_fields)==0) 1755 1756 sim = get_sim() 1757 1758 sim.create_directories() 1759 1760 sim.save_image() 1761 1762 imagefile = os.path.join(sim.imlocdir,sim.sim_image) 1763 assert(os.path.exists(imagefile)) 1764 1765 image = obj() 1766 image.load(imagefile) 1767 assert(len(image)==nsave) 1768 for field in SimulationImage.save_fields: 1769 assert(field in image) 1770 assert(field in sim) 1771 assert(value_eq(image[field],sim[field])) 1772 #end for 1773 1774 orig = obj() 1775 for field in SimulationImage.load_fields: 1776 orig[field] = sim[field] 1777 del sim[field] 1778 #end for 1779 sim.sim_image = orig.sim_image 1780 sim.load_image() 1781 for field in SimulationImage.load_fields: 1782 assert(field in sim) 1783 assert(value_eq(sim[field],orig[field])) 1784 #end for 1785 1786 restore_nexus() 1787 1788 Simulation.clear_all_sims() 1789#end def test_save_load_image 1790 1791 1792 1793def test_load_analyzer_image(): 1794 import os 1795 from simulation import Simulation 1796 1797 tpath = testing.setup_unit_test_output_directory('simulation','test_save_load_analyzer_image',divert=True) 1798 1799 sim = get_test_sim() 1800 1801 if not os.path.exists(sim.imresdir): 1802 os.makedirs(sim.imresdir) 1803 #end if 1804 1805 analyzer_file = os.path.join(sim.imresdir,sim.analyzer_image) 1806 1807 a = sim.analyzer_type(None) 1808 assert(not a.analysis_performed) 1809 a.analyze() 1810 assert(a.analysis_performed) 1811 a.save(analyzer_file) 1812 assert(os.path.exists(analyzer_file)) 1813 1814 a2 = sim.load_analyzer_image() 1815 assert(isinstance(a2,sim.analyzer_type)) 1816 assert(a2.analysis_performed) 1817 assert(object_eq(a2,a)) 1818 1819 restore_nexus() 1820 1821 Simulation.clear_all_sims() 1822#end def test_load_analyzer_image 1823 1824 1825 1826def test_save_attempt(): 1827 import os 1828 from simulation import Simulation 1829 1830 tpath = testing.setup_unit_test_output_directory('simulation','test_save_attempt',divert=True) 1831 1832 sim = get_test_sim() 1833 1834 sim.create_directories() 1835 1836 files = (sim.infile,sim.outfile,sim.errfile) 1837 1838 assert(sim.attempt_files()==files) 1839 for file in files: 1840 open(os.path.join(sim.locdir,file),'w').write('made an attempt') 1841 #end for 1842 1843 attempt_dir = os.path.join(sim.locdir,'{}_attempt1'.format(sim.identifier)) 1844 assert(not os.path.exists(attempt_dir)) 1845 sim.save_attempt() 1846 assert(os.path.exists(attempt_dir)) 1847 for file in files: 1848 assert(not os.path.exists(os.path.join(sim.locdir,file))) 1849 assert(os.path.exists(os.path.join(attempt_dir,file))) 1850 #end for 1851 1852 restore_nexus() 1853 1854 Simulation.clear_all_sims() 1855#end def test_save_attempt 1856 1857 1858 1859def test_write_inputs(): 1860 import os 1861 from simulation import Simulation,input_template 1862 1863 tpath = testing.setup_unit_test_output_directory('simulation','test_write_inputs',divert=True) 1864 1865 template = ''' 1866name = "$name" 1867a = $a 1868''' 1869 1870 input_ref = ''' 1871name = "input_name" 1872a = 1 1873''' 1874 1875 si = input_template(text=template) 1876 si.assign(name='input_name',a=1) 1877 1878 s = get_test_sim( 1879 input = si, 1880 ) 1881 s.create_directories() 1882 1883 input_file = os.path.join(s.locdir,s.infile) 1884 image_file = os.path.join(s.imlocdir,s.sim_image) 1885 input_image_file = os.path.join(s.imlocdir,s.input_image) 1886 1887 assert(not s.setup) 1888 assert(not os.path.exists(input_file)) 1889 assert(not os.path.exists(image_file)) 1890 assert(not os.path.exists(input_image_file)) 1891 1892 s.write_inputs() 1893 1894 assert(s.setup) 1895 assert(os.path.exists(input_file)) 1896 assert(os.path.exists(image_file)) 1897 assert(os.path.exists(input_image_file)) 1898 1899 assert(open(input_file,'r').read()==input_ref) 1900 1901 s.setup = False 1902 s.load_image() 1903 assert(s.setup) 1904 1905 restore_nexus() 1906 1907 Simulation.clear_all_sims() 1908 1909#end def test_write_inputs 1910 1911 1912 1913def test_send_files(): 1914 import os 1915 from nexus_base import nexus_core 1916 from simulation import Simulation 1917 1918 tpath = testing.setup_unit_test_output_directory('simulation','test_send_files',divert=True) 1919 1920 # make fake data files 1921 data_file1 = 'data_file1.txt' 1922 data_file2 = 'data_file2.txt' 1923 1924 open(os.path.join(tpath,data_file1),'w').write('data1') 1925 open(os.path.join(tpath,data_file2),'w').write('data2') 1926 1927 data_files = [data_file1,data_file2] 1928 1929 s = get_test_sim( 1930 files = data_files, 1931 ) 1932 s.infile = None 1933 1934 assert(s.locdir==s.remdir) 1935 assert(s.files==set(data_files)) 1936 1937 s.create_directories() 1938 1939 loc_data_file1 = os.path.join(s.locdir,data_file1) 1940 loc_data_file2 = os.path.join(s.locdir,data_file2) 1941 1942 assert(not s.sent_files) 1943 assert(not os.path.exists(loc_data_file1)) 1944 assert(not os.path.exists(loc_data_file2)) 1945 1946 s.send_files() 1947 1948 assert(s.sent_files) 1949 assert(os.path.exists(loc_data_file1)) 1950 assert(os.path.exists(loc_data_file2)) 1951 1952 assert(open(loc_data_file1,'r').read()=='data1') 1953 assert(open(loc_data_file2,'r').read()=='data2') 1954 1955 s.sent_files = False 1956 s.load_image() 1957 assert(s.sent_files) 1958 1959 restore_nexus() 1960 1961 Simulation.clear_all_sims() 1962 1963#end def test_send_files 1964 1965 1966 1967def test_submit(): 1968 from machines import job 1969 from simulation import Simulation 1970 1971 tpath = testing.setup_unit_test_output_directory('simulation','test_submit',divert=True) 1972 1973 s = get_test_sim( 1974 job = job(machine='ws1',app_command='echo run'), 1975 ) 1976 1977 j = s.job 1978 m = j.get_machine() 1979 1980 s.create_directories() 1981 1982 assert(not s.submitted) 1983 assert(not s.job.submitted) 1984 assert(j.internal_id not in m.jobs) 1985 assert(j.internal_id not in m.waiting) 1986 1987 s.submit() 1988 1989 assert(s.submitted) 1990 assert(s.job.submitted) 1991 assert(j.internal_id in m.jobs) 1992 assert(j.internal_id in m.waiting) 1993 1994 restore_nexus() 1995 1996 Simulation.clear_all_sims() 1997 1998#end def test_submit 1999 2000 2001 2002def test_update_process_id(): 2003 from simulation import Simulation 2004 2005 tpath = testing.setup_unit_test_output_directory('simulation','test_update_process_id',divert=True) 2006 2007 s = get_test_sim() 2008 j = s.job 2009 2010 s.create_directories() 2011 2012 ref_pid = 0 2013 2014 assert(s.process_id is None) 2015 assert(j.system_id is None) 2016 2017 j.system_id = ref_pid 2018 2019 s.update_process_id() 2020 2021 assert(s.process_id==ref_pid) 2022 2023 s.process_id = None 2024 s.load_image() 2025 assert(s.process_id==ref_pid) 2026 2027 restore_nexus() 2028 2029 Simulation.clear_all_sims() 2030 2031#end def test_update_process_id 2032 2033 2034 2035def test_check_status(): 2036 import os 2037 from simulation import Simulation 2038 2039 tpath = testing.setup_unit_test_output_directory('simulation','test_check_status',divert=True) 2040 2041 s = get_test_sim() 2042 j = s.job 2043 2044 assert(not s.finished) 2045 assert(not j.finished) 2046 2047 s.check_status() 2048 assert(not s.finished) 2049 2050 s.create_directories() 2051 2052 open(os.path.join(s.locdir,s.outfile),'w').write('out') 2053 open(os.path.join(s.locdir,s.errfile),'w').write('err') 2054 j.finished = True 2055 2056 s.check_status() 2057 2058 assert(s.finished) 2059 2060 s.finished = False 2061 s.load_image() 2062 assert(s.finished) 2063 2064 restore_nexus() 2065 2066 Simulation.clear_all_sims() 2067 2068#end def test_check_status 2069 2070 2071 2072def test_get_output(): 2073 import os 2074 from simulation import Simulation 2075 2076 tpath = testing.setup_unit_test_output_directory('simulation','test_get_output',divert=True) 2077 2078 s = get_test_sim() 2079 2080 s.create_directories() 2081 2082 remote_image = os.path.join(s.imremdir,s.sim_image) 2083 results_image = os.path.join(s.imresdir,s.sim_image) 2084 2085 assert(not os.path.exists(remote_image)) 2086 assert(not os.path.exists(results_image)) 2087 assert(not s.finished) 2088 2089 assert(value_eq(s.get_output_files(),[])) 2090 2091 files = [s.infile,s.outfile,s.errfile] 2092 2093 loc_files = [] 2094 res_files = [] 2095 2096 for file in files: 2097 loc_files.append(os.path.join(s.locdir,file)) 2098 res_files.append(os.path.join(s.resdir,file)) 2099 #end for 2100 2101 for loc_file in loc_files: 2102 open(loc_file,'w').write('contents') 2103 #end for 2104 s.finished = True 2105 2106 assert(not s.got_output) 2107 for loc_file,res_file in zip(loc_files,res_files): 2108 assert(os.path.exists(loc_file)) 2109 if s.resdir!=s.locdir: 2110 assert(not os.path.exists(res_file)) 2111 else: 2112 assert(os.path.exists(res_file)) 2113 #end if 2114 #end for 2115 2116 s.get_output() 2117 2118 assert(s.got_output) 2119 for loc_file,res_file in zip(loc_files,res_files): 2120 assert(os.path.exists(loc_file)) 2121 assert(os.path.exists(res_file)) 2122 #end for 2123 2124 s.got_output = False 2125 s.load_image() 2126 assert(s.got_output) 2127 2128 restore_nexus() 2129 2130 Simulation.clear_all_sims() 2131 2132#end def test_get_output 2133 2134 2135 2136def test_analyze(): 2137 import os 2138 from simulation import Simulation 2139 2140 tpath = testing.setup_unit_test_output_directory('simulation','test_analyze',divert=True) 2141 2142 s = get_test_sim() 2143 2144 s.create_directories() 2145 2146 assert(not s.finished) 2147 s.finished = True 2148 2149 analyzer_image = os.path.join(s.imresdir,s.analyzer_image) 2150 2151 assert(not s.analyzed) 2152 assert(not os.path.exists(analyzer_image)) 2153 2154 s.analyze() 2155 2156 assert(s.analyzed) 2157 assert(os.path.exists(analyzer_image)) 2158 2159 s.analyzed = False 2160 s.load_image() 2161 assert(s.analyzed) 2162 2163 restore_nexus() 2164 2165 Simulation.clear_all_sims() 2166 2167#end def test_analyze 2168 2169 2170 2171def test_progress(): 2172 import os 2173 from nexus_base import nexus_core 2174 from simulation import Simulation,input_template 2175 2176 tpath = testing.setup_unit_test_output_directory('simulation','test_progress',divert=True) 2177 2178 assert(nexus_core.mode==nexus_core.modes.stages) 2179 assert(len(nexus_core.stages)==0) 2180 2181 nexus_core.stages = list(nexus_core.primary_modes) 2182 nexus_core.stages_set = set(nexus_core.stages) 2183 2184 primary_modes = ['setup','send_files','submit','get_output','analyze'] 2185 assert(value_eq(nexus_core.stages,primary_modes)) 2186 assert(value_eq(nexus_core.stages_set,set(primary_modes))) 2187 2188 2189 template = ''' 2190name = "$name" 2191a = $a 2192''' 2193 si = input_template(text=template) 2194 si.assign(name='input_name',a=1) 2195 2196 s = get_test_sim(input=si) 2197 2198 indicators = [ 2199 'created_directories', 2200 'got_dependencies', 2201 'setup', 2202 'sent_files', 2203 'submitted', 2204 'finished', 2205 'failed', 2206 'got_output', 2207 'analyzed', 2208 ] 2209 2210 inds = obj() 2211 2212 # first progression 2213 # directory creation 2214 # dependency processing 2215 # input file write 2216 # auxilliary file transfer 2217 # job submission 2218 assert(not s.created_directories) 2219 assert(not s.got_dependencies) 2220 assert(not s.setup) 2221 assert(not s.sent_files) 2222 assert(not s.submitted) 2223 assert(not s.finished) 2224 assert(not s.got_output) 2225 assert(not s.analyzed) 2226 assert(s.files==set()) 2227 assert(s.job.status==0) 2228 assert(not os.path.exists(s.locdir)) 2229 assert(not os.path.exists(s.remdir)) 2230 assert(not os.path.exists(s.resdir)) 2231 assert(not os.path.exists(s.imlocdir)) 2232 assert(not os.path.exists(s.imremdir)) 2233 assert(not os.path.exists(s.imresdir)) 2234 2235 s.progress() 2236 2237 assert(s.created_directories) 2238 assert(s.got_dependencies) 2239 assert(s.setup) 2240 assert(s.sent_files) 2241 assert(s.submitted) 2242 assert(not s.finished) 2243 assert(not s.got_output) 2244 assert(not s.analyzed) 2245 assert(s.files==set([s.infile])) 2246 assert(s.job.status==1) 2247 assert(os.path.exists(s.locdir)) 2248 assert(os.path.exists(s.remdir)) 2249 assert(os.path.exists(s.imlocdir)) 2250 assert(os.path.exists(s.imremdir)) 2251 assert(os.path.exists(os.path.join(s.locdir,s.infile))) 2252 assert(os.path.exists(os.path.join(s.imlocdir,s.sim_image))) 2253 assert(os.path.exists(os.path.join(s.imlocdir,s.input_image))) 2254 assert(not os.path.exists(os.path.join(s.locdir,s.outfile))) 2255 assert(not os.path.exists(os.path.join(s.locdir,s.errfile))) 2256 assert(not os.path.exists(os.path.join(s.imlocdir,s.analyzer_image))) 2257 if s.resdir!=s.locdir: 2258 assert(not os.path.exists(s.resdir)) 2259 assert(not os.path.exists(s.imresdir)) 2260 else: 2261 assert(os.path.exists(s.resdir)) 2262 assert(os.path.exists(s.imresdir)) 2263 #end if 2264 2265 # check image 2266 inds.transfer_from(s,indicators) 2267 s.reset_indicators() 2268 s.load_image() 2269 assert(s.setup) 2270 assert(s.sent_files) 2271 assert(not s.submitted) # submitted is not stored yet 2272 assert(not s.finished) 2273 assert(not s.got_output) 2274 assert(not s.analyzed) 2275 s.transfer_from(inds,indicators) 2276 2277 2278 # simulate job completion 2279 # create output and error files 2280 # set job status to finished 2281 open(os.path.join(s.locdir,s.outfile),'w').write('out') 2282 open(os.path.join(s.locdir,s.errfile),'w').write('err') 2283 s.job.finished = True 2284 2285 assert(os.path.exists(os.path.join(s.locdir,s.outfile))) 2286 assert(os.path.exists(os.path.join(s.locdir,s.errfile))) 2287 2288 2289 # second progression 2290 # status check 2291 # output file transfer 2292 # output analysis 2293 assert(s.created_directories) 2294 assert(s.got_dependencies) 2295 assert(s.setup) 2296 assert(s.sent_files) 2297 assert(s.submitted) 2298 assert(not s.finished) 2299 assert(not s.got_output) 2300 assert(not s.analyzed) 2301 2302 s.progress() 2303 2304 assert(s.created_directories) 2305 assert(s.got_dependencies) 2306 assert(s.setup) 2307 assert(s.sent_files) 2308 assert(s.submitted) 2309 assert(s.finished) 2310 assert(s.got_output) 2311 assert(s.analyzed) 2312 assert(os.path.exists(s.resdir)) 2313 assert(os.path.exists(s.imresdir)) 2314 assert(os.path.exists(os.path.join(s.resdir,s.infile))) 2315 assert(os.path.exists(os.path.join(s.resdir,s.errfile))) 2316 assert(os.path.exists(os.path.join(s.resdir,s.outfile))) 2317 assert(os.path.exists(os.path.join(s.imresdir,s.sim_image))) 2318 assert(os.path.exists(os.path.join(s.imresdir,s.input_image))) 2319 assert(os.path.exists(os.path.join(s.imresdir,s.analyzer_image))) 2320 if s.resdir!=s.locdir: 2321 assert(not os.path.exists(os.path.join(s.imlocdir,s.analyzer_image))) 2322 else: 2323 assert(os.path.exists(os.path.join(s.imlocdir,s.analyzer_image))) 2324 #end if 2325 2326 # check image 2327 inds.transfer_from(s,indicators) 2328 s.reset_indicators() 2329 s.load_image() 2330 assert(s.setup) 2331 assert(s.sent_files) 2332 assert(s.submitted) 2333 assert(s.finished) 2334 assert(s.got_output) 2335 assert(s.analyzed) 2336 s.transfer_from(inds,indicators) 2337 2338 2339 # attempt third progression 2340 # nothing should happen 2341 sbef = s.copy() 2342 sbef.input.template = s.input.template 2343 2344 s.progress() 2345 2346 assert(object_eq(s,sbef)) 2347 2348 2349 restore_nexus() 2350 2351 Simulation.clear_all_sims() 2352 2353#end def test_progress 2354 2355 2356 2357def test_execute(): 2358 import os 2359 from machines import job 2360 from simulation import Simulation 2361 2362 tpath = testing.setup_unit_test_output_directory('simulation','test_execute',divert=True) 2363 2364 s = get_test_sim( 2365 job = job(machine='ws1',app_command='echo run'), 2366 ) 2367 2368 s.create_directories() 2369 2370 outfile = os.path.join(s.locdir,s.outfile) 2371 errfile = os.path.join(s.locdir,s.errfile) 2372 2373 assert(not s.submitted) 2374 assert(not s.job.finished) 2375 assert(s.job.status==0) 2376 assert(not os.path.exists(outfile)) 2377 assert(not os.path.exists(errfile)) 2378 2379 s.execute() 2380 2381 assert(s.submitted) 2382 assert(s.job.finished) 2383 assert(s.job.status==4) 2384 assert(os.path.exists(outfile)) 2385 assert(os.path.exists(errfile)) 2386 assert(open(outfile,'r').read().strip()=='run') 2387 assert(open(errfile,'r').read().strip()=='') 2388 2389 restore_nexus() 2390 2391 Simulation.clear_all_sims() 2392 2393#end def test_execute 2394 2395 2396 2397def test_reset_wait_ids(): 2398 from simulation import Simulation 2399 2400 for i in range(n_test_workflows): 2401 sims = get_test_workflow(i) 2402 for s in sims: 2403 s.wait_ids = None 2404 #end for 2405 for s in sims: 2406 if len(s.dependencies)==0: 2407 s.reset_wait_ids() 2408 #end if 2409 #end for 2410 for s in sims: 2411 assert(isinstance(s.wait_ids,set)) 2412 assert(s.wait_ids==s.dependency_ids) 2413 #end for 2414 #end for 2415 2416 Simulation.clear_all_sims() 2417#end def test_reset_wait_ids 2418 2419 2420 2421def test_check_subcascade(): 2422 from simulation import Simulation 2423 2424 def finish(sim): 2425 sim.finished = True 2426 #end def finish 2427 2428 for i in range(n_test_workflows): 2429 sims = get_test_workflow(i) 2430 2431 # no cascades are finished 2432 for s in sims: 2433 assert(not s.finished) 2434 #end for 2435 for s in sims: 2436 if len(s.dependencies)==0: 2437 finished = s.check_subcascade() 2438 assert(isinstance(finished,bool)) 2439 assert(not finished) 2440 #end if 2441 #end for 2442 2443 # all cascades are finished 2444 for s in sims: 2445 s.finished = True 2446 #end for 2447 for s in sims: 2448 if len(s.dependencies)==0: 2449 finished = s.check_subcascade() 2450 #end if 2451 #end for 2452 2453 # only a single cascade is finished 2454 for s in sims: 2455 s.finished = False 2456 #end for 2457 single = None 2458 for s in sims: 2459 if len(s.dependencies)==0: 2460 if single is None: 2461 single = s 2462 #end if 2463 #end if 2464 #end for 2465 single.traverse_full_cascade(finish) 2466 for s in sims: 2467 if len(s.dependencies)==0: 2468 finished = s.check_subcascade() 2469 if id(s)==id(single): 2470 if not finished: 2471 from simulation import graph_sims 2472 for sim in sims: 2473 if sim.finished: 2474 sim.block = True 2475 #end if 2476 #end for 2477 graph_sims(sims.list()) 2478 #end if 2479 assert(finished) 2480 else: 2481 assert(not finished) 2482 #end if 2483 #end if 2484 #end for 2485 2486 # all simulations are finished except one 2487 # not all cascades are finished 2488 for s in sims: 2489 s.finished = True 2490 #end for 2491 n = 0 2492 for key in sorted(sims.keys()): 2493 n+=1 2494 if 2*n>len(sims): 2495 sims[key].finished = False 2496 #end if 2497 #end for 2498 finished = True 2499 for s in sims: 2500 if len(s.dependencies)==0: 2501 finished &= s.check_subcascade() 2502 #end if 2503 #end for 2504 assert(not finished) 2505 #end for 2506 2507 Simulation.clear_all_sims() 2508#end def test_check_subcascade 2509 2510 2511 2512def test_block_dependents(): 2513 from simulation import Simulation 2514 2515 def assert_blocked(sim): 2516 assert(sim.block) 2517 assert(sim.block_subcascade) 2518 #end def assert_blocked 2519 2520 for i in range(n_test_workflows): 2521 sims = get_test_workflow(i) 2522 for s in sims: 2523 if len(s.dependencies)==0: 2524 s.block_dependents() 2525 s.traverse_full_cascade(assert_blocked) 2526 #end if 2527 #end for 2528 #end for 2529 2530 Simulation.clear_all_sims() 2531#end def test_block_dependents 2532 2533 2534 2535def test_reconstruct_cascade(): 2536 import os 2537 from simulation import Simulation 2538 2539 tpath = testing.setup_unit_test_output_directory('simulation','test_reconstruct_cascade',divert=True) 2540 2541 sims = get_test_workflow(2) 2542 assert(len(sims)==7) 2543 2544 for s in sims: 2545 imagefile = os.path.join(s.imlocdir,s.sim_image) 2546 assert(not os.path.exists(imagefile)) 2547 assert(not s.loaded) 2548 assert(not s.submitted) 2549 assert(not s.finished) 2550 assert(s.process_id is None) 2551 assert(s.job.system_id is None) 2552 #end for 2553 2554 for s in sims: 2555 s.create_directories() 2556 s.save_image() 2557 #end for 2558 2559 for s in sims: 2560 imagefile = os.path.join(s.imlocdir,s.sim_image) 2561 assert(os.path.exists(imagefile)) 2562 assert(not s.loaded) 2563 assert(not s.submitted) 2564 assert(not s.finished) 2565 assert(s.process_id is None) 2566 assert(s.job.system_id is None) 2567 #end for 2568 2569 sims.s1.reconstruct_cascade() 2570 2571 for s in sims: 2572 imagefile = os.path.join(s.imlocdir,s.sim_image) 2573 assert(os.path.exists(imagefile)) 2574 assert(s.loaded) 2575 assert(not s.submitted) 2576 assert(not s.finished) 2577 assert(s.process_id is None) 2578 assert(s.job.system_id is None) 2579 #end for 2580 2581 Simulation.clear_all_sims() 2582 2583 2584 sims = get_test_workflow(2) 2585 2586 def get_process_id(): 2587 get_process_id.current += 1 2588 return get_process_id.current 2589 #end def get_process_id 2590 get_process_id.current = 0 2591 2592 def finish(s): 2593 s.setup = True 2594 s.sent_files = True 2595 s.submitted = True 2596 s.finished = True 2597 s.failed = True 2598 s.got_output = True 2599 s.analyzed = True 2600 s.process_id = get_process_id() 2601 #end def finish 2602 2603 def clear(s): 2604 s.loaded = False 2605 s.setup = False 2606 s.sent_files = False 2607 s.submitted = False 2608 s.finished = False 2609 s.failed = False 2610 s.got_output = False 2611 s.analyzed = False 2612 s.process_id = None 2613 s.job.system_id = None 2614 #end def clear 2615 2616 def finished(s): 2617 f = True 2618 f &= s.setup 2619 f &= s.sent_files 2620 f &= s.submitted 2621 f &= s.finished 2622 f &= s.failed 2623 f &= s.got_output 2624 f &= s.analyzed 2625 f &= isinstance(s.process_id,int) 2626 return f 2627 #end def finished 2628 2629 def empty(s): 2630 e = True 2631 e &= not s.setup 2632 e &= not s.sent_files 2633 e &= not s.submitted 2634 e &= not s.finished 2635 e &= not s.failed 2636 e &= not s.got_output 2637 e &= not s.analyzed 2638 e &= s.process_id is None 2639 e &= s.job.system_id is None 2640 return e 2641 #end def empty 2642 2643 def cleared(s): 2644 return empty(s) and not s.loaded 2645 #end def cleared 2646 2647 for s in sims: 2648 assert(cleared(s)) 2649 #end for 2650 2651 finish(sims.s1) 2652 finish(sims.s2) 2653 finish(sims.s3) 2654 2655 s = sims.s41 2656 s.got_dependencies = True 2657 s.setup = True 2658 s.sent_files = True 2659 s.submitted = True 2660 s.finished = True 2661 s.process_id = get_process_id() 2662 2663 s = sims.s42 2664 s.got_dependencies = True 2665 s.setup = True 2666 s.sent_files = True 2667 s.submitted = True 2668 s.process_id = get_process_id() 2669 2670 for s in sims: 2671 s.create_directories() 2672 s.save_image() 2673 clear(s) 2674 #end for 2675 2676 for s in sims: 2677 assert(cleared(s)) 2678 #end for 2679 2680 sims.s1.reconstruct_cascade() 2681 2682 for s in sims: 2683 assert(s.loaded) 2684 #end for 2685 2686 assert(finished(sims.s1)) 2687 assert(finished(sims.s2)) 2688 assert(finished(sims.s3)) 2689 2690 s = sims.s41 2691 assert(not finished(s) and not empty(s)) 2692 assert(s.got_dependencies) 2693 assert(s.setup ) 2694 assert(s.sent_files ) 2695 assert(s.submitted ) 2696 assert(s.finished ) 2697 assert(not s.failed ) 2698 assert(not s.got_output ) 2699 assert(not s.analyzed ) 2700 assert(s.process_id==4 ) 2701 assert(s.job.system_id is None ) 2702 2703 s = sims.s42 2704 assert(not finished(s) and not empty(s)) 2705 assert(s.got_dependencies) 2706 assert(s.setup ) 2707 assert(s.sent_files ) 2708 assert(s.submitted ) 2709 assert(not s.finished ) 2710 assert(not s.failed ) 2711 assert(not s.got_output ) 2712 assert(not s.analyzed ) 2713 assert(s.process_id==5 ) 2714 assert(s.job.system_id==5) 2715 2716 assert(empty(sims.s51)) 2717 assert(empty(sims.s52)) 2718 2719 restore_nexus() 2720 2721 Simulation.clear_all_sims() 2722#end def test_reconstruct_cascade 2723 2724 2725 2726def test_traverse_cascade(): 2727 from simulation import Simulation 2728 2729 def count_visits(sim,visit_counts): 2730 i = sim.simid 2731 if i not in visit_counts: 2732 visit_counts[i] = 1 2733 else: 2734 visit_counts[i] += 1 2735 #end if 2736 #end def count_visits 2737 2738 for i in range(n_test_workflows): 2739 sims = get_test_workflow(i) 2740 counts = dict() 2741 for s in sims: 2742 if len(s.dependencies)==0: 2743 s.traverse_cascade(count_visits,counts) 2744 #end if 2745 #end for 2746 assert(len(counts)==len(sims)) 2747 for s in sims: 2748 assert(s.simid in counts) 2749 assert(counts[s.simid]==1) 2750 #end for 2751 #end for 2752 2753 Simulation.clear_all_sims() 2754#end def test_traverse_cascade 2755 2756 2757 2758def test_traverse_full_cascade(): 2759 from simulation import Simulation 2760 2761 def finish(sim): 2762 sim.finished = True 2763 #end def finish 2764 2765 for i in range(n_test_workflows): 2766 sims = get_test_workflow(i) 2767 for s in sims: 2768 assert(not s.finished) 2769 #end for 2770 for s in sims: 2771 if len(s.dependencies)==0: 2772 s.traverse_full_cascade(finish) 2773 #end if 2774 #end for 2775 for s in sims: 2776 assert(s.finished) 2777 #end for 2778 #end for 2779 2780 Simulation.clear_all_sims() 2781#end def test_traverse_full_cascade 2782 2783 2784 2785def test_write_dependents(): 2786 from simulation import Simulation 2787 2788 divert_nexus_log() 2789 2790 for i in range(n_test_workflows): 2791 sims = get_test_workflow(i) 2792 for s in sims: 2793 if len(s.dependencies)==0: 2794 s.write_dependents() 2795 #end if 2796 #end for 2797 #end for 2798 2799 restore_nexus_log() 2800 2801 Simulation.clear_all_sims() 2802#end def test_write_dependents 2803 2804 2805 2806def test_generate_simulation(): 2807 from generic import NexusError 2808 from simulation import Simulation,GenericSimulation 2809 from simulation import generate_simulation 2810 2811 try: 2812 sim = generate_simulation(sim_type='unknown') 2813 raise FailedTest 2814 except NexusError: 2815 None 2816 except FailedTest: 2817 failed() 2818 except Exception as e: 2819 failed(str(e)) 2820 #end try 2821 2822 sim = generate_simulation() 2823 assert(isinstance(sim,Simulation)) 2824 assert(isinstance(sim,GenericSimulation)) 2825 2826 Simulation.clear_all_sims() 2827#end def test_generate_simulation 2828 2829 2830 2831if versions.matplotlib_available and versions.pydot_available: 2832 def test_graph_sims(): 2833 from simulation import Simulation,graph_sims 2834 2835 sims = get_test_workflow(3) 2836 2837 graph_sims(sims.list(),display=False,exit=False) 2838 2839 Simulation.clear_all_sims() 2840 #end def test_graph_sims 2841#end if 2842