1!--------------------------------------------------------------------------------------------------! 2! CP2K: A general program to perform molecular dynamics simulations ! 3! Copyright (C) 2000 - 2020 CP2K developers group ! 4!--------------------------------------------------------------------------------------------------! 5 6! ************************************************************************************************** 7!> \brief module contains the master routine handling the tree creation, 8!> communication with workers and task distribution 9!> For each idle working group the master creates a new global tree 10!> element, and if neccessay a related sub tree element, 11!> OR find the next element to calculate the exact energy. 12!> Goal is to keep at least the exact energy calculation working groups 13!> as busy as possible. 14!> Master also checks for incomming results and update the tree and the 15!> acceptance ratios. 16!> \par History 17!> 11.2012 created [Mandes Schoenherr] 18!> \author Mandes 19! ************************************************************************************************** 20 21MODULE tmc_master 22 USE cell_types, ONLY: init_cell 23 USE cp_external_control, ONLY: external_control 24 USE cp_log_handling, ONLY: cp_to_string 25 USE cp_para_types, ONLY: cp_para_env_type 26 USE global_types, ONLY: global_environment_type 27 USE kinds, ONLY: dp,& 28 int_8 29 USE machine, ONLY: m_flush,& 30 m_memory,& 31 m_walltime 32 USE tmc_calculations, ONLY: get_subtree_efficiency 33 USE tmc_cancelation, ONLY: free_cancelation_list 34 USE tmc_dot_tree, ONLY: create_dot_color,& 35 create_global_tree_dot_color,& 36 finalize_draw_tree,& 37 init_draw_trees 38 USE tmc_file_io, ONLY: print_restart_file,& 39 write_element_in_file 40 USE tmc_messages, ONLY: communicate_atom_types,& 41 recv_msg,& 42 send_msg,& 43 stop_whole_group,& 44 tmc_message 45 USE tmc_move_handle, ONLY: check_moves,& 46 print_move_types 47 USE tmc_stati, ONLY: & 48 TMC_CANCELING_MESSAGE, TMC_CANCELING_RECEIPT, TMC_STATUS_FAILED, & 49 TMC_STATUS_WAIT_FOR_NEW_TASK, TMC_STATUS_WORKER_INIT, TMC_STAT_ANALYSIS_REQUEST, & 50 TMC_STAT_ANALYSIS_RESULT, TMC_STAT_APPROX_ENERGY_REQUEST, TMC_STAT_APPROX_ENERGY_RESULT, & 51 TMC_STAT_ENERGY_REQUEST, TMC_STAT_ENERGY_RESULT, TMC_STAT_INIT_ANALYSIS, & 52 TMC_STAT_MD_REQUEST, TMC_STAT_MD_RESULT, TMC_STAT_NMC_REQUEST, TMC_STAT_NMC_RESULT, & 53 TMC_STAT_SCF_STEP_ENER_RECEIVE, TMC_STAT_START_CONF_REQUEST, TMC_STAT_START_CONF_RESULT 54 USE tmc_tree_acceptance, ONLY: check_acceptance_of_depending_subtree_nodes,& 55 check_elements_for_acc_prob_update,& 56 tree_update 57 USE tmc_tree_build, ONLY: create_new_gt_tree_node,& 58 deallocate_sub_tree_node,& 59 finalize_init,& 60 finalize_trees,& 61 init_tree_mod,& 62 remove_all_trees 63 USE tmc_tree_search, ONLY: count_nodes_in_trees,& 64 count_prepared_nodes_in_trees,& 65 search_next_energy_calc 66 USE tmc_tree_types, ONLY: & 67 elem_array_type, elem_list_type, global_tree_type, status_accepted, & 68 status_calc_approx_ener, status_calculate_MD, status_calculate_NMC_steps, & 69 status_calculate_energy, status_calculated, status_cancel_ener, status_cancel_nmc, & 70 status_canceled_ener, status_canceled_nmc, status_created, status_rejected, tree_type 71 USE tmc_types, ONLY: tmc_env_type 72#include "../base/base_uses.f90" 73 74 IMPLICIT NONE 75 76 PRIVATE 77 78 CHARACTER(len=*), PARAMETER, PRIVATE :: moduleN = 'tmc_master' 79 80 PUBLIC :: do_tmc_master 81 82 INTEGER, PARAMETER :: DEBUG = 0 83 84CONTAINS 85 86! ************************************************************************************************** 87!> \brief send cancel request to all workers processing elements in the list 88!> \param cancel_list list with elements to cancel 89!> \param work_list list with all elements processed by working groups 90!> \param cancel_count counter of canceled elements 91!> \param para_env communication environment 92!> \param tmc_env ... 93!> \author Mandes 12.2012 94! ************************************************************************************************** 95 SUBROUTINE cancel_calculations(cancel_list, work_list, cancel_count, & 96 para_env, tmc_env) 97 TYPE(elem_list_type), POINTER :: cancel_list 98 TYPE(elem_array_type), DIMENSION(:), POINTER :: work_list 99 INTEGER :: cancel_count 100 TYPE(cp_para_env_type), POINTER :: para_env 101 TYPE(tmc_env_type), POINTER :: tmc_env 102 103 CHARACTER(LEN=*), PARAMETER :: routineN = 'cancel_calculations', & 104 routineP = moduleN//':'//routineN 105 106 INTEGER :: i, stat, wg 107 TYPE(elem_list_type), POINTER :: tmp_element 108 109 IF (.NOT. ASSOCIATED(cancel_list)) RETURN 110 NULLIFY (tmp_element) 111 112 CPASSERT(ASSOCIATED(tmc_env)) 113 CPASSERT(ASSOCIATED(tmc_env%params)) 114 CPASSERT(ASSOCIATED(tmc_env%m_env)) 115 CPASSERT(ASSOCIATED(work_list)) 116 CPASSERT(ASSOCIATED(para_env)) 117 118 stat = TMC_STATUS_FAILED 119 wg = -1 120 cancel_elem_loop: DO 121 ! find certain working group calculating this element 122 working_elem_loop: DO i = 1, SIZE(work_list) 123 ! in special cases element could be distributed to several working groups, 124 ! but all, except of one, should already be in canceling process 125 IF ((.NOT. work_list(i)%canceled) .AND. & 126 ASSOCIATED(work_list(i)%elem)) THEN 127 IF (ASSOCIATED(cancel_list%elem, work_list(i)%elem)) THEN 128 stat = TMC_CANCELING_MESSAGE 129 wg = i 130 EXIT working_elem_loop 131 END IF 132 END IF 133 END DO working_elem_loop 134 135 CPASSERT(wg .GE. 0) 136 CPASSERT(stat .NE. TMC_STATUS_FAILED) 137 CPASSERT(work_list(wg)%elem%stat .NE. status_calc_approx_ener) 138 139 IF (DEBUG .GE. 1) & 140 WRITE (tmc_env%m_env%io_unit, *) & 141 "TMC|master: cancel group "//cp_to_string(wg) 142 CALL tmc_message(msg_type=stat, send_recv=send_msg, dest=wg, & 143 para_env=para_env, tmc_params=tmc_env%params) 144 work_list(wg)%canceled = .TRUE. 145 146 ! counting the amount of canceled elements 147 cancel_count = cancel_count + 1 148 149 ! delete element from canceling list 150 IF (.NOT. ASSOCIATED(cancel_list%next)) THEN 151 DEALLOCATE (cancel_list) 152 cancel_list => NULL() 153 EXIT cancel_elem_loop 154 ELSE 155 tmp_element => cancel_list%next 156 DEALLOCATE (cancel_list) 157 cancel_list => tmp_element 158 END IF 159 END DO cancel_elem_loop 160 END SUBROUTINE cancel_calculations 161 162! ************************************************************************************************** 163!> \brief send analysis request to a worker 164!> \param ana_list list with elements to be analysed 165!> \param ana_worker_info ... 166!> \param para_env communication environment 167!> \param tmc_env ... 168!> \author Mandes 12.2012 169! ************************************************************************************************** 170 SUBROUTINE send_analysis_tasks(ana_list, ana_worker_info, para_env, tmc_env) 171 TYPE(elem_list_type), POINTER :: ana_list 172 TYPE(elem_array_type), DIMENSION(:), POINTER :: ana_worker_info 173 TYPE(cp_para_env_type), POINTER :: para_env 174 TYPE(tmc_env_type), POINTER :: tmc_env 175 176 CHARACTER(LEN=*), PARAMETER :: routineN = 'send_analysis_tasks', & 177 routineP = moduleN//':'//routineN 178 179 INTEGER :: dest, stat, wg 180 TYPE(elem_list_type), POINTER :: list_tmp 181 182 NULLIFY (list_tmp) 183 184 CPASSERT(ASSOCIATED(ana_worker_info)) 185 CPASSERT(ASSOCIATED(para_env)) 186 187 wg_loop: DO wg = 1, SIZE(ana_worker_info) 188 IF (.NOT. ASSOCIATED(ana_list)) EXIT wg_loop 189 IF (.NOT. ana_worker_info(wg)%busy) THEN 190 stat = TMC_STAT_ANALYSIS_REQUEST 191 dest = wg 192 CALL tmc_message(msg_type=stat, send_recv=send_msg, dest=dest, & 193 para_env=para_env, tmc_params=tmc_env%params, & 194 list_elem=ana_list) 195 IF (.NOT. ASSOCIATED(ana_list%next)) THEN 196 DEALLOCATE (ana_list) 197 ana_list => NULL() 198 ELSE 199 list_tmp => ana_list%next 200 DEALLOCATE (ana_list) 201 ana_list => list_tmp 202 END IF 203 END IF 204 END DO wg_loop 205 END SUBROUTINE send_analysis_tasks 206 207! ************************************************************************************************** 208!> \brief global master handling tree creation and communication/work 209!> distribution with workers 210!> \param tmc_env structure for storing all the tmc parameters 211!> \param globenv global environment for external control 212!> \author Mandes 11.2012 213! ************************************************************************************************** 214 SUBROUTINE do_tmc_master(tmc_env, globenv) 215 TYPE(tmc_env_type), POINTER :: tmc_env 216 TYPE(global_environment_type), POINTER :: globenv 217 218 CHARACTER(LEN=*), PARAMETER :: routineN = 'do_tmc_master', routineP = moduleN//':'//routineN 219 220 INTEGER :: cancel_count, handle, last_output, reactivation_cc_count, & 221 reactivation_ener_count, restart_count, restarted_elem_nr, stat, walltime_delay, & 222 walltime_offset, wg, worker_counter 223 INTEGER(KIND=int_8) :: mem 224 INTEGER, DIMENSION(6) :: nr_of_job 225 INTEGER, DIMENSION(:), POINTER :: tree_elem_counters, tree_elem_heads 226 LOGICAL :: external_stop, flag, l_update_tree 227 REAL(KIND=dp) :: run_time_start 228 REAL(KIND=dp), DIMENSION(4) :: worker_timings_aver 229 REAL(KIND=dp), DIMENSION(:), POINTER :: efficiency 230 TYPE(elem_array_type), DIMENSION(:), POINTER :: ana_worker_info, worker_info 231 TYPE(global_tree_type), POINTER :: gt_elem_tmp 232 TYPE(tree_type), POINTER :: init_conf 233 234 external_stop = .FALSE. 235 restarted_elem_nr = 0 236 NULLIFY (init_conf, worker_info, ana_worker_info, gt_elem_tmp, tree_elem_counters) 237 238 CPASSERT(ASSOCIATED(tmc_env)) 239 240 CPASSERT(tmc_env%tmc_comp_set%group_nr == 0) 241 CPASSERT(ASSOCIATED(tmc_env%tmc_comp_set)) 242 CPASSERT(ASSOCIATED(tmc_env%tmc_comp_set%para_env_m_w)) 243 244 CPASSERT(ASSOCIATED(tmc_env%m_env)) 245 246 !-- run time measurment, to end just in time 247 ! start the timing 248 CALL timeset(routineN, handle) 249 run_time_start = m_walltime() 250 walltime_delay = 0 251 walltime_offset = 20 ! default value the whole program needs to finalize 252 253 ! initialize the different modules 254 IF (tmc_env%params%DRAW_TREE) & 255 CALL init_draw_trees(tmc_params=tmc_env%params) 256 257 !-- initialize variables 258 ! nr_of_job: counting the different task send / received 259 ! (1:NMC submitted, 2:energies submitted, 3:NMC finished 4:energy finished, 5:NMC canceled, 6:energy canceled) 260 nr_of_job(:) = 0 261 worker_counter = -1 262 reactivation_ener_count = 0 263 reactivation_cc_count = 0 264 cancel_count = 0 265 tmc_env%m_env%result_count = 0 266 l_update_tree = .FALSE. 267 restart_count = 1 268 last_output = -1 269 ! average timings 270 ! (1:calculated NMC, 2:calculated ener, 3:canceled NMC, 4: canceled ener) 271 worker_timings_aver(:) = 0.0_dp 272 ! remembers state of workers and their actual configurations 273 ! the actual working group, communicating with 274 ALLOCATE (worker_info(tmc_env%tmc_comp_set%para_env_m_w%num_pe - 1)) 275 ALLOCATE (ana_worker_info(tmc_env%tmc_comp_set%para_env_m_ana%num_pe - 1)) 276 277 ! get the start configuration form the first (exact energy) worker, 278 ! master should/could have no Force environment 279 stat = TMC_STAT_START_CONF_REQUEST 280 wg = 1 281 CALL tmc_message(msg_type=stat, send_recv=send_msg, dest=wg, & 282 para_env=tmc_env%tmc_comp_set%para_env_m_w, & 283 tmc_params=tmc_env%params, & 284 wait_for_message=.TRUE.) 285 !-- wait for start configuration results and number of dimensions 286 !-- get start configuration (init_conf element should not be allocated already) 287 CALL tmc_message(msg_type=stat, send_recv=recv_msg, dest=wg, & 288 para_env=tmc_env%tmc_comp_set%para_env_m_w, & 289 tmc_params=tmc_env%params, & 290 elem=init_conf, success=flag, wait_for_message=.TRUE.) 291 IF (stat .NE. TMC_STAT_START_CONF_RESULT) & 292 CALL cp_abort(__LOCATION__, & 293 "receiving start configuration failed, received stat "// & 294 cp_to_string(stat)) 295 ! get the atom names from first energy worker 296 CALL communicate_atom_types(atoms=tmc_env%params%atoms, & 297 source=1, & 298 para_env=tmc_env%tmc_comp_set%para_env_m_first_w) 299 300 CALL init_cell(cell=tmc_env%params%cell) 301 302 ! check the configuration consitency with selected moves 303 CALL check_moves(tmc_params=tmc_env%params, & 304 move_types=tmc_env%params%move_types, & 305 mol_array=init_conf%mol) 306 IF (ASSOCIATED(tmc_env%params%nmc_move_types)) & 307 CALL check_moves(tmc_params=tmc_env%params, & 308 move_types=tmc_env%params%nmc_move_types, & 309 mol_array=init_conf%mol) 310 311 ! set initial configuration 312 ! set initial random number generator seed (rng seed) 313 ! initialize the tree structure espacially for parallel tmepering, 314 ! seting the subtrees 315 CALL init_tree_mod(start_elem=init_conf, tmc_env=tmc_env, & 316 job_counts=nr_of_job, & 317 worker_timings=worker_timings_aver) 318 319 ! init restart counter (espacially for restart case) 320 IF (tmc_env%m_env%restart_out_step .NE. 0) THEN 321 restart_count = INT(tmc_env%m_env%result_count(0)/ & 322 REAL(tmc_env%m_env%restart_out_step, KIND=dp)) + 1 323 END IF 324 restarted_elem_nr = tmc_env%m_env%result_count(0) 325 326!TODO check conf and cell of both input files (cell has to be equal, 327! because it is used as reference cell for scaling the cell) 328 ! communicate the reference cell size 329 DO wg = 1, tmc_env%tmc_comp_set%para_env_m_w%num_pe - 1 330 stat = TMC_STATUS_WORKER_INIT 331 CALL tmc_message(msg_type=stat, send_recv=send_msg, dest=wg, & 332 para_env=tmc_env%tmc_comp_set%para_env_m_w, & 333 tmc_params=tmc_env%params) 334 END DO 335 336 ! send the atom informations to all analysis workers 337 IF (tmc_env%tmc_comp_set%para_env_m_ana%num_pe .GT. 1) THEN 338 DO wg = 1, tmc_env%tmc_comp_set%para_env_m_ana%num_pe - 1 339 stat = TMC_STAT_INIT_ANALYSIS 340 CALL tmc_message(msg_type=stat, send_recv=send_msg, dest=wg, & 341 para_env=tmc_env%tmc_comp_set%para_env_m_ana, & 342 result_count=tmc_env%m_env%result_count, & 343 tmc_params=tmc_env%params, & 344 elem=init_conf, & 345 wait_for_message=.TRUE.) 346 END DO 347 CALL communicate_atom_types(atoms=tmc_env%params%atoms, & 348 source=0, & 349 para_env=tmc_env%tmc_comp_set%para_env_m_ana) 350 END IF 351 352 CALL deallocate_sub_tree_node(tree_elem=init_conf) 353 354 ! regtest output 355 IF (tmc_env%params%print_test_output .OR. DEBUG .GT. 0) & 356 WRITE (tmc_env%m_env%io_unit, *) "TMC|first_global_tree_rnd_nr_X= ", & 357 tmc_env%m_env%gt_head%rnd_nr 358 359 ! calculate the approx energy of the first element (later the exact) 360 IF (tmc_env%m_env%gt_head%conf(1)%elem%stat .EQ. status_calc_approx_ener) THEN 361 wg = 1 362 IF (tmc_env%tmc_comp_set%group_cc_nr .GT. 0) & 363 wg = tmc_env%tmc_comp_set%group_ener_nr + 1 364 stat = TMC_STAT_APPROX_ENERGY_REQUEST 365 CALL tmc_message(msg_type=stat, send_recv=send_msg, dest=wg, & 366 para_env=tmc_env%tmc_comp_set%para_env_m_w, & 367 tmc_params=tmc_env%params, & 368 elem=tmc_env%m_env%gt_head%conf(1)%elem) 369 worker_info(wg)%busy = .TRUE. 370 worker_info(wg)%elem => tmc_env%m_env%gt_head%conf(1)%elem 371 init_conf => tmc_env%m_env%gt_head%conf(1)%elem 372 ELSE IF (tmc_env%m_env%gt_head%conf(1)%elem%stat .EQ. status_created) THEN 373 init_conf => tmc_env%m_env%gt_head%conf(1)%elem 374 ! calculation will be done automatically, 375 ! by searching the next conf for energy calculation 376 END IF 377 !-- START WORK --! 378 !-- distributing work: 379 ! 1. receive incoming results 380 ! 2. check new results in tree 381 ! 3. if idle worker, create new tree element and send them to worker 382 task_loop: DO 383 ! ======================================================================= 384 !-- RECEIVING ALL incoming messages and handling them 385 ! results of tree node 1 is distributed to all other subtree nodes 386 ! ======================================================================= 387 worker_request_loop: DO 388 wg = 1 389 flag = .FALSE. 390 CALL tmc_message(msg_type=stat, send_recv=recv_msg, dest=wg, & 391 para_env=tmc_env%tmc_comp_set%para_env_m_w, & 392 tmc_params=tmc_env%params, & 393 elem_array=worker_info(:), success=flag) 394 395 IF (flag .EQV. .FALSE.) EXIT worker_request_loop 396 ! messages from worker group could be faster then the canceling request 397 IF (worker_info(wg)%canceled .AND. (stat .NE. TMC_CANCELING_RECEIPT)) THEN 398 IF (DEBUG .GE. 1) & 399 WRITE (tmc_env%m_env%io_unit, *) & 400 "TMC|master: recv stat "//cp_to_string(stat)// & 401 " of canceled worker group" 402 CYCLE worker_request_loop 403 END IF 404 405 ! in case of parallel tempering canceled element could be reactivated, 406 ! calculated faster and deleted 407 IF (.NOT. ASSOCIATED(worker_info(wg)%elem)) & 408 CALL cp_abort(__LOCATION__, & 409 "no tree elem exist when receiving stat "// & 410 cp_to_string(stat)//"of group"//cp_to_string(wg)) 411 412 IF (DEBUG .GE. 1) & 413 WRITE (tmc_env%m_env%io_unit, *) & 414 "TMC|master: received stat "//cp_to_string(stat)// & 415 " of sub tree "//cp_to_string(worker_info(wg)%elem%sub_tree_nr)// & 416 " elem"//cp_to_string(worker_info(wg)%elem%nr)// & 417 " with stat"//cp_to_string(worker_info(wg)%elem%stat)// & 418 " of group"//cp_to_string(wg)//" group canceled ", worker_info(wg)%canceled 419 SELECT CASE (stat) 420 ! -- FAILED -------------------------- 421 CASE (TMC_STATUS_FAILED) 422 EXIT task_loop 423 ! -- CANCEL_RECEIPT ------------------ 424 CASE (TMC_CANCELING_RECEIPT) 425 ! worker should got cancel message before 426 CPASSERT(worker_info(wg)%canceled) 427 worker_info(wg)%canceled = .FALSE. 428 worker_info(wg)%busy = .FALSE. 429 430 IF (ASSOCIATED(worker_info(wg)%elem)) THEN 431 SELECT CASE (worker_info(wg)%elem%stat) 432 CASE (status_cancel_ener) 433 !-- timings 434 worker_timings_aver(4) = (worker_timings_aver(4)*nr_of_job(6) + & 435 (m_walltime() - worker_info(wg)%start_time))/REAL(nr_of_job(6) + 1, KIND=dp) 436 nr_of_job(6) = nr_of_job(6) + 1 437 438 worker_info(wg)%elem%stat = status_canceled_ener 439 worker_info(wg)%elem%potential = 8000.0_dp 440 IF (tmc_env%params%DRAW_TREE) THEN 441 CALL create_dot_color(tree_element=worker_info(wg)%elem, & 442 tmc_params=tmc_env%params) 443 END IF 444 CASE (status_cancel_nmc) 445 !-- timings 446 worker_timings_aver(3) = (worker_timings_aver(3)*nr_of_job(5) + & 447 (m_walltime() - worker_info(wg)%start_time))/REAL(nr_of_job(5) + 1, KIND=dp) 448 nr_of_job(5) = nr_of_job(5) + 1 449 450 worker_info(wg)%elem%stat = status_canceled_nmc 451 worker_info(wg)%elem%potential = 8000.0_dp 452 IF (tmc_env%params%DRAW_TREE) THEN 453 CALL create_dot_color(tree_element=worker_info(wg)%elem, & 454 tmc_params=tmc_env%params) 455 END IF 456 CASE DEFAULT 457 ! the subtree element is again in use (reactivated) 458 END SELECT 459 worker_info(wg)%elem => NULL() 460 END IF 461 ! -- START_CONF_RESULT --------------- 462 CASE (TMC_STAT_START_CONF_RESULT) 463 ! start configuration should already be handeled 464 CPABORT("") 465 ! -- ENERGY RESULT ----------------- 466 CASE (TMC_STAT_APPROX_ENERGY_RESULT) 467 nr_of_job(3) = nr_of_job(3) + 1 468 worker_info(wg)%busy = .FALSE. 469 worker_info(wg)%elem%stat = status_created 470 IF (tmc_env%params%DRAW_TREE) THEN 471 CALL create_dot_color(tree_element=worker_info(wg)%elem, & 472 tmc_params=tmc_env%params) 473 END IF 474 worker_info(wg)%elem => NULL() 475 ! nothing to do, the approximate potential 476 ! should be updated in the message interface 477 ! -- NMC / MD RESULT ----------------- 478 CASE (TMC_STAT_NMC_RESULT, TMC_STAT_MD_RESULT) 479 IF (.NOT. worker_info(wg)%canceled) worker_info(wg)%busy = .FALSE. 480 !-- timings for Nested Monte Carlo calculation 481 worker_timings_aver(1) = (worker_timings_aver(1)*nr_of_job(3) + & 482 (m_walltime() - worker_info(wg)%start_time))/REAL(nr_of_job(3) + 1, KIND=dp) 483 nr_of_job(3) = nr_of_job(3) + 1 484 485 worker_info(wg)%start_time = m_walltime() - worker_info(wg)%start_time 486 CALL set_walltime_delay(worker_info(wg)%start_time, walltime_delay) 487 worker_info(wg)%elem%stat = status_created 488 IF (tmc_env%params%DRAW_TREE) THEN 489 CALL create_dot_color(tree_element=worker_info(wg)%elem, & 490 tmc_params=tmc_env%params) 491 END IF 492 !-- send energy request 493 ! in case of one singe input file, energy is already calculated 494 IF (tmc_env%params%NMC_inp_file .EQ. "") THEN 495 worker_info(wg)%elem%potential = worker_info(wg)%elem%e_pot_approx 496 worker_info(wg)%elem%stat = status_calculated 497 ! check acceptance of depending nodes 498 IF (.NOT. (ASSOCIATED(worker_info(wg)%elem, init_conf))) THEN 499 CALL check_acceptance_of_depending_subtree_nodes(tree_elem=worker_info(wg)%elem, & 500 tmc_env=tmc_env) 501 END IF 502 IF (tmc_env%params%DRAW_TREE) THEN 503 CALL create_dot_color(tree_element=worker_info(wg)%elem, & 504 tmc_params=tmc_env%params) 505 END IF 506 !-- CANCELING the calculations of the elements, which are definetively not needed anymore 507 CALL cancel_calculations(cancel_list=tmc_env%m_env%cancelation_list, & 508 work_list=worker_info, & 509 para_env=tmc_env%tmc_comp_set%para_env_m_w, & 510 tmc_env=tmc_env, & 511 cancel_count=cancel_count) 512 worker_info(wg)%elem => NULL() 513 ELSE 514 ! if all working groups are equal, the same group calculates the energy 515 IF (tmc_env%tmc_comp_set%group_cc_nr .LE. 0 & 516 .AND. (.NOT. worker_info(wg)%canceled)) THEN 517 worker_info(wg)%elem%stat = status_calculate_energy 518 stat = TMC_STAT_ENERGY_REQUEST 519 ! immediately send energy request 520 CALL tmc_message(msg_type=stat, send_recv=send_msg, dest=wg, & 521 para_env=tmc_env%tmc_comp_set%para_env_m_w, & 522 tmc_params=tmc_env%params, & 523 elem=worker_info(wg)%elem) 524 worker_info(wg)%busy = .TRUE. 525 nr_of_job(2) = nr_of_job(2) + 1 526 IF (tmc_env%params%DRAW_TREE) THEN 527 CALL create_dot_color(tree_element=worker_info(wg)%elem, & 528 tmc_params=tmc_env%params) 529 END IF 530 !-- set start time for energy calculation 531 worker_info(wg)%start_time = m_walltime() 532 ELSE 533 worker_info(wg)%elem => NULL() 534 END IF 535 END IF 536 ! -- ENERGY RESULT -------------------- 537 CASE (TMC_STAT_ENERGY_RESULT) 538 !-- timings 539 worker_timings_aver(2) = (worker_timings_aver(2)*nr_of_job(4) + & 540 (m_walltime() - worker_info(wg)%start_time))/REAL(nr_of_job(4) + 1, KIND=dp) 541 nr_of_job(4) = nr_of_job(4) + 1 542 543 worker_info(wg)%start_time = m_walltime() - worker_info(wg)%start_time 544 CALL set_walltime_delay(worker_info(wg)%start_time, walltime_delay) 545 546 IF (.NOT. worker_info(wg)%canceled) & 547 worker_info(wg)%busy = .FALSE. 548 ! the first node in tree is always accepted.!. 549 IF (ASSOCIATED(worker_info(wg)%elem, init_conf)) THEN 550 !-- distribute energy of first element to all subtrees 551 CALL finalize_init(gt_tree_ptr=tmc_env%m_env%gt_head, & 552 tmc_env=tmc_env) 553 IF (tmc_env%params%DRAW_TREE) THEN 554 CALL create_global_tree_dot_color(gt_tree_element=tmc_env%m_env%gt_act, & 555 tmc_params=tmc_env%params) 556 CALL create_dot_color(tree_element=worker_info(wg)%elem, & 557 tmc_params=tmc_env%params) 558 END IF 559 init_conf => NULL() 560 ELSE 561 worker_info(wg)%elem%stat = status_calculated 562 IF (tmc_env%params%DRAW_TREE) & 563 CALL create_dot_color(worker_info(wg)%elem, & 564 tmc_params=tmc_env%params) 565 ! check acceptance of depending nodes 566 ! first (initial) configuration do not have to be checked 567 CALL check_acceptance_of_depending_subtree_nodes(tree_elem=worker_info(wg)%elem, & 568 tmc_env=tmc_env) 569 END IF 570 !-- write out all configurations (not only Markov Chain) e.g. for fitting 571 IF (tmc_env%params%all_conf_file_name .NE. "") THEN 572 CALL write_element_in_file(elem=worker_info(wg)%elem, & 573 file_name=tmc_env%params%all_conf_file_name, & 574 tmc_params=tmc_env%params, & 575 conf_nr=nr_of_job(4)) 576 END IF 577 578 !-- CANCELING the calculations of the elements, 579 ! which are definetively not needed anymore 580 CALL cancel_calculations(cancel_list=tmc_env%m_env%cancelation_list, & 581 work_list=worker_info, & 582 para_env=tmc_env%tmc_comp_set%para_env_m_w, & 583 tmc_env=tmc_env, & 584 cancel_count=cancel_count) 585 IF (DEBUG .GE. 9) & 586 WRITE (tmc_env%m_env%io_unit, *) & 587 "TMC|master: handled energy result of sub tree ", & 588 worker_info(wg)%elem%sub_tree_nr, " elem ", worker_info(wg)%elem%nr, & 589 " with stat", worker_info(wg)%elem%stat 590 worker_info(wg)%elem => NULL() 591 592 !-- SCF ENERGY ----------------------- 593 CASE (TMC_STAT_SCF_STEP_ENER_RECEIVE) 594 IF (.NOT. (ASSOCIATED(worker_info(wg)%elem, init_conf)) .AND. & 595 worker_info(wg)%elem%stat .NE. status_cancel_ener .AND. & 596 worker_info(wg)%elem%stat .NE. status_cancel_nmc) THEN 597 ! update the acceptance probability and the canceling list 598 CALL check_elements_for_acc_prob_update(tree_elem=worker_info(wg)%elem, & 599 tmc_env=tmc_env) 600 END IF 601 ! cancel inlikely elements 602 CALL cancel_calculations(cancel_list=tmc_env%m_env%cancelation_list, & 603 work_list=worker_info, & 604 para_env=tmc_env%tmc_comp_set%para_env_m_w, & 605 tmc_env=tmc_env, & 606 cancel_count=cancel_count) 607 CASE (TMC_STAT_ANALYSIS_RESULT) 608 ana_worker_info(wg)%busy = .FALSE. 609 ana_worker_info(wg)%elem => NULL() 610 CASE DEFAULT 611 CPABORT("received message with unknown info/stat type") 612 END SELECT 613 END DO worker_request_loop 614 !-- do tree update (check new results) 615 CALL tree_update(tmc_env=tmc_env, result_acc=flag, & 616 something_updated=l_update_tree) 617 IF (DEBUG .GE. 2 .AND. l_update_tree) & 618 WRITE (tmc_env%m_env%io_unit, *) & 619 "TMC|master: tree updated "//cp_to_string(l_update_tree)// & 620 " of with gt elem "//cp_to_string(tmc_env%m_env%gt_act%nr)// & 621 " with stat"//cp_to_string(tmc_env%m_env%gt_act%stat) 622 623 CALL send_analysis_tasks(ana_list=tmc_env%m_env%analysis_list, & 624 ana_worker_info=ana_worker_info, & 625 para_env=tmc_env%tmc_comp_set%para_env_m_ana, & 626 tmc_env=tmc_env) 627 628 ! ======================================================================= 629 !-- ALL CALCULATIONS DONE (check) --- 630 ! ======================================================================= 631 ! if enough configurations are sampled or walltime is exeeded, 632 ! finish building trees 633!TODO set correct logger para_env to use this 634 CALL external_control(should_stop=external_stop, flag="TMC", globenv=globenv) 635 IF ((ANY(tmc_env%m_env%result_count(1:) .GE. tmc_env%m_env%num_MC_elem) & 636 .AND. flag) .OR. & 637 (m_walltime() - run_time_start .GT. & 638 tmc_env%m_env%walltime - walltime_delay - walltime_offset) .OR. & 639 external_stop) THEN 640 WRITE (tmc_env%m_env%io_unit, FMT="(/,T2,A)") REPEAT("=", 79) 641 ! calculations NOT finished, walltime exceeded 642 IF (.NOT. ANY(tmc_env%m_env%result_count(1:) & 643 .GE. tmc_env%m_env%num_MC_elem)) THEN 644 WRITE (tmc_env%m_env%io_unit, *) "Walltime exceeded.", & 645 m_walltime() - run_time_start, " of ", tmc_env%m_env%walltime - walltime_delay - walltime_offset, & 646 "(incl. delay", walltime_delay, "and offset", walltime_offset, ") left" 647 ELSE 648 ! calculations finished 649 IF (tmc_env%params%print_test_output) & 650 WRITE (tmc_env%m_env%io_unit, *) "Total energy: ", & 651 tmc_env%m_env%result_list(1)%elem%potential 652 END IF 653 IF (tmc_env%m_env%restart_out_step .NE. 0) & 654 CALL print_restart_file(tmc_env=tmc_env, job_counts=nr_of_job, & 655 timings=worker_timings_aver) 656 EXIT task_loop 657 END IF 658 659 ! ======================================================================= 660 ! update the rest of the tree (canceling and deleting elements) 661 ! ======================================================================= 662 IF (l_update_tree) THEN 663 IF (DEBUG .GE. 2) & 664 WRITE (tmc_env%m_env%io_unit, *) & 665 "TMC|master: start remove elem and cancel calculation" 666 !-- CLEANING tree nodes beside the path through the tree from 667 ! end_of_clean_tree to tree_ptr 668 ! --> getting back the end of clean tree 669 CALL remove_all_trees(working_elem_list=worker_info, tmc_env=tmc_env) 670 !-- CANCELING the calculations of the elements, 671 ! which are definetively not needed anymore 672 ! elements are added to canceling list if no global tree reference 673 ! exist anymore 674 CALL cancel_calculations(cancel_list=tmc_env%m_env%cancelation_list, & 675 work_list=worker_info, & 676 cancel_count=cancel_count, & 677 para_env=tmc_env%tmc_comp_set%para_env_m_w, & 678 tmc_env=tmc_env) 679 END IF 680 681 ! ===================================================================== 682 !-- NEW TASK (if worker not busy submit next task) 683 ! ===================================================================== 684 worker_counter = worker_counter + 1 685 wg = MODULO(worker_counter, tmc_env%tmc_comp_set%para_env_m_w%num_pe - 1) + 1 686 687 IF (DEBUG .GE. 16 .AND. ALL(worker_info(:)%busy)) & 688 WRITE (tmc_env%m_env%io_unit, *) "all workers are busy" 689 690 IF (.NOT. worker_info(wg)%busy) THEN 691 IF (DEBUG .GE. 13) & 692 WRITE (tmc_env%m_env%io_unit, *) & 693 "TMC|master: search new task for worker ", wg 694 ! no group separation 695 IF (tmc_env%tmc_comp_set%group_cc_nr .LE. 0) THEN 696 ! search next element to calculate the energy 697 CALL search_next_energy_calc(gt_head=tmc_env%m_env%gt_act, & 698 new_gt_elem=gt_elem_tmp, stat=stat, & 699 react_count=reactivation_ener_count) 700 IF (stat .EQ. TMC_STATUS_WAIT_FOR_NEW_TASK) THEN 701 CALL create_new_gt_tree_node(tmc_env=tmc_env, stat=stat, & 702 new_elem=gt_elem_tmp, & 703 reactivation_cc_count=reactivation_cc_count) 704 END IF 705 ELSEIF (wg .GT. tmc_env%tmc_comp_set%group_ener_nr) THEN 706 ! specialized groups (groups for exact energy and groups for configurational change) 707 ! creating new element (configurational change group) 708 !-- crate new node, configurational change is handled in tmc_tree module 709 CALL create_new_gt_tree_node(tmc_env=tmc_env, stat=stat, & 710 new_elem=gt_elem_tmp, & 711 reactivation_cc_count=reactivation_cc_count) 712 ! element could be already created, hence CC worker has nothing to do for this element 713 ! in next round he will get a task 714 IF (stat .EQ. status_created .OR. stat .EQ. status_calculate_energy) & 715 stat = TMC_STATUS_WAIT_FOR_NEW_TASK 716 ELSE 717 ! search next element to calculate the energy 718 CALL search_next_energy_calc(gt_head=tmc_env%m_env%gt_act, & 719 new_gt_elem=gt_elem_tmp, stat=stat, & 720 react_count=reactivation_ener_count) 721 END IF 722 723 IF (DEBUG .GE. 10) & 724 WRITE (tmc_env%m_env%io_unit, *) & 725 "TMC|master: send task with elem stat "//cp_to_string(stat)// & 726 " to group "//cp_to_string(wg) 727 ! MESSAGE settings: status informations and task for communication 728 SELECT CASE (stat) 729 CASE (TMC_STATUS_WAIT_FOR_NEW_TASK) 730 CYCLE task_loop 731 CASE (TMC_STATUS_FAILED) 732 !STOP "in creating new task, status failed should be handled before" 733 CYCLE task_loop 734 CASE (status_calculated, status_accepted, status_rejected) 735 CYCLE task_loop 736 CASE (status_calc_approx_ener) 737 ! e.g. after volume move, we need the approximate potential for 2 potential check of following NMC nodes 738 stat = TMC_STAT_APPROX_ENERGY_REQUEST 739 CALL tmc_message(msg_type=stat, send_recv=send_msg, dest=wg, & 740 para_env=tmc_env%tmc_comp_set%para_env_m_w, & 741 tmc_params=tmc_env%params, & 742 elem=gt_elem_tmp%conf(gt_elem_tmp%mv_conf)%elem) 743 nr_of_job(1) = nr_of_job(1) + 1 744 CASE (status_created, status_calculate_energy) 745 ! in case of parallel tempering the node can be already be calculating (related to another global tree node 746 !-- send task to calculate system property 747 gt_elem_tmp%conf(gt_elem_tmp%mv_conf)%elem%stat = status_calculate_energy 748 IF (tmc_env%params%DRAW_TREE) & 749 CALL create_dot_color(tree_element=gt_elem_tmp%conf(gt_elem_tmp%mv_conf)%elem, & 750 tmc_params=tmc_env%params) 751 stat = TMC_STAT_ENERGY_REQUEST 752 CALL tmc_message(msg_type=stat, send_recv=send_msg, dest=wg, & 753 para_env=tmc_env%tmc_comp_set%para_env_m_w, & 754 tmc_params=tmc_env%params, & 755 elem=gt_elem_tmp%conf(gt_elem_tmp%mv_conf)%elem) 756 nr_of_job(2) = nr_of_job(2) + 1 757 CASE (status_calculate_MD) 758 stat = TMC_STAT_MD_REQUEST 759 CALL tmc_message(msg_type=stat, send_recv=send_msg, dest=wg, & 760 para_env=tmc_env%tmc_comp_set%para_env_m_w, & 761 tmc_params=tmc_env%params, & 762 elem=gt_elem_tmp%conf(gt_elem_tmp%mv_conf)%elem) 763! temperature=tmc_env%params%Temp(gt_elem_tmp%mv_conf), & 764 nr_of_job(1) = nr_of_job(1) + 1 765 CASE (status_calculate_NMC_steps) 766 !-- send information of element, which should be calculated 767 stat = TMC_STAT_NMC_REQUEST 768 CALL tmc_message(msg_type=stat, send_recv=send_msg, dest=wg, & 769 para_env=tmc_env%tmc_comp_set%para_env_m_w, & 770 tmc_params=tmc_env%params, & 771 elem=gt_elem_tmp%conf(gt_elem_tmp%mv_conf)%elem) 772 nr_of_job(1) = nr_of_job(1) + 1 773 CASE (status_cancel_nmc, status_cancel_ener) 774 ! skip that task until receipt is received 775 ! no status update 776 CASE DEFAULT 777 CALL cp_abort(__LOCATION__, & 778 "new task of tree element"// & 779 cp_to_string(gt_elem_tmp%nr)// & 780 "has unknown status"//cp_to_string(stat)) 781 END SELECT 782 worker_info(wg)%elem => gt_elem_tmp%conf(gt_elem_tmp%mv_conf)%elem 783 worker_info(wg)%busy = .TRUE. 784 ! set timer for maximum calculation time recognition 785 worker_info(wg)%start_time = m_walltime() 786 787 !===================== write out info after x requested tasks========== 788 IF (nr_of_job(4) .GT. last_output .AND. & 789 (MODULO(nr_of_job(4), tmc_env%m_env%info_out_step_size) .EQ. 0) .AND. & 790 (stat .NE. TMC_STATUS_FAILED)) THEN 791 last_output = nr_of_job(4) 792 WRITE (tmc_env%m_env%io_unit, FMT="(/,T2,A)") REPEAT("-", 79) 793 WRITE (tmc_env%m_env%io_unit, *) & 794 "Tasks submitted: E ", nr_of_job(2), ", cc", nr_of_job(1) 795 WRITE (tmc_env%m_env%io_unit, *) & 796 "Results received: E ", nr_of_job(4), ", cc", nr_of_job(3) 797 WRITE (tmc_env%m_env%io_unit, *) & 798 "Configurations used:", tmc_env%m_env%result_count(0), & 799 ", sub trees", tmc_env%m_env%result_count(1:) 800 801 CALL print_move_types(init=.FALSE., file_io=tmc_env%m_env%io_unit, & 802 tmc_params=tmc_env%params) 803 ALLOCATE (tree_elem_counters(0:SIZE(tmc_env%params%Temp))) 804 ALLOCATE (tree_elem_heads(0:SIZE(tmc_env%params%Temp))) 805 CALL count_nodes_in_trees(global_tree_ptr=tmc_env%m_env%gt_act, & 806 end_of_clean_trees=tmc_env%m_env%st_clean_ends, & 807 counters=tree_elem_counters, head_elements_nr=tree_elem_heads) 808 WRITE (tmc_env%m_env%io_unit, *) "nodes in trees", tree_elem_counters(:) 809 WRITE (tmc_env%m_env%io_unit, *) "tree heads ", tree_elem_heads(:) 810 IF (tmc_env%params%NMC_inp_file .NE. "") THEN 811 CALL count_prepared_nodes_in_trees(global_tree_ptr=tmc_env%m_env%gt_act, & 812 counters=tree_elem_counters) 813 WRITE (tmc_env%m_env%io_unit, FMT=*) & 814 "ener prepared ", tree_elem_counters 815 END IF 816 IF (tmc_env%params%SPECULATIVE_CANCELING) & 817 WRITE (tmc_env%m_env%io_unit, *) & 818 "canceled cc|E: ", nr_of_job(5:6), & 819 ", reactivated: cc ", & 820 reactivation_cc_count, & 821 ", reactivated: E ", & 822 reactivation_ener_count 823 WRITE (tmc_env%m_env%io_unit, FMT='(A,2F10.2)') & 824 " Average time for cc/ener calc ", & 825 worker_timings_aver(1), worker_timings_aver(2) 826 IF (tmc_env%params%SPECULATIVE_CANCELING) & 827 WRITE (tmc_env%m_env%io_unit, FMT='(A,2F10.2)') & 828 " Average time until cancel cc/ener calc ", & 829 worker_timings_aver(3), worker_timings_aver(4) 830 IF (tmc_env%params%esimate_acc_prob) & 831 WRITE (tmc_env%m_env%io_unit, *) & 832 "Estimate correct (acc/Nacc) | wrong (acc/nacc)", & 833 tmc_env%m_env%estim_corr_wrong(1), & 834 tmc_env%m_env%estim_corr_wrong(3), " | ", & 835 tmc_env%m_env%estim_corr_wrong(2), & 836 tmc_env%m_env%estim_corr_wrong(4) 837 WRITE (tmc_env%m_env%io_unit, *) & 838 "Time: ", INT(m_walltime() - run_time_start), "of", & 839 INT(tmc_env%m_env%walltime - walltime_delay - walltime_offset), & 840 "sec needed. " 841 CALL m_memory(mem) 842 WRITE (tmc_env%m_env%io_unit, *) & 843 "Memory used: ", INT(mem/(1024*1024), KIND=KIND(0)), "MiBytes" 844 CALL m_flush(tmc_env%m_env%io_unit) 845 DEALLOCATE (tree_elem_heads) 846 DEALLOCATE (tree_elem_counters) 847 END IF 848 !===================== write out restart file after x results============ 849 IF (tmc_env%m_env%restart_out_step .GT. 0 .AND. & 850 tmc_env%m_env%result_count(0) .GT. & 851 restart_count*tmc_env%m_env%restart_out_step) THEN 852 CALL print_restart_file(tmc_env=tmc_env, job_counts=nr_of_job, & 853 timings=worker_timings_aver) 854 restart_count = restart_count + 1 855 END IF 856 857 END IF !worker busy? 858 END DO task_loop 859 860 ! -- END OF WORK (enough configurations are calculated or walltime exceeded 861 WRITE (tmc_env%m_env%io_unit, FMT="(/,T2,A)") REPEAT("=", 79) 862 WRITE (UNIT=tmc_env%m_env%io_unit, FMT="(T2,A,T35,A,T80,A)") "=", & 863 "finalizing TMC", "=" 864 WRITE (tmc_env%m_env%io_unit, *) "acceptance rates:" 865 CALL print_move_types(init=.FALSE., file_io=tmc_env%m_env%io_unit, & 866 tmc_params=tmc_env%params) 867 WRITE (tmc_env%m_env%io_unit, FMT="(/,T2,A)") REPEAT("-", 79) 868 ! program efficiency result outputs 869 ALLOCATE (efficiency(0:tmc_env%params%nr_temp)) 870 CALL get_subtree_efficiency(tmc_env=tmc_env, eff=efficiency) 871 WRITE (tmc_env%m_env%io_unit, *) "Efficiencies:" 872 WRITE (tmc_env%m_env%io_unit, FMT="(A,F5.2,A,1000F5.2)") & 873 " (MC elements/calculated configuration) global:", & 874 efficiency(0), " sub tree(s): ", efficiency(1:) 875 DEALLOCATE (efficiency) 876 IF (tmc_env%tmc_comp_set%group_cc_nr .GT. 0) & 877 WRITE (tmc_env%m_env%io_unit, FMT="(A,1000F5.2)") & 878 " (MC elements/created configuration) :", & 879 tmc_env%m_env%result_count(:)/REAL(nr_of_job(3), KIND=dp) 880 WRITE (tmc_env%m_env%io_unit, FMT="(A,1000F5.2)") & 881 " (MC elements/energy calculated configuration):", & 882 tmc_env%m_env%result_count(:)/REAL(nr_of_job(4), KIND=dp) 883 IF (tmc_env%params%NMC_inp_file .NE. "") THEN 884 WRITE (tmc_env%m_env%io_unit, *) & 885 "Amount of canceled elements (E/cc):", & 886 tmc_env%m_env%count_cancel_ener, tmc_env%m_env%count_cancel_NMC 887 WRITE (tmc_env%m_env%io_unit, *) & 888 " reactivated E ", reactivation_ener_count 889 WRITE (tmc_env%m_env%io_unit, *) & 890 " reactivated cc ", reactivation_cc_count 891 END IF 892 WRITE (tmc_env%m_env%io_unit, FMT="(A,F10.2)") & 893 " computing time of one Markov chain element ", & 894 (m_walltime() - run_time_start)/REAL(tmc_env%m_env%result_count(0) - & 895 restarted_elem_nr, KIND=dp) 896 WRITE (tmc_env%m_env%io_unit, FMT="(A,F10.2)") " TMC run time[s]: ", m_walltime() - run_time_start 897 WRITE (tmc_env%m_env%io_unit, FMT="(/,T2,A)") REPEAT("=", 79) 898 899 !-- FINALIZE 900 WRITE (tmc_env%m_env%io_unit, *) "stopping workers" 901 CALL stop_whole_group(para_env=tmc_env%tmc_comp_set%para_env_m_w, & 902 worker_info=worker_info, & 903 tmc_params=tmc_env%params) 904 DEALLOCATE (worker_info) 905 CALL stop_whole_group(para_env=tmc_env%tmc_comp_set%para_env_m_ana, & 906 worker_info=ana_worker_info, & 907 tmc_params=tmc_env%params) 908 DEALLOCATE (ana_worker_info) 909 910 !-- deallocating everything in tree module 911 CALL finalize_trees(tmc_env=tmc_env) 912 913 CALL free_cancelation_list(tmc_env%m_env%cancelation_list) 914 915 ! -- write final configuration 916 IF (tmc_env%params%DRAW_TREE) & 917 CALL finalize_draw_tree(tmc_params=tmc_env%params) 918 919 WRITE (tmc_env%m_env%io_unit, *) "TMC master: all work done." 920 921 ! end the timing 922 CALL timestop(handle) 923 924 END SUBROUTINE do_tmc_master 925 926! ************************************************************************************************** 927!> \brief routine sets the walltime delay, to the maximum calculation time 928!> hence the program can stop with a proper finailze 929!> \param time actual calculation time 930!> \param walltime_delay the actual biggest calculation time 931!> \author Mandes 12.2012 932! ************************************************************************************************** 933 SUBROUTINE set_walltime_delay(time, walltime_delay) 934 REAL(KIND=dp) :: time 935 INTEGER :: walltime_delay 936 937 CHARACTER(LEN=*), PARAMETER :: routineN = 'set_walltime_delay', & 938 routineP = moduleN//':'//routineN 939 940 CPASSERT(time .GE. 0.0_dp) 941 942 IF (time .GT. walltime_delay) THEN 943 walltime_delay = INT(time) + 1 944 END IF 945 END SUBROUTINE set_walltime_delay 946 947END MODULE tmc_master 948