1 /* _______________________________________________________________________
2
3 DAKOTA: Design Analysis Kit for Optimization and Terascale Applications
4 Copyright 2014-2020 National Technology & Engineering Solutions of Sandia, LLC (NTESS).
5 This software is distributed under the GNU Lesser General Public License.
6 For more information, see the README file in the top Dakota directory.
7 _______________________________________________________________________ */
8
9 //- Class: ProcessApplicInterface
10 //- Description: Class implementation
11 //- Owner: Mike Eldred
12
13 #include "DakotaResponse.hpp"
14 #include "ParamResponsePair.hpp"
15 #include "ProcessApplicInterface.hpp"
16 #include "ProblemDescDB.hpp"
17 #include "ParallelLibrary.hpp"
18 #include "WorkdirHelper.hpp"
19 #include <algorithm>
20 #include <boost/filesystem/fstream.hpp>
21
22 /*
23 Work directory TODO
24
25 * Doc: we will search for drivers in PATH, workdir (.), RUNDIR
26
27 * Remove legacy utilities (once concepts migrated)
28
29 - In general review cases with race conditions such as single dir
30 getting created / removed for each eval.
31
32 - Verify creation/removal in parallel runs (eval_comm_rank 0?);
33 are there scenarios where we should create once ahead of time?
34
35 - Enforce tagging when asynch is possible
36
37 - Challenge of shared vs. distinct filesystems
38
39 - Verify template files exist at parse and that workdir parent exists
40
41 - Verify behavior when directory exists
42
43 - Allow recursive copy to descend to overwrite leaf nodes when
44 directories already exist
45
46 - Old code setting permissions mask 0700
47
48 - Workdir with multiple analysis components (per analysis)
49
50 - Workdirs shared for each unique concurrent eval (not per eval ID)
51
52 - Evaluate environment variables
53
54 - How to manage drivers that want just param names in the work
55 dir?!? I think arg_adjust is removing the directory args that
56 aren't needed.
57
58 - Consider making the class members for directories and files bfs::paths
59
60 - Behavior of file_save when directory not saved
61
62 - Error checking:
63 * directory was created
64 * directory path is a directory
65 * directory has rwx for user
66 * population worked
67 * try/catch around all fs operations
68
69 - Verify correct handling of relative vs. absolute files/dirs
70
71 - Enforce that first argument must be an executable program for
72 all drivers; at least for fork
73
74 - Historical behaviors / features to consider
75 * Template dirs on PATH: likely no longer
76
77 * Allowed FOO=zorch and would set that in the environment; could
78 allow separate env var specification; otherwise likely remove
79
80 - TODO: pass environment to exec as separate pointer
81
82 TESTING NEEDS
83 - Allow nested quotes in driver, at least one level:
84 analysis_driver = 'ad.sh "-opt foo -opt1 goo"' p.in.1 r.out.1
85
86 - Env vars will be carried along for now, not expanded before
87 eval; set some helpful env vars before the eval.
88 */
89
90
91 namespace Dakota {
92
93 // Regular expressions used to substitute the names of parameters and resutls
94 // files into driver strings
95 boost::regex PARAMS_TOKEN("\\{PARAMETERS\\}");
96 boost::regex RESULTS_TOKEN("\\{RESULTS\\}");
97
98 /// Substitute parameters and results file names into driver strings
substitute_params_and_results(const String & driver,const String & params,const String & results)99 String substitute_params_and_results(const String &driver, const String ¶ms, const String &results) {
100 String params_subbed = boost::regex_replace(driver, PARAMS_TOKEN, params);
101 String results_subbed = boost::regex_replace(params_subbed, RESULTS_TOKEN, results);
102 return results_subbed;
103 }
104
105
106 ProcessApplicInterface::
ProcessApplicInterface(const ProblemDescDB & problem_db)107 ProcessApplicInterface(const ProblemDescDB& problem_db):
108 ApplicationInterface(problem_db),
109 fileTagFlag(problem_db.get_bool("interface.application.file_tag")),
110 fileSaveFlag(problem_db.get_bool("interface.application.file_save")),
111 commandLineArgs(!problem_db.get_bool("interface.application.verbatim")),
112 apreproFlag(problem_db.get_bool("interface.application.aprepro")),
113 resultsFileFormat(problem_db.get_ushort("interface.application.results_file_format")),
114 multipleParamsFiles(false),
115 iFilterName(problem_db.get_string("interface.application.input_filter")),
116 oFilterName(problem_db.get_string("interface.application.output_filter")),
117 programNames(problem_db.get_sa("interface.application.analysis_drivers")),
118 specifiedParamsFileName(
119 problem_db.get_string("interface.application.parameters_file")),
120 specifiedResultsFileName(
121 problem_db.get_string("interface.application.results_file")),
122 allowExistingResults(problem_db.get_bool("interface.allow_existing_results")),
123 useWorkdir(problem_db.get_bool("interface.useWorkdir")),
124 workDirName(problem_db.get_string("interface.workDir")),
125 dirTag(problem_db.get_bool("interface.dirTag")),
126 dirSave(problem_db.get_bool("interface.dirSave")),
127 linkFiles(problem_db.get_sa("interface.linkFiles")),
128 copyFiles(problem_db.get_sa("interface.copyFiles")),
129 templateReplace(problem_db.get_bool("interface.templateReplace"))
130 {
131 // When using work directory, relative analysis drivers starting
132 // with . or .. may need to be converted to absolute so they work
133 // from the work directory. While inefficient, we convert them in
134 // place as strings (could leave tokenized for Fork/Spawn, but
135 // assemble the string for SysCall Interfaces
136 if (useWorkdir) {
137 StringArray::iterator pn_it = programNames.begin();
138 StringArray::iterator pn_end = programNames.end();
139 for ( ; pn_it != pn_end; ++pn_it)
140 if (WorkdirHelper::resolve_driver_path(*pn_it) &&
141 outputLevel >= DEBUG_OUTPUT)
142 Cout << "Adjusted relative analysis_driver to absolute path:\n "
143 << *pn_it << std::endl;
144 }
145
146 size_t num_programs = programNames.size();
147 if (num_programs > 1 && !analysisComponents.empty())
148 multipleParamsFiles = true;
149
150 // RATIONALE: While a user might truly want concurrent evaluations
151 // with non-unique work_directory and/or parameters/results files,
152 // it could too easily lead to errors or surprising results.
153 // Moreover, with asynch it's not clear which eval would make the
154 // directory. We therefore force tagging on either the directory or
155 // the parameters/results files.
156
157 // EDGE CASE: For MPI parallelism we could allow non-tagged dirs /
158 // files since nodes may not share a common filesystem. However, we
159 // consider this an edge case and prefer to be robust, requiring
160 // unique files/dirs.
161
162 // BMA TODO: extend this to the message-passing concurrent
163 // evaluation case in init_communicators. Can't rely on
164 // asynchLocalEvalConcurrency because this is set per parallel
165 // configuration in set_communicators.
166
167 bool require_unique =
168 (interface_synchronization() == ASYNCHRONOUS_INTERFACE) &&
169 (asynchLocalEvalConcSpec != 1) &&
170 !batchEval;
171
172 if (require_unique) {
173 if (useWorkdir) {
174 if (!dirTag && !workDirName.empty()) {
175 Cout << "\nWarning: Concurrent local evaluations with named "
176 << "work_directory require\n directory_tag; "
177 << "enabling directory_tag." << std::endl;
178 dirTag = true;
179 }
180 // even with a work_directory, there might be absolute
181 // params/results paths...
182 bfs::path spf_path(specifiedParamsFileName);
183 bfs::path srf_path(specifiedResultsFileName);
184 // an empty path is not absolute, so no need to check empty
185 bool exist_abs_filenames =
186 spf_path.is_absolute() || srf_path.is_absolute();
187 if (!fileTagFlag && exist_abs_filenames) {
188 Cout << "\nWarning: Concurrent local evaluations with absolute named "
189 << "parameters_file or\n results_file require file_tag; "
190 << "enabling file_tag." << std::endl;
191 fileTagFlag = true;
192 }
193 }
194 else if (!fileTagFlag &&
195 (!specifiedParamsFileName.empty() ||
196 !specifiedResultsFileName.empty()) ) {
197 Cout << "\nWarning: Concurrent local evaluations with named "
198 << "parameters_file or\n results_file require file_tag; "
199 << "enabling file_tag." << std::endl;
200 fileTagFlag = true;
201 }
202 }
203
204 }
205
206
~ProcessApplicInterface()207 ProcessApplicInterface::~ProcessApplicInterface()
208 { /* empty dtor */ }
209
210
211 // -------------------------------------------------------
212 // Begin derived functions for evaluation level schedulers
213 // -------------------------------------------------------
214 void ProcessApplicInterface::
derived_map(const Variables & vars,const ActiveSet & set,Response & response,int fn_eval_id)215 derived_map(const Variables& vars, const ActiveSet& set, Response& response,
216 int fn_eval_id)
217 {
218 // This function may be executed by a multiprocessor evalComm.
219
220 define_filenames(final_eval_id_tag(fn_eval_id)); // all evalComm
221 if (evalCommRank == 0)
222 write_parameters_files(vars, set, response, fn_eval_id);
223
224 // execute the simulator application -- blocking call
225 create_evaluation_process(BLOCK);
226
227 try {
228 if (evalCommRank == 0)
229 read_results_files(response, fn_eval_id, final_eval_id_tag(fn_eval_id));
230 }
231
232 // catch exception for ResultsFileError or TabularDataTruncated;
233 // TODO: need to verify that tabular failure can happen
234 catch(const FileReadException& fr_except) {
235 // a FileReadException means detection of an incomplete file/data
236 // set. In the synchronous case, there is no potential for an incomplete
237 // file resulting from a race condition -> echo the error and abort.
238 Cerr << fr_except.what() << std::endl;
239 abort_handler(INTERFACE_ERROR);
240 }
241
242 // Do not call manage_failure() from the following catch since the
243 // recursion of calling derived_map again would be confusing at
244 // best. The approach here is to have catch(FunctionEvalFailure)
245 // rethrow the exception to an outer catch (either the catch within
246 // manage_failure or a catch that calls manage_failure). An
247 // alternative solution would be to eliminate the try block above
248 // and move all catches to the higher level of try { derived_map() }
249 // - this would be simpler to understand but would replicate
250 // catch(FileReadException) in map, manage_failure, and serve. By
251 // having catch(FileReadException) here and having
252 // catch(FunctionEvalFailure) rethrow, we eliminate unnecessary
253 // proliferation of catch(FileReadException).
254 catch(const FunctionEvalFailure& fneval_except) {
255 // failure capture exception thrown by response.read()
256 //Cout << "Rethrowing FunctionEvalFailure; message: "
257 // << fneval_except.what() << std::endl;
258 throw; // from this catch to the outer one in manage_failure
259 }
260 }
261
262
derived_map_asynch(const ParamResponsePair & pair)263 void ProcessApplicInterface::derived_map_asynch(const ParamResponsePair& pair)
264 {
265 // This function may not be executed by a multiprocessor evalComm.
266 if(!batchEval) {
267 int fn_eval_id = pair.eval_id();
268 define_filenames(final_eval_id_tag(fn_eval_id)); // all evalComm
269 write_parameters_files(pair.variables(), pair.active_set(),
270 pair.response(), fn_eval_id);
271 // execute the simulator application -- nonblocking call
272 pid_t pid = create_evaluation_process(FALL_THROUGH);
273 // bind process id with eval id for use in synchronization
274 map_bookkeeping(pid, fn_eval_id);
275 }
276 }
277
278
wait_local_evaluations(PRPQueue & prp_queue)279 void ProcessApplicInterface::wait_local_evaluations(PRPQueue& prp_queue)
280 {
281 if (batchEval) wait_local_evaluation_batch(prp_queue);
282 else wait_local_evaluation_sequence(prp_queue);
283 }
284
285
test_local_evaluations(PRPQueue & prp_queue)286 void ProcessApplicInterface::test_local_evaluations(PRPQueue& prp_queue)
287 {
288 if (batchEval) test_local_evaluation_batch(prp_queue);
289 else test_local_evaluation_sequence(prp_queue);
290 }
291
292
wait_local_evaluation_batch(PRPQueue & prp_queue)293 void ProcessApplicInterface::wait_local_evaluation_batch(PRPQueue& prp_queue)
294 {
295
296 batchIdCntr++;
297 // define_filenames sets paramsFileWritten and resultsFileWritten, taking
298 // into consideration all the myriad settings the user could have provided,
299 // hierarchical tagging, etc.
300 String batch_id_tag = final_batch_id_tag();
301 define_filenames(batch_id_tag);
302 if(!allowExistingResults)
303 std::remove(resultsFileWritten.c_str());
304 std::vector<String> an_comps;
305 if(!analysisComponents.empty())
306 copy_data(analysisComponents, an_comps);
307 std::remove(paramsFileWritten.c_str()); //
308 for(const auto & pair : prp_queue) {
309 int fn_eval_id = pair.eval_id();
310 fullEvalId = final_eval_id_tag(fn_eval_id); // must be set for eval ID to
311 // appear in params file
312 write_parameters_file(pair.variables(), pair.active_set(),
313 pair.response(), programNames[0], an_comps,
314 paramsFileWritten, false /*append to file*/);
315 }
316
317 // In this case, individual jobs have not been launched and we launch the
318 // user's analysis driver once for the complete batch:
319 create_evaluation_process(BLOCK);
320
321 // Response.read() expects results for a single evaluation to be present in a
322 // stream, but the results file for a batch evaluation will contain multiple.
323 // The strategy here is to consume the results file one evaluation at a time,
324 // placing the content for one evaluation into a stringstream, which will be
325 // passed to Response.read(). The # character at the beginning of a line is
326 // presumed to separate evaluations (content that follows on that line will
327 // be dropped), and the evaluations are presumed to be in the same order as
328 // prp_queue. Probably not a very robust way of doing things, but for purposes
329 // of prototyping, it'll do.
330
331 bfs::ifstream results_file(resultsFileWritten);
332 if (!results_file) {
333 Cerr << "\nError: cannot open results file " << resultsFileWritten
334 << " for batch " << std::to_string(batchIdCntr) << std::endl;
335 abort_handler(INTERFACE_ERROR); // will clean up files unless file_save was specified
336 }
337 Response response;
338 for(auto & pair : prp_queue) {
339 std::stringstream eval_ss;
340 while(true) {
341 String eval_buffer;
342 std::getline(results_file, eval_buffer);
343 if(results_file.eof())
344 break;
345 if(eval_buffer[0] == '#') {
346 if(eval_ss.str().empty())
347 continue;
348 else
349 break;
350 } else {
351 eval_ss << eval_buffer << std::endl;
352 }
353 }
354 response = pair.response();
355 // the read operation errors out for improperly formatted data
356 try {
357 response.read(eval_ss, resultsFileFormat);
358 }
359 catch(const FunctionEvalFailure & fneval_except) {
360 manage_failure(pair.variables(), response.active_set(), response, pair.eval_id());
361 }
362 catch(const FileReadException& fr_except) {
363 throw FileReadException("Error(s) encountered reading batch results file " +
364 resultsFileWritten + " for Evaluation " + std::to_string(pair.eval_id())
365 + ":\n" + fr_except.what());
366 }
367 completionSet.insert(pair.eval_id());
368 }
369 results_file.close();
370 file_and_workdir_cleanup(paramsFileWritten, resultsFileWritten, createdDir, batch_id_tag);
371
372 }
373
test_local_evaluation_batch(PRPQueue & prp_queue)374 void ProcessApplicInterface::test_local_evaluation_batch(PRPQueue& prp_queue)
375 { wait_local_evaluation_batch(prp_queue); }
376
377
378 // ------------------------
379 // Begin file I/O utilities
380 // ------------------------
define_filenames(const String & eval_id_tag)381 void ProcessApplicInterface::define_filenames(const String& eval_id_tag)
382 {
383 // Define modified file names by handling Unix temp file and tagging options.
384 // These names are used in writing the parameters file, in defining the
385 // Command Shell behavior in SysCallProcessApplicInterface.cpp, in
386 // reading the results file, and in file cleanup.
387
388 // Different analysis servers _must_ share the same parameters and results
389 // file root names. If temporary file names are not used, then each evalComm
390 // proc can define the same names without need for interproc communication.
391 // However, if temporary file names are used, then evalCommRank 0 must define
392 // the names and broadcast them to other evalComm procs.
393
394 // Behavior has been simplified to tag with either hierarchical tag,
395 // e.g. 4.9.2, or just eval id tag, .2. If directory_tag, workdirs
396 // are tagged, if file_tag, directories are tagged, or both if both
397 // specified.
398
399 const ParallelConfiguration& pc = parallelLib.parallel_configuration();
400 int eval_comm_rank = parallelLib.ie_parallel_level_defined()
401 ? pc.ie_parallel_level().server_communicator_rank() : 0;
402 int analysis_servers = parallelLib.ea_parallel_level_defined()
403 ? pc.ea_parallel_level().num_servers() : 1;
404
405 // technically don't need to broadcast in some useWorkdir subcases
406 // like absolute params/results filenames
407 bool dynamic_filenames = specifiedParamsFileName.empty() ||
408 specifiedResultsFileName.empty() || useWorkdir;
409
410 bool bcast_flag = ( analysis_servers > 1 && dynamic_filenames );
411
412 if (eval_comm_rank == 0 || !bcast_flag) {
413
414 // eval_tag_prefix will be empty if no hierarchical tagging
415 fullEvalId = eval_id_tag;
416
417 // establish the workdir name first, since parameters files might go in it
418 bool wd_created = false;
419 if (useWorkdir) {
420 // curWorkdir is used by Fork/SysCall arg_adjust
421 curWorkdir = get_workdir_name();
422 // TODO: Create with 0700 mask?
423 wd_created = WorkdirHelper::create_directory(curWorkdir, DIR_PERSIST);
424 // copy/link tolerate empty items
425 WorkdirHelper::copy_items(copyFiles, curWorkdir, templateReplace);
426 WorkdirHelper::link_items(linkFiles, curWorkdir, templateReplace);
427 }
428
429 // non-empty createdDir communicates to write_parameters_files that
430 // the directory was created specifically for this eval and should
431 // be removed when results collected
432 if (wd_created)
433 createdDir = curWorkdir;
434 else
435 createdDir.clear();
436
437 // define paramsFleName for use in write_parameters_files
438
439 bfs::path params_path(specifiedParamsFileName);
440
441 // no user spec -> use temp files, possibly in workdir; generate relative
442 if (specifiedParamsFileName.empty())
443 params_path = WorkdirHelper::system_tmp_file("dakota_params");
444
445 // append tag for all cases (previously temp files weren't tagged)
446 if (fileTagFlag)
447 params_path = WorkdirHelper::concat_path(params_path, fullEvalId);
448
449 // TODO: do we want to modify the parameters file here or later?
450 // Need to differentiate where the file is written, removed,
451 // etc. from the name of it as passed to the analysis driver
452
453 // could we consider the tmpdir to be a workdir...?
454
455 // if a relative path, prepend with workdir or tmp, else use absolute name
456 // BMA TODO: don't use native string here...
457 paramsFileName = paramsFileWritten = params_path.string();
458 if (params_path.is_relative()) {
459 if (useWorkdir) {
460 paramsFileWritten = (curWorkdir / params_path).string();
461 if (outputLevel >= DEBUG_OUTPUT)
462 Cout << "\nAdjusting parameters_file to " << paramsFileName
463 << " due to work_directory usage." << std::endl;
464 }
465 else if (specifiedParamsFileName.empty()) {
466 // historically temp files go in /tmp or C:\temp, not rundir
467 paramsFileName = (WorkdirHelper::system_tmp_path()/params_path).string();
468 paramsFileWritten = paramsFileName;
469 }
470 else
471 paramsFileName = params_path.string();
472 }
473 else
474 paramsFileName = params_path.string();
475
476
477 // define resultsFileName for use in write_parameters_files
478
479 bfs::path results_path(specifiedResultsFileName);
480
481 // no user spec -> use temp files, possibly in workdir; generate relative
482 if (specifiedResultsFileName.empty())
483 results_path = WorkdirHelper::system_tmp_file("dakota_results");
484
485 // append tag for all cases (previously temp files weren't tagged)
486 if (fileTagFlag)
487 results_path = WorkdirHelper::concat_path(results_path, fullEvalId);
488
489 // if a relative path, prepend with workdir or tmp, else use absolute name
490 // BMA TODO: don't use native string here...
491 resultsFileName = resultsFileWritten = results_path.string();
492 if (results_path.is_relative()) {
493 if (useWorkdir) {
494 resultsFileWritten = (curWorkdir / results_path).string();
495 if (outputLevel >= DEBUG_OUTPUT)
496 Cout << "\nAdjusting results_file to " << resultsFileName
497 << " due to work_directory usage." <<std::endl;
498 }
499 else if (specifiedResultsFileName.empty()) {
500 // historically temp files go in /tmp or C:\temp, not rundir
501 resultsFileName = (WorkdirHelper::system_tmp_path()/results_path).string();
502 resultsFileWritten = resultsFileName;
503 }
504 else
505 resultsFileName = results_path.string();
506 }
507 else
508 resultsFileName = results_path.string();
509
510 }
511
512 // Not broadcasting curWorkdir as we don't want other ranks to
513 // remove it later.
514
515 if (bcast_flag) {
516 // Note that evalComm and evalAnalysisIntraComm are equivalent in this case
517 // since multiprocessor analyses are forbidden when using system calls/forks
518 // (enforced by ApplicationInterface::init_communicators()).
519 if (eval_comm_rank == 0) {
520 // pack the buffer with root file names for the evaluation
521 MPIPackBuffer send_buffer;
522 send_buffer << paramsFileName << resultsFileName << fullEvalId;
523 // bcast buffer length so that other procs can allocate MPIUnpackBuffer
524 int buffer_len = send_buffer.size();
525 parallelLib.bcast_e(buffer_len);
526 // broadcast actual buffer
527 parallelLib.bcast_e(send_buffer);
528 }
529 else {
530 // receive incoming buffer length and allocate space for MPIUnpackBuffer
531 int buffer_len;
532 parallelLib.bcast_e(buffer_len);
533 // receive incoming buffer
534 MPIUnpackBuffer recv_buffer(buffer_len);
535 parallelLib.bcast_e(recv_buffer);
536 recv_buffer >> paramsFileName >> resultsFileName >> fullEvalId;
537 }
538 }
539 }
540
541
542 void ProcessApplicInterface::
write_parameters_files(const Variables & vars,const ActiveSet & set,const Response & response,const int id)543 write_parameters_files(const Variables& vars, const ActiveSet& set,
544 const Response& response, const int id)
545 {
546 PathTriple file_names(paramsFileWritten, resultsFileWritten, createdDir);
547
548 // If a new evaluation, insert the modified file names into map for use in
549 // identifying the proper file names within read_results_files for the case
550 // of asynch fn evaluations. If a replacement eval (e.g., failure capturing
551 // with retry or continuation), then overwrite old file names with new ones.
552 std::map<int, PathTriple>::iterator map_iter = fileNameMap.find(id);
553 if (map_iter != fileNameMap.end()) {
554 // remove old files. This is only necessary for the tmp file case, since
555 // all other cases (including tagged) will overwrite the old files. Since
556 // we don't want to save tmp files, don't bother to check fileSaveFlag.
557 // Also don't check allow existing results since they may be bogus.
558 const bfs::path& pfile_path = (map_iter->second).get<0>();
559 WorkdirHelper::recursive_remove(pfile_path, FILEOP_WARN);
560 const bfs::path& rfile_path = (map_iter->second).get<1>();
561 WorkdirHelper::recursive_remove(rfile_path, FILEOP_WARN);
562 // replace file names in map, avoiding 2nd lookup
563 map_iter->second = file_names;
564 }
565 else // insert new filenames.
566 fileNameMap[id] = file_names;
567
568 /* Uncomment for filename debugging
569 Cout << "\nFile name entries at fn. eval. " << id << '\n';
570 for (map_iter = fileNameMap.begin(); map_iter != fileNameMap.end();map_iter++)
571 Cout << map_iter->first << " " << (map_iter->second).first << " "
572 << (map_iter->second).second << '\n';
573 Cout << std::endl;
574 */
575
576 // Write paramsFileName without prog_num tag if there's an input filter or if
577 // multiple sets of analysisComponents are not used.
578 size_t num_programs = programNames.size();
579 if (!multipleParamsFiles || !iFilterName.empty()) {
580 std::string prog; // empty default indicates generic attribution of AC_i
581 // populate prog string in cases where attribution is clear
582 if (num_programs == 1 && iFilterName.empty())
583 prog = programNames[0]; // this file corresponds to sole analysis driver
584 else if (!iFilterName.empty() && multipleParamsFiles)
585 prog = iFilterName; // this file corresponds to ifilter
586 std::vector<String> all_an_comps;
587 if (!analysisComponents.empty())
588 copy_data(analysisComponents, all_an_comps);
589 if (!allowExistingResults)
590 std::remove(resultsFileWritten.c_str());
591 write_parameters_file(vars, set, response, prog, all_an_comps,
592 paramsFileWritten);
593 }
594 // If analysis components are used for multiple analysis drivers, then the
595 // parameters filename is tagged with program number, e.g. params.in.20.2
596 // provides the parameters for the 2nd analysis for the 20th fn eval.
597 if (multipleParamsFiles) { // append prog counter
598 for (size_t i=0; i<num_programs; ++i) {
599 std::string prog_num("." + std::to_string(i+1));
600 std::string tag_results_fname = resultsFileWritten + prog_num;
601 std::string tag_params_fname = paramsFileWritten + prog_num;
602 if (!allowExistingResults)
603 std::remove(tag_results_fname.c_str());
604 write_parameters_file(vars, set, response, programNames[i],
605 analysisComponents[i], tag_params_fname);
606 }
607 }
608 }
609
610
611 void ProcessApplicInterface::
write_parameters_file(const Variables & vars,const ActiveSet & set,const Response & response,const std::string & prog,const std::vector<String> & an_comps,const std::string & params_fname,const bool file_mode_out)612 write_parameters_file(const Variables& vars, const ActiveSet& set,
613 const Response& response, const std::string& prog,
614 const std::vector<String>& an_comps,
615 const std::string& params_fname,
616 const bool file_mode_out)
617 {
618 // Write the parameters file
619 std::ofstream parameter_stream;
620 if(file_mode_out) // params for one evaluation per file
621 parameter_stream.open(params_fname.c_str());
622 else // params for multiple evaluations per file (batch mode)
623 parameter_stream.open(params_fname.c_str(), std::ios_base::app);
624
625 using std::setw;
626 if (!parameter_stream) {
627 Cerr << "\nError: cannot create parameters file " << params_fname
628 << std::endl;
629 abort_handler(IO_ERROR);
630 }
631 StringMultiArrayConstView acv_labels = vars.all_continuous_variable_labels();
632 SizetMultiArrayConstView acv_ids = vars.all_continuous_variable_ids();
633 const ShortArray& asv = set.request_vector();
634 const SizetArray& dvv = set.derivative_vector();
635 const StringArray& resp_labels = response.function_labels();
636 size_t i, asv_len = asv.size(), dvv_len = dvv.size(),
637 ac_len = an_comps.size();
638 StringArray asv_labels(asv_len), dvv_labels(dvv_len), ac_labels(ac_len);
639 build_labels(asv_labels, "ASV_");
640 build_labels(dvv_labels, "DVV_");
641 build_labels(ac_labels, "AC_");
642 for (i=0; i<asv_len; ++i)
643 asv_labels[i] += ":" + resp_labels[i];
644 for (i=0; i<dvv_len; ++i) {
645 size_t acv_index = find_index(acv_ids, dvv[i]);
646 if (acv_index != _NPOS)
647 dvv_labels[i] += ":" + acv_labels[acv_index];
648 }
649 if (!prog.empty()) // empty string passed if multiple attributions possible
650 for (i=0; i<ac_len; ++i)
651 ac_labels[i] += ":" + prog; // attribution to particular program
652 int prec = write_precision; // for restoration
653 //write_precision = 16; // 17 total digits: full double precision
654 write_precision = 15; // 16 total digits: preserves desirable roundoff in
655 // last digit
656 int w = write_precision+7;
657 if (apreproFlag) {
658 std::string sp20(" ");
659 parameter_stream << sp20 << "{ DAKOTA_VARS = " << setw(w) << vars.tv()
660 << " }\n";
661 vars.write_aprepro(parameter_stream);
662 parameter_stream << sp20 << "{ DAKOTA_FNS = " << setw(w) << asv_len
663 << " }\n"; //<< setiosflags(ios::left);
664 array_write_aprepro(parameter_stream, asv, asv_labels);
665 parameter_stream << sp20 << "{ DAKOTA_DER_VARS = " << setw(w) << dvv_len
666 << " }\n";
667 array_write_aprepro(parameter_stream, dvv, dvv_labels);
668 parameter_stream << sp20 << "{ DAKOTA_AN_COMPS = " << setw(w) << ac_len
669 << " }\n";
670 array_write_aprepro(parameter_stream, an_comps, ac_labels);
671 // write full eval ID tag, without leading period, converting . to :
672 String full_eval_id(fullEvalId);
673 full_eval_id.erase(0,1);
674 boost::algorithm::replace_all(full_eval_id, String("."), String(":"));
675 parameter_stream << sp20 << "{ DAKOTA_EVAL_ID = " << setw(w)
676 << full_eval_id << " }\n";
677 //parameter_stream << resetiosflags(ios::adjustfield);
678 }
679 else {
680 std::string sp21(" ");
681 parameter_stream << sp21 << setw(w) << vars.tv() << " variables\n" << vars
682 << sp21 << setw(w) << asv_len << " functions\n";
683 //<< setiosflags(ios::left);
684 array_write(parameter_stream, asv, asv_labels);
685 parameter_stream << sp21 << setw(w) << dvv_len << " derivative_variables\n";
686 array_write(parameter_stream, dvv, dvv_labels);
687 parameter_stream << sp21 << setw(w) << ac_len << " analysis_components\n";
688 array_write(parameter_stream, an_comps, ac_labels);
689 // write full eval ID tag, without leading period
690 String full_eval_id(fullEvalId);
691 full_eval_id.erase(0,1);
692 boost::algorithm::replace_all(full_eval_id, String("."), String(":"));
693 parameter_stream << sp21 << setw(w) << full_eval_id << " eval_id\n";
694 //parameter_stream << resetiosflags(ios::adjustfield);
695 }
696 write_precision = prec; // restore
697
698 // Explicit flush and close added 3/96 to prevent Solaris problem of input
699 // filter reading file before the write was completed.
700 parameter_stream.flush();
701 parameter_stream.close();
702 }
703
704
705 void ProcessApplicInterface::
read_results_files(Response & response,const int id,const String & eval_id_tag)706 read_results_files(Response& response, const int id, const String& eval_id_tag)
707 {
708 // Retrieve parameters & results file names using fn. eval. id. A map of
709 // filenames is used because the names of tmp files must be available here
710 // and asynch_recv operations can perform output filtering out of order
711 // (which rules out making the file names into attributes of
712 // ProcessApplicInterface or rebuilding the file name from the root name
713 // plus the counter).
714 std::map<int, PathTriple>::iterator map_iter = fileNameMap.find(id);
715 const bfs::path& params_path = (map_iter->second).get<0>();
716 const bfs::path& results_path = (map_iter->second).get<1>();
717 const bfs::path& workdir_path = (map_iter->second).get<2>();
718
719 // If multiple analysis programs are used, then results_filename is tagged
720 // with program number (to match ProcessApplicInterface::spawn_evaluation)
721 // e.g. results.out.20.2 is the 2nd analysis results from the 20th fn eval.
722
723 // Read the results file(s). If there's more than one program, then partial
724 // responses must be overlaid to create the total response. If an output
725 // filter is used, then it has the responsibility to perform the overlay and
726 // map results.out.[eval#].[1->num_programs] to results.out.[eval#]. If no
727 // output filter is used, then perform the overlay here.
728 size_t num_programs = programNames.size();
729 if (num_programs > 1 && oFilterName.empty()) {
730 response.reset();
731 Response partial_response = response.copy();
732 for (size_t i=0; i<num_programs; ++i) {
733 const std::string prog_num("." + std::to_string(i+1));
734 bfs::path prog_tagged_results
735 = WorkdirHelper::concat_path(results_path, prog_num);
736 read_results_file(partial_response, prog_tagged_results, id);
737 response.overlay(partial_response);
738 }
739 }
740 else
741 read_results_file(response,results_path,id);
742
743 file_and_workdir_cleanup(params_path, results_path, workdir_path, eval_id_tag);
744 // Remove the evaluation which has been processed from the bookkeeping
745 fileNameMap.erase(map_iter);
746 }
747
748
read_results_file(Response & response,const bfs::path & results_path,const int id)749 void ProcessApplicInterface::read_results_file(Response &response,
750 const bfs::path &results_path,
751 const int id) {
752 /// Helper for read_results_files that opens the results file at
753 /// results_path and reads it, handling various errors/exceptions.
754 bfs::ifstream recovery_stream(results_path);
755 if (!recovery_stream) {
756 Cerr << "\nError: cannot open results file " << results_path
757 << " for evaluation " << std::to_string(id) << std::endl;
758 abort_handler(INTERFACE_ERROR); // will clean up files unless file_save was specified
759 }
760 try {
761 response.read(recovery_stream,resultsFileFormat);
762 }
763 catch(const FileReadException& fr_except) {
764 throw FileReadException("Error(s) encountered reading results file " +
765 results_path.string() + " for Evaluation " +
766 std::to_string(id) + ":\n" + fr_except.what());
767 }
768 }
769
770
771 /** Move specified params and results files to unique tagged versions
772 when needed */
autotag_files(const bfs::path & params_path,const bfs::path & results_path,const String & eval_id_tag) const773 void ProcessApplicInterface::autotag_files(const bfs::path& params_path,
774 const bfs::path& results_path,
775 const String& eval_id_tag
776 //, const bfs::path dest_dir
777 ) const
778
779 {
780 size_t num_programs = programNames.size();
781
782 if ((!specifiedParamsFileName.empty() || !specifiedParamsFileName.empty()) &&
783 !suppressOutput && outputLevel > NORMAL_OUTPUT)
784 Cout << "Files with nonunique names will be tagged for file_save:\n";
785
786 if (!specifiedParamsFileName.empty()) {
787 bfs::path eval_tagged_params =
788 WorkdirHelper::concat_path(params_path, eval_id_tag);
789
790 if (!multipleParamsFiles || !iFilterName.empty()) {
791 if (!suppressOutput && outputLevel > NORMAL_OUTPUT)
792 Cout << "Moving " << params_path << " to " << eval_tagged_params << '\n';
793 WorkdirHelper::rename(params_path, eval_tagged_params, FILEOP_WARN);
794 }
795 if (multipleParamsFiles) { // append program counters to old/new strings
796 for (size_t i=0; i<num_programs; ++i) {
797 const std::string prog_num("." + std::to_string(i+1));
798 const bfs::path prog_tagged_old =
799 WorkdirHelper::concat_path(params_path, prog_num);
800 const bfs::path eval_prog_tagged_new =
801 WorkdirHelper::concat_path(eval_tagged_params, prog_num);
802 if (!suppressOutput && outputLevel > NORMAL_OUTPUT)
803 Cout << "Moving " << prog_tagged_old << " to " << eval_prog_tagged_new
804 << '\n';
805 WorkdirHelper::
806 rename(prog_tagged_old, eval_prog_tagged_new, FILEOP_WARN);
807 }
808 }
809 }
810
811 if (!specifiedResultsFileName.empty()) {
812 bfs::path eval_tagged_results =
813 WorkdirHelper::concat_path(results_path, eval_id_tag);
814
815 if (num_programs == 1 || !oFilterName.empty()) {
816 if (!suppressOutput && outputLevel > NORMAL_OUTPUT)
817 Cout << "Moving " << results_path << " to "
818 << eval_tagged_results << '\n';
819 WorkdirHelper::rename(results_path, eval_tagged_results, FILEOP_WARN);
820 }
821 if (num_programs > 1) { // append program counters to old/new strings
822 for (size_t i=0; i<num_programs; ++i) {
823 const std::string prog_num("." + std::to_string(i+1));
824 const bfs::path prog_tagged_old =
825 WorkdirHelper::concat_path(results_path, prog_num);
826 const bfs::path eval_prog_tagged_new =
827 WorkdirHelper::concat_path(eval_tagged_results, prog_num);
828 if (!suppressOutput && outputLevel > NORMAL_OUTPUT)
829 Cout << "Moving " << prog_tagged_old << " to " << eval_prog_tagged_new
830 << '\n';
831 WorkdirHelper::rename(prog_tagged_old, eval_prog_tagged_new, FILEOP_WARN);
832 }
833 }
834 }
835
836 }
837
838 void ProcessApplicInterface::
remove_params_results_files(const bfs::path & params_path,const bfs::path & results_path) const839 remove_params_results_files(const bfs::path& params_path,
840 const bfs::path& results_path) const
841 {
842 size_t num_programs = programNames.size();
843
844 if (!suppressOutput && outputLevel > NORMAL_OUTPUT) {
845 Cout << "Removing " << params_path;
846 if (multipleParamsFiles) {
847 if (!iFilterName.empty())
848 Cout << " and " << params_path;
849 Cout << ".[1-" << num_programs << ']';
850 }
851 Cout << " and " << results_path;
852 if (num_programs > 1) {
853 if (!oFilterName.empty())
854 Cout << " and " << results_path;
855 Cout << ".[1-" << num_programs << ']';
856 }
857 Cout << '\n';
858 }
859
860 if (!multipleParamsFiles || !iFilterName.empty())
861 WorkdirHelper::recursive_remove(params_path, FILEOP_WARN);
862
863 if (multipleParamsFiles) {
864 for (size_t i=0; i<num_programs; ++i) {
865 const std::string prog_num("." + std::to_string(i+1));
866 const bfs::path tagged_params =
867 WorkdirHelper::concat_path(params_path, prog_num);
868 WorkdirHelper::recursive_remove(tagged_params, FILEOP_WARN);
869 }
870 }
871
872 if (num_programs == 1 || !oFilterName.empty())
873 WorkdirHelper::recursive_remove(results_path, FILEOP_WARN);
874
875 if (num_programs > 1)
876 for (size_t i=0; i<num_programs; ++i) {
877 const std::string prog_num("." + std::to_string(i+1));
878 const bfs::path tagged_results =
879 WorkdirHelper::concat_path(results_path, prog_num);
880 WorkdirHelper::recursive_remove(tagged_results, FILEOP_WARN);
881 }
882 }
883
884
885 /** Remove any files and directories still referenced in the fileNameMap */
file_cleanup() const886 void ProcessApplicInterface::file_cleanup() const
887 {
888 if (fileSaveFlag && dirSave)
889 return;
890
891 std::map<int, PathTriple>::const_iterator
892 file_name_map_it = fileNameMap.begin(),
893 file_name_map_end = fileNameMap.end();
894 for(; file_name_map_it != file_name_map_end; ++file_name_map_it) {
895 const bfs::path& parfile = (file_name_map_it->second).get<0>();
896 const bfs::path& resfile = (file_name_map_it->second).get<1>();
897 const bfs::path& wd_path = (file_name_map_it->second).get<2>();
898 if (!fileSaveFlag) {
899 if (!multipleParamsFiles || !iFilterName.empty()) {
900 WorkdirHelper::recursive_remove(parfile, FILEOP_SILENT);
901 WorkdirHelper::recursive_remove(resfile, FILEOP_SILENT);
902 }
903 if (multipleParamsFiles) {
904 size_t i, num_programs = programNames.size();
905 for(i=1; i<=num_programs; ++i) {
906 std::string prog_num("." + std::to_string(i));
907 bfs::path pname = WorkdirHelper::concat_path(parfile, prog_num);
908 WorkdirHelper::recursive_remove(pname, FILEOP_SILENT);
909 bfs::path rname = WorkdirHelper::concat_path(resfile, prog_num);
910 WorkdirHelper::recursive_remove(rname, FILEOP_SILENT);
911 }
912 }
913 }
914 // a non-empty entry here indicates the directory was created for this eval
915 if (!dirSave && !wd_path.empty())
916 WorkdirHelper::recursive_remove(wd_path, FILEOP_SILENT);
917 }
918 }
919
920
921 // get the current work directory name
get_workdir_name()922 bfs::path ProcessApplicInterface::get_workdir_name()
923 {
924 // PDH suggets making in rundir instead of tmp area...
925 bfs::path wd_name = workDirName.empty() ?
926 ( WorkdirHelper::system_tmp_path() /
927 WorkdirHelper::system_tmp_file("dakota_work") ) :
928 workDirName;
929
930 // we allow tagging of tmp dirs in case the user's script needs the tag
931 if (dirTag)
932 return WorkdirHelper::concat_path(wd_name, fullEvalId);
933
934 return wd_name;
935 }
936
937
938 /** Guidance: environment (PATH, current directory) should be set
939 immediately before Dakota spawns a process and reset immediately
940 afterwards (except fork which never returns) */
prepare_process_environment()941 void ProcessApplicInterface::prepare_process_environment()
942 {
943 // If not using workdir, just put . and startupPWD on PATH. If
944 // using workdir, also put the absolute path to the workdir on the
945 // PATH (. should suffice; this is conservative). It doesn't help
946 // to prepend a relative workdir path, since we will change
947 // directory into it, so the helper makes the path absolute.
948 if (useWorkdir) {
949 if (outputLevel >= DEBUG_OUTPUT)
950 Cout << "Prepending environment PATH with work_directory "
951 << curWorkdir << "." << std::endl;
952 WorkdirHelper::set_preferred_path(curWorkdir);
953 if (outputLevel >= VERBOSE_OUTPUT)
954 Cout << "Changing directory to " << curWorkdir << std::endl;
955 WorkdirHelper::change_directory(curWorkdir);
956 }
957 else
958 WorkdirHelper::set_preferred_path();
959
960 WorkdirHelper::set_environment("DAKOTA_PARAMETERS_FILE", paramsFileName);
961 WorkdirHelper::set_environment("DAKOTA_RESULTS_FILE", resultsFileName);
962 }
963
964
965 /** Undo anything done prior to spawn */
reset_process_environment()966 void ProcessApplicInterface::reset_process_environment()
967 {
968 // BMA TODO: consider unsetting environment variables previously set
969
970 // No need to reset in non-workdir case, as long as PATH doesn't get
971 // multiply appended to
972 if (useWorkdir) {
973 if (outputLevel >= VERBOSE_OUTPUT)
974 Cout << "Changing directory back to " << WorkdirHelper::startup_pwd()
975 << std::endl;
976 if (outputLevel >= DEBUG_OUTPUT)
977 Cout << "Resetting environment PATH." << std::endl;
978 WorkdirHelper::reset();
979 }
980 }
981
982 void ProcessApplicInterface::
file_and_workdir_cleanup(const bfs::path & params_path,const bfs::path & results_path,const bfs::path & workdir_path,const String & tag) const983 file_and_workdir_cleanup(const bfs::path ¶ms_path,
984 const bfs::path &results_path,
985 const bfs::path &workdir_path,
986 const String &tag) const
987 {
988
989 // remove the workdir if in the map and we're not saving
990 bool removing_workdir = (!workdir_path.empty() && !dirSave);
991 if (fileSaveFlag) {
992
993 // Prevent overwriting of files with reused names for which a file_save
994 // request has been given. Assume tmp files always unique.
995
996 // Cases (work in progress):
997 // * no workdir --> tag if needed (old behavior)
998 // * workdir, shared, saved --> tag if needed
999 // * a workdir can be unique via dir_tag or tmp files
1000 // - workdir, unique, saved --> done no tag
1001 // - workdir, not saved --> tag and move to rundir
1002 // * when workdir and absolute path to files tag in abs path
1003
1004 if ( useWorkdir ) {
1005 if ( dirSave ) {
1006 // if saving the directory, just need to make sure filenames are unique
1007 // use legacy tagging mechanism within the workdir
1008 // TODO: don't need to tag if workdir is unique per eval...
1009 if (!fileTagFlag && !dirTag && !workDirName.empty())
1010 autotag_files(params_path, results_path, tag);
1011 }
1012 else {
1013 // work_directory getting removed; unique tag the files into the rundir
1014 // take the filename off the path and use with rundir
1015 // TODO: do we even need to support this?
1016 // TODO: distinguish between params in vs. not in (absolute
1017 // path) the workdir
1018 // autotag_files(params_path, results_path, eval_id_tag,
1019 // bfs::current_path());
1020 }
1021 }
1022 else {
1023 // simple case; no workdir --> old behavior, tagging if needed
1024 // in place (whether relative or absolute)
1025 if (!fileTagFlag)
1026 autotag_files(params_path, results_path, tag);
1027 }
1028 }
1029 else
1030 remove_params_results_files(params_path, results_path);
1031
1032 // Now that files are handled, conditionally remove the work directory
1033 if (removing_workdir) {
1034 if (outputLevel > NORMAL_OUTPUT)
1035 Cout << "Removing work_directory " << workdir_path << std::endl;
1036 WorkdirHelper::recursive_remove(workdir_path, FILEOP_ERROR);
1037 }
1038
1039 }
1040
1041 } // namespace Dakota
1042