1 /*
2 * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
3 * University Research and Technology
4 * Corporation. All rights reserved.
5 * Copyright (c) 2004-2008 The University of Tennessee and The University
6 * of Tennessee Research Foundation. All rights
7 * reserved.
8 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9 * University of Stuttgart. All rights reserved.
10 * Copyright (c) 2004-2005 The Regents of the University of California.
11 * All rights reserved.
12 * Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
13 * Copyright (c) 2016-2017 Intel, Inc. All rights reserved.
14 * Copyright (c) 2017 IBM Corporation. All rights reserved.
15 * Copyright (c) 2017 Research Organization for Information Science
16 * and Technology (RIST). All rights reserved.
17 * $COPYRIGHT$
18 *
19 * Additional copyrights may follow
20 *
21 * $HEADER$
22 *
23 * These symbols are in a file by themselves to provide nice linker
24 * semantics. Since linkers generally pull in symbols by object
25 * files, keeping these symbols as the only symbols in this file
26 * prevents utility programs such as "ompi_info" from having to import
27 * entire components just to query their version and parameters.
28 */
29
30 #include "orte_config.h"
31 #include "orte/constants.h"
32
33 #include <stdlib.h>
34 #ifdef HAVE_UNISTD_H
35 #include <unistd.h>
36 #endif
37 #include <errno.h>
38 #include <sys/types.h>
39 #ifdef HAVE_SYS_WAIT_H
40 #include <sys/wait.h>
41 #endif
42 #include <signal.h>
43 #ifdef HAVE_UTIL_H
44 #include <util.h>
45 #endif
46 #ifdef HAVE_PTY_H
47 #include <pty.h>
48 #endif
49 #ifdef HAVE_FCNTL_H
50 #include <fcntl.h>
51 #endif
52 #ifdef HAVE_TERMIOS_H
53 #include <termios.h>
54 # ifdef HAVE_TERMIO_H
55 # include <termio.h>
56 # endif
57 #endif
58 #ifdef HAVE_LIBUTIL_H
59 #include <libutil.h>
60 #endif
61
62 #include "opal/util/opal_pty.h"
63 #include "opal/util/opal_environ.h"
64 #include "opal/util/os_dirpath.h"
65 #include "opal/util/output.h"
66 #include "opal/util/argv.h"
67
68 #include "orte/mca/errmgr/errmgr.h"
69 #include "orte/util/name_fns.h"
70 #include "orte/runtime/orte_globals.h"
71
72 #include "orte/mca/iof/iof.h"
73 #include "orte/mca/iof/base/base.h"
74 #include "orte/mca/iof/base/iof_base_setup.h"
75
76 int
orte_iof_base_setup_prefork(orte_iof_base_io_conf_t * opts)77 orte_iof_base_setup_prefork(orte_iof_base_io_conf_t *opts)
78 {
79 int ret = -1;
80
81 fflush(stdout);
82
83 /* first check to make sure we can do ptys */
84 #if OPAL_ENABLE_PTY_SUPPORT
85 if (opts->usepty) {
86 /**
87 * It has been reported that on MAC OS X 10.4 and prior one cannot
88 * safely close the writing side of a pty before completly reading
89 * all data inside.
90 * There seems to be two issues: first all pending data is
91 * discarded, and second it randomly generate kernel panics.
92 * Apparently this issue was fixed in 10.5 so by now we use the
93 * pty exactly as we use the pipes.
94 * This comment is here as a reminder.
95 */
96 ret = opal_openpty(&(opts->p_stdout[0]), &(opts->p_stdout[1]),
97 (char*)NULL, (struct termios*)NULL, (struct winsize*)NULL);
98 }
99 #else
100 opts->usepty = 0;
101 #endif
102
103 if (ret < 0) {
104 opts->usepty = 0;
105 if (pipe(opts->p_stdout) < 0) {
106 ORTE_ERROR_LOG(ORTE_ERR_SYS_LIMITS_PIPES);
107 return ORTE_ERR_SYS_LIMITS_PIPES;
108 }
109 }
110 if (opts->connect_stdin) {
111 if (pipe(opts->p_stdin) < 0) {
112 ORTE_ERROR_LOG(ORTE_ERR_SYS_LIMITS_PIPES);
113 return ORTE_ERR_SYS_LIMITS_PIPES;
114 }
115 }
116 if( !orte_iof_base.redirect_app_stderr_to_stdout ) {
117 if (pipe(opts->p_stderr) < 0) {
118 ORTE_ERROR_LOG(ORTE_ERR_SYS_LIMITS_PIPES);
119 return ORTE_ERR_SYS_LIMITS_PIPES;
120 }
121 }
122 #if OPAL_PMIX_V1
123 if (pipe(opts->p_internal) < 0) {
124 ORTE_ERROR_LOG(ORTE_ERR_SYS_LIMITS_PIPES);
125 return ORTE_ERR_SYS_LIMITS_PIPES;
126 }
127 #endif
128 return ORTE_SUCCESS;
129 }
130
131
132 int
orte_iof_base_setup_child(orte_iof_base_io_conf_t * opts,char *** env)133 orte_iof_base_setup_child(orte_iof_base_io_conf_t *opts, char ***env)
134 {
135 int ret;
136 #if OPAL_PMIX_V1
137 char *str;
138 #endif
139
140 if (opts->connect_stdin) {
141 close(opts->p_stdin[1]);
142 }
143 close(opts->p_stdout[0]);
144 if( !orte_iof_base.redirect_app_stderr_to_stdout ) {
145 close(opts->p_stderr[0]);
146 }
147 #if OPAL_PMIX_V1
148 close(opts->p_internal[0]);
149 #endif
150
151 if (opts->usepty) {
152 /* disable echo */
153 struct termios term_attrs;
154 if (tcgetattr(opts->p_stdout[1], &term_attrs) < 0) {
155 return ORTE_ERR_PIPE_SETUP_FAILURE;
156 }
157 term_attrs.c_lflag &= ~ (ECHO | ECHOE | ECHOK |
158 ECHOCTL | ECHOKE | ECHONL);
159 term_attrs.c_iflag &= ~ (ICRNL | INLCR | ISTRIP | INPCK | IXON);
160 term_attrs.c_oflag &= ~ (
161 #ifdef OCRNL
162 /* OS X 10.3 does not have this
163 value defined */
164 OCRNL |
165 #endif
166 ONLCR);
167 if (tcsetattr(opts->p_stdout[1], TCSANOW, &term_attrs) == -1) {
168 return ORTE_ERR_PIPE_SETUP_FAILURE;
169 }
170 ret = dup2(opts->p_stdout[1], fileno(stdout));
171 if (ret < 0) {
172 return ORTE_ERR_PIPE_SETUP_FAILURE;
173 }
174 if( orte_iof_base.redirect_app_stderr_to_stdout ) {
175 ret = dup2(opts->p_stdout[1], fileno(stderr));
176 if (ret < 0) {
177 return ORTE_ERR_PIPE_SETUP_FAILURE;
178 }
179 }
180 close(opts->p_stdout[1]);
181 } else {
182 if(opts->p_stdout[1] != fileno(stdout)) {
183 ret = dup2(opts->p_stdout[1], fileno(stdout));
184 if (ret < 0) {
185 return ORTE_ERR_PIPE_SETUP_FAILURE;
186 }
187 if( orte_iof_base.redirect_app_stderr_to_stdout ) {
188 ret = dup2(opts->p_stdout[1], fileno(stderr));
189 if (ret < 0) {
190 return ORTE_ERR_PIPE_SETUP_FAILURE;
191 }
192 }
193 close(opts->p_stdout[1]);
194 }
195 }
196 if (opts->connect_stdin) {
197 if(opts->p_stdin[0] != fileno(stdin)) {
198 ret = dup2(opts->p_stdin[0], fileno(stdin));
199 if (ret < 0) {
200 return ORTE_ERR_PIPE_SETUP_FAILURE;
201 }
202 close(opts->p_stdin[0]);
203 }
204 } else {
205 int fd;
206
207 /* connect input to /dev/null */
208 fd = open("/dev/null", O_RDONLY, 0);
209 if(fd != fileno(stdin)) {
210 dup2(fd, fileno(stdin));
211 close(fd);
212 }
213 }
214
215 if(opts->p_stderr[1] != fileno(stderr)) {
216 if( !orte_iof_base.redirect_app_stderr_to_stdout ) {
217 ret = dup2(opts->p_stderr[1], fileno(stderr));
218 if (ret < 0) return ORTE_ERR_PIPE_SETUP_FAILURE;
219 close(opts->p_stderr[1]);
220 }
221 }
222
223 #if OPAL_PMIX_V1
224 if (!orte_map_stddiag_to_stderr && !orte_map_stddiag_to_stdout ) {
225 /* Set an environment variable that the new child process can use
226 to get the fd of the pipe connected to the INTERNAL IOF tag. */
227 asprintf(&str, "%d", opts->p_internal[1]);
228 if (NULL != str) {
229 opal_setenv("OPAL_OUTPUT_STDERR_FD", str, true, env);
230 free(str);
231 }
232 } else if( orte_map_stddiag_to_stdout ) {
233 opal_setenv("OPAL_OUTPUT_INTERNAL_TO_STDOUT", "1", true, env);
234 }
235 #endif
236
237 return ORTE_SUCCESS;
238 }
239
240
241 int
orte_iof_base_setup_parent(const orte_process_name_t * name,orte_iof_base_io_conf_t * opts)242 orte_iof_base_setup_parent(const orte_process_name_t* name,
243 orte_iof_base_io_conf_t *opts)
244 {
245 int ret;
246
247 /* connect stdin endpoint */
248 if (opts->connect_stdin) {
249 /* and connect the pty to stdin */
250 ret = orte_iof.pull(name, ORTE_IOF_STDIN, opts->p_stdin[1]);
251 if(ORTE_SUCCESS != ret) {
252 ORTE_ERROR_LOG(ret);
253 return ret;
254 }
255 }
256
257 /* connect read ends to IOF */
258 ret = orte_iof.push(name, ORTE_IOF_STDOUT, opts->p_stdout[0]);
259 if(ORTE_SUCCESS != ret) {
260 ORTE_ERROR_LOG(ret);
261 return ret;
262 }
263
264 if( !orte_iof_base.redirect_app_stderr_to_stdout ) {
265 ret = orte_iof.push(name, ORTE_IOF_STDERR, opts->p_stderr[0]);
266 if(ORTE_SUCCESS != ret) {
267 ORTE_ERROR_LOG(ret);
268 return ret;
269 }
270 }
271
272 #if OPAL_PMIX_V1
273 ret = orte_iof.push(name, ORTE_IOF_STDDIAG, opts->p_internal[0]);
274 if(ORTE_SUCCESS != ret) {
275 ORTE_ERROR_LOG(ret);
276 return ret;
277 }
278 #endif
279
280 return ORTE_SUCCESS;
281 }
282
orte_iof_base_setup_output_files(const orte_process_name_t * dst_name,orte_job_t * jobdat,orte_iof_proc_t * proct)283 int orte_iof_base_setup_output_files(const orte_process_name_t* dst_name,
284 orte_job_t *jobdat,
285 orte_iof_proc_t *proct)
286 {
287 int rc;
288 char *dirname, *outdir, *outfile;
289 int np, numdigs, fdout, i;
290 char *p, **s;
291 bool usejobid = true;
292
293 /* see if we are to output to a file */
294 dirname = NULL;
295 if (orte_get_attribute(&jobdat->attributes, ORTE_JOB_OUTPUT_TO_FILE, (void**)&dirname, OPAL_STRING) &&
296 NULL != dirname) {
297 np = jobdat->num_procs / 10;
298 /* determine the number of digits required for max vpid */
299 numdigs = 1;
300 while (np > 0) {
301 numdigs++;
302 np = np / 10;
303 }
304 /* check for a conditional in the directory name */
305 if (NULL != (p = strchr(dirname, ':'))) {
306 *p = '\0';
307 ++p;
308 /* could me more than one directive */
309 s = opal_argv_split(p, ',');
310 for (i=0; NULL != s[i]; i++) {
311 if (0 == strcasecmp(s[i], "nojobid")) {
312 usejobid = false;
313 } else if (0 == strcasecmp(s[i], "nocopy")) {
314 proct->copy = false;
315 }
316 }
317 }
318
319 /* construct the directory where the output files will go */
320 if (usejobid) {
321 asprintf(&outdir, "%s/%d/rank.%0*lu", dirname,
322 (int)ORTE_LOCAL_JOBID(proct->name.jobid),
323 numdigs, (unsigned long)proct->name.vpid);
324 } else {
325 asprintf(&outdir, "%s/rank.%0*lu", dirname,
326 numdigs, (unsigned long)proct->name.vpid);
327 }
328 /* ensure the directory exists */
329 if (OPAL_SUCCESS != (rc = opal_os_dirpath_create(outdir, S_IRWXU|S_IRGRP|S_IXGRP))) {
330 ORTE_ERROR_LOG(rc);
331 free(outdir);
332 return rc;
333 }
334 if (NULL != proct->revstdout && NULL == proct->revstdout->sink) {
335 /* setup the stdout sink */
336 asprintf(&outfile, "%s/stdout", outdir);
337 fdout = open(outfile, O_CREAT|O_RDWR|O_TRUNC, 0644);
338 free(outfile);
339 if (fdout < 0) {
340 /* couldn't be opened */
341 ORTE_ERROR_LOG(ORTE_ERR_FILE_OPEN_FAILURE);
342 return ORTE_ERR_FILE_OPEN_FAILURE;
343 }
344 /* define a sink to that file descriptor */
345 ORTE_IOF_SINK_DEFINE(&proct->revstdout->sink, dst_name,
346 fdout, ORTE_IOF_STDOUT,
347 orte_iof_base_write_handler);
348 }
349
350 if (NULL != proct->revstderr && NULL == proct->revstderr->sink) {
351 /* if they asked for stderr to be combined with stdout, then we
352 * only create one file and tell the IOF to put both streams
353 * into it. Otherwise, we create separate files for each stream */
354 if (orte_get_attribute(&jobdat->attributes, ORTE_JOB_MERGE_STDERR_STDOUT, NULL, OPAL_BOOL)) {
355 /* just use the stdout sink */
356 OBJ_RETAIN(proct->revstdout->sink);
357 proct->revstdout->sink->tag = ORTE_IOF_STDMERGE; // show that it is merged
358 proct->revstderr->sink = proct->revstdout->sink;
359 } else {
360 asprintf(&outfile, "%s/stderr", outdir);
361 fdout = open(outfile, O_CREAT|O_RDWR|O_TRUNC, 0644);
362 free(outfile);
363 if (fdout < 0) {
364 /* couldn't be opened */
365 ORTE_ERROR_LOG(ORTE_ERR_FILE_OPEN_FAILURE);
366 return ORTE_ERR_FILE_OPEN_FAILURE;
367 }
368 /* define a sink to that file descriptor */
369 ORTE_IOF_SINK_DEFINE(&proct->revstderr->sink, dst_name,
370 fdout, ORTE_IOF_STDERR,
371 orte_iof_base_write_handler);
372 }
373 }
374 #if OPAL_PMIX_V1
375 if (NULL != proct->revstddiag && NULL == proct->revstddiag->sink) {
376 /* always tie the sink for stddiag to stderr */
377 OBJ_RETAIN(proct->revstderr->sink);
378 proct->revstddiag->sink = proct->revstderr->sink;
379 }
380 #endif
381 }
382
383 return ORTE_SUCCESS;
384 }
385