1 /* begin_generated_IBM_copyright_prolog */
2 /* */
3 /* This is an automatically generated copyright prolog. */
4 /* After initializing, DO NOT MODIFY OR MOVE */
5 /* --------------------------------------------------------------- */
6 /* Licensed Materials - Property of IBM */
7 /* Blue Gene/Q 5765-PER 5765-PRP */
8 /* */
9 /* (C) Copyright IBM Corp. 2011, 2012 All Rights Reserved */
10 /* US Government Users Restricted Rights - */
11 /* Use, duplication, or disclosure restricted */
12 /* by GSA ADP Schedule Contract with IBM Corp. */
13 /* */
14 /* --------------------------------------------------------------- */
15 /* */
16 /* end_generated_IBM_copyright_prolog */
17 /* (C)Copyright IBM Corp. 2007, 2011 */
18 /**
19 * \file src/mpidi_env.c
20 * \brief Read Env Vars
21 */
22
23 #include <mpidimpl.h>
24
25 /** \page env_vars Environment Variables
26 ***************************************************************************
27 * Pt2Pt Options *
28 ***************************************************************************
29 *
30 * - PAMID_EAGER -
31 * - PAMID_RZV - Sets the cutoff for the switch to the rendezvous protocol.
32 * The two options are identical. This takes an argument, in bytes,
33 * to switch from the eager protocol to the rendezvous protocol for point-to-point
34 * messaging. Increasing the limit might help for larger partitions
35 * and if most of the communication is nearest neighbor.
36 * - Default is 2049 bytes.
37 *
38 * - PAMID_EAGER_LOCAL -
39 * - PAMID_RZV_LOCAL - Sets the cutoff for the switch to the rendezvous protocol
40 * when the destination rank is local. The two options are identical. This
41 * takes an argument, in bytes, to switch from the eager protocol to the
42 * rendezvous protocol for point-to-point messaging. The default value
43 * effectively disables the eager protocol for local transfers.
44 * - Default is 0 bytes.
45 *
46 * - PAMID_RMA_PENDING - Maximum outstanding RMA requests.
47 * Limits number of PAMI_Request objects allocated by MPI Onesided operations.
48 * - Default is 1000.
49 *
50 * - PAMID_SHMEM_PT2PT - Determines if intranode point-to-point communication will
51 * use the optimized shared memory protocols.
52 * - Default is 1.
53 ***************************************************************************
54 * High-level Options *
55 ***************************************************************************
56 *
57 * - PAMID_THREAD_MULTIPLE - Specifies the messaging execution environment.
58 * It specifically selects whether there can be multiple independent
59 * communications occurring in parallel, driven by internal communications
60 * threads.
61 * - 0: The application threads drive the communications. No additional
62 * internal communications threads are used.
63 * - 1: There can be multiple independent communications occurring in
64 * parallel, driven by internal communications threads.
65 * Defaults to 0 when the application is linked with one of the "legacy"
66 * MPICH libraries (gcc.legacy, xl.legacy, xl.legacy.ndebug), or when
67 * MPI_Init_thread() is called without MPI_THREAD_MULTIPLE, and defaults
68 * to 1 when the application is linked with one of the "non-legacy" MPICH
69 * libraries (gcc, xl, xl.ndebug) and MPI_Init_thread() is called with
70 * MPI_THREAD_MULTIPLE.
71 * - NOTE: This environment variable has the same effect as setting
72 * MPICH_THREADLEVEL_DEFAULT=multiple
73 *
74 * - PAMID_CONTEXT_MAX - This variable sets the maximum allowable number
75 * of contexts. Contexts are a method of dividing hardware resources
76 * among a parallel active messaging interface (PAMI) client (for example,
77 * MPI) to set how many parallel operations can occur at one time. Contexts
78 * are similar to channels in a communications system. The practical
79 * maximum is usually 64 processes per node. The default depends on the
80 * processes per node.
81 *
82 * - PAMID_CONTEXT_POST - This variable is required to take advantage of
83 * the parallelism of multiple contexts. This variable can increase
84 * latency, but it is the only way to allow parallelism among contexts.
85 * - 0: Only one parallel communications context can be used. Each
86 * operation runs in the application thread.
87 * - 1: Multiple parallel communications contexts can be used. An
88 * operation is posted to one of the contexts, and communications
89 * for that context are driven by communications threads.
90 * The default value is 1 when using the gcc, xl, and xl.ndebug libraries
91 * and MPI_Init_thread(... MPI_THREAD_MULTIPLE ...), and 0 otherwise.
92 *
93 * - PAMID_ASYNC_PROGRESS - This variable enables or disables the async
94 * progress extension.
95 *
96 * - PAMID_COLLECTIVES - Controls whether optimized collectives are used.
97 * Possible values:
98 * - 0 - Optimized collectives are not used.
99 * - 1 - Optimized collectives are used.
100 * If this is set to 0, only MPICH point-to-point based collectives will be used.
101 *
102 * - PAMID_COLLECTIVES_SELECTION - Turns on optimized collective selection. If this
103 * is not on, only the generic PGAS "always works" algorithms will be selected. This
104 * can still be better than PAMID_COLLECTIVES=0. Additionally, setting this off
105 * still allows environment variable selection of individual collectives protocols.
106 * - 0 - Optimized collective selection is not used.
107 * - 1 - Optimized collective selection is used. (default)
108 *
109 * - PAMID_VERBOSE - Increases the amount of information dumped during an
110 * MPI_Abort() call and during varoius MPI function calls. Possible values:
111 * - 0 - No additional information is dumped.
112 * - 1 - Useful information is printed from MPI_Init(). This option does
113 * NOT impact performance (other than a tiny bit during MPI_Init()) and
114 * is highly recommended for all applications. It will display which PAMID_
115 * and BG_ env vars were used. However, it does NOT guarantee you typed the
116 * env vars in correctly :)
117 * - 2 - This turns on a lot of verbose output for collective selection,
118 * including a list of which protocols are viable for each communicator
119 * creation. This can be a lot of output, but typically only at
120 * communicator creation. Additionally, if PAMID_STATISTICS are on,
121 * queue depths for point-to-point operations will be printed for each node
122 * during MPI_Finalize();
123 * - 3 - This turns on a lot of per-node information (plus everything
124 * at the verbose=2 level). This can be a lot of information and is
125 * rarely recommended.
126 * - Default is 0. However, setting to 1 is recommended.
127 *
128 * - PAMID_STATISTICS - Turns on statistics printing for the message layer
129 * such as the maximum receive queue depth. Possible values:
130 * - 0 - No statistics are printedcmf_bcas.
131 * - 1 - Statistics are printed.
132 * - Default is 0.
133 *
134 ***************************************************************************
135 ***************************************************************************
136 ** Collective Options **
137 ***************************************************************************
138 ***************************************************************************
139 *
140 ***************************************************************************
141 * General Collectives Options *
142 ***************************************************************************
143 *
144 * - PAMID_NUMREQUESTS - Sets the number of outstanding asynchronous
145 * broadcasts to have before a barrier is called. This is mostly
146 * used in allgather/allgatherv using asynchronous broadcasts.
147 * Higher numbers can help on larger partitions and larger
148 * message sizes. This is also used for asynchronous broadcasts.
149 * After every {PAMID_NUMREQUESTS} async bcasts, the "glue" will call
150 * a barrier. See PAMID_BCAST and PAMID_ALLGATHER(V) for more information
151 * - Default is 32.
152 *
153 ***************************************************************************
154 * "Safety" Options *
155 ***************************************************************************
156 *
157 * - PAMID_SAFEALLGATHER - Some optimized allgather protocols require
158 * contiguous datatypes and similar datatypes on all nodes. To verify
159 * this is true, we must do an allreduce at the beginning of the
160 * allgather call. If the application uses "well behaved" datatypes, you can
161 * set this option to skip over the allreduce. This is most useful in
162 * irregular subcommunicators where the allreduce can be expensive and in
163 * applications calling MPI_Allgather() with simple/regular datatypes.
164 * Datatypes must also be 16-byte aligned to use the optimized
165 * broadcast-based allgather for larger allgather sendcounts. See
166 * PAMID_PREALLREDUCE for more options
167 * Possible values:
168 * - N - Perform the preallreduce.
169 * - Y - Skip the preallreduce. Setting this with "unsafe" datatypes will
170 * yield unpredictable results, usually hangs.
171 * - Default is N.
172 *
173 * - PAMID_SAFEALLGATHERV - The optimized allgatherv protocols require
174 * contiguous datatypes and similar datatypes on all nodes. Allgatherv
175 * also requires continuous displacements. To verify
176 * this is true, we must do an allreduce at the beginning of the
177 * allgatherv call. If the application uses "well behaved" datatypes and
178 * displacements, you can set this option to skip over the allreduce.
179 * This is most useful in irregular subcommunicators where the allreduce
180 * can be expensive and in applications calling MPI_Allgatherv() with
181 * simple/regular datatypes.
182 * Datatypes must also be 16-byte aligned to use the optimized
183 * broadcast-based allgather for larger allgather sendcounts. See
184 * PAMID_PREALLREDUCE for more options
185 * Possible values:
186 * - N - Perform the allreduce.
187 * - Y - Skip the allreduce. Setting this with "unsafe" datatypes will
188 * yield unpredictable results, usually hangs.
189 * - Default is N.
190 *
191 * - PAMID_SAFESCATTERV - The optimized scatterv protocol requires
192 * contiguous datatypes and similar datatypes on all nodes. It
193 * also requires continuous displacements. To verify
194 * this is true, we must do an allreduce at the beginning of the
195 * scatterv call. If the application uses "well behaved" datatypes and
196 * displacements, you can set this option to skip over the allreduce.
197 * This is most useful in irregular subcommunicators where the allreduce
198 * can be expensive. We have seen more applications with strange datatypes
199 * passed to scatterv than allgather/allgatherv/bcast/allreduce, so it is
200 * more likely you need to leave this at the default setting. However, we
201 * encourage you to try turning this off and seeing if your application
202 * completes successfully.
203 * Possible values:
204 * - N - Perform the allreduce.
205 * - Y - Skip the allreduce. Setting this with "unsafe" datatypes will
206 * yield unpredictable results, usually hangs.
207 * - Default is N.
208 *
209 * - PAMID_PREALLREDUCE - Controls the protocol used for the pre-allreducing
210 * employed by allgather(v), and scatterv. This option
211 * is independant from PAMID_ALLREDUCE. The available protocols can be
212 * determined with PAMID_VERBOSE=2. If collective selection is on, we will
213 * attempt to use the fastest protocol for the given communicator.
214 *
215 ***************************************************************************
216 * Specific Collectives Switches *
217 ***************************************************************************
218 *
219 * - PAMID_ALLGATHER - Controls the protocol used for allgather.
220 * Possible values:
221 * - MPICH - Turn off all optimizations for allgather and use the MPICH
222 * point-to-point protocol. This can be very bad on larger partitions.
223 * - Other options can be determined by running with PAMID_VERBOSE=2.
224 * TODO: Implement the NO options
225 * NOTE: Individual options may be turned off by prefixing them with "NO".
226 * For example, PAMID_ALLGATHER=NO{specific protocol name} will prevent the
227 * use of the specified protocol.
228 *
229 * - PAMID_ALLGATHERV - Controls the protocol used for allgatherv.
230 * Possible values:
231 * - MPICH - Turn off all optimizations for allgather and use the MPICH
232 * point-to-point protocol. This can be very bad on larger partitions.
233 * - Other options can be determined by running with PAMID_VERBOSE=2.
234 * TODO: Implement the NO options
235 * NOTE: Individual options may be turned off by prefixing them with "NO".
236 * For example, PAMID_ALLGATHERV=NO{specific protocol name} will prevent the
237 * use of the specified protocol.
238 *
239 * - PAMID_ALLREDUCE - Controls the protocol used for allreduce.
240 * Possible values:
241 * - MPICH - Turn off all optimizations for allreduce and use the MPICH
242 * point-to-point protocol.
243 * - Other options can be determined by running with PAMID_VERBOSE=2.
244 * Note: Many allreduce algorithms are in the "must query" category and
245 * might or might not be available for a specific callsite.
246 * TODO: Implement the NO options
247 * NOTE: Individual options may be turned off by prefixing them with "NO".
248 * For example, PAMID_ALLREDUCE=NO{specific protocol name} will prevent the
249 * use of the specified protocol.
250 *
251 * - PAMID_ALLTOALL -
252 * - PAMID_ALLTOALLV - Controls the protocol used for
253 * alltoall/alltoallv Possible values:
254 * - MPICH - Turn off all optimizations and use the MPICH
255 * point-to-point protocol.
256 * - Other options can be determined by running with PAMID_VERBOSE=2.
257 * TODO: Implement the NO options
258 * NOTE: Individual options may be turned off by prefixing them with "NO".
259 * For example, PAMID_ALLTOALL=NO{specific protocol name} will prevent the
260 * use of the specified protocol.
261 *
262 *
263 * - PAMID_BARRIER - Controls the protocol used for barriers.
264 * Possible values:
265 * - MPICH - Turn off optimized barriers and use the MPICH
266 * point-to-point protocol.
267 * - Other options can be determined by running with PAMID_VERBOSE=2.
268 * TODO: Implement the NO options
269 * NOTE: Individual options may be turned off by prefixing them with "NO".
270 * For example, PAMID_BARRIER=NO{specific protocol name} will prevent the
271 * use of the specified protocol.
272 *
273 * - PAMID_BCAST - Controls the protocol used for broadcast. Possible values:
274 * - MPICH - Turn off all optimizations for broadcast and use the MPICH
275 * point-to-point protocol.
276 * - Other options can be determined by running with PAMID_VERBOSE=2.
277 * TODO: Implement the NO options
278 * NOTE: Individual options may be turned off by prefixing them with "NO".
279 * For example, PAMID_BCAST=NO{specific protocol name} will prevent the
280 * use of the specified protocol.
281 *
282 * - PAMID_GATHER - Controls the protocol used for gather.
283 * Possible values:
284 * - MPICH - Use the MPICH point-to-point protocol.
285 * - Other options can be determined by running with PAMID_VERBOSE=2.
286 * TODO: Implement the NO options
287 * NOTE: Individual options may be turned off by prefixing them with "NO".
288 * For example, PAMID_GATHER=NO{specific protocol name} will prevent the
289 * use of the specified protocol.
290 *
291 * - PAMID_REDUCE - Controls the protocol used for reduce.
292 * Possible values:
293 * - MPICH - Turn off all optimizations and use the MPICH point-to-point
294 * protocol.
295 * - Other options can be determined by running with PAMID_VERBOSE=2.
296 * TODO: Implement the NO options
297 * NOTE: Individual options may be turned off by prefixing them with "NO".
298 * For example, PAMID_REDUCE=NO{specific protocol name} will prevent the
299 * use of the specified protocol.
300 *
301 * - PAMID_REDUCE_SCATTER - Controls the protocol used for reduce_scatter
302 * operations. The options for PAMID_SCATTERV and PAMID_REDUCE can change the
303 * behavior of reduce_scatter. Possible values:
304 * - MPICH - Use the MPICH point-to-point protocol.
305 * - Other options can be determined by running with PAMID_VERBOSE=2.
306 * TODO: Implement the NO options
307 * NOTE: Individual options may be turned off by prefixing them with "NO".
308 * For example, PAMID_REDUCESCATTER=NO{specific protocol name} will prevent the
309 * use of the specified protocol.
310 *
311 * - PAMID_SCATTER - Controls the protocol used for scatter.
312 * Possible values:
313 * - MPICH - Use the MPICH point-to-point protocol.
314 * - Other options can be determined by running with PAMID_VERBOSE=2.
315 * TODO: Implement the NO options
316 * NOTE: Individual options may be turned off by prefixing them with "NO".
317 * For example, PAMID_SCATTER=NO{specific protocol name} will prevent the
318 * use of the specified protocol.
319 *
320 * - PAMID_SCATTERV - Controls the protocol used for scatterv.
321 * Possible values:
322 * - MPICH - Use the MPICH point-to-point protocol.
323 * - Other options can be determined by running with PAMID_VERBOSE=2.
324 * TODO: Implement the NO options
325 * NOTE: Individual options may be turned off by prefixing them with "NO".
326 * For example, PAMID_SCATTERV=NO{specific protocol name} will prevent the
327 * use of the specified protocol.
328 *
329 *
330 */
331 #if (MPIDI_STATISTICS || MPIDI_PRINTENV)
332 int prtStat=0;
333 int prtEnv=0;
334 int numTasks=0;
335 MPIX_stats_t *mpid_statp=NULL;
336 extern MPIDI_printenv_t *mpich_env;
337 #endif
338
339 #define ENV_Deprecated(a, b, c, d, e) ENV_Deprecated__(a, b, c, d, e)
340 static inline void
ENV_Deprecated__(char * name[],unsigned num_supported,unsigned * deprecated,int rank,int NA)341 ENV_Deprecated__(char* name[], unsigned num_supported, unsigned* deprecated, int rank, int NA)
342 {
343 if (name == NULL) return;
344
345 unsigned i;
346 char * env;
347
348 for (i=0; i<num_supported; ++i)
349 {
350 if (name[i] == NULL) return;
351
352 if (NA)
353 {
354 env = getenv(name[i]);
355 if (env != NULL)
356 {
357 if (rank == 0)
358 {
359 if (*deprecated == 0)
360 fprintf (stderr, "\n");
361
362 fprintf (stderr, "The environment variable \"%s\" is not applicable.\n", name[i]);
363 }
364 *deprecated = 1;
365 }
366 }
367 }
368
369 for (i=num_supported; name[i] != NULL; ++i)
370 {
371 env = getenv(name[i]);
372 if (env != NULL)
373 {
374 if (rank == 0)
375 {
376 if (*deprecated == 0)
377 fprintf (stderr, "\n");
378
379 if (NA)
380 fprintf (stderr, "The environment variable \"%s\" is deprecated.\n", name[i]);
381 else
382 {
383 char supported[10240];
384 int n, index = 0;
385 index += snprintf(&supported[index], 10240-index-1, "\"%s\"", name[0]);
386 for (n=1; n<num_supported; ++n)
387 index += snprintf(&supported[index], 10240-index-1, " or \"%s\"", name[n]);
388
389 fprintf (stderr, "The environment variable \"%s\" is deprecated. Consider using %s instead.\n", name[i], supported);
390 }
391 }
392 *deprecated = 1;
393 }
394 }
395 }
396
397 #define ENV_Unsigned(a, b, c, d, e) ENV_Unsigned__(a, b, #b, c, d, e, 0)
398
399 #define ENV_Unsigned_NA(a, b, c, d, e) ENV_Unsigned__(a, b, #b, c, d, e, 1)
400
401 static inline void
ENV_Unsigned__(char * name[],unsigned * val,char * string,unsigned num_supported,unsigned * deprecated,int rank,int NA)402 ENV_Unsigned__(char* name[], unsigned* val, char* string, unsigned num_supported, unsigned* deprecated, int rank, int NA)
403 {
404 /* Check for deprecated environment variables. */
405 ENV_Deprecated(name, num_supported, deprecated, rank, NA);
406
407 char * env;
408
409 unsigned i=0;
410 for (;; ++i) {
411 if (name[i] == NULL)
412 return;
413 env = getenv(name[i]);
414 if (env != NULL)
415 break;
416 }
417
418 *val = atoi(env);
419 if (MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_ALL && rank == 0)
420 fprintf(stderr, "%s = %u\n", string, *val);
421 }
422
423
424 #define ENV_Char(a, b) ENV_Char__(a, b, #b)
425 static inline void
ENV_Char__(char * name[],unsigned * val,char * string)426 ENV_Char__(char* name[], unsigned* val, char* string)
427 {
428 char * env;
429 unsigned i=0;
430 for (;; ++i) {
431 if (name[i] == NULL)
432 return;
433 env = getenv(name[i]);
434 if (env != NULL)
435 break;
436 }
437
438 if ((env[0]=='y')|| (env[0]=='Y')|| (env[0]=='p')|| (env[0]=='P') || (env[0]=='F')|| (env[0]=='f'))
439 *val = 1;
440 /*This may seem redundant; however,
441 in some cases we need to force val=0 if value = no/none*/
442 if ((env[0]=='n')|| (env[0]=='N'))
443 *val = 0;
444 if (MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_ALL)
445 fprintf(stderr, "%s = %u\n", string, *val);
446 }
447
448 #define ENV_Char2(a, b) ENV_Char2__(a, b, #b)
449 static inline void
ENV_Char2__(char * name[],char ** val,char * string)450 ENV_Char2__(char* name[], char** val, char* string)
451 {
452 unsigned i=0;
453 for (;; ++i) {
454 if (name[i] == NULL)
455 return;
456 *val = getenv(name[i]);
457 if (*val != NULL)
458 break;
459 }
460
461 if (MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_ALL)
462 fprintf(stderr, "%s = %s\n", string, *val);
463 }
464
465 /** \brief Checks the Environment variables at initialization and stores the results.
466 *
467 * \param [in] rank The process rank; used to limit output
468 * \param [in] requested The thread model requested by the user
469 */
470 void
MPIDI_Env_setup(int rank,int requested)471 MPIDI_Env_setup(int rank, int requested)
472 {
473 unsigned found_deprecated_env_var = 0;
474
475 /* Set defaults for various environment variables */
476
477 /* Set the verbosity level. When set, this will print information at finalize. */
478 {
479 char* names[] = {"PAMID_VERBOSE", "PAMI_VERBOSE", NULL};
480 ENV_Unsigned(names, &MPIDI_Process.verbose, 1, &found_deprecated_env_var, rank);
481 }
482
483 /* Enable statistics collection. */
484 {
485 char* names[] = {"PAMID_STATISTICS", "PAMI_STATISTICS", NULL};
486 ENV_Unsigned(names, &MPIDI_Process.statistics, 1, &found_deprecated_env_var, rank);
487 }
488
489 /* "Globally" set the optimization flag for low-level collectives in geometry creation.
490 * This is probably temporary. metadata should set this flag likely.
491 */
492 {
493 /* THIS ENVIRONMENT VARIABLE NEEDS TO BE DOCUMENTED ABOVE */
494 char *names[] = {"PAMID_OPTIMIZED_SUBCOMMS", "PAMI_OPTIMIZED_SUBCOMMS", NULL};
495 ENV_Unsigned(names, &MPIDI_Process.optimized.subcomms, 1, &found_deprecated_env_var, rank);
496 TRACE_ERR("MPIDI_Process.optimized.subcomms=%u\n", MPIDI_Process.optimized.subcomms);
497 }
498
499 /* Set the threads values from a single override. */
500 {
501 unsigned value = (unsigned)-1;
502 char* names[] = {"PAMID_THREAD_MULTIPLE", NULL};
503 ENV_Unsigned(names, &value, 1, &found_deprecated_env_var, rank);
504 #if (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_PER_OBJECT)
505 if (value == 1) /* force on */
506 {
507 /* The default value for context post should be enabled only if the
508 * default async progress mode is the 'locked' mode.
509 */
510 MPIDI_Process.perobj.context_post.requested =
511 (ASYNC_PROGRESS_MODE_DEFAULT == ASYNC_PROGRESS_MODE_LOCKED);
512
513 MPIDI_Process.async_progress.mode = ASYNC_PROGRESS_MODE_DEFAULT;
514 MPIDI_Process.avail_contexts = MPIDI_MAX_CONTEXTS;
515 }
516 else if (value == 0) /* force off */
517 {
518 MPIDI_Process.perobj.context_post.requested = 0;
519 MPIDI_Process.async_progress.mode = ASYNC_PROGRESS_MODE_DISABLED;
520 MPIDI_Process.avail_contexts = 1;
521 }
522 else if (requested != MPI_THREAD_MULTIPLE)
523 {
524 /* The PAMID_THREAD_MULTIPLE override was not set, yet the application
525 * requested a thread mode other than MPI_THREAD_MULTIPLE. Therefore,
526 * assume that the application prefers the 'latency optimization' over
527 * the 'throughput optimization' and set the async progress configuration
528 * to disable context post.
529 */
530 MPIDI_Process.perobj.context_post.requested = 0;
531 #ifdef BGQ_SUPPORTS_TRIGGER_ASYNC_PROGRESS
532 MPIDI_Process.async_progress.mode = ASYNC_PROGRESS_MODE_TRIGGER;
533 MPIDI_Process.avail_contexts = MPIDI_MAX_CONTEXTS;
534 #else
535 /* BGQ does not support the 'trigger' style of async progress. Until
536 * this is implemented, set the async progress configuration to the
537 * 'single threaded' defaults.
538 */
539 MPIDI_Process.async_progress.mode = ASYNC_PROGRESS_MODE_DISABLED;
540 MPIDI_Process.avail_contexts = 1;
541 #endif
542 }
543 #else
544 /* The only valid async progress mode when using the 'global' mpich lock
545 * mode is the 'trigger' async progress mode. Also, the 'global' mpich lock
546 * mode only supports a single context.
547 *
548 * See discussions in mpich2/src/mpid/pamid/src/mpid_init.c and
549 * src/mpid/pamid/src/mpid_progress.h for more information.
550 */
551 if (value == 1) /* force on */
552 {
553 MPIDI_Process.async_progress.mode = ASYNC_PROGRESS_MODE_TRIGGER;
554 MPIDI_Process.avail_contexts = 1;
555 }
556 else if (value == 0) /* force off */
557 {
558 MPIDI_Process.async_progress.mode = ASYNC_PROGRESS_MODE_DISABLED;
559 MPIDI_Process.avail_contexts = 1;
560 }
561 #endif
562 }
563
564 /* Set the upper-limit of number of PAMI Contexts. */
565 {
566 unsigned value = (unsigned)-1;
567 char *names[] = {"PAMID_CONTEXT_MAX", "PAMI_MAXCONTEXT", "PAMI_MAXCONTEXTS", "PAMI_MAX_CONTEXT", "PAMI_MAX_CONTEXTS", NULL};
568 ENV_Unsigned(names, &value, 1, &found_deprecated_env_var, rank);
569
570 if (value != -1)
571 {
572 #if (MPIU_THREAD_GRANULARITY != MPIU_THREAD_GRANULARITY_PER_OBJECT)
573 /* The 'global' mpich lock mode only supports a single context.
574 * See discussion in mpich2/src/mpid/pamid/src/mpid_init.c for more
575 * information.
576 */
577 if (value > 1)
578 {
579 found_deprecated_env_var++;
580 if (MPIDI_Process.verbose >= MPIDI_VERBOSE_SUMMARY_0 && rank == 0)
581 fprintf(stderr, "The environment variable \"PAMID_CONTEXT_MAX\" is invalid as this mpich2 library was configured.\n");
582 }
583 #endif
584 if (value == 0)
585 {
586 if (MPIDI_Process.verbose >= MPIDI_VERBOSE_SUMMARY_0 && rank == 0)
587 fprintf(stderr, "The environment variable \"PAMID_CONTEXT_MAX\" must specify a value > 0. The number of contexts will be set to 1\n");
588
589 value=1;
590 }
591
592 MPIDI_Process.avail_contexts=value;
593 }
594
595 TRACE_ERR("MPIDI_Process.avail_contexts=%u\n", MPIDI_Process.avail_contexts);
596 }
597
598 /* Enable context post. */
599 {
600 unsigned value = (unsigned)-1;
601 char *names[] = {"PAMID_CONTEXT_POST", "PAMI_CONTEXT_POST", NULL};
602 ENV_Unsigned(names, &value, 1, &found_deprecated_env_var, rank);
603 if (value != -1)
604 {
605 #if (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_PER_OBJECT)
606 MPIDI_Process.perobj.context_post.requested = (value > 0);
607 #else
608 found_deprecated_env_var++;
609 if (MPIDI_Process.verbose >= MPIDI_VERBOSE_SUMMARY_0 && rank == 0)
610 fprintf(stderr, "The environment variable \"PAMID_CONTEXT_POST\" is invalid as this mpich2 library was configured.\n");
611 #endif
612 }
613 TRACE_ERR("MPIDI_Process.perobj.context_post.requested=%u\n", MPIDI_Process.perobj.context_post.requested);
614 }
615
616 /* Enable/Disable asynchronous progress. */
617 {
618 /* THIS ENVIRONMENT VARIABLE NEEDS TO BE DOCUMENTED ABOVE */
619 unsigned value = (unsigned)-1;
620 char *names[] = {"PAMID_ASYNC_PROGRESS", "PAMID_COMMTHREADS", "PAMI_COMMTHREAD", "PAMI_COMMTHREADS", "PAMI_COMM_THREAD", "PAMI_COMM_THREADS", NULL};
621 ENV_Unsigned(names, &value, 1, &found_deprecated_env_var, rank);
622
623 if (value != (unsigned)-1)
624 {
625 if (value != ASYNC_PROGRESS_MODE_DISABLED)
626 {
627 #if (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_PER_OBJECT)
628 if (value == ASYNC_PROGRESS_MODE_LOCKED &&
629 MPIDI_Process.perobj.context_post.requested == 0)
630 {
631 /* The 'locked' async progress mode requires context post.
632 *
633 * See discussion in src/mpid/pamid/src/mpid_progress.h for more
634 * information.
635 */
636 found_deprecated_env_var++;
637 if (MPIDI_Process.verbose >= MPIDI_VERBOSE_SUMMARY_0 && rank == 0)
638 fprintf(stderr, "The environment variable \"PAMID_ASYNC_PROGRESS=1\" requires \"PAMID_CONTEXT_POST=1\".\n");
639 }
640
641 #else /* (MPIU_THREAD_GRANULARITY != MPIU_THREAD_GRANULARITY_PER_OBJECT) */
642 if (value == ASYNC_PROGRESS_MODE_LOCKED)
643 {
644 /* The only valid async progress mode when using the 'global' mpich
645 * lock mode is the 'trigger' async progress mode.
646 *
647 * See discussion in src/mpid/pamid/src/mpid_progress.h for more
648 * information.
649 */
650 found_deprecated_env_var++;
651 if (MPIDI_Process.verbose >= MPIDI_VERBOSE_SUMMARY_0 && rank == 0)
652 fprintf(stderr, "The environment variable \"PAMID_ASYNC_PROGRESS=1\" is invalid as this mpich2 library was configured.\n");
653
654 }
655 #endif
656 }
657
658 MPIDI_Process.async_progress.mode = value;
659 }
660 TRACE_ERR("MPIDI_Process.async_progress.mode=%u\n", MPIDI_Process.async_progress.mode);
661 }
662
663 /* Determine short limit */
664 {
665 /* THIS ENVIRONMENT VARIABLE NEEDS TO BE DOCUMENTED ABOVE */
666 char* names[] = {"PAMID_SHORT", "MP_S_SHORT_LIMIT", "PAMI_SHORT", NULL};
667 ENV_Unsigned(names, &MPIDI_Process.short_limit, 2, &found_deprecated_env_var, rank);
668 }
669
670 /* Determine eager limit */
671 {
672 char* names[] = {"PAMID_EAGER", "PAMID_RZV", "MP_EAGER_LIMIT", "PAMI_RVZ", "PAMI_RZV", "PAMI_EAGER", NULL};
673 ENV_Unsigned(names, &MPIDI_Process.eager_limit, 3, &found_deprecated_env_var, rank);
674 }
675
676 /* Determine 'local' eager limit */
677 {
678 char* names[] = {"PAMID_RZV_LOCAL", "PAMID_EAGER_LOCAL", "MP_EAGER_LIMIT_LOCAL", "PAMI_RVZ_LOCAL", "PAMI_RZV_LOCAL", "PAMI_EAGER_LOCAL", NULL};
679 ENV_Unsigned(names, &MPIDI_Process.eager_limit_local, 3, &found_deprecated_env_var, rank);
680 }
681
682 /* Set the maximum number of outstanding RDMA requests */
683 {
684 char* names[] = {"PAMID_RMA_PENDING", "MP_RMA_PENDING", "PAMI_RMA_PENDING", NULL};
685 ENV_Unsigned(names, &MPIDI_Process.rma_pending, 2, &found_deprecated_env_var, rank);
686 }
687
688 /* Set the status of the optimized collectives */
689 {
690 char* names[] = {"PAMID_COLLECTIVES", "PAMI_COLLECTIVE", "PAMI_COLLECTIVES", NULL};
691 ENV_Unsigned(names, &MPIDI_Process.optimized.collectives, 1, &found_deprecated_env_var, rank);
692 TRACE_ERR("MPIDI_Process.optimized.collectives=%u\n", MPIDI_Process.optimized.collectives);
693 }
694
695 /* First, if MP_COLLECTIVE_OFFLOAD is "on", then we want PE (FCA) collectives */
696 {
697 unsigned temp;
698 temp = 0;
699 char* names[] = {"MP_COLLECTIVE_OFFLOAD", NULL};
700 ENV_Char(names, &temp);
701 if(temp)
702 MPIDI_Process.optimized.collectives = MPID_COLL_FCA;
703 }
704
705 /* However, MP_MP_PAMI_FOR can be set to "none" in which case we don't want PE (FCA) collectives */
706 {
707 char *env = getenv("MP_MPI_PAMI_FOR");
708 if(env != NULL)
709 {
710 if(strncasecmp(env, "N", 1) == 1)
711 MPIDI_Process.optimized.collectives = 0;
712 }
713 }
714
715 /* Set the status for optimized selection of collectives */
716 {
717 char* names[] = {"PAMID_COLLECTIVES_SELECTION", NULL};
718 ENV_Unsigned(names, &MPIDI_Process.optimized.select_colls, 1, &found_deprecated_env_var, rank);
719 TRACE_ERR("MPIDI_Process.optimized.select_colls=%u\n", MPIDI_Process.optimized.select_colls);
720 }
721
722
723 /* Set the status of the optimized shared memory point-to-point functions */
724 {
725 char* names[] = {"PAMID_SHMEM_PT2PT", "MP_SHMEM_PT2PT", "PAMI_SHMEM_PT2PT", NULL};
726 ENV_Unsigned(names, &MPIDI_Process.shmem_pt2pt, 2, &found_deprecated_env_var, rank);
727 }
728
729 /* Check for deprecated collectives environment variables. These variables are
730 * used in src/mpid/pamid/src/comm/mpid_selectcolls.c */
731 {
732 unsigned tmp;
733 {
734 char* names[] = {"PAMID_COLLECTIVE_ALLGATHER", "PAMI_ALLGATHER", NULL};
735 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
736 }
737 {
738 char* names[] = {"PAMID_COLLECTIVE_ALLGATHER_PREALLREDUCE", "PAMI_ALLGATHER_PREALLREDUCE", NULL};
739 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
740 }
741 {
742 char* names[] = {"PAMID_COLLECTIVE_ALLGATHERV", "PAMI_ALLGATHERV", NULL};
743 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
744 }
745 {
746 char* names[] = {"PAMID_COLLECTIVE_ALLGATHERV_PREALLREDUCE", "PAMI_ALLGATHERV_PREALLREDUCE", NULL};
747 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
748 }
749 {
750 char* names[] = {"PAMID_COLLECTIVE_ALLREDUCE", "PAMI_ALLREDUCE", NULL};
751 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
752 }
753 {
754 char* names[] = {"PAMID_COLLECTIVE_ALLREDUCE_PREALLREDUCE", "PAMI_ALLREDUCE_PREALLREDUCE", NULL};
755 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
756 }
757 {
758 char* names[] = {"PAMID_COLLECTIVE_ALLTOALL", "PAMI_ALLTOALL", NULL};
759 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
760 }
761 {
762 char* names[] = {"PAMID_COLLECTIVE_ALLTOALLV", "PAMI_ALLTOALLV", NULL};
763 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
764 }
765 {
766 char* names[] = {"PAMID_COLLECTIVE_ALLTOALLV_INT", "PAMI_ALLTOALLV_INT", NULL};
767 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
768 }
769 {
770 char* names[] = {"PAMID_COLLECTIVE_BARRIER", "PAMI_BARRIER", NULL};
771 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
772 }
773 {
774 char* names[] = {"PAMID_COLLECTIVE_BCAST", "PAMI_BCAST", NULL};
775 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
776 }
777 {
778 char* names[] = {"PAMID_COLLECTIVE_BCAST_PREALLREDUCE", "PAMI_BCAST_PREALLREDUCE", NULL};
779 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
780 }
781 {
782 char* names[] = {"PAMID_COLLECTIVE_GATHER", "PAMI_GATHER", NULL};
783 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
784 }
785 {
786 char* names[] = {"PAMID_COLLECTIVE_GATHERV", "PAMI_GATHERV", NULL};
787 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
788 }
789 {
790 char* names[] = {"PAMID_COLLECTIVE_REDUCE", "PAMI_REDUCE", NULL};
791 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
792 }
793 {
794 char* names[] = {"PAMID_COLLECTIVE_SCAN", "PAMI_SCAN", NULL};
795 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
796 }
797 {
798 char* names[] = {"PAMID_COLLECTIVE_SCATTER", "PAMI_SCATTER", NULL};
799 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
800 }
801 {
802 char* names[] = {"PAMID_COLLECTIVE_SCATTERV", "PAMI_SCATTERV", NULL};
803 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
804 }
805 {
806 char* names[] = {"PAMID_COLLECTIVE_SCATTERV_PREALLREDUCE", "PAMI_SCATTERV_PREALLREDUCE", NULL};
807 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
808 }
809 {
810 char* names[] = {"PAMID_CORE_ON_ABORT", "PAMI_COREONABORT", "PAMI_COREONMPIABORT", "PAMI_COREONMPIDABORT", NULL};
811 ENV_Unsigned(names, &tmp, 1, &found_deprecated_env_var, rank);
812 }
813 #if (MPIDI_STATISTICS || MPIDI_PRINTENV)
814 mpich_env=(MPIDI_printenv_t *) MPIU_Malloc(sizeof(MPIDI_printenv_t)+1);
815 mpid_statp=(MPIX_stats_t *) MPIU_Malloc(sizeof(MPIX_stats_t)+1);
816 memset((void *)mpich_env,0,sizeof(MPIDI_printenv_t));
817 memset((void *)mpid_statp,0,sizeof(MPIX_stats_t));
818 /* If MP_STATISTICS is set, each task prints statistics data at the end of an MPI jobs */
819 {
820 char* names[] = {"MP_STATISTICS", NULL};
821 ENV_Char(names, &MPIDI_Process.mp_statistics);
822 if (MPIDI_Process.mp_statistics) {
823 prtStat=1;
824 }
825 }
826 /* if MP_INFOLEVEL is greater than or equal to 1, the library will display the banner */
827 {
828 char* names[] = {"MP_INFOLEVEL", NULL};
829 ENV_Unsigned(names, &MPIDI_Process.mp_infolevel, 1, &found_deprecated_env_var, rank);
830 }
831 /* If MP_PRINTENV is set, task 0 prints out the env. info after MPI_Init */
832 {
833 char* names[] = {"MP_PRINTENV", NULL};
834 ENV_Char(names, &MPIDI_Process.mp_printenv);
835 if (MPIDI_Process.mp_printenv) {
836 prtEnv=1;
837 }
838 }
839 /* MP_CSS_INTERRUPT */
840 {
841 char* names[] = {"MP_CSS_INTERRUPT", NULL};
842 ENV_Char(names, &mpich_env->interrupts);
843 if (mpich_env->interrupts == 1) /* force on */
844 {
845 MPIDI_Process.perobj.context_post.requested = 0;
846 MPIDI_Process.async_progress.mode = ASYNC_PROGRESS_MODE_TRIGGER;
847 #if (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_PER_OBJECT)
848 MPIDI_Process.avail_contexts = MPIDI_MAX_CONTEXTS;
849 #else
850 MPIDI_Process.avail_contexts = 1;
851 #endif
852 }
853 else if (mpich_env->interrupts == 0) /* force off */
854 {
855 MPIDI_Process.perobj.context_post.requested = 0;
856 MPIDI_Process.async_progress.mode = ASYNC_PROGRESS_MODE_DISABLED;
857 MPIDI_Process.avail_contexts = 1;
858 }
859 }
860 /* MP_POLLING_INTERVAL */
861 {
862 char* names[] = {"MP_POLLING_INTERVAL", NULL};
863 ENV_Unsigned(names,(unsigned int *) &mpich_env->polling_interval,1, &found_deprecated_env_var, rank);
864 }
865
866 /* MP_RETRANSMIT_INTERVAL */
867 {
868 char* names[] = {"MP_RETRANSMIT_INTERVAL", NULL};
869 ENV_Unsigned(names, &mpich_env->retransmit_interval,1, &found_deprecated_env_var, rank);
870 }
871
872 /* MP_S_USE_PAMI_GET */
873 {
874 char* names[] = {"MP_S_USE_PAMI_GET", NULL};
875 ENV_Char(names, &MPIDI_Process.mp_s_use_pami_get);
876 }
877 #endif
878 }
879 /* Exit if any deprecated environment variables were specified. */
880 if (found_deprecated_env_var)
881 {
882 if (rank == 0)
883 {
884 // Only rank 0 prints and exits. sleep to make sure message is sent before exiting.
885 // Other ranks will proceed, but will be waiting in a barrier in context create
886 // when the exit occurs.
887 fprintf (stderr, "\n"); fflush(stderr); sleep(1);
888 exit(1);
889 }
890 }
891 }
892