1 /*****************************************************************************\ 2 ** setup.h - MPI/PMI2 plugin setup 3 ***************************************************************************** 4 * Copyright (C) 2011-2012 National University of Defense Technology. 5 * Written by Hongjia Cao <hjcao@nudt.edu.cn>. 6 * All rights reserved. 7 * Portions copyright (C) 2015 Mellanox Technologies Inc. 8 * Written by Artem Y. Polyakov <artemp@mellanox.com>. 9 * All rights reserved. 10 * 11 * This file is part of Slurm, a resource management program. 12 * For details, see <https://slurm.schedmd.com/>. 13 * Please also read the included file: DISCLAIMER. 14 * 15 * Slurm is free software; you can redistribute it and/or modify it under 16 * the terms of the GNU General Public License as published by the Free 17 * Software Foundation; either version 2 of the License, or (at your option) 18 * any later version. 19 * 20 * In addition, as a special exception, the copyright holders give permission 21 * to link the code of portions of this program with the OpenSSL library under 22 * certain conditions as described in each individual source file, and 23 * distribute linked combinations including the two. You must obey the GNU 24 * General Public License in all respects for all of the code used other than 25 * OpenSSL. If you modify file(s) with this exception, you may extend this 26 * exception to your version of the file(s), but you are not obligated to do 27 * so. If you do not wish to do so, delete this exception statement from your 28 * version. If you delete this exception statement from all source files in 29 * the program, then also delete it here. 30 * 31 * Slurm is distributed in the hope that it will be useful, but WITHOUT ANY 32 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 33 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 34 * details. 35 * 36 * You should have received a copy of the GNU General Public License along 37 * with Slurm; if not, write to the Free Software Foundation, Inc., 38 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 39 \*****************************************************************************/ 40 41 #ifndef _SETUP_H 42 #define _SETUP_H 43 44 #include <inttypes.h> 45 46 #include "slurm/slurm_errno.h" 47 48 #include "src/common/slurm_xlator.h" 49 #include "src/common/pack.h" 50 #include "src/common/slurm_opt.h" 51 #include "src/common/slurm_mpi.h" 52 #include "src/common/xstring.h" 53 #include "src/slurmd/slurmstepd/slurmstepd_job.h" 54 #include "src/srun/libsrun/debugger.h" 55 #include "src/srun/libsrun/opt.h" 56 57 #include "tree.h" 58 59 typedef struct pmi2_job_info { 60 uint32_t jobid; /* Current Slurm job id */ 61 uint32_t stepid; /* Current step id (or NO_VAL) */ 62 uint32_t nnodes; /* number of nodes in current job step */ 63 uint32_t nodeid; /* relative position of this node in job */ 64 uint32_t ntasks; /* total number of tasks in current job */ 65 uint32_t ltasks; /* number of tasks on *this* (local) node */ 66 uint32_t *gtids; /* global task ids of the tasks */ 67 uint32_t spawn_seq; /* seq of spawn. 0 if not spawned */ 68 69 int pmi_debugged; /* whether output verbose PMI messages */ 70 char *step_nodelist; /* list of nodes in this job step */ 71 char *proc_mapping; /* processor mapping */ 72 char *pmi_jobid; /* PMI job id */ 73 char *spawner_jobid; /* spawner pmi job id */ 74 char **job_env; /* environment of job. use in stepd */ 75 76 MPIR_PROCDESC *MPIR_proctable; /* used only in srun */ 77 slurm_opt_t *srun_opt; /* used only in srun */ 78 char *resv_ports; /* MPI reserved ports */ 79 } pmi2_job_info_t; 80 81 typedef struct pmi2_tree_info { 82 char *this_node; /* this nodename */ 83 char *parent_node; /* parent nodename */ 84 int parent_id; /* parent nodeid */ 85 int num_children; /* number of children stepds */ 86 int depth; /* depth in tree */ 87 int max_depth; /* max depth of the tree */ 88 uint16_t pmi_port; /* PMI2 comm port of this srun */ 89 slurm_addr_t *srun_addr; /* PMI2 comm address parent srun */ 90 uint32_t *children_kvs_seq; /* sequence number of children nodes */ 91 } pmi2_tree_info_t; 92 93 94 extern pmi2_job_info_t job_info; 95 extern pmi2_tree_info_t tree_info; 96 extern char tree_sock_addr[]; 97 extern int tree_sock; 98 extern int *task_socks; 99 #define STEPD_PMI_SOCK(lrank) task_socks[lrank * 2] 100 #define TASK_PMI_SOCK(lrank) task_socks[lrank * 2 + 1] 101 102 extern bool in_stepd(void); 103 extern int pmi2_setup_stepd(const stepd_step_rec_t *job, char ***env); 104 extern int pmi2_setup_srun(const mpi_plugin_client_info_t *job, char ***env); 105 extern void pmi2_cleanup_stepd(void); 106 107 #endif /* _SETUP_H */ 108