1 /* 2 * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana 3 * University Research and Technology 4 * Corporation. All rights reserved. 5 * Copyright (c) 2004-2006 The University of Tennessee and The University 6 * of Tennessee Research Foundation. All rights 7 * reserved. 8 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 9 * University of Stuttgart. All rights reserved. 10 * Copyright (c) 2004-2005 The Regents of the University of California. 11 * All rights reserved. 12 * Copyright (c) 2006-2013 Los Alamos National Security, LLC. 13 * All rights reserved. 14 * Copyright (c) 2010-2011 Cisco Systems, Inc. All rights reserved. 15 * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. 16 * Copyright (c) 2014 Mellanox Technologies, Inc. 17 * All rights reserved. 18 * Copyright (c) 2014 Research Organization for Information Science 19 * and Technology (RIST). All rights reserved. 20 * $COPYRIGHT$ 21 * 22 * Additional copyrights may follow 23 * 24 * $HEADER$ 25 */ 26 27 #ifndef _PMIX_SERVER_INTERNAL_H_ 28 #define _PMIX_SERVER_INTERNAL_H_ 29 30 #include "orte_config.h" 31 #include "orte/types.h" 32 33 #ifdef HAVE_SYS_SOCKET_H 34 #include <sys/socket.h> 35 #endif 36 #ifdef HAVE_SYS_UN_H 37 #include <sys/un.h> 38 #endif 39 40 #include "opal/types.h" 41 #include "opal/class/opal_hotel.h" 42 #include "opal/mca/base/base.h" 43 #include "opal/mca/event/event.h" 44 #include "opal/mca/pmix/pmix.h" 45 #include "opal/util/proc.h" 46 #include "opal/sys/atomic.h" 47 48 #include "orte/mca/grpcomm/base/base.h" 49 #include "orte/runtime/orte_globals.h" 50 #include "orte/util/threads.h" 51 52 BEGIN_C_DECLS 53 54 #define ORTED_PMIX_MIN_DMX_TIMEOUT 10 55 #define ORTE_ADJUST_TIMEOUT(a) \ 56 do { \ 57 (a)->timeout = (2 * orte_process_info.num_daemons) / 1000; \ 58 if ((a)->timeout < ORTED_PMIX_MIN_DMX_TIMEOUT) { \ 59 (a)->timeout = ORTED_PMIX_MIN_DMX_TIMEOUT; \ 60 } \ 61 } while(0) 62 63 /* object for tracking requests so we can 64 * correctly route the eventual reply */ 65 typedef struct { 66 opal_object_t super; 67 opal_event_t ev; 68 char *operation; 69 int status; 70 int timeout; 71 int room_num; 72 int remote_room_num; 73 opal_pmix_data_range_t range; 74 orte_process_name_t proxy; 75 orte_process_name_t target; 76 orte_job_t *jdata; 77 opal_buffer_t msg; 78 opal_pmix_op_cbfunc_t opcbfunc; 79 opal_pmix_modex_cbfunc_t mdxcbfunc; 80 opal_pmix_spawn_cbfunc_t spcbfunc; 81 opal_pmix_lookup_cbfunc_t lkcbfunc; 82 opal_pmix_release_cbfunc_t rlcbfunc; 83 void *cbdata; 84 } pmix_server_req_t; 85 OBJ_CLASS_DECLARATION(pmix_server_req_t); 86 87 /* object for thread-shifting server operations */ 88 typedef struct { 89 opal_object_t super; 90 opal_event_t ev; 91 int status; 92 opal_process_name_t proc; 93 const char *msg; 94 void *server_object; 95 opal_list_t *procs; 96 opal_list_t *eprocs; 97 opal_list_t *info; 98 opal_pmix_op_cbfunc_t cbfunc; 99 opal_pmix_info_cbfunc_t infocbfunc; 100 opal_pmix_tool_connection_cbfunc_t toolcbfunc; 101 void *cbdata; 102 } orte_pmix_server_op_caddy_t; 103 OBJ_CLASS_DECLARATION(orte_pmix_server_op_caddy_t); 104 105 typedef struct { 106 opal_object_t super; 107 orte_grpcomm_signature_t *sig; 108 opal_pmix_modex_cbfunc_t cbfunc; 109 void *cbdata; 110 } orte_pmix_mdx_caddy_t; 111 OBJ_CLASS_DECLARATION(orte_pmix_mdx_caddy_t); 112 113 #define ORTE_DMX_REQ(p, cf, ocf, ocd) \ 114 do { \ 115 pmix_server_req_t *_req; \ 116 _req = OBJ_NEW(pmix_server_req_t); \ 117 (void)asprintf(&_req->operation, "DMDX: %s:%d", __FILE__, __LINE__); \ 118 _req->target = (p); \ 119 _req->mdxcbfunc = (ocf); \ 120 _req->cbdata = (ocd); \ 121 opal_event_set(orte_event_base, &(_req->ev), \ 122 -1, OPAL_EV_WRITE, (cf), _req); \ 123 opal_event_set_priority(&(_req->ev), ORTE_MSG_PRI); \ 124 ORTE_POST_OBJECT(_req); \ 125 opal_event_active(&(_req->ev), OPAL_EV_WRITE, 1); \ 126 } while(0); 127 128 #define ORTE_SPN_REQ(j, cf, ocf, ocd) \ 129 do { \ 130 pmix_server_req_t *_req; \ 131 _req = OBJ_NEW(pmix_server_req_t); \ 132 (void)asprintf(&_req->operation, "SPAWN: %s:%d", __FILE__, __LINE__); \ 133 _req->jdata = (j); \ 134 _req->spcbfunc = (ocf); \ 135 _req->cbdata = (ocd); \ 136 opal_event_set(orte_event_base, &(_req->ev), \ 137 -1, OPAL_EV_WRITE, (cf), _req); \ 138 opal_event_set_priority(&(_req->ev), ORTE_MSG_PRI); \ 139 ORTE_POST_OBJECT(_req); \ 140 opal_event_active(&(_req->ev), OPAL_EV_WRITE, 1); \ 141 } while(0); 142 143 #define ORTE_PMIX_OPERATION(p, i, fn, cf, cb) \ 144 do { \ 145 orte_pmix_server_op_caddy_t *_cd; \ 146 _cd = OBJ_NEW(orte_pmix_server_op_caddy_t); \ 147 _cd->procs = (p); \ 148 _cd->info = (i); \ 149 _cd->cbfunc = (cf); \ 150 _cd->cbdata = (cb); \ 151 opal_event_set(orte_event_base, &(_cd->ev), -1, \ 152 OPAL_EV_WRITE, (fn), _cd); \ 153 opal_event_set_priority(&(_cd->ev), ORTE_MSG_PRI); \ 154 ORTE_POST_OBJECT(_cd); \ 155 opal_event_active(&(_cd->ev), OPAL_EV_WRITE, 1); \ 156 } while(0); 157 158 #define ORTE_PMIX_THREADSHIFT(p, s, st, m, pl, fn, cf, cb) \ 159 do { \ 160 orte_pmix_server_op_caddy_t *_cd; \ 161 _cd = OBJ_NEW(orte_pmix_server_op_caddy_t); \ 162 _cd->proc.jobid = (p)->jobid; \ 163 _cd->proc.vpid = (p)->vpid; \ 164 _cd->server_object = (s); \ 165 _cd->status = (st); \ 166 _cd->msg = (m); \ 167 _cd->procs = (pl); \ 168 _cd->cbfunc = (cf); \ 169 _cd->cbdata = (cb); \ 170 opal_event_set(orte_event_base, &(_cd->ev), -1, \ 171 OPAL_EV_WRITE, (fn), _cd); \ 172 opal_event_set_priority(&(_cd->ev), ORTE_MSG_PRI); \ 173 ORTE_POST_OBJECT(_cd); \ 174 opal_event_active(&(_cd->ev), OPAL_EV_WRITE, 1); \ 175 } while(0); 176 177 /* define the server module functions */ 178 extern int pmix_server_client_connected_fn(opal_process_name_t *proc, void* server_object, 179 opal_pmix_op_cbfunc_t cbfunc, void *cbdata); 180 extern int pmix_server_client_finalized_fn(opal_process_name_t *proc, void* server_object, 181 opal_pmix_op_cbfunc_t cbfunc, void *cbdata); 182 extern int pmix_server_abort_fn(opal_process_name_t *proc, void *server_object, 183 int status, const char msg[], 184 opal_list_t *procs_to_abort, 185 opal_pmix_op_cbfunc_t cbfunc, void *cbdata); 186 extern int pmix_server_fencenb_fn(opal_list_t *procs, opal_list_t *info, 187 char *data, size_t ndata, 188 opal_pmix_modex_cbfunc_t cbfunc, void *cbdata); 189 extern int pmix_server_dmodex_req_fn(opal_process_name_t *proc, opal_list_t *info, 190 opal_pmix_modex_cbfunc_t cbfunc, void *cbdata); 191 extern int pmix_server_publish_fn(opal_process_name_t *proc, 192 opal_list_t *info, 193 opal_pmix_op_cbfunc_t cbfunc, void *cbdata); 194 extern int pmix_server_lookup_fn(opal_process_name_t *proc, char **keys, 195 opal_list_t *info, 196 opal_pmix_lookup_cbfunc_t cbfunc, void *cbdata); 197 extern int pmix_server_unpublish_fn(opal_process_name_t *proc, char **keys, 198 opal_list_t *info, 199 opal_pmix_op_cbfunc_t cbfunc, void *cbdata); 200 extern int pmix_server_spawn_fn(opal_process_name_t *requestor, 201 opal_list_t *job_info, opal_list_t *apps, 202 opal_pmix_spawn_cbfunc_t cbfunc, void *cbdata); 203 extern int pmix_server_connect_fn(opal_list_t *procs, opal_list_t *info, 204 opal_pmix_op_cbfunc_t cbfunc, void *cbdata); 205 extern int pmix_server_disconnect_fn(opal_list_t *procs, opal_list_t *info, 206 opal_pmix_op_cbfunc_t cbfunc, void *cbdata); 207 extern int pmix_server_register_events_fn(opal_list_t *info, 208 opal_pmix_op_cbfunc_t cbfunc, 209 void *cbdata); 210 extern int pmix_server_deregister_events_fn(opal_list_t *info, 211 opal_pmix_op_cbfunc_t cbfunc, 212 void *cbdata); 213 extern int pmix_server_notify_event(int code, opal_process_name_t *source, 214 opal_list_t *info, 215 opal_pmix_op_cbfunc_t cbfunc, void *cbdata); 216 extern int pmix_server_query_fn(opal_process_name_t *requestor, 217 opal_list_t *queries, 218 opal_pmix_info_cbfunc_t cbfunc, void *cbdata); 219 extern void pmix_tool_connected_fn(opal_list_t *info, 220 opal_pmix_tool_connection_cbfunc_t cbfunc, 221 void *cbdata); 222 223 extern void pmix_server_log_fn(opal_process_name_t *requestor, 224 opal_list_t *info, 225 opal_list_t *directives, 226 opal_pmix_op_cbfunc_t cbfunc, 227 void *cbdata); 228 229 extern int pmix_server_alloc_fn(const opal_process_name_t *requestor, 230 opal_pmix_alloc_directive_t dir, 231 opal_list_t *info, 232 opal_pmix_info_cbfunc_t cbfunc, 233 void *cbdata); 234 235 extern int pmix_server_job_ctrl_fn(const opal_process_name_t *requestor, 236 opal_list_t *targets, 237 opal_list_t *info, 238 opal_pmix_info_cbfunc_t cbfunc, 239 void *cbdata); 240 241 /* declare the RML recv functions for responses */ 242 extern void pmix_server_launch_resp(int status, orte_process_name_t* sender, 243 opal_buffer_t *buffer, 244 orte_rml_tag_t tg, void *cbdata); 245 246 extern void pmix_server_keyval_client(int status, orte_process_name_t* sender, 247 opal_buffer_t *buffer, 248 orte_rml_tag_t tg, void *cbdata); 249 250 extern void pmix_server_notify(int status, orte_process_name_t* sender, 251 opal_buffer_t *buffer, 252 orte_rml_tag_t tg, void *cbdata); 253 254 /* exposed shared variables */ 255 typedef struct { 256 bool initialized; 257 int verbosity; 258 int output; 259 opal_hotel_t reqs; 260 int num_rooms; 261 int timeout; 262 bool wait_for_server; 263 orte_process_name_t server; 264 opal_list_t notifications; 265 bool pubsub_init; 266 bool session_server; 267 bool system_server; 268 bool legacy; 269 } pmix_server_globals_t; 270 271 extern pmix_server_globals_t orte_pmix_server_globals; 272 273 END_C_DECLS 274 275 #endif /* PMIX_SERVER_INTERNAL_H_ */ 276