1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ 2 /* 3 * Copyright (c) 2015-2020 Intel, Inc. All rights reserved. 4 * Copyright (c) 2015 Artem Y. Polyakov <artpol84@gmail.com>. 5 * All rights reserved. 6 * Copyright (c) 2015 Mellanox Technologies, Inc. 7 * All rights reserved. 8 * Copyright (c) 2016-2020 IBM Corporation. All rights reserved. 9 * Copyright (c) 2016-2018 Research Organization for Information Science 10 * and Technology (RIST). All rights reserved. 11 * $COPYRIGHT$ 12 */ 13 14 #ifndef PMIX_SERVER_OPS_H 15 #define PMIX_SERVER_OPS_H 16 17 #include <unistd.h> 18 #ifdef HAVE_SYS_TYPES_H 19 #include <sys/types.h> 20 #endif 21 22 #include "src/include/pmix_config.h" 23 #include "src/include/types.h" 24 #include "include/pmix_common.h" 25 26 #include "src/class/pmix_hotel.h" 27 #include "include/pmix_server.h" 28 #include "src/threads/threads.h" 29 #include "src/include/pmix_globals.h" 30 #include "src/util/hash.h" 31 32 #define PMIX_IOF_HOTEL_SIZE 256 33 #define PMIX_IOF_MAX_STAY 300000000 34 35 typedef struct { 36 pmix_object_t super; 37 pmix_event_t ev; 38 pmix_server_trkr_t *trk; 39 } pmix_trkr_caddy_t; 40 PMIX_CLASS_DECLARATION(pmix_trkr_caddy_t); 41 42 typedef struct { 43 pmix_object_t super; 44 pmix_event_t ev; 45 pmix_lock_t lock; 46 pmix_peer_t *peer; 47 char *nspace; 48 pmix_status_t status; 49 pmix_status_t *codes; 50 size_t ncodes; 51 pmix_proc_t proc; 52 pmix_proc_t *procs; 53 size_t nprocs; 54 uid_t uid; 55 gid_t gid; 56 void *server_object; 57 int nlocalprocs; 58 pmix_info_t *info; 59 size_t ninfo; 60 char **keys; 61 pmix_app_t *apps; 62 size_t napps; 63 pmix_iof_channel_t channels; 64 pmix_byte_object_t *bo; 65 size_t nbo; 66 /* timestamp receipt of the notification so we 67 * can evict the oldest one if we get overwhelmed */ 68 time_t ts; 69 /* what room of the hotel they are in */ 70 int room; 71 pmix_op_cbfunc_t opcbfunc; 72 pmix_dmodex_response_fn_t cbfunc; 73 pmix_setup_application_cbfunc_t setupcbfunc; 74 pmix_lookup_cbfunc_t lkcbfunc; 75 pmix_spawn_cbfunc_t spcbfunc; 76 void *cbdata; 77 } pmix_setup_caddy_t; 78 PMIX_CLASS_DECLARATION(pmix_setup_caddy_t); 79 80 /* define a callback function returning inventory */ 81 typedef void (*pmix_inventory_cbfunc_t)(pmix_status_t status, 82 pmix_list_t *inventory, 83 void *cbdata); 84 85 /* define an object for rolling up the inventory*/ 86 typedef struct { 87 pmix_object_t super; 88 pmix_lock_t lock; 89 pmix_event_t ev; 90 pmix_status_t status; 91 int requests; 92 int replies; 93 pmix_list_t payload; // list of pmix_kval_t containing the replies 94 pmix_info_t *info; 95 size_t ninfo; 96 pmix_inventory_cbfunc_t cbfunc; 97 pmix_info_cbfunc_t infocbfunc; 98 pmix_op_cbfunc_t opcbfunc; 99 void *cbdata; 100 } pmix_inventory_rollup_t; 101 PMIX_CLASS_DECLARATION(pmix_inventory_rollup_t); 102 103 typedef struct { 104 pmix_list_item_t super; 105 pmix_setup_caddy_t *cd; 106 } pmix_dmdx_remote_t; 107 PMIX_CLASS_DECLARATION(pmix_dmdx_remote_t); 108 109 typedef struct { 110 pmix_list_item_t super; 111 pmix_proc_t proc; // id of proc whose data is being requested 112 pmix_list_t loc_reqs; // list of pmix_dmdx_request_t elem is keeping track of 113 // all local ranks that are interested in this namespace-rank 114 pmix_info_t *info; // array of info structs for this request 115 size_t ninfo; // number of info structs 116 } pmix_dmdx_local_t; 117 PMIX_CLASS_DECLARATION(pmix_dmdx_local_t); 118 119 typedef struct { 120 pmix_list_item_t super; 121 pmix_event_t ev; 122 bool event_active; 123 pmix_dmdx_local_t *lcd; 124 pmix_modex_cbfunc_t cbfunc; // cbfunc to be executed when data is available 125 void *cbdata; 126 } pmix_dmdx_request_t; 127 PMIX_CLASS_DECLARATION(pmix_dmdx_request_t); 128 129 /* event/error registration book keeping */ 130 typedef struct { 131 pmix_list_item_t super; 132 pmix_peer_t *peer; 133 bool enviro_events; 134 pmix_proc_t *affected; 135 size_t naffected; 136 } pmix_peer_events_info_t; 137 PMIX_CLASS_DECLARATION(pmix_peer_events_info_t); 138 139 typedef struct { 140 pmix_list_item_t super; 141 pmix_list_t peers; // list of pmix_peer_events_info_t 142 int code; 143 } pmix_regevents_info_t; 144 PMIX_CLASS_DECLARATION(pmix_regevents_info_t); 145 146 typedef struct { 147 pmix_list_item_t super; 148 char *grpid; 149 pmix_proc_t *members; 150 size_t nmbrs; 151 } pmix_group_t; 152 PMIX_CLASS_DECLARATION(pmix_group_t); 153 154 typedef struct { 155 pmix_list_item_t super; 156 pmix_group_t *grp; 157 pmix_rank_t rank; 158 size_t idx; 159 } pmix_group_caddy_t; 160 PMIX_CLASS_DECLARATION(pmix_group_caddy_t); 161 162 typedef struct { 163 pmix_list_item_t super; 164 pmix_proc_t source; 165 pmix_iof_channel_t channel; 166 pmix_byte_object_t *bo; 167 pmix_info_t *info; 168 size_t ninfo; 169 } pmix_iof_cache_t; 170 PMIX_CLASS_DECLARATION(pmix_iof_cache_t); 171 172 typedef struct { 173 pmix_list_t nspaces; // list of pmix_nspace_t for the nspaces we know about 174 pmix_pointer_array_t clients; // array of pmix_peer_t local clients 175 pmix_list_t collectives; // list of active pmix_server_trkr_t 176 pmix_list_t remote_pnd; // list of pmix_dmdx_remote_t awaiting arrival of data fror servicing remote req's 177 pmix_list_t local_reqs; // list of pmix_dmdx_local_t awaiting arrival of data from local neighbours 178 pmix_list_t gdata; // cache of data given to me for passing to all clients 179 pmix_list_t events; // list of pmix_regevents_info_t registered events 180 pmix_list_t groups; // list of pmix_group_t group memberships 181 pmix_list_t iof; // IO to be forwarded to clients 182 size_t max_iof_cache; // max number of IOF messages to cache 183 bool tool_connections_allowed; 184 char *tmpdir; // temporary directory for this server 185 char *system_tmpdir; // system tmpdir 186 bool fence_localonly_opt; // local-only fence optimization 187 // verbosity for server get operations 188 int get_output; 189 int get_verbose; 190 // verbosity for server connect operations 191 int connect_output; 192 int connect_verbose; 193 // verbosity for server fence operations 194 int fence_output; 195 int fence_verbose; 196 // verbosity for server pub operations 197 int pub_output; 198 int pub_verbose; 199 // verbosity for server spawn operations 200 int spawn_output; 201 int spawn_verbose; 202 // verbosity for server event operations 203 int event_output; 204 int event_verbose; 205 // verbosity for server iof operations 206 int iof_output; 207 int iof_verbose; 208 // verbosity for basic server functions 209 int base_output; 210 int base_verbose; 211 212 } pmix_server_globals_t; 213 214 #define PMIX_GDS_CADDY(c, p, t) \ 215 do { \ 216 (c) = PMIX_NEW(pmix_server_caddy_t); \ 217 (c)->hdr.tag = (t); \ 218 PMIX_RETAIN((p)); \ 219 (c)->peer = (p); \ 220 } while (0) 221 222 #define PMIX_SETUP_COLLECTIVE(c, t) \ 223 do { \ 224 (c) = PMIX_NEW(pmix_trkr_caddy_t); \ 225 (c)->trk = (t); \ 226 } while (0) 227 228 #define PMIX_EXECUTE_COLLECTIVE(c, t, f) \ 229 do { \ 230 PMIX_SETUP_COLLECTIVE(c, t); \ 231 pmix_event_assign(&((c)->ev), pmix_globals.evbase, -1, \ 232 EV_WRITE, (f), (c)); \ 233 pmix_event_active(&((c)->ev), EV_WRITE, 1); \ 234 } while (0) 235 236 237 238 PMIX_EXPORT bool pmix_server_trk_update(pmix_server_trkr_t *trk); 239 240 PMIX_EXPORT void pmix_pending_nspace_requests(pmix_namespace_t *nptr); 241 PMIX_EXPORT pmix_status_t pmix_pending_resolve(pmix_namespace_t *nptr, pmix_rank_t rank, 242 pmix_status_t status, pmix_dmdx_local_t *lcd); 243 244 245 PMIX_EXPORT pmix_status_t pmix_server_abort(pmix_peer_t *peer, pmix_buffer_t *buf, 246 pmix_op_cbfunc_t cbfunc, void *cbdata); 247 248 PMIX_EXPORT pmix_status_t pmix_server_commit(pmix_peer_t *peer, pmix_buffer_t *buf); 249 250 PMIX_EXPORT pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd, 251 pmix_buffer_t *buf, 252 pmix_modex_cbfunc_t modexcbfunc, 253 pmix_op_cbfunc_t opcbfunc); 254 255 PMIX_EXPORT pmix_status_t pmix_server_get(pmix_buffer_t *buf, 256 pmix_modex_cbfunc_t cbfunc, 257 void *cbdata); 258 259 PMIX_EXPORT pmix_status_t pmix_server_publish(pmix_peer_t *peer, 260 pmix_buffer_t *buf, 261 pmix_op_cbfunc_t cbfunc, 262 void *cbdata); 263 264 PMIX_EXPORT pmix_status_t pmix_server_lookup(pmix_peer_t *peer, 265 pmix_buffer_t *buf, 266 pmix_lookup_cbfunc_t cbfunc, 267 void *cbdata); 268 269 PMIX_EXPORT pmix_status_t pmix_server_unpublish(pmix_peer_t *peer, 270 pmix_buffer_t *buf, 271 pmix_op_cbfunc_t cbfunc, 272 void *cbdata); 273 274 PMIX_EXPORT pmix_status_t pmix_server_spawn(pmix_peer_t *peer, 275 pmix_buffer_t *buf, 276 pmix_spawn_cbfunc_t cbfunc, 277 void *cbdata); 278 279 PMIX_EXPORT pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd, 280 pmix_buffer_t *buf, 281 pmix_op_cbfunc_t cbfunc); 282 283 PMIX_EXPORT pmix_status_t pmix_server_disconnect(pmix_server_caddy_t *cd, 284 pmix_buffer_t *buf, 285 pmix_op_cbfunc_t cbfunc); 286 287 PMIX_EXPORT pmix_status_t pmix_server_notify_error(pmix_status_t status, 288 pmix_proc_t procs[], size_t nprocs, 289 pmix_proc_t error_procs[], size_t error_nprocs, 290 pmix_info_t info[], size_t ninfo, 291 pmix_op_cbfunc_t cbfunc, void *cbdata); 292 293 PMIX_EXPORT pmix_status_t pmix_server_register_events(pmix_peer_t *peer, 294 pmix_buffer_t *buf, 295 pmix_op_cbfunc_t cbfunc, 296 void *cbdata); 297 298 PMIX_EXPORT void pmix_server_deregister_events(pmix_peer_t *peer, 299 pmix_buffer_t *buf); 300 301 PMIX_EXPORT pmix_status_t pmix_server_query(pmix_peer_t *peer, 302 pmix_buffer_t *buf, 303 pmix_info_cbfunc_t cbfunc, 304 void *cbdata); 305 306 PMIX_EXPORT pmix_status_t pmix_server_log(pmix_peer_t *peer, 307 pmix_buffer_t *buf, 308 pmix_op_cbfunc_t cbfunc, 309 void *cbdata); 310 311 PMIX_EXPORT pmix_status_t pmix_server_alloc(pmix_peer_t *peer, 312 pmix_buffer_t *buf, 313 pmix_info_cbfunc_t cbfunc, 314 void *cbdata); 315 316 PMIX_EXPORT pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer, 317 pmix_buffer_t *buf, 318 pmix_info_cbfunc_t cbfunc, 319 void *cbdata); 320 321 PMIX_EXPORT pmix_status_t pmix_server_monitor(pmix_peer_t *peer, 322 pmix_buffer_t *buf, 323 pmix_info_cbfunc_t cbfunc, 324 void *cbdata); 325 326 PMIX_EXPORT pmix_status_t pmix_server_get_credential(pmix_peer_t *peer, 327 pmix_buffer_t *buf, 328 pmix_credential_cbfunc_t cbfunc, 329 void *cbdata); 330 331 PMIX_EXPORT pmix_status_t pmix_server_validate_credential(pmix_peer_t *peer, 332 pmix_buffer_t *buf, 333 pmix_validation_cbfunc_t cbfunc, 334 void *cbdata); 335 336 PMIX_EXPORT pmix_status_t pmix_server_iofreg(pmix_peer_t *peer, 337 pmix_buffer_t *buf, 338 pmix_op_cbfunc_t cbfunc, 339 void *cbdata); 340 341 PMIX_EXPORT pmix_status_t pmix_server_iofstdin(pmix_peer_t *peer, 342 pmix_buffer_t *buf, 343 pmix_op_cbfunc_t cbfunc, 344 void *cbdata); 345 346 PMIX_EXPORT pmix_status_t pmix_server_iofdereg(pmix_peer_t *peer, 347 pmix_buffer_t *buf, 348 pmix_op_cbfunc_t cbfunc, 349 void *cbdata); 350 351 PMIX_EXPORT pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer, 352 pmix_buffer_t *buf, 353 pmix_op_cbfunc_t cbfunc, 354 void *cbdata); 355 PMIX_EXPORT void pmix_server_execute_collective(int sd, short args, void *cbdata); 356 357 PMIX_EXPORT pmix_status_t pmix_server_initialize(void); 358 359 PMIX_EXPORT void pmix_server_message_handler(struct pmix_peer_t *pr, 360 pmix_ptl_hdr_t *hdr, 361 pmix_buffer_t *buf, void *cbdata); 362 363 PMIX_EXPORT void pmix_server_purge_events(pmix_peer_t *peer, 364 pmix_proc_t *proc); 365 366 PMIX_EXPORT extern pmix_server_module_t pmix_host_server; 367 PMIX_EXPORT extern pmix_server_globals_t pmix_server_globals; 368 369 370 #endif // PMIX_SERVER_OPS_H 371