1 /* -*- Mode: C; c-basic-offset:4 ; -*- */
2 /*
3  *  (C) 2001 by Argonne National Laboratory.
4  *      See COPYRIGHT in top-level directory.
5  */
6 
7 #if !defined(MPIDU_SOCK_H_INCLUDED)
8 #define MPIDU_SOCK_H_INCLUDED
9 
10 #if defined(__cplusplus)
11 #if !defined(CPLUSPLUS_BEGIN)
12 #define CPLUSPLUS_BEGIN extern "C" {
13 #define CPLUSPLUS_END }
14 #endif
15 #else
16 #define CPLUSPLUS_BEGIN
17 #define CPLUSPLUS_END
18 #endif
19 
20 CPLUSPLUS_BEGIN
21 
22 /* Load just the utility definitions that we need */
23 #include "mpichconf.h"
24 #include "mpibase.h"
25 #include "mpiutil.h"
26 #include "mpitypedefs.h"
27 #include "mpich_param_vals.h"
28 /* implementation specific header file */
29 #include "mpidu_socki.h"
30 
31 
32 /*D
33 MPIDU_SOCK_ERR - Extended error classes specific to the Sock module
34 
35 Notes:
36 The actual meaning of these error classes is defined by each function.
37 
38 Module:
39 Utility-Sock
40 D*/
41 /* FIXME: This is not the right way to add error values to an MPICH module.
42    Note that (a) the last class values are not respected by the error handling
43    code, (b) the entire point of codes and classes is to provide a
44    natural grouping of codes to a class, (c) this approach can only be used
45    by one module and hence breaks any component design, and (d) this is
46    what the MPI dynamic error codes and classes was designed for. */
47 #define MPIDU_SOCK_SUCCESS		MPI_SUCCESS
48 #define MPIDU_SOCK_ERR_FAIL		MPICH_ERR_LAST_CLASS + 1
49 #define MPIDU_SOCK_ERR_INIT		MPICH_ERR_LAST_CLASS + 2
50 #define MPIDU_SOCK_ERR_NOMEM		MPICH_ERR_LAST_CLASS + 3
51 #define MPIDU_SOCK_ERR_BAD_SET		MPICH_ERR_LAST_CLASS + 4
52 #define MPIDU_SOCK_ERR_BAD_SOCK		MPICH_ERR_LAST_CLASS + 5
53 #define MPIDU_SOCK_ERR_BAD_HOST		MPICH_ERR_LAST_CLASS + 6
54 #define MPIDU_SOCK_ERR_BAD_HOSTNAME     MPICH_ERR_LAST_CLASS + 7
55 #define MPIDU_SOCK_ERR_BAD_PORT		MPICH_ERR_LAST_CLASS + 8
56 #define MPIDU_SOCK_ERR_BAD_BUF		MPICH_ERR_LAST_CLASS + 9
57 #define MPIDU_SOCK_ERR_BAD_LEN		MPICH_ERR_LAST_CLASS + 10
58 #define MPIDU_SOCK_ERR_SOCK_CLOSED	MPICH_ERR_LAST_CLASS + 11
59 #define MPIDU_SOCK_ERR_CONN_CLOSED	MPICH_ERR_LAST_CLASS + 12
60 #define MPIDU_SOCK_ERR_CONN_FAILED	MPICH_ERR_LAST_CLASS + 13
61 #define MPIDU_SOCK_ERR_INPROGRESS	MPICH_ERR_LAST_CLASS + 14
62 #define MPIDU_SOCK_ERR_TIMEOUT		MPICH_ERR_LAST_CLASS + 15
63 #define MPIDU_SOCK_ERR_INTR		MPICH_ERR_LAST_CLASS + 16
64 #define MPIDU_SOCK_ERR_NO_NEW_SOCK	MPICH_ERR_LAST_CLASS + 17
65 
66 
67 /*E
68 MPIDU_Sock_op_t - enumeration of posted operations that can be completed by the Sock module
69 
70 Notes:
71 MPIDU_SOCK_OP_ACCEPT is different that the other operations.  When returned by MPIDU_Sock_wait(), operations other than
72 MPIDU_SOCK_OP_ACCEPT mark the completion of a previously posted operation.  MPIDU_SOCK_OP_ACCEPT indicates that a new connection is
73 being formed and that MPIDU_Sock_accept() should be called.
74 
75 Module:
76 Utility-Sock
77 E*/
78 typedef enum MPIDU_Sock_op
79 {
80     MPIDU_SOCK_OP_READ,
81     MPIDU_SOCK_OP_WRITE,
82     MPIDU_SOCK_OP_ACCEPT,
83     MPIDU_SOCK_OP_CONNECT,
84     MPIDU_SOCK_OP_CLOSE,
85     MPIDU_SOCK_OP_WAKEUP
86 } MPIDU_Sock_op_t;
87 
88 
89 /*S
90 MPIDU_Sock_event_t - event structure returned by MPIDU_Sock_wait() describing the operation that completed
91 
92 Fields:
93 + op_type - type of operation that completed
94 . num_bytes - number of bytes transferred (if appropriate)
95 . user_ptr - user pointer associated with the sock on which this operation completed
96 - error - a MPI error code with a Sock extended error class
97 
98 Notes:
99 The num_bytes field is only used when a posted read or write operation completes.
100 
101 Module:
102 Utility-Sock
103 S*/
104 typedef struct MPIDU_Sock_event
105 {
106     MPIDU_Sock_op_t op_type;
107     MPIU_Size_t num_bytes;
108     void * user_ptr;
109     int error;
110 } MPIDU_Sock_event_t;
111 
112 
113 /*@
114 MPIDU_Sock_init - initialize the Sock communication library
115 
116 Return value: a MPI error code with a Sock extended error class
117 + MPI_SUCCESS - initialization completed successfully
118 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
119 - MPIDU_SOCK_ERR_FAIL - other failure; initialization failed
120 
121 Notes:
122 The Sock module may be initialized multiple times.  The implementation should perform reference counting if necessary.
123 
124 Module:
125 Utility-Sock
126 @*/
127 int MPIDU_Sock_init(void);
128 
129 
130 /*@
131 MPIDU_Sock_finalize - shutdown the Sock communication library
132 
133 Return value: a MPI error code with a Sock extended error class
134 + MPI_SUCCESS - shutdown completed successfully
135 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
136 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
137 - MPIDU_SOCK_ERR_FAIL - other failure; shutdown failed
138 
139 Notes:
140 <BRT> What are the semantics of finalize?  Is it responsible for releasing any resources (socks and sock sets) that the calling
141 code(s) leaked?  Should it block until all OS resources are released?
142 
143 Module:
144 Utility-Sock
145 @*/
146 int MPIDU_Sock_finalize(void);
147 
148 
149 /*@
150 MPIDU_Sock_get_host_description - obtain a description of the host's
151 communication capabilities
152 
153 Input Parameters:
154 + myRank - Rank of this process in its MPI_COMM_WORLD.  This can be used
155   to find environment variables that are specific for this process.
156 . host_description - character array in which the function can store a string
157   describing the communication capabilities of the host
158 - len - length of the character array
159 
160 Return value: a MPI error code with a Sock extended error class
161 + MPI_SUCCESS - description successfully obtained and placed in host_description
162 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
163 . MPIDU_SOCK_ERR_BAD_LEN - len parameter is less than zero
164 . MPIDU_SOCK_ERR_BAD_HOST - host_description parameter not big enough to
165   store required information
166 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
167 - MPIDU_SOCK_ERR_FAIL - unable to obtain network interface information from OS
168 
169 Notes:
170 The host description string returned by the function is defined by the
171 implementation and should not be interpreted by the
172 application.  This string is to be supplied to MPIDU_Sock_post_connect() when
173 one wishes to form a connection with this host.
174 
175 Module:
176 Utility-Sock
177 @*/
178 int MPIDU_Sock_get_host_description(int myRank, char * host_description, int len);
179 
180 
181 /*@
182 MPIDU_Sock_hostname_to_host_description - convert a host name to a description of the host's communication capabilities
183 
184 Input Parameters:
185 + hostname - host name string
186 . host_description - character array in which the function can store a string describing the communication capabilities of the host
187 - len - length of host_description
188 
189 Return value: a MPI error code with a Sock extended error class
190 + MPI_SUCCESS - description successfully obtained and placed in host_description
191 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
192 . MPIDU_SOCK_ERR_BAD_LEN - len parameter is less than zero
193 . MPIDU_SOCK_ERR_BAD_HOSTNAME - hostname parameter not valid
194 . MPIDU_SOCK_ERR_BAD_HOST - host_description parameter not big enough to store required information
195 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
196 - MPIDU_SOCK_ERR_FAIL - unable to obtain network interface information from OS
197 
198 Notes:
199 The host description string returned by the function is defined by the implementation and should not be interpreted by the
200 application.  This string is to be supplied to MPIDU_Sock_post_connect() when one wishes to form a connection with the host
201 specified by hostname.
202 
203 Module:
204 Utility-Sock
205 @*/
206 int MPIDU_Sock_hostname_to_host_description(char *hostname, char * host_description, int len);
207 
208 /*@
209 MPIDU_Sock_create_set - create a new sock set object
210 
211 Output Parameter:
212 . set - pointer to the new sock set object
213 
214 Return value: a MPI error code with a Sock extended error class
215 + MPI_SUCCESS - new sock set successfully create
216 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
217 . MPIDU_SOCK_ERR_BAD_SET - pointer to the sock set object is bad
218 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
219 - MPIDU_SOCK_ERR_FAIL - other failure
220 
221 Notes:
222 A sock set contains zero or more sock objects.  Each sock object belongs to a single sock set and is bound to that set for the life
223 of that object.
224 
225 Module:
226 Utility-Sock
227 @*/
228 int MPIDU_Sock_create_set(MPIDU_Sock_set_t * set);
229 
230 
231 /*@
232 MPIDU_Sock_destroy_set - destroy an existing sock set, releasing an internal resource associated with that set
233 
234 Input Parameter:
235 . set - set to be destroyed
236 
237 Return value: a MPI error code with a Sock extended error class
238 + MPI_SUCCESS - sock set successfully destroyed
239 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
240 . MPIDU_SOCK_ERR_BAD_SET - invalid sock set
241 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
242 - MPIDU_SOCK_ERR_FAIL - unable to destroy the sock set (<BRT> because it still contained active sock objects?)
243 
244 Notes:
245 <BRT> What are the semantics for destroying a sock set that still contains active sock objects?  sock objects by definition
246 cannot exist outside of a set.
247 
248 It is consider erroneous to destroy a set that still contains sock objects or is being operated upon with an of the Sock routines.
249 
250 Module:
251 Utility-Sock
252 @*/
253 int MPIDU_Sock_destroy_set(MPIDU_Sock_set_t set);
254 
255 
256 /*@
257 MPIDU_Sock_native_to_sock - convert a native file descriptor/handle to a sock object
258 
259 Input Parameters:
260 + set - sock set to which the new sock should be added
261 . fd - native file descriptor
262 - user_ptr - user pointer to be associated with the new sock
263 
264 Output Parameter:
265 . sock - new sock object
266 
267 Return value: a MPI error code with a Sock extended error class
268 + MPI_SUCCESS - sock successfully created
269 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
270 . MPIDU_SOCK_ERR_BAD_SET - invalid sock set
271 . MPIDU_SOCK_ERR_BAD_NATIVE_FD - invalid native file descriptor
272 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
273 - MPIDU_SOCK_ERR_FAIL - other failure; listener sock could not be created
274 
275 Notes:
276 The constraints on which file descriptors/handles may be converted to a sock object are defined by the implementation.  The
277 implementation may return MPIDU_SOCK_ERR_BAD_NATIVE_FD if the descriptor/handle cannot be used with the implementation.  It is
278 possible, however, that the conversion of an inappropriate descriptor/handle may complete successfully but the sock object may not
279 function properly.
280 
281 Thread safety:
282 The addition of a new sock object to the sock set may occur while other threads are performing operations on the same sock set.
283 Thread safety of simultaneously operations on the same sock set must be guaranteed by the Sock implementation.
284 
285 @*/
286 int MPIDU_Sock_native_to_sock(MPIDU_Sock_set_t set, MPIDU_SOCK_NATIVE_FD fd, void * user_ptr, MPIDU_Sock_t * sock);
287 
288 
289 /*@
290 MPIDU_Sock_listen - establish a listener sock
291 
292 Input Parameters:
293 + set - sock set to which the listening sock should be added
294 . user_ptr - user pointer associated with the new sock object
295 - port - desired port (or zero if a specific port is not desired)
296 
297 Output Parameters:
298 + port - port assigned to the listener
299 - sock - new listener sock
300 
301 Return value: a MPI error code with a Sock extended error class
302 + MPI_SUCCESS - listener sock successfully established
303 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
304 . MPIDU_SOCK_ERR_BAD_SET - invalid sock set
305 . MPIDU_SOCK_ERR_BAD_PORT - port number is out of range or pointer to port parameter is invalid
306 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
307 - MPIDU_SOCK_ERR_FAIL - other failure; listener sock could not be created
308 
309 Events generated:
310 . MPIDU_SOCK_OP_ACCEPT - each time a new connection is being formed and needs to be accepted (with MPIDU_Sock_accept())
311 
312 Event errors:
313 + MPI_SUCCESS - new sock waiting to be accepted
314 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
315 - MPIDU_SOCK_ERR_FAIL - other failure?
316 
317 Notes:
318 While not a post routine, this routine can generate events.  In fact,
319 unlike the post routine, many MPIDU_SOCK_OP_ACCEPT events can
320 be generated from a listener (typically one per incoming connection attempt).
321 
322 The implementation may generate an event as soon it is notified that a
323 new connection is forming.  In such an implementation,
324 MPIDU_Sock_accept() may be responsible for finalizing the connection.
325 It is also possible that the connection may fail to
326 complete, causing MPIDU_Sock_accept() to be unable to obtain a sock
327 despite the event notification.
328 
329 The environment variable MPICH_PORT_RANGE=min:max may be used to
330 restrict the ports mpich processes listen on.
331 
332 Thread safety:
333 The addition of the listener sock object to the sock set may occur
334 while other threads are performing operations on the same sock
335 set.  Thread safety of simultaneously operations on the same sock set
336 must be guaranteed by the Sock implementation.
337 
338 Module:
339 Utility-Sock
340 @*/
341 int MPIDU_Sock_listen(MPIDU_Sock_set_t set, void * user_ptr, int * port, MPIDU_Sock_t * sock);
342 
343 
344 /*@
345 MPIDU_Sock_accept - obtain the sock object associated with a new connection
346 
347 Input Parameters:
348 + listener_sock - listener sock object from which to obtain the new connection
349 . set - sock set to which the new sock object should be added
350 - user_ptr - user pointer associated with the new sock object
351 
352 Output Parameter:
353 . sock - sock object for the new connection
354 
355 Return value: a MPI error code with a Sock extended error class
356 + MPI_SUCCESS - new connection successfully established and associated with new sock objecta
357 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
358 . MPIDU_SOCK_ERR_NO_NEW_SOCK - no new connection was available
359 . MPIDU_SOCK_ERR_BAD_SOCK - invalid listener sock or bad pointer to new sock object
360 . MPIDU_SOCK_ERR_BAD_SET - invalid sock set
361 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
362 - MPIDU_SOCK_ERR_FAIL - failed to acquire a new connection
363 
364 Notes:
365 In the event of a connection failure, MPIDU_Sock_accept() may fail to acquire and return a new sock despite any
366 MPIDU_SOCK_OP_ACCEPT event notification.  On the other hand, MPIDU_Sock_accept() may return a sock for which the underlying
367 connection has already failed.  (The Sock implementation may be unaware of the failure until read/write operations are performed.)
368 
369 Thread safety:
370 The addition of the new sock object to the sock set may occur while other threads are performing operations on the same sock set.
371 Thread safety of simultaneously operations on the same sock set must be guaranteed by the Sock implementation.
372 
373 MPIDU_Sock_accept() may fail to return a new sock if multiple threads call MPIDU_Sock_accept() and queue of new connections is
374 depleted.  In this case, MPIDU_SOCK_ERR_NO_SOCK is returned.
375 
376 Module:
377 Utility-Sock
378 @*/
379 int MPIDU_Sock_accept(MPIDU_Sock_t listener_sock, MPIDU_Sock_set_t set, void * user_ptr, MPIDU_Sock_t * sock);
380 
381 
382 /*@
383 MPIDU_Sock_post_connect - request that a new connection be formed
384 
385 Input Parameters:
386 + set - sock set to which the new sock object should be added
387 . user_ptr - user pointer associated with the new sock object
388 . host_description - string containing the communication capabilities of the listening host
389 + port - port number of listener sock on the listening host
390 
391 Output Parameter:
392 . sock - new sock object associated with the connection request
393 
394 Return value: a MPI error code with a Sock extended error class
395 + MPI_SUCCESS - request to form new connection successfully posted
396 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
397 . MPIDU_SOCK_ERR_BAD_SET - invalid sock set
398 . MPIDU_SOCK_ERR_BAD_HOST - host description string is not valid
399 . MPIDU_SOCK_ERR_BAD_PORT - port number is out of range
400 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
401 - MPIDU_SOCK_ERR_FAIL - other failure attempting to post connection request
402 
403 Events generated:
404 . MPIDU_SOCK_OP_CONNECT
405 
406 Event errors: a MPI error code with a Sock extended error class
407 + MPI_SUCCESS -  successfully established
408 . MPIDU_SOCK_ERR_CONN_FAILED - failed to connect to the remote host
409 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
410 - MPIDU_SOCK_ERR_FAIL - other error?
411 
412 <BRT> Any other event errors?  Does the sock channel require finer granularity?
413 
414 Notes:
415 The host description of the listening host is supplied MPIDU_Sock_get_host_description().  The intention is that the description
416 contain an enumeration of interface information so that the MPIDU_Sock_connect() can try each of the interfaces until it succeeds
417 in forming a connection.  Having a complete set of interface information also allows a particular interface be used selected by the
418 user at runtime using the MPICH_NETMASK.  <BRT> The name of the environment variable seems wrong.  Perhaps MPICH_INTERFACE?  We
419 should ask the Systems group.
420 
421 Thread safety:
422 The addition of the new sock object to the sock set may occur while other threads are performing operations on the same sock set.
423 Thread safety of simultaneously operations on the same sock set must be guaranteed by the Sock implementation.
424 
425 Module:
426 Utility-Sock
427 @*/
428 int MPIDU_Sock_post_connect(MPIDU_Sock_set_t set, void * user_ptr, char * host_description, int port, MPIDU_Sock_t * sock);
429 
430 /*S
431   MPIDU_Sock_ifaddr_t - Structure to hold an Internet address.
432 
433 + len - Length of the address.  4 for IPv4, 16 for IPv6.
434 - ifaddr - Address bytes (as bytes, not characters)
435 
436 S*/
437 typedef struct MPIDU_Sock_ifaddr_t {
438     int len, type;
439     unsigned char ifaddr[16];
440 } MPIDU_Sock_ifaddr_t;
441 
442 /*@ MPIDU_Sock_post_connect_ifaddr - Post a connection given an interface
443   address (bytes, not string).
444 
445   This is the basic routine.  MPIDU_Sock_post_connect converts the
446   host description into the ifaddr and calls this routine.
447   @*/
448 int MPIDU_Sock_post_connect_ifaddr( MPIDU_Sock_set_t sock_set,
449 				    void * user_ptr,
450 				    MPIDU_Sock_ifaddr_t *ifaddr, int port,
451 				    MPIDU_Sock_t * sockp);
452 
453 
454 /*@
455 MPIDU_Sock_set_user_ptr - change the user pointer associated with a sock object
456 
457 Input Parameters:
458 + sock - sock object
459 - user_ptr - new user pointer
460 
461 Return value: a MPI error code with a Sock extended error class
462 + MPI_SUCCESS - user pointer successfully updated
463 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
464 . MPIDU_SOCK_ERR_BAD_SOCK - invalid sock object
465 - MPIDU_SOCK_ERR_FAIL - other failure?
466 
467 Module:
468 Utility-Sock
469 @*/
470 int MPIDU_Sock_set_user_ptr(MPIDU_Sock_t sock, void * user_ptr);
471 
472 
473 /*@
474 MPIDU_Sock_post_close - request that an existing connection be closed
475 
476 Input Parameter:
477 . sock - sock object to be closed
478 
479 Return value: a MPI error code with a Sock extended error class
480 + MPI_SUCCESS - request to close the connection was successfully posted
481 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
482 . MPIDU_SOCK_ERR_BAD_SOCK - invalid sock object or close already posted
483 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
484 - MPIDU_SOCK_ERR_FAIL - other failure?
485 
486 Events generated:
487 + MPIDU_SOCK_OP_CLOSE
488 . MPIDU_SOCK_OP_READ
489 - MPIDU_SOCK_OP_WRITE
490 
491 Event errors: a MPI error code with a Sock extended error class
492 + MPI_SUCCESS -  successfully established
493 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
494 - MPIDU_SOCK_ERR_FAIL - an error occurred closing the sock; sock object is still destroyed
495 
496 Notes:
497 If any other operations are posted on the specified sock, they will be terminated.  An appropriate event will be generated for each
498 terminated operation.  All such events will be delivered by MPIDU_Sock_wait() prior to the delivery of the MPIDU_SOCK_OP_CLOSE
499 event.
500 
501 The sock object is destroyed just prior to the MPIDU_SOCK_OP_CLOSE event being returned by MPIDU_Sock_wait().  Any oustanding
502 references to the sock object held by the application should be considered invalid and not used again.
503 
504 Thread safety:
505 MPIDU_Sock_post_close() may be called while another thread is calling or blocking in MPIDU_Sock_wait() specifying the same sock set
506 to which this sock belongs.  If another thread is blocking MPIDU_Sock_wait() and the close operation causes the sock set to become
507 empty, then MPIDU_Sock_wait() will return with an error.
508 
509 Calling any of the immediate or post routines during or after the call to MPIDU_Sock_post_close() is consider an application error.
510 The result of doing so is undefined.  The application should coordinate the closing of a sock with the activities of other threads
511 to ensure that simultaneous calls do not occur.
512 
513 Module:
514 Utility-Sock
515 @*/
516 int MPIDU_Sock_post_close(MPIDU_Sock_t sock);
517 
518 
519 /*E
520 MPIDU_Sock_progress_update_func_t - progress update callback functions
521 
522 If a pointer to a function of this type is passed to one of the post read or write functions, the implementation must honor the
523 following rules:
524 
525 1) The sock progress engine will call this function when partial data has been read or written for the posted operation.
526 
527 2) All progress_update calls must complete before completion notification is signalled.  In other words, MPIDU_Sock_wait() will not
528 return until all progress_update calls have completed.
529 
530 Notes:
531 <BRT> We need to define ordering and delivery from multiple threads.  Do the handlers have to be thread safe?  If so, then the
532 internal progress engine could block on an application routine.
533 
534 Module:
535 Utility-Sock
536 E*/
537 typedef int (* MPIDU_Sock_progress_update_func_t)(MPIU_Size_t num_bytes, void * user_ptr);
538 
539 
540 /*@
541 MPIDU_Sock_post_read - request that data be read from a sock
542 
543 Input Parameters:
544 + sock - sock object from which data is to be read
545 . buf - buffer into which the data should be placed
546 . minlen - minimum number of bytes to read
547 . maxlen - maximum number of bytes to read
548 + upate_fn - application progress update function (may be NULL)
549 
550 Return value: a MPI error code with a Sock extended error class
551 + MPI_SUCCESS - request to read was successfully posted
552 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
553 . MPIDU_SOCK_ERR_BAD_SOCK - invalid sock object
554 . MPIDU_SOCK_ERR_BAD_LEN - length parameters must be greater than zero and maxlen must be greater than minlen
555 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
556 . MPIDU_SOCK_ERR_INPROGRESS - this operation overlapped with another like operation already in progress
557 - MPIDU_SOCK_ERR_FAIL - other error attempting to post read
558 
559 Events generated:
560 . MPIDU_SOCK_OP_READ
561 
562 Event errors: a MPI error code with a Sock extended error class
563 + MPI_SUCCESS -  successfully established
564 . MPIDU_SOCK_ERR_BAD_BUF - using the buffer described by buf and maxlen resulted in a memory fault
565 . MPIDU_SOCK_ERR_SOCK_CLOSED - the sock object was closed locally
566 . MPIDU_SOCK_ERR_CONN_CLOSED - the connection was closed by the peer
567 . MPIDU_SOCK_ERR_CONN_FAILED - the connection failed
568 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
569 - MPIDU_SOCK_ERR_FAIL - other error completing the read
570 
571 Notes:
572 Only one read operation may be posted at a time.  Furthermore, an immediate read may not be performed while a posted write is
573 outstanding.  This is considered to be an application error, and the results of doing so are undefined.  The implementation may
574 choose to catch the error and return MPIDU_SOCK_ERR_INPROGRESS, but it is not required to do so.
575 
576 If MPIDU_Sock_post_close() is called before the posted read operation completes, the read operation will be terminated and a
577 MPIDU_SOCK_OP_READ event containing a MPIDU_SOCK_ERR_SOCK_CLOSED error will be generated.  This event will be returned by
578 MPIDU_Sock_wait() prior to the MPIDU_SOCK_OP_CLOSE event.
579 
580 Thread safety:
581 MPIDU_Sock_post_read() may be called while another thread is attempting to perform an immediate write or post a write operation on
582 the same sock.  MPIDU_Sock_post_read() may also be called while another thread is calling or blocking in MPIDU_Sock_wait() on the
583 same sock set to which the specified sock belongs.
584 
585 MPIDU_Sock_post_write() may not be called while another thread is performing an immediate read on the same sock.  This is
586 considered to be an application error, and the results of doing so are undefined.  The implementation may choose to catch the error
587 and return MPIDU_SOCK_ERR_INPROGRESS, but it is not required to do so.
588 
589 Calling MPIDU_Sock_post_read() during or after the call to MPIDU_Sock_post_close() is consider an application error.  The result of
590 doing so is undefined.  The application should coordinate the closing of a sock with the activities of other threads to ensure that
591 one thread is not attempting to post a new operation while another thread is attempting to close the sock.
592 
593 Module:
594 Utility-Sock
595 @*/
596 int MPIDU_Sock_post_read(MPIDU_Sock_t sock, void * buf, MPIU_Size_t minbr, MPIU_Size_t maxbr,
597                          MPIDU_Sock_progress_update_func_t fn);
598 
599 
600 /*@
601 MPIDU_Sock_post_readv - request that a vector of data be read from a sock
602 
603 Input Parameters:
604 + sock - sock object from which the data is to read
605 . iov - I/O vector describing buffers into which the data is placed
606 . iov_n - number of elements in I/O vector (must be less than MPID_IOV_LIMIT)
607 + upate_fn - application progress update function (may be NULL)
608 
609 Return value: a MPI error code with a Sock extended error class
610 + MPI_SUCCESS - request to read was successfully posted
611 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
612 . MPIDU_SOCK_ERR_BAD_SOCK - invalid sock object
613 . MPIDU_SOCK_ERR_BAD_LEN - iov_n is out of range
614 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
615 . MPIDU_SOCK_ERR_INPROGRESS - this operation overlapped with another like operation already in progress
616 - MPIDU_SOCK_ERR_FAIL - other error attempting to post read
617 
618 Events generated:
619 . MPIDU_SOCK_OP_READ
620 
621 Event errors: a MPI error code with a Sock extended error class
622 + MPI_SUCCESS -  successfully established
623 . MPIDU_SOCK_ERR_BAD_BUF - using the buffer described by iov and iov_n resulted in a memory fault
624 . MPIDU_SOCK_ERR_SOCK_CLOSED - the sock object was closed locally
625 . MPIDU_SOCK_ERR_CONN_CLOSED - the connection was closed by the peer
626 . MPIDU_SOCK_ERR_CONN_FAILED - the connection failed
627 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
628 - MPIDU_SOCK_ERR_FAIL - other error completing the read
629 
630 Notes:
631 Only one read operation may be posted at a time.  Furthermore, an immediate read may not be performed while a posted write is
632 outstanding.  This is considered to be an application error, and the results of doing so are undefined.  The implementation may
633 choose to catch the error and return MPIDU_SOCK_ERR_INPROGRESS, but it is not required to do so.
634 
635 If MPIDU_Sock_post_close() is called before the posted read operation completes, the read operation will be terminated and a
636 MPIDU_SOCK_OP_READ event containing a MPIDU_SOCK_ERR_SOCK_CLOSED error will be generated.  This event will be returned by
637 MPIDU_Sock_wait() prior to the MPIDU_SOCK_OP_CLOSE event.
638 
639 Thread safety:
640 MPIDU_Sock_post_readv() may be called while another thread is attempting to perform an immediate write or post a write operation on
641 the same sock.  MPIDU_Sock_post_readv() may also be called while another thread is calling or blocking in MPIDU_Sock_wait() on the
642 same sock set to which the specified sock belongs.
643 
644 MPIDU_Sock_post_readv() may not be called while another thread is performing an immediate read on the same sock.  This is
645 considered to be an application error, and the results of doing so are undefined.  The implementation may choose to catch the error
646 and return MPIDU_SOCK_ERR_INPROGRESS, but it is not required to do so.
647 
648 Calling MPIDU_Sock_post_readv() during or after the call to MPIDU_Sock_post_close() is consider an application error.  The result
649 of doing so is undefined.  The application should coordinate the closing of a sock with the activities of other threads to ensure
650 that one thread is not attempting to post a new operation while another thread is attempting to close the sock.
651 
652 Module:
653 Utility-Sock
654 @*/
655 int MPIDU_Sock_post_readv(MPIDU_Sock_t sock, MPID_IOV * iov, int iov_n, MPIDU_Sock_progress_update_func_t fn);
656 
657 
658 /*@
659 MPIDU_Sock_post_write - request that data be written to a sock
660 
661 Input Parameters:
662 + sock - sock object which the data is to be written
663 . buf - buffer containing the data
664 . minlen - minimum number of bytes to write
665 . maxlen - maximum number of bytes to write
666 + upate_fn - application progress aupdate function (may be NULL)
667 
668 Return value: a MPI error code with a Sock extended error class
669 + MPI_SUCCESS - request to write was successfully posted
670 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
671 . MPIDU_SOCK_ERR_BAD_SOCK - invalid sock object
672 . MPIDU_SOCK_ERR_BAD_LEN - length parameters must be greater than zero and maxlen must be greater than minlen
673 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
674 . MPIDU_SOCK_ERR_INPROGRESS - this operation overlapped with another like operation already in progress
675 - MPIDU_SOCK_ERR_FAIL - other error attempting to post write
676 
677 Events generated:
678 . MPIDU_SOCK_OP_WRITE
679 
680 Event errors: a MPI error code with a Sock extended error class
681 + MPI_SUCCESS -  successfully established
682 . MPIDU_SOCK_ERR_BAD_BUF - using the buffer described by buf and maxlen resulted in a memory fault
683 . MPIDU_SOCK_ERR_SOCK_CLOSED - the sock object was closed locally
684 . MPIDU_SOCK_ERR_CONN_CLOSED - the connection was closed by the peer
685 . MPIDU_SOCK_ERR_CONN_FAILED - the connection failed
686 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
687 - MPIDU_SOCK_ERR_FAIL - other error completing the write
688 
689 Notes:
690 Only one write operation may be posted at a time.  Furthermore, an immediate write may not be performed while a posted write is
691 outstanding.  This is considered to be an application error, and the results of doing so are undefined.  The implementation may
692 choose to catch the error and return MPIDU_SOCK_ERR_INPROGRESS, but it is not required to do so.
693 
694 If MPIDU_Sock_post_close() is called before the posted write operation completes, the write operation will be terminated and a
695 MPIDU_SOCK_OP_WRITE event containing a MPIDU_SOCK_ERR_SOCK_CLOSED error will be generated.  This event will be returned by
696 MPIDU_Sock_wait() prior to the MPIDU_SOCK_OP_CLOSE event.
697 
698 Thread safety:
699 MPIDU_Sock_post_write() may be called while another thread is attempting to perform an immediate read or post a read operation on
700 the same sock.  MPIDU_Sock_post_write() may also be called while another thread is calling or blocking in MPIDU_Sock_wait() on the
701 same sock set to which the specified sock belongs.
702 
703 MPIDU_Sock_post_write() may not be called while another thread is performing an immediate write on the same sock.  This is
704 considered to be an application error, and the results of doing so are undefined.  The implementation may choose to catch the error
705 and return MPIDU_SOCK_ERR_INPROGRESS, but it is not required to do so.
706 
707 Calling MPIDU_Sock_post_write() during or after the call to MPIDU_Sock_post_close() is consider an application error.  The result
708 of doing so is undefined.  The application should coordinate the closing of a sock with the activities of other threads to ensure
709 that one thread is not attempting to post a new operation while another thread is attempting to close the sock.  <BRT> Do we really
710 need this flexibility?
711 
712 Module:
713 Utility-Sock
714 @*/
715 int MPIDU_Sock_post_write(MPIDU_Sock_t sock, void * buf, MPIU_Size_t min, MPIU_Size_t max,
716 			  MPIDU_Sock_progress_update_func_t fn);
717 
718 
719 /*@
720 MPIDU_Sock_post_writev - request that a vector of data be written to a sock
721 
722 Input Parameters:
723 + sock - sock object which the data is to be written
724 . iov - I/O vector describing buffers of data to be written
725 . iov_n - number of elements in I/O vector (must be less than MPID_IOV_LIMIT)
726 + upate_fn - application progress update function (may be NULL)
727 
728 Return value: a MPI error code with a Sock extended error class
729 + MPI_SUCCESS - request to write was successfully posted
730 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
731 . MPIDU_SOCK_ERR_BAD_SOCK - invalid sock object
732 . MPIDU_SOCK_ERR_BAD_LEN - iov_n is out of range
733 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
734 . MPIDU_SOCK_ERR_INPROGRESS - this operation overlapped with another like operation already in progress
735 - MPIDU_SOCK_ERR_FAIL - other error attempting to post write
736 
737 Events generated:
738 . MPIDU_SOCK_OP_WRITE
739 
740 Event errors: a MPI error code with a Sock extended error class
741 + MPI_SUCCESS -  successfully established
742 . MPIDU_SOCK_ERR_BAD_BUF - using the buffer described by iov and iov_n resulted in a memory fault
743 . MPIDU_SOCK_ERR_SOCK_CLOSED - the sock object was closed locally
744 . MPIDU_SOCK_ERR_CONN_CLOSED - the connection was closed by the peer
745 . MPIDU_SOCK_ERR_CONN_FAILED - the connection failed
746 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
747 - MPIDU_SOCK_ERR_FAIL - other error completing the write
748 
749 Notes:
750 Only one write operation may be posted at a time.  Furthermore, an immediate write may not be performed while a posted write is
751 outstanding.  This is considered to be an application error, and the results of doing so are undefined.  The implementation may
752 choose to catch the error and return MPIDU_SOCK_ERR_INPROGRESS, but it is not required to do so.
753 
754 If MPIDU_Sock_post_close() is called before the posted write operation completes, the write operation will be terminated and a
755 MPIDU_SOCK_OP_WRITE event containing a MPIDU_SOCK_ERR_SOCK_CLOSED error will be generated.  This event will be returned by
756 MPIDU_Sock_wait() prior to the MPIDU_SOCK_OP_CLOSE event.
757 
758 Thread safety:
759 MPIDU_Sock_post_writev() may be called while another thread is attempting to perform an immediate read or post a read operation on
760 the same sock.  MPIDU_Sock_post_writev() may also be called while another thread is calling or blocking in MPIDU_Sock_wait() on the
761 same sock set to which the specified sock belongs.
762 
763 MPIDU_Sock_post_writev() may not be called while another thread is performing an immediate write on the same sock.  This is
764 considered to be an application error, and the results of doing so are undefined.  The implementation may choose to catch the error
765 and return MPIDU_SOCK_ERR_INPROGRESS, but it is not required to do so.
766 
767 Calling MPIDU_Sock_post_writev() during or after the call to MPIDU_Sock_post_close() is consider an application error.  The result
768 of doing so is undefined.  The application should coordinate the closing of a sock with the activities of other threads to ensure
769 that one thread is not attempting to post a new operation while another thread is attempting to close the sock.
770 
771 Module:
772 Utility-Sock
773 @*/
774 int MPIDU_Sock_post_writev(MPIDU_Sock_t sock, MPID_IOV * iov, int iov_n, MPIDU_Sock_progress_update_func_t fn);
775 
776 
777 /*@
778 MPIDU_Sock_wait - wait for an event
779 
780 Input Parameters:
781 + set - sock set upon which to wait for an event
782 - timeout - timeout in milliseconds (<0 for infinity)
783 
784 Output Parameter:
785 . event - pointer to the event structure to be populated
786 
787 Return value: a MPI error code with a Sock extended error class
788 + MPI_SUCCESS - a new event was returned
789 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
790 . MPIDU_SOCK_ERR_BAD_SET - invalid sock set object
791 . MPIDU_SOCK_ERR_TIMEOUT - a timeout occurred
792 . MPIDU_SOCK_ERR_INTR - the routine was interrupted by a call to MPIDU_Sock_wakeup()
793 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
794 - MPIDU_SOCK_ERR_FAIL - other error (are there any?)
795 
796 Notes:
797 MPIDU_Sock_wakeup() can be called from another thread to force MPIDU_Sock_wait() to return with a MPIDU_SOCK_ERR_INTR error.
798 MPIDU_Sock_wakeup() may not be called from within a progress update function or any function directly or indirectly called by a
799 progress update function.
800 
801 Thread safety:
802 New operations may be posted to sock contained in the specified sock set while another thread is calling or blocking in
803 MPIDU_Sock_wait().  These operations should complete as though they were posted before MPIDU_Sock_wait() was called.
804 
805 Module:
806 Utility-Sock
807 @*/
808 int MPIDU_Sock_wait(MPIDU_Sock_set_t set, int timeout, MPIDU_Sock_event_t * event);
809 
810 
811 /*@
812 MPIDU_Sock_wakeup - wakeup a MPIDU_Sock_wait blocking in another thread
813 
814 Input Parameter:
815 . set - sock set upon which to wait for an event
816 
817 Return value: a MPI error code with a Sock extended error class
818 + MPI_SUCCESS - wakeup request successfully processed
819 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
820 . MPIDU_SOCK_ERR_BAD_SET - invalid sock set object
821 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory (is this possible?)
822 - MPIDU_SOCK_ERR_FAIL - other error (are there any?)
823 
824 Notes:
825 This routine forces a MPIDU_Sock_wait() blocking in another thread to wakeup and return a MPIDU_SOCK_ERR_INTR error.
826 MPIDU_Sock_wakeup() may not be called from within a progress update function or any function directly or indirectly called by a
827 progress update function.
828 
829 The implementation should strive to only wakeup a MPIDU_Sock_wait() that is already blocking; however, it is acceptable (although
830 undesireable) for it wakeup a MPIDU_Sock_wait() that is called in the future.
831 
832 Module:
833 Utility-Sock
834 @*/
835 int MPIDU_Sock_wakeup(MPIDU_Sock_set_t set);
836 
837 /*@
838 MPIDU_Sock_read - perform an immediate read
839 
840 Input Parameters:
841 + sock - sock object from which data is to be read
842 . buf - buffer into which the data should be placed
843 - len - maximum number of bytes to read
844 
845 Output Parameter:
846 . num_read - number of bytes actually read
847 
848 Return value: a MPI error code with a Sock extended error class
849 + MPI_SUCCESS - no error encountered during the read operation
850 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
851 . MPIDU_SOCK_ERR_BAD_SOCK - invalid sock object
852 . MPIDU_SOCK_ERR_BAD_BUF - using the buffer described by buf and len resulted in a memory fault
853 . MPIDU_SOCK_ERR_BAD_LEN - length parameter must be greater than zero
854 . MPIDU_SOCK_ERR_SOCK_CLOSED - the sock object was closed locally
855 . MPIDU_SOCK_ERR_CONN_CLOSED - the connection was closed by the peer
856 . MPIDU_SOCK_ERR_CONN_FAILED - the connection failed
857 . MPIDU_SOCK_ERR_INPROGRESS - this operation overlapped with another like operation already in progress
858 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
859 - MPIDU_SOCK_ERR_FAIL - other error attempting to post read
860 
861 Notes:
862 
863 An immediate read may not be performed while a posted read is outstanding on the same sock.  This is considered to be an
864 application error, and the results of doing so are undefined.  The implementation may choose to catch the error and return
865 MPIDU_SOCK_ERR_INPROGRESS, but it is not required to do so.
866 
867 Thread safety:
868 MPIDU_Sock_read() may be called while another thread is attempting to perform an immediate write or post a write operation on the
869 same sock.  MPIDU_Sock_read() may also be called while another thread is calling or blocking in MPIDU_Sock_wait() on the same sock
870 set to which the specified sock belongs.
871 
872 A immediate read may not be performed if another thread is performing an immediate read on the same sock.  This is considered to be
873 an application error, and the results of doing so are undefined.  The implementation may choose to catch the error and return
874 MPIDU_SOCK_ERR_INPROGRESS, but it is not required to do so.
875 
876 Calling MPIDU_Sock_read() during or after the call to MPIDU_Sock_post_close() is consider an application error.  The result of
877 doing so is undefined, although the implementation may choose to return MPIDU_SOCK_ERR_SOCK_CLOSED if it is able to catch the
878 error.  The application should coordinate the closing of a sock with the activities of other threads to ensure that one thread is
879 not attempting to perform an immediate read while another thread is attempting to close the sock.
880 
881 Module:
882 Utility-Sock
883 @*/
884 int MPIDU_Sock_read(MPIDU_Sock_t sock, void * buf, MPIU_Size_t len, MPIU_Size_t * num_read);
885 
886 
887 /*@
888 MPIDU_Sock_readv - perform an immediate vector read
889 
890 Input Parameters:
891 + sock - sock object from which data is to be read
892 . iov - I/O vector describing buffers into which the data is placed
893 - iov_n - number of elements in I/O vector (must be less than MPID_IOV_LIMIT)
894 
895 Output Parameter:
896 . num_read - number of bytes actually read
897 
898 Return value: a MPI error code with a Sock extended error class
899 + MPI_SUCCESS - no error encountered during the read operation
900 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
901 . MPIDU_SOCK_ERR_BAD_SOCK - invalid sock object
902 . MPIDU_SOCK_ERR_BAD_BUF - using the buffer described by iov and iov_n resulted in a memory fault
903 . MPIDU_SOCK_ERR_BAD_LEN - iov_n parameter must be greater than zero and not greater than MPID_IOV_LIMIT
904 . MPIDU_SOCK_ERR_SOCK_CLOSED - the sock object was closed locally
905 . MPIDU_SOCK_ERR_CONN_CLOSED - the connection was closed by the peer
906 . MPIDU_SOCK_ERR_CONN_FAILED - the connection failed
907 . MPIDU_SOCK_ERR_INPROGRESS - this operation overlapped with another like operation already in progress
908 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
909 - MPIDU_SOCK_ERR_FAIL - other error attempting to post read
910 
911 Notes:
912 
913 An immediate read may not be performed while a posted read is outstanding on the same sock.  This is considered to be an
914 application error, and the results of doing so are undefined.  The implementation may choose to catch the error and return
915 MPIDU_SOCK_ERR_INPROGRESS, but it is not required to do so.
916 
917 Thread safety:
918 MPIDU_Sock_read() may be called while another thread is attempting to perform an immediate write or post a write operation on the
919 same sock.  MPIDU_Sock_read() may also be called while another thread is calling or blocking in MPIDU_Sock_wait() on the same sock
920 set to which the specified sock belongs.
921 
922 A immediate read may not be performed if another thread is performing an immediate read on the same sock.  This is considered to be
923 an application error, and the results of doing so are undefined.  The implementation may choose to catch the error and return
924 MPIDU_SOCK_ERR_INPROGRESS, but it is not required to do so.
925 
926 Calling MPIDU_Sock_read() during or after the call to MPIDU_Sock_post_close() is consider an application error.  The result of
927 doing so is undefined, although the implementation may choose to return MPIDU_SOCK_ERR_SOCK_CLOSED if it is able to catch the
928 error.  The application should coordinate the closing of a sock with the activities of other threads to ensure that one thread is
929 not attempting to perform an immediate read while another thread is attempting to close the sock.
930 
931 Module:
932 Utility-Sock
933 @*/
934 int MPIDU_Sock_readv(MPIDU_Sock_t sock, MPID_IOV * iov, int iov_n, MPIU_Size_t * num_read);
935 
936 
937 /*@
938 MPIDU_Sock_write - perform an immediate write
939 
940 Input Parameters:
941 + sock - sock object to which data is to be written
942 . buf - buffer containing the data to be written
943 - len - maximum number of bytes to written
944 
945 Output Parameter:
946 . num_written - actual number of bytes written
947 
948 Return value: a MPI error code with a Sock extended error class
949 + MPI_SUCCESS - no error encountered during the write operation
950 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
951 . MPIDU_SOCK_ERR_BAD_SOCK - invalid sock object
952 . MPIDU_SOCK_ERR_BAD_BUF - using the buffer described by buf and len resulted in a memory fault
953 . MPIDU_SOCK_ERR_BAD_LEN - length parameter must be greater than zero
954 . MPIDU_SOCK_ERR_SOCK_CLOSED - the sock object was closed locally
955 . MPIDU_SOCK_ERR_CONN_CLOSED - the connection was closed by the peer
956 . MPIDU_SOCK_ERR_CONN_FAILED - the connection failed
957 . MPIDU_SOCK_ERR_INPROGRESS - this operation overlapped with another like operation already in progress
958 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
959 - MPIDU_SOCK_ERR_FAIL - other error attempting to perform the write
960 
961 Notes:
962 An immediate write may not be performed while a posted write is outstanding on the same sock.  This is considered to be an
963 application error, and the results of doing so are undefined.  The implementation may choose to catch the error and return
964 MPIDU_SOCK_ERR_INPROGRESS, but it is not required to do so.
965 
966 Thread safety:
967 MPIDU_Sock_write() may be called while another thread is attempting to perform an immediate read or post a read operation on the
968 same sock.  MPIDU_Sock_write() may also be called while another thread is calling or blocking in MPIDU_Sock_wait() on the same sock
969 set to which the specified sock belongs.
970 
971 A immediate write may not be performed if another thread is performing an immediate write on the same sock.  This is considered to
972 be an application error, and the results of doing so are undefined.  The implementation may choose to catch the error and return
973 MPIDU_SOCK_ERR_INPROGRESS, but it is not required to do so.
974 
975 Calling MPIDU_Sock_write() during or after the call to MPIDU_Sock_post_close() is consider to be an application error.  The result
976 of doing so is undefined, although the implementation may choose to return MPIDU_SOCK_ERR_SOCK_CLOSED if it is able to catch the
977 error.  The application should coordinate the closing of a sock with the activities of other threads to ensure that one thread is
978 not attempting to perform an immediate write while another thread is attempting to close the sock.
979 
980 Module:
981 Utility-Sock
982 @*/
983 int MPIDU_Sock_write(MPIDU_Sock_t sock, void * buf, MPIU_Size_t len, MPIU_Size_t * num_written);
984 
985 
986 /*@
987 MPIDU_Sock_writev - perform an immediate vector write
988 
989 Input Parameters:
990 + sock - sock object to which data is to be written
991 . iov - I/O vector describing buffers of data to be written
992 - iov_n - number of elements in I/O vector (must be less than MPID_IOV_LIMIT)
993 
994 Output Parameter:
995 . num_written - actual number of bytes written
996 
997 Return value: a MPI error code with a Sock extended error class
998 + MPI_SUCCESS - no error encountered during the write operation
999 . MPIDU_SOCK_ERR_INIT - Sock module not initialized
1000 . MPIDU_SOCK_ERR_BAD_SOCK - invalid sock object
1001 . MPIDU_SOCK_ERR_BAD_BUF - using the buffer described by iov and iov_n resulted in a memory fault
1002 . MPIDU_SOCK_ERR_BAD_LEN - iov_n parameter must be greater than zero and not greater than MPID_IOV_LIMIT
1003 . MPIDU_SOCK_ERR_SOCK_CLOSED - the sock object was closed locally
1004 . MPIDU_SOCK_ERR_CONN_CLOSED - the connection was closed by the peer
1005 . MPIDU_SOCK_ERR_CONN_FAILED - the connection failed
1006 . MPIDU_SOCK_ERR_INPROGRESS - this operation overlapped with another like operation already in progress
1007 . MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
1008 - MPIDU_SOCK_ERR_FAIL - other error attempting to perform the write
1009 
1010 Notes:
1011 An immediate write may not be performed while a posted write is outstanding on the same sock.  This is considered to be an
1012 application error, and the results of doing so are undefined.  The implementation may choose to catch the error and return
1013 MPIDU_SOCK_ERR_INPROGRESS, but it is not required to do so.
1014 
1015 Thread safety:
1016 MPIDU_Sock_write() may be called while another thread is attempting to perform an immediate read or post a read operation on the
1017 same sock.  MPIDU_Sock_write() may also be called while another thread is calling or blocking in MPIDU_Sock_wait() on the same sock
1018 set to which the specified sock belongs.
1019 
1020 A immediate write may not be performed if another thread is performing an immediate write on the same sock.  This is considered to
1021 be an application error, and the results of doing so are undefined.  The implementation may choose to catch the error and return
1022 MPIDU_SOCK_ERR_INPROGRESS, but it is not required to do so.
1023 
1024 Calling MPIDU_Sock_write() during or after the call to MPIDU_Sock_post_close() is consider to be an application error.  The result
1025 of doing so is undefined, although the implementation may choose to return MPIDU_SOCK_ERR_SOCK_CLOSED if it is able to catch the
1026 error.  The application should coordinate the closing of a sock with the activities of other threads to ensure that one thread is
1027 not attempting to perform an immediate write while another thread is attempting to close the sock.
1028 
1029 Module:
1030 Utility-Sock
1031 @*/
1032 int MPIDU_Sock_writev(MPIDU_Sock_t sock, MPID_IOV * iov, int iov_n, MPIU_Size_t * num_written);
1033 
1034 
1035 /*@
1036 MPIDU_Sock_get_sock_id - get an integer identifier for a sock object
1037 
1038 Input Parameter:
1039 . sock - sock object
1040 
1041 Return value: an integer that uniquely identifies the sock object
1042 
1043 Notes:
1044 The integer is unique relative to all other open sock objects in the local process.  The integer may later be reused for a
1045 different sock once the current object is closed and destroyed.
1046 
1047 This function does not return an error code.  Passing in an invalid sock object has undefined results (garbage in, garbage out).
1048 
1049 Module:
1050 Utility-Sock
1051 @*/
1052 int MPIDU_Sock_get_sock_id(MPIDU_Sock_t sock);
1053 
1054 
1055 /*@
1056 MPIDU_Sock_get_sock_set_id - get an integer identifier for a sock set object
1057 
1058 Input Parameter:
1059 . sock set - sock set object
1060 
1061 Return value: an integer that uniquely identifies the sock set object
1062 
1063 Notes:
1064 
1065 The integer is unique relative to all other sock set objects currently existing in the local process.  The integer may later be
1066 reused for a different sock set once the current object destroyed.
1067 
1068 This function does not return an error code.  Passing in an invalid sock set object has undefined results (garbage in, garbage
1069 out).
1070 
1071 Module:
1072 Utility-Sock
1073 @*/
1074 int MPIDU_Sock_get_sock_set_id(MPIDU_Sock_set_t set);
1075 
1076 
1077 /*@
1078 MPIDU_Sock_get_error_class_string - get a generic error string from an error code
1079 
1080 Input Parameter:
1081 + error - sock error
1082 - length - length of error string
1083 
1084 Output Parameter:
1085 . error_string - error string
1086 
1087 Return value: a string representation of the sock error
1088 
1089 Notes:
1090 
1091 The returned string is the generic error message for the supplied error code.
1092 
1093 Module:
1094 Utility-Sock
1095 @*/
1096 int MPIDU_Sock_get_error_class_string(int error, char *error_string, size_t length);
1097 
1098 
1099 CPLUSPLUS_END
1100 
1101 #endif /* !defined(MPIDU_SOCK_H_INCLUDED) */
1102