1 /*
2  * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
3  *                         University Research and Technology
4  *                         Corporation.  All rights reserved.
5  * Copyright (c) 2004-2011 The University of Tennessee and The University
6  *                         of Tennessee Research Foundation.  All rights
7  *                         reserved.
8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9  *                         University of Stuttgart.  All rights reserved.
10  * Copyright (c) 2004-2005 The Regents of the University of California.
11  *                         All rights reserved.
12  * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
13  *                         All rights reserved.
14  * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
15  * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
16  * Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
17  * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
18  * $COPYRIGHT$
19  *
20  * Additional copyrights may follow
21  *
22  * $HEADER$
23  *
24  */
25 
26 #define _GNU_SOURCE
27 #include <stdbool.h>
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <unistd.h>
31 #include <time.h>
32 
33 #include <pmix.h>
34 #include "examples.h"
35 
36 static pmix_proc_t myproc;
37 
38 /* this is the event notification function we pass down below
39  * when registering for general events - i.e.,, the default
40  * handler. We don't technically need to register one, but it
41  * is usually good practice to catch any events that occur */
notification_fn(size_t evhdlr_registration_id,pmix_status_t status,const pmix_proc_t * source,pmix_info_t info[],size_t ninfo,pmix_info_t results[],size_t nresults,pmix_event_notification_cbfunc_fn_t cbfunc,void * cbdata)42 static void notification_fn(size_t evhdlr_registration_id,
43                             pmix_status_t status,
44                             const pmix_proc_t *source,
45                             pmix_info_t info[], size_t ninfo,
46                             pmix_info_t results[], size_t nresults,
47                             pmix_event_notification_cbfunc_fn_t cbfunc,
48                             void *cbdata)
49 {
50     if (NULL != cbfunc) {
51         cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
52     }
53 }
54 
55 /* event handler registration is done asynchronously because it
56  * may involve the PMIx server registering with the host RM for
57  * external events. So we provide a callback function that returns
58  * the status of the request (success or an error), plus a numerical index
59  * to the registered event. The index is used later on to deregister
60  * an event handler - if we don't explicitly deregister it, then the
61  * PMIx server will do so when it see us exit */
evhandler_reg_callbk(pmix_status_t status,size_t evhandler_ref,void * cbdata)62 static void evhandler_reg_callbk(pmix_status_t status,
63                                  size_t evhandler_ref,
64                                  void *cbdata)
65 {
66     mylock_t *lock = (mylock_t*)cbdata;
67 
68     if (PMIX_SUCCESS != status) {
69         fprintf(stderr, "Client %s:%d EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n",
70                    myproc.nspace, myproc.rank, status, (unsigned long)evhandler_ref);
71     }
72     lock->status = status;
73     lock->evhandler_ref = evhandler_ref;
74     DEBUG_WAKEUP_THREAD(lock);
75 }
76 
main(int argc,char ** argv)77 int main(int argc, char **argv)
78 {
79     pmix_status_t rc;
80     pmix_value_t value;
81     pmix_value_t *val, *vptr;
82     pmix_proc_t proc;
83     uint32_t nprocs, n, k;
84     pmix_info_t *info;
85     bool flag;
86     mylock_t mylock;
87     pmix_data_array_t da, *dptr;
88 
89     /* init us - note that the call to "init" includes the return of
90      * any job-related info provided by the RM. This includes any
91      * debugger flag instructing us to stop-in-init. If such a directive
92      * is included, then the process will be stopped in this call until
93      * the "debugger release" notification arrives */
94     if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) {
95         fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %d\n", myproc.nspace, myproc.rank, rc);
96         exit(0);
97     }
98     fprintf(stderr, "Client ns %s rank %d: Running\n", myproc.nspace, myproc.rank);
99 
100 
101     /* register our default event handler - again, this isn't strictly
102      * required, but is generally good practice */
103     DEBUG_CONSTRUCT_LOCK(&mylock);
104     PMIx_Register_event_handler(NULL, 0, NULL, 0,
105                                 notification_fn, evhandler_reg_callbk, (void*)&mylock);
106     DEBUG_WAIT_THREAD(&mylock);
107     rc = mylock.status;
108     DEBUG_DESTRUCT_LOCK(&mylock);
109 
110     if (PMIX_SUCCESS != rc) {
111         fprintf(stderr, "[%s:%d] Default handler registration failed\n", myproc.nspace, myproc.rank);
112         goto done;
113     }
114 
115     /* job-related info is found in our nspace, assigned to the
116      * wildcard rank as it doesn't relate to a specific rank. Setup
117      * a name to retrieve such values */
118     PMIX_PROC_CONSTRUCT(&proc);
119     (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN);
120     proc.rank = PMIX_RANK_WILDCARD;
121 
122     /* get our job size */
123     if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_JOB_SIZE, NULL, 0, &val))) {
124         fprintf(stderr, "Client ns %s rank %d: PMIx_Get universe size failed: %d\n", myproc.nspace, myproc.rank, rc);
125         goto done;
126     }
127     nprocs = val->data.uint32;
128     PMIX_VALUE_RELEASE(val);
129     fprintf(stderr, "Client %s:%d job size %d\n", myproc.nspace, myproc.rank, nprocs);
130 
131     /* put a data array of pmix_value's */
132     val = (pmix_value_t*)malloc(32 * sizeof(pmix_value_t));
133     for (n=0; n < 32; n++) {
134         val[n].type = PMIX_UINT64;
135         val[n].data.uint64 = 2*n;
136     }
137     da.type = PMIX_VALUE;
138     da.size = 32;
139     da.array = val;
140     value.type = PMIX_DATA_ARRAY;
141     value.data.darray = &da;
142     rc = PMIx_Put(PMIX_GLOBAL, "test-key", &value);
143     if (PMIX_SUCCESS != rc) {
144         fprintf(stderr, "Client ns %s rank %d: PMIx_Put failed: %d\n", myproc.nspace, myproc.rank, rc);
145         goto done;
146     }
147     free(val);
148 
149     /* push the data to our PMIx server */
150     if (PMIX_SUCCESS != (rc = PMIx_Commit())) {
151         fprintf(stderr, "Client ns %s rank %d: PMIx_Commit failed: %d\n", myproc.nspace, myproc.rank, rc);
152         goto done;
153     }
154 
155     /* call fence to synchronize with our peers - instruct
156      * the fence operation to collect and return all "put"
157      * data from our peers */
158     PMIX_INFO_CREATE(info, 1);
159     flag = true;
160     PMIX_INFO_LOAD(info, PMIX_COLLECT_DATA, &flag, PMIX_BOOL);
161     if (PMIX_SUCCESS != (rc = PMIx_Fence(&proc, 1, info, 1))) {
162         fprintf(stderr, "Client ns %s rank %d: PMIx_Fence failed: %d\n", myproc.nspace, myproc.rank, rc);
163         goto done;
164     }
165     PMIX_INFO_FREE(info, 1);
166 
167     /* check the returned data */
168     for (n=0; n < nprocs; n++) {
169         proc.rank = n;
170         if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, "test-key", NULL, 0, &val))) {
171             fprintf(stderr, "Client ns %s rank %d: PMIx_Get on rank %u failed: %d\n",
172                     myproc.nspace, myproc.rank, proc.rank, rc);
173             goto done;
174         }
175         if (PMIX_DATA_ARRAY != val->type) {
176             fprintf(stderr, "Client ns %s rank %d: PMIx_Get on rank %u returned wrong type: %d\n",
177                     myproc.nspace, myproc.rank, proc.rank, val->type);
178             PMIX_VALUE_RELEASE(val);
179             goto done;
180         }
181         dptr = val->data.darray;
182         if (NULL == dptr) {
183             fprintf(stderr, "Client ns %s rank %d: PMIx_Get %d returned NULL array\n",
184                     myproc.nspace, myproc.rank, proc.rank);
185             PMIX_VALUE_RELEASE(val);
186             goto done;
187         }
188         if (PMIX_VALUE != dptr->type) {
189             fprintf(stderr, "Client ns %s rank %d: PMIx_Get %d returned wrong array value type %d\n",
190                     myproc.nspace, myproc.rank, proc.rank, dptr->type);
191             PMIX_VALUE_RELEASE(val);
192             goto done;
193         }
194         if (32 != dptr->size) {
195             fprintf(stderr, "Client ns %s rank %d: PMIx_Get %d returned wrong array value size %d\n",
196                     myproc.nspace, myproc.rank, proc.rank, (int)dptr->size);
197             PMIX_VALUE_RELEASE(val);
198             goto done;
199         }
200         vptr = (pmix_value_t*)dptr->array;
201         for (k=0; k < 32; k++) {
202             if (PMIX_UINT64 != vptr[k].type) {
203                 fprintf(stderr, "Client ns %s rank %d: PMIx_Get %d returned wrong type: %d\n",
204                         myproc.nspace, myproc.rank, proc.rank, vptr[k].type);
205                 PMIX_VALUE_RELEASE(val);
206                 goto done;
207             }
208             if (2*k != vptr[k].data.uint64) {
209                 fprintf(stderr, "Client ns %s rank %d: PMIx_Get %d returned wrong value: %lu\n",
210                         myproc.nspace, myproc.rank, proc.rank, (unsigned long)vptr[k].data.uint64);
211                 PMIX_VALUE_RELEASE(val);
212                 goto done;
213             }
214         }
215     }
216 
217  done:
218     /* finalize us */
219     fprintf(stderr, "Client ns %s rank %d: Finalizing\n", myproc.nspace, myproc.rank);
220     if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) {
221         fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc);
222     } else {
223         fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank);
224     }
225     fflush(stderr);
226     return(0);
227 }
228