1 /*
2 * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
3 * University Research and Technology
4 * Corporation. All rights reserved.
5 * Copyright (c) 2004-2011 The University of Tennessee and The University
6 * of Tennessee Research Foundation. All rights
7 * reserved.
8 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9 * University of Stuttgart. All rights reserved.
10 * Copyright (c) 2004-2005 The Regents of the University of California.
11 * All rights reserved.
12 * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
13 * All rights reserved.
14 * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved.
15 * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved.
16 * Copyright (c) 2013-2019 Intel, Inc. All rights reserved.
17 * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved.
18 * $COPYRIGHT$
19 *
20 * Additional copyrights may follow
21 *
22 * $HEADER$
23 *
24 */
25
26 #define _GNU_SOURCE
27 #include <stdbool.h>
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <unistd.h>
31 #include <time.h>
32
33 #include <pmix.h>
34 #include "examples.h"
35
36 static pmix_proc_t myproc;
37
38 /* this is the event notification function we pass down below
39 * when registering for general events - i.e.,, the default
40 * handler. We don't technically need to register one, but it
41 * is usually good practice to catch any events that occur */
notification_fn(size_t evhdlr_registration_id,pmix_status_t status,const pmix_proc_t * source,pmix_info_t info[],size_t ninfo,pmix_info_t results[],size_t nresults,pmix_event_notification_cbfunc_fn_t cbfunc,void * cbdata)42 static void notification_fn(size_t evhdlr_registration_id,
43 pmix_status_t status,
44 const pmix_proc_t *source,
45 pmix_info_t info[], size_t ninfo,
46 pmix_info_t results[], size_t nresults,
47 pmix_event_notification_cbfunc_fn_t cbfunc,
48 void *cbdata)
49 {
50 if (NULL != cbfunc) {
51 cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
52 }
53 }
54
55 /* event handler registration is done asynchronously because it
56 * may involve the PMIx server registering with the host RM for
57 * external events. So we provide a callback function that returns
58 * the status of the request (success or an error), plus a numerical index
59 * to the registered event. The index is used later on to deregister
60 * an event handler - if we don't explicitly deregister it, then the
61 * PMIx server will do so when it see us exit */
evhandler_reg_callbk(pmix_status_t status,size_t evhandler_ref,void * cbdata)62 static void evhandler_reg_callbk(pmix_status_t status,
63 size_t evhandler_ref,
64 void *cbdata)
65 {
66 mylock_t *lock = (mylock_t*)cbdata;
67
68 if (PMIX_SUCCESS != status) {
69 fprintf(stderr, "Client %s:%d EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n",
70 myproc.nspace, myproc.rank, status, (unsigned long)evhandler_ref);
71 }
72 lock->status = status;
73 lock->evhandler_ref = evhandler_ref;
74 DEBUG_WAKEUP_THREAD(lock);
75 }
76
main(int argc,char ** argv)77 int main(int argc, char **argv)
78 {
79 pmix_status_t rc;
80 pmix_value_t value;
81 pmix_value_t *val, *vptr;
82 pmix_proc_t proc;
83 uint32_t nprocs, n, k;
84 pmix_info_t *info;
85 bool flag;
86 mylock_t mylock;
87 pmix_data_array_t da, *dptr;
88
89 /* init us - note that the call to "init" includes the return of
90 * any job-related info provided by the RM. This includes any
91 * debugger flag instructing us to stop-in-init. If such a directive
92 * is included, then the process will be stopped in this call until
93 * the "debugger release" notification arrives */
94 if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) {
95 fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %d\n", myproc.nspace, myproc.rank, rc);
96 exit(0);
97 }
98 fprintf(stderr, "Client ns %s rank %d: Running\n", myproc.nspace, myproc.rank);
99
100
101 /* register our default event handler - again, this isn't strictly
102 * required, but is generally good practice */
103 DEBUG_CONSTRUCT_LOCK(&mylock);
104 PMIx_Register_event_handler(NULL, 0, NULL, 0,
105 notification_fn, evhandler_reg_callbk, (void*)&mylock);
106 DEBUG_WAIT_THREAD(&mylock);
107 rc = mylock.status;
108 DEBUG_DESTRUCT_LOCK(&mylock);
109
110 if (PMIX_SUCCESS != rc) {
111 fprintf(stderr, "[%s:%d] Default handler registration failed\n", myproc.nspace, myproc.rank);
112 goto done;
113 }
114
115 /* job-related info is found in our nspace, assigned to the
116 * wildcard rank as it doesn't relate to a specific rank. Setup
117 * a name to retrieve such values */
118 PMIX_PROC_CONSTRUCT(&proc);
119 (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN);
120 proc.rank = PMIX_RANK_WILDCARD;
121
122 /* get our job size */
123 if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_JOB_SIZE, NULL, 0, &val))) {
124 fprintf(stderr, "Client ns %s rank %d: PMIx_Get universe size failed: %d\n", myproc.nspace, myproc.rank, rc);
125 goto done;
126 }
127 nprocs = val->data.uint32;
128 PMIX_VALUE_RELEASE(val);
129 fprintf(stderr, "Client %s:%d job size %d\n", myproc.nspace, myproc.rank, nprocs);
130
131 /* put a data array of pmix_value's */
132 val = (pmix_value_t*)malloc(32 * sizeof(pmix_value_t));
133 for (n=0; n < 32; n++) {
134 val[n].type = PMIX_UINT64;
135 val[n].data.uint64 = 2*n;
136 }
137 da.type = PMIX_VALUE;
138 da.size = 32;
139 da.array = val;
140 value.type = PMIX_DATA_ARRAY;
141 value.data.darray = &da;
142 rc = PMIx_Put(PMIX_GLOBAL, "test-key", &value);
143 if (PMIX_SUCCESS != rc) {
144 fprintf(stderr, "Client ns %s rank %d: PMIx_Put failed: %d\n", myproc.nspace, myproc.rank, rc);
145 goto done;
146 }
147 free(val);
148
149 /* push the data to our PMIx server */
150 if (PMIX_SUCCESS != (rc = PMIx_Commit())) {
151 fprintf(stderr, "Client ns %s rank %d: PMIx_Commit failed: %d\n", myproc.nspace, myproc.rank, rc);
152 goto done;
153 }
154
155 /* call fence to synchronize with our peers - instruct
156 * the fence operation to collect and return all "put"
157 * data from our peers */
158 PMIX_INFO_CREATE(info, 1);
159 flag = true;
160 PMIX_INFO_LOAD(info, PMIX_COLLECT_DATA, &flag, PMIX_BOOL);
161 if (PMIX_SUCCESS != (rc = PMIx_Fence(&proc, 1, info, 1))) {
162 fprintf(stderr, "Client ns %s rank %d: PMIx_Fence failed: %d\n", myproc.nspace, myproc.rank, rc);
163 goto done;
164 }
165 PMIX_INFO_FREE(info, 1);
166
167 /* check the returned data */
168 for (n=0; n < nprocs; n++) {
169 proc.rank = n;
170 if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, "test-key", NULL, 0, &val))) {
171 fprintf(stderr, "Client ns %s rank %d: PMIx_Get on rank %u failed: %d\n",
172 myproc.nspace, myproc.rank, proc.rank, rc);
173 goto done;
174 }
175 if (PMIX_DATA_ARRAY != val->type) {
176 fprintf(stderr, "Client ns %s rank %d: PMIx_Get on rank %u returned wrong type: %d\n",
177 myproc.nspace, myproc.rank, proc.rank, val->type);
178 PMIX_VALUE_RELEASE(val);
179 goto done;
180 }
181 dptr = val->data.darray;
182 if (NULL == dptr) {
183 fprintf(stderr, "Client ns %s rank %d: PMIx_Get %d returned NULL array\n",
184 myproc.nspace, myproc.rank, proc.rank);
185 PMIX_VALUE_RELEASE(val);
186 goto done;
187 }
188 if (PMIX_VALUE != dptr->type) {
189 fprintf(stderr, "Client ns %s rank %d: PMIx_Get %d returned wrong array value type %d\n",
190 myproc.nspace, myproc.rank, proc.rank, dptr->type);
191 PMIX_VALUE_RELEASE(val);
192 goto done;
193 }
194 if (32 != dptr->size) {
195 fprintf(stderr, "Client ns %s rank %d: PMIx_Get %d returned wrong array value size %d\n",
196 myproc.nspace, myproc.rank, proc.rank, (int)dptr->size);
197 PMIX_VALUE_RELEASE(val);
198 goto done;
199 }
200 vptr = (pmix_value_t*)dptr->array;
201 for (k=0; k < 32; k++) {
202 if (PMIX_UINT64 != vptr[k].type) {
203 fprintf(stderr, "Client ns %s rank %d: PMIx_Get %d returned wrong type: %d\n",
204 myproc.nspace, myproc.rank, proc.rank, vptr[k].type);
205 PMIX_VALUE_RELEASE(val);
206 goto done;
207 }
208 if (2*k != vptr[k].data.uint64) {
209 fprintf(stderr, "Client ns %s rank %d: PMIx_Get %d returned wrong value: %lu\n",
210 myproc.nspace, myproc.rank, proc.rank, (unsigned long)vptr[k].data.uint64);
211 PMIX_VALUE_RELEASE(val);
212 goto done;
213 }
214 }
215 }
216
217 done:
218 /* finalize us */
219 fprintf(stderr, "Client ns %s rank %d: Finalizing\n", myproc.nspace, myproc.rank);
220 if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) {
221 fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc);
222 } else {
223 fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank);
224 }
225 fflush(stderr);
226 return(0);
227 }
228