1 #if HAVE_CONFIG_H
2 # include "config.h"
3 #endif
4
5 /* $Id: simple.c,v 1.1.2.1 2007-06-20 17:41:40 vinod Exp $ */
6
7 #if HAVE_STDIO_H
8 # include <stdio.h>
9 #endif
10 #if HAVE_STDLIB_H
11 # include <stdlib.h>
12 #endif
13 #if HAVE_UNISTD_H
14 # include <unistd.h>
15 #elif HAVE_WINDOWS_H
16 # include <windows.h>
17 # define sleep(x) Sleep(1000*(x))
18 #endif
19 #if HAVE_ASSERT_H
20 # include <assert.h>
21 #endif
22
23 #include "armci.h"
24 #include "message.h"
25
26 #define DIM1 5
27 #define DIM2 3
28 #ifdef __sun
29 /* Solaris has shared memory shortages in the default system configuration */
30 # define DIM3 6
31 # define DIM4 5
32 # define DIM5 4
33 #elif defined(__alpha__)
34 # define DIM3 8
35 # define DIM4 5
36 # define DIM5 6
37 #else
38 # define DIM3 8
39 # define DIM4 9
40 # define DIM5 7
41 #endif
42 #define DIM6 3
43 #define DIM7 2
44
45
46 #define OFF 1
47 #define EDIM1 (DIM1+OFF)
48 #define EDIM2 (DIM2+OFF)
49 #define EDIM3 (DIM3+OFF)
50 #define EDIM4 (DIM4+OFF)
51 #define EDIM5 (DIM5+OFF)
52 #define EDIM6 (DIM6+OFF)
53 #define EDIM7 (DIM7+OFF)
54
55 #define DIMS 4
56 #define MAXDIMS 7
57 #define MAX_DIM_VAL 50
58 #define LOOP 200
59
60 #define BASE 100.
61 #define MAXPROC 128
62 #define TIMES 100
63
64 #ifdef CRAY
65 # define ELEMS 800
66 #else
67 # define ELEMS 200
68 #endif
69
70
71 /***************************** macros ************************/
72 #define COPY(src, dst, bytes) memcpy((dst),(src),(bytes))
73 #define ARMCI_MAX(a,b) (((a) >= (b)) ? (a) : (b))
74 #define ARMCI_MIN(a,b) (((a) <= (b)) ? (a) : (b))
75 #define ARMCI_ABS(a) (((a) <0) ? -(a) : (a))
76
77 /***************************** global data *******************/
78 int me, nproc;
79 void* work[MAXPROC]; /* work array for propagating addresses */
80
81
82
83 #ifdef MSG_COMMS_PVM
pvm_init(int argc,char * argv[])84 void pvm_init(int argc, char *argv[])
85 {
86 int mytid, mygid, ctid[MAXPROC];
87 int np, i;
88
89 mytid = pvm_mytid();
90 if((argc != 2) && (argc != 1)) goto usage;
91 if(argc == 1) np = 1;
92 if(argc == 2)
93 if((np = atoi(argv[1])) < 1) goto usage;
94 if(np > MAXPROC) goto usage;
95
96 mygid = pvm_joingroup(MPGROUP);
97
98 if(np > 1)
99 if (mygid == 0)
100 i = pvm_spawn(argv[0], argv+1, 0, "", np-1, ctid);
101
102 while(pvm_gsize(MPGROUP) < np) sleep(1);
103
104 /* sync */
105 pvm_barrier(MPGROUP, np);
106
107 printf("PVM initialization done!\n");
108
109 return;
110
111 usage:
112 fprintf(stderr, "usage: %s <nproc>\n", argv[0]);
113 pvm_exit();
114 exit(-1);
115 }
116 #endif
117
create_array(void * a[],int elem_size,int ndim,int dims[])118 void create_array(void *a[], int elem_size, int ndim, int dims[])
119 {
120 int bytes=elem_size, i, rc;
121
122 assert(ndim<=MAXDIMS);
123 for(i=0;i<ndim;i++)bytes*=dims[i];
124
125 rc = ARMCI_Malloc(a, bytes);
126 assert(rc==0);
127
128 assert(a[me]);
129
130 }
131
destroy_array(void * ptr[])132 void destroy_array(void *ptr[])
133 {
134 int check;
135
136 armci_msg_barrier();
137 check = !ARMCI_Free(ptr[me]);
138 assert(check);
139 }
140
141 #define MAXELEMS 1000
142 #define MAX_REQUESTS MAXELEMS
143
test_aggregate(int dryrun)144 void test_aggregate(int dryrun) {
145
146 int i, j, rc, bytes, elems[2] = {MAXPROC, MAXELEMS};
147 double *ddst_put[MAXPROC];
148 double *ddst_get[MAXPROC];
149 double *dsrc[MAXPROC];
150 armci_hdl_t aggr_hdl_put[MAXPROC];
151 armci_hdl_t aggr_hdl_get[MAXPROC];
152 armci_hdl_t hdl_put[MAXELEMS];
153 armci_hdl_t hdl_get[MAXELEMS];
154 armci_giov_t darr;
155 void *src_ptr[MAX_REQUESTS], *dst_ptr[MAX_REQUESTS];
156 int start = 0, end = 0;
157 double start_time;
158
159 create_array((void**)ddst_put, sizeof(double),2, elems);
160 create_array((void**)ddst_get, sizeof(double),2, elems);
161 create_array((void**)dsrc, sizeof(double),1, &elems[1]);
162
163 for(i=0; i<elems[1]; i++) dsrc[me][i]=i*1.001*(me+1);
164 for(i=0; i<elems[0]*elems[1]; i++) {
165 ddst_put[me][i]=0.0;
166 ddst_get[me][i]=0.0;
167 }
168
169 armci_msg_barrier();
170
171 /* only proc 0 does the work */
172 if(me == 0) {
173 if(!dryrun)printf("Transferring %d doubles (Not an array of %d doubles)\n", MAXELEMS, MAXELEMS);
174
175 /* initializing non-blocking handles */
176 for(i=0; i<elems[1]; i++) ARMCI_INIT_HANDLE(&hdl_put[i]);
177 for(i=0; i<elems[1]; i++) ARMCI_INIT_HANDLE(&hdl_get[i]);
178
179 /* aggregate handles */
180 for(i=0; i<nproc; i++) ARMCI_INIT_HANDLE(&aggr_hdl_put[i]);
181 for(i=0; i<nproc; i++) ARMCI_INIT_HANDLE(&aggr_hdl_get[i]);
182 for(i=0; i<nproc; i++) ARMCI_SET_AGGREGATE_HANDLE(&aggr_hdl_put[i]);
183 for(i=0; i<nproc; i++) ARMCI_SET_AGGREGATE_HANDLE(&aggr_hdl_get[i]);
184
185 bytes = sizeof(double);
186
187 /* **************** PUT **************** */
188 /* register put */
189 start_time=armci_timer();
190 start = 0; end = elems[1];
191 for(i=1; i<nproc; i++) {
192 for(j=start; j<end; j++) {
193 ARMCI_NbPutValueDouble(dsrc[me][j], &ddst_put[i][me*elems[1]+j], i,
194 &hdl_put[j]);
195 }
196 for(j=start; j<end; j++) ARMCI_Wait(&hdl_put[j]);
197 }
198 if(!dryrun)printf("%d: Value Put time = %.2es\n", me, armci_timer()-start_time);
199
200 /* vector put */
201 start_time=armci_timer();
202 for(i=1; i<nproc; i++) {
203 for(j=start; j<end; j++) {
204 src_ptr[j] = (void *)&dsrc[me][j];
205 dst_ptr[j] = (void *)&ddst_put[i][me*elems[1]+j];
206 }
207 darr.src_ptr_array = src_ptr;
208 darr.dst_ptr_array = dst_ptr;
209 darr.bytes = sizeof(double);
210 darr.ptr_array_len = elems[1];
211 if((rc=ARMCI_NbPutV(&darr, 1, i, &hdl_put[i])))
212 ARMCI_Error("armci_nbputv failed\n",rc);
213 }
214 for(i=1; i<nproc; i++) ARMCI_Wait(&hdl_put[i]);
215 if(!dryrun)printf("%d: Vector Put time = %.2es\n", me, armci_timer()-start_time);
216
217 /* regular put */
218 start_time=armci_timer();
219 for(i=1; i<nproc; i++) {
220 for(j=start; j<end; j++) {
221 if((rc=ARMCI_NbPut(&dsrc[me][j], &ddst_put[i][me*elems[1]+j], bytes,
222 i, &hdl_put[j])))
223 ARMCI_Error("armci_nbput failed\n",rc);
224 }
225 for(j=start; j<end; j++) ARMCI_Wait(&hdl_put[j]);
226 }
227 if(!dryrun)printf("%d: Regular Put time = %.2es\n", me, armci_timer()-start_time);
228
229 /* aggregate put */
230 start_time=armci_timer();
231 for(i=1; i<nproc; i++) {
232 for(j=start; j<end; j++) {
233 if((rc=ARMCI_NbPut(&dsrc[me][j], &ddst_put[i][me*elems[1]+j], bytes,
234 i, &aggr_hdl_put[i])))
235 ARMCI_Error("armci_nbput failed\n",rc);
236 }
237 }
238 for(i=1; i<nproc; i++) ARMCI_Wait(&aggr_hdl_put[i]);
239 if(!dryrun)printf("%d: Aggregate Put time = %.2es\n\n", me, armci_timer()-start_time);
240
241
242 /* **************** GET **************** */
243
244 /* vector get */
245 start_time=armci_timer();
246 for(i=1; i<nproc; i++) {
247 for(j=start; j<end; j++) {
248 src_ptr[j] = (void *)&dsrc[i][j];
249 dst_ptr[j] = (void *)&ddst_get[me][i*elems[1]+j];
250 }
251 darr.src_ptr_array = src_ptr;
252 darr.dst_ptr_array = dst_ptr;
253 darr.bytes = sizeof(double);
254 darr.ptr_array_len = elems[1];
255 if((rc=ARMCI_NbGetV(&darr, 1, i, &hdl_get[i])))
256 ARMCI_Error("armci_nbgetv failed\n",rc);
257 ARMCI_Wait(&hdl_get[i]);
258 }
259 if(!dryrun)printf("%d: Vector Get time = %.2es\n", me, armci_timer()-start_time);
260
261 /* regular get */
262 start_time=armci_timer();
263 for(i=1; i<nproc; i++) {
264 for(j=start; j<end; j++) {
265 if((rc=ARMCI_NbGet(&dsrc[i][j], &ddst_get[me][i*elems[1]+j], bytes,
266 i, &hdl_get[j])))
267 ARMCI_Error("armci_nbget failed\n",rc);
268 }
269 for(j=start; j<end; j++) ARMCI_Wait(&hdl_get[j]);
270 }
271 if(!dryrun)printf("%d: Regular Get time = %.2es\n", me, armci_timer()-start_time);
272
273 /* aggregate get */
274 start_time=armci_timer();
275 for(i=1; i<nproc; i++) {
276 for(j=start; j<end; j++) {
277 ARMCI_NbGet(&dsrc[i][j], &ddst_get[me][i*elems[1]+j], bytes,
278 i, &aggr_hdl_get[i]);
279 }
280 }
281 for(i=1; i<nproc; i++) ARMCI_Wait(&aggr_hdl_get[i]);
282 if(!dryrun)printf("%d: Aggregate Get time = %.2es\n", me, armci_timer()-start_time);
283 }
284
285 armci_msg_barrier();
286 ARMCI_AllFence();
287 armci_msg_barrier();
288
289 /* Verify */
290 if(!(me==0))
291 for(j=0; j<elems[1]; j++) {
292 if( ARMCI_ABS(ddst_put[me][j]-j*1.001) > 0.1) {
293 ARMCI_Error("aggregate put failed...1", 0);
294 }
295 }
296 armci_msg_barrier();
297 if(!dryrun) {
298 if(me==0) {
299 printf("\n aggregate put ..O.K.\n"); fflush(stdout);
300 }
301 }
302
303 if(me==0) {
304 for(i=1; i<nproc; i++) {
305 for(j=0; j<elems[1]; j++) {
306 if( ARMCI_ABS(ddst_get[me][i*elems[1]+j]-j*1.001*(i+1)) > 0.1) {
307 ARMCI_Error("aggregate get failed...1", 0);
308 }
309 }
310 }
311 }
312 armci_msg_barrier();
313 if(!dryrun) {
314 if(me==0) {
315 printf(" aggregate get ..O.K.\n"); fflush(stdout);
316 }
317 }
318
319 ARMCI_AllFence();
320 armci_msg_barrier();
321
322 if(!dryrun)if(me==0){printf("O.K.\n"); fflush(stdout);}
323 destroy_array((void **)ddst_put);
324 destroy_array((void **)ddst_get);
325 destroy_array((void **)dsrc);
326 }
327
328
main(int argc,char * argv[])329 int main(int argc, char* argv[])
330 {
331
332 armci_msg_init(&argc, &argv);
333 nproc = armci_msg_nproc();
334 me = armci_msg_me();
335
336 /* printf("nproc = %d, me = %d\n", nproc, me);*/
337
338 if(nproc>MAXPROC && me==0)
339 ARMCI_Error("Test works for up to %d processors\n",MAXPROC);
340
341 if(me==0){
342 printf("ARMCI test program (%d processes)\n",nproc);
343 fflush(stdout);
344 sleep(1);
345 }
346
347 ARMCI_Init();
348
349 if(me==0){
350 printf("\nAggregate put/get requests\n\n");
351 fflush(stdout);
352 }
353 test_aggregate(1); /* cold start */
354 test_aggregate(0); /* warm start */
355
356 ARMCI_AllFence();
357 armci_msg_barrier();
358 if(me==0){printf("\nSuccess!!\n"); fflush(stdout);}
359 sleep(2);
360
361 armci_msg_barrier();
362 ARMCI_Finalize();
363 armci_msg_finalize();
364 return(0);
365 }
366