1 #if HAVE_CONFIG_H
2 # include "config.h"
3 #endif
4
5 /* $Id: testnotify.c,v 1.1.2.1 2007-06-20 17:42:09 vinod Exp $ */
6
7 #if HAVE_STDIO_H
8 # include <stdio.h>
9 #endif
10 #if HAVE_STDLIB_H
11 # include <stdlib.h>
12 #endif
13 #if HAVE_STRINGS_H
14 # include <strings.h>
15 #endif
16 #if HAVE_ASSERT_H
17 # include <assert.h>
18 #endif
19 #if HAVE_UNISTD_H
20 # include <unistd.h>
21 #endif
22
23 #define DEBUG__
24
25 #include "armci.h"
26 #include "message.h"
27
28 #define DIM1 5
29 #define DIM2 3
30 #ifdef __sun
31 /* Solaris has shared memory shortages in the default system configuration */
32 # define DIM3 6
33 # define DIM4 5
34 # define DIM5 4
35 #elif defined(__alpha__)
36 # define DIM3 8
37 # define DIM4 5
38 # define DIM5 6
39 #else
40 # define DIM3 8
41 # define DIM4 9
42 # define DIM5 7
43 #endif
44 #define DIM6 3
45 #define DIM7 2
46
47
48 #define OFF 1
49 #define EDIM1 (DIM1+OFF)
50 #define EDIM2 (DIM2+OFF)
51 #define EDIM3 (DIM3+OFF)
52 #define EDIM4 (DIM4+OFF)
53 #define EDIM5 (DIM5+OFF)
54 #define EDIM6 (DIM6+OFF)
55 #define EDIM7 (DIM7+OFF)
56
57 #define DIMS 4
58 #define MAXDIMS 7
59 #define MAX_DIM_VAL 50
60 #define LOOP 200
61
62 #define BASE 100.
63 #define MAXPROC 128
64 #define TIMES 100
65
66 #ifdef CRAY
67 # define ELEMS 800
68 #else
69 # define ELEMS 200
70 #endif
71
72
73
74 /***************************** macros ************************/
75 #define COPY(src, dst, bytes) memcpy((dst),(src),(bytes))
76 #define ARMCI_MAX(a,b) (((a) >= (b)) ? (a) : (b))
77 #define ARMCI_MIN(a,b) (((a) <= (b)) ? (a) : (b))
78 #define ARMCI_ABS(a) (((a) <0) ? -(a) : (a))
79
80 /***************************** global data *******************/
81 int me, nproc;
82 void* work[MAXPROC]; /* work array for propagating addresses */
83
84
85
86 /*\ generate random range for a section of multidimensional array
87 \*/
get_range(int ndim,int dims[],int lo[],int hi[])88 void get_range(int ndim, int dims[], int lo[], int hi[])
89 {
90 int dim;
91 for(dim=0; dim <ndim;dim++){
92 int toss1, toss2;
93 toss1 = rand()%dims[dim];
94 toss2 = rand()%dims[dim];
95 if(toss1<toss2){
96 lo[dim]=toss1;
97 hi[dim]=toss2;
98 }else {
99 hi[dim]=toss1;
100 lo[dim]=toss2;
101 }
102 }
103 }
104
105
106
107 /*\ generates a new random range similar to the input range for an array with specified dimensions
108 \*/
new_range(int ndim,int dims[],int lo[],int hi[],int new_lo[],int new_hi[])109 void new_range(int ndim, int dims[], int lo[], int hi[],int new_lo[], int new_hi[])
110 {
111 int dim;
112 for(dim=0; dim <ndim;dim++){
113 int toss, range;
114 int diff = hi[dim] -lo[dim]+1;
115 assert(diff <= dims[dim]);
116 range = dims[dim]-diff;
117 toss = (range > 0)? rand()%range : lo[dim];
118 new_lo[dim] = toss;
119 new_hi[dim] = toss + diff -1;
120 assert(new_hi[dim] < dims[dim]);
121 assert(diff == (new_hi[dim] -new_lo[dim]+1));
122 }
123 }
124
125
126 /*\ print range of ndim dimensional array with two strings before and after
127 \*/
print_range(char * pre,int ndim,int lo[],int hi[],char * post)128 void print_range(char *pre,int ndim, int lo[], int hi[], char* post)
129 {
130 int i;
131
132 printf("%s[",pre);
133 for(i=0;i<ndim;i++){
134 printf("%d:%d",lo[i],hi[i]);
135 if(i==ndim-1)printf("] %s",post);
136 else printf(",");
137 }
138 }
139
140 /*\ print subscript of ndim dimensional array with two strings before and after
141 \*/
print_subscript(char * pre,int ndim,int subscript[],char * post)142 void print_subscript(char *pre,int ndim, int subscript[], char* post)
143 {
144 int i;
145
146 printf("%s [",pre);
147 for(i=0;i<ndim;i++){
148 printf("%d",subscript[i]);
149 if(i==ndim-1)printf("] %s",post);
150 else printf(",");
151 }
152 }
153
154
155 /*\ initialize array: a[i,j,k,..]=i+100*j+10000*k+ ...
156 \*/
init(double * a,int ndim,int elems,int dims[])157 void init(double *a, int ndim, int elems, int dims[])
158 {
159 int idx[MAXDIMS];
160 int i,dim;
161
162 for(i=0; i<elems; i++){
163 int Index = i;
164 double field, val;
165
166 for(dim = 0; dim < ndim; dim++){
167 idx[dim] = Index%dims[dim];
168 Index /= dims[dim];
169 }
170
171 field=1.; val=0.;
172 for(dim=0; dim< ndim;dim++){
173 val += field*idx[dim];
174 field *= BASE;
175 }
176 a[i] = val;
177 /* printf("(%d,%d,%d)=%6.0f",idx[0],idx[1],idx[2],val); */
178 }
179 }
180
181
182 /*\ compute Index from subscript
183 * assume that first subscript component changes first
184 \*/
Index(int ndim,int subscript[],int dims[])185 int Index(int ndim, int subscript[], int dims[])
186 {
187 int idx = 0, i, factor=1;
188 for(i=0;i<ndim;i++){
189 idx += subscript[i]*factor;
190 factor *= dims[i];
191 }
192 return idx;
193 }
194
195
update_subscript(int ndim,int subscript[],int lo[],int hi[],int dims[])196 void update_subscript(int ndim, int subscript[], int lo[], int hi[], int dims[])
197 {
198 int i;
199 for(i=0;i<ndim;i++){
200 if(subscript[i] < hi[i]) { subscript[i]++; return; }
201 subscript[i] = lo[i];
202 }
203 }
204
205
206
compare_patches(double eps,int ndim,double * patch1,int lo1[],int hi1[],int dims1[],double * patch2,int lo2[],int hi2[],int dims2[])207 void compare_patches(double eps, int ndim, double *patch1, int lo1[], int hi1[],
208 int dims1[],double *patch2, int lo2[], int hi2[],
209 int dims2[])
210
211 {
212 int i,j, elems=1;
213 int subscr1[MAXDIMS], subscr2[MAXDIMS];
214 double diff,max;
215
216 for(i=0;i<ndim;i++){
217 int diff = hi1[i]-lo1[i];
218 assert(diff == (hi2[i]-lo2[i]));
219 assert(diff < dims1[i]);
220 assert(diff < dims2[i]);
221 elems *= diff+1;
222 subscr1[i]= lo1[i];
223 subscr2[i]=lo2[i];
224 }
225 for(j=0; j< elems; j++){
226 int idx1=0, idx2=0, offset1=0, offset2=0;
227 idx1 = Index(ndim, subscr1, dims1);
228 idx2 = Index(ndim, subscr2, dims2);
229 if(j==0){
230 offset1 =idx1;
231 offset2 =idx2;
232 }
233 idx1 -= offset1;
234 idx2 -= offset2;
235 diff = patch1[idx1] - patch2[idx2];
236 max = ARMCI_MAX(ARMCI_ABS(patch1[idx1]),ARMCI_ABS(patch2[idx2]));
237 if(max == 0. || max <eps) max = 1.;
238 if(eps < ARMCI_ABS(diff)/max){
239 char msg[48];
240 sprintf(msg,"(proc=%d):%f",me,patch1[idx1]);
241 print_subscript("ERROR: a",ndim,subscr1,msg);
242 sprintf(msg,"%f\n",patch2[idx2]);
243 print_subscript(" b",ndim,subscr2,msg);
244 fflush(stdout);
245 sleep(1);
246 ARMCI_Error("Bailing out",0);
247 }
248 update_subscript(ndim, subscr1, lo1,hi1, dims1);
249 update_subscript(ndim, subscr2, lo2,hi2, dims2);
250 }
251 }
252
253
create_array(void * a[],int elem_size,int ndim,int dims[])254 void create_array(void *a[], int elem_size, int ndim, int dims[])
255 {
256 armci_size_t bytes=elem_size;
257 int i, rc;
258
259 assert(ndim<=MAXDIMS);
260 for(i=0;i<ndim;i++)bytes*=dims[i];
261
262 rc = ARMCI_Malloc(a, bytes);
263 assert(rc==0);
264
265 #ifdef DEBUG_
266 printf("%d after malloc ndim=%d b=%d ptr=%p\n",me,ndim,(int) bytes,a[me]);
267 fflush(stdout);
268 #endif
269
270 assert(a[me]);
271 bzero(a[me],bytes);
272 }
273
destroy_array(void * ptr[])274 void destroy_array(void *ptr[])
275 {
276 int check;
277 armci_msg_barrier();
278 check = !ARMCI_Free(ptr[me]) ;
279 assert(check);
280 }
281
282
get_elems(int ndim,int * stride,int * dims,int size)283 int get_elems(int ndim, int *stride, int *dims, int size)
284 {
285 int elems=1,i;
286 stride[0]=size;
287 for(i=0;i<ndim;i++){
288 stride[i] *= dims[i];
289 if(i<ndim-1){
290 stride[i+1] = stride[i];
291 }
292 elems *= dims[i];
293 }
294 return elems;
295 }
296
297
298 int dimsB[MAXDIMS]={30,EDIM2,EDIM3,EDIM4,EDIM5,EDIM6,EDIM7};
299 #define BATCH 5
300
test_notify(int ndim)301 void test_notify(int ndim)
302 {
303 int lo[MAXDIMS], hi[MAXDIMS], count[MAXDIMS];
304 int stride[MAXDIMS];
305 int dim,elems;
306 int i,Idx=1,idx=0;
307 void *b[MAXPROC], *a[MAXPROC];
308 int left = (me+nproc-1) % nproc;
309 int right = (me+1) % nproc;
310 int loopcnt=1, less=2, strl; /* less>1 takes a partial plane */
311
312
313 /* create shared and local arrays */
314 create_array(b, sizeof(double),ndim,dimsB);
315 create_array(a, sizeof(double),ndim,dimsB);
316
317 elems = get_elems(ndim, stride, dimsB, sizeof(double));
318 init((double*)a[me], ndim, elems, dimsB);
319
320 for(i=0; i<ndim; i++){
321 lo[i]=0;
322 hi[i]= (less > dimsB[i]) ? dimsB[i]-1: dimsB[i]-less;
323 count[i]=hi[i]-lo[i]+1;
324 }
325 count[0]*=sizeof(double);
326
327 for(i=0; i<ndim-1; i++)Idx *= dimsB[i];
328
329 ARMCI_Barrier();
330 if(me==0){
331 printf("--------array[%d",dimsB[0]);
332 for(dim=1;dim<ndim;dim++)printf(",%d",dimsB[dim]);
333 printf("]--------\n");
334 fflush(stdout);
335 }
336
337 ARMCI_Barrier();
338 loopcnt = (ndim>1)? dimsB[ndim-1] : 1;
339 strl = (ndim>1)? ndim-2: 0; /* strides of the subpatch to transfer */
340
341 for(i=0;i<loopcnt;i++){
342 int wc;
343
344 if(me==0){
345
346 ARMCI_PutS((double*)a[me]+idx, stride,
347 (double*)b[left]+idx, stride, count, strl, left);
348 #if DEBUG_
349 printf("%d-%d: ps=%p pd=%p i=%d idx=%d count=%d\n",me,left,(double*)
350 a[me]+idx, (double*)b[left]+idx,i, idx,count[0]); fflush(stdout);
351 #endif
352 (void)armci_notify(left);
353 (void)armci_notify_wait(right,&wc);
354
355 } else{
356
357
358 (void)armci_notify_wait(right,&wc);
359 ARMCI_PutS((double*)b[me]+idx, stride,
360 (double*)b[left]+idx, stride, count, strl, left);
361 #if DEBUG_
362 printf("%d: ps=%p pd=%p i=%d idx=%d count=%d\n",me,(double*)b[me]+idx,
363 (double*)b[left]+idx,i, idx,count[0]); fflush(stdout);
364 #endif
365 (void)armci_notify(left);
366 }
367
368 idx += Idx; /* advance to the next slab */
369 }
370
371 ARMCI_Barrier();
372
373 if(me==0){
374 compare_patches(0.,ndim,(double*)a[0],lo,hi,dimsB,
375 (double*)b[0],lo,hi,dimsB);
376 printf("OK\n");
377 }
378
379 ARMCI_Barrier();
380 destroy_array(b);
381 destroy_array(a);
382 }
383
384
main(int argc,char * argv[])385 int main(int argc, char* argv[])
386 {
387 int ndim;
388
389 armci_msg_init(&argc, &argv);
390 nproc = armci_msg_nproc();
391 me = armci_msg_me();
392
393 ARMCI_Init();
394
395 armci_msg_barrier();
396 if(me==0){
397 printf("\nTesting armci_notify\n");
398 fflush(stdout);
399 sleep(1);
400 }
401 armci_msg_barrier();
402
403 for(ndim=1; ndim<=MAXDIMS; ndim++) test_notify(ndim);
404 armci_msg_barrier();
405
406 ARMCI_Finalize();
407 armci_msg_finalize();
408 return(0);
409 }
410