1 /*********************************************************************
2  *
3  *  Copyright (C) 2015, Northwestern University and Argonne National Laboratory
4  *  See COPYRIGHT notice in top-level directory.
5  *
6  *********************************************************************/
7 /* $Id: ivarn.c 2744 2016-12-28 16:25:22Z wkliao $ */
8 
9 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
10  * This example tests using calls to ncmpi_iput_varn_int(),
11  * ncmpi_iput_varn_float(), ncmpi_iput_varn_double() to write a sequence of
12  * requests with arbitrary array indices and lengths. Note that the request IDs
13  * in the argument array_of_requests[] of ncmpi_wait_all() are in an arbitray
14  * order (instead in an increasing order).
15  *
16  * The compile and run commands are given below, together with an ncmpidump of
17  * the output file.
18  *
19  *    % mpicc -O2 -o ivarn ivarn.c -lpnetcdf
20  *    % mpiexec -n 4 ./ivarn /pvfs2/wkliao/testfile.nc
21  *    % ncmpidump /pvfs2/wkliao/testfile.nc
22  *    netcdf testfile {
23  *    // file format: CDF-5 (big variables)
24  *    dimensions:
25  *             netcdf testfile {
26  *    dimensions:
27  *            dim000001 = 16 ;
28  *            time = UNLIMITED ; // (1 currently)
29  *    variables:
30  *            int vari0001(time, dim000001) ;
31  *            float varr0001(time, dim000001) ;
32  *            double vard0001(time, dim000001) ;
33  *            int vari0002(time, dim000001) ;
34  *            float varr0002(time, dim000001) ;
35  *            double vard0002(time, dim000001) ;
36  *    data:
37  *
38  *     vari0001 =
39  *      1, _, 3, 4, 5, 6, 7, _, 9, 10, 11, 12, 13, 14, _, 16 ;
40  *
41  *     varr0001 =
42  *      1.1, _, 3.1, 4.1, 5.1, 6.1, 7.1, _, 9.1, 10.1, 11.1, 12.1, 13.1, 14.1, _, 16.1 ;
43  *
44  *     vard0001 =
45  *      1.3, _, 3.3, 4.3, 5.3, 6.3, 7.3, _, 9.3, 10.3, 11.3, 12.3, 13.3, 14.3, _, 16.3 ;
46  *
47  *     vari0002 =
48  *      1, _, 3, 4, 5, 6, 7, _, 9, 10, 11, 12, 13, 14, _, 16 ;
49  *
50  *     varr0002 =
51  *      1.2, _, 3.2, 4.2, 5.2, 6.2, 7.2, _, 9.2, 10.2, 11.2, 12.2, 13.2, 14.2, _, 16.2 ;
52  *
53  *     vard0002 =
54  *      1.4, _, 3.4, 4.4, 5.4, 6.4, 7.4, _, 9.4, 10.4, 11.4, 12.4, 13.4, 14.4, _, 16.4 ;
55  *    }
56  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
57 
58 #include <stdio.h>
59 #include <stdlib.h>
60 #include <string.h> /* strcpy() */
61 #include <libgen.h> /* basename() */
62 #include <mpi.h>
63 #include <pnetcdf.h>
64 
65 #include <testutils.h>
66 
67 #define LEN 16
68 #define ERR {if(err!=NC_NOERR)printf("Error at line=%d: %s\n", __LINE__, ncmpi_strerror(err));}
69 
70 static
check_int_buf(int * buffer)71 int check_int_buf(int *buffer)
72 {
73     int i, nprocs;
74     int expected[LEN];
75 
76     MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
77 
78     for (i=0; i<LEN; i++) expected[i] = i+1;
79     expected[1] = expected[7] = expected[14] = NC_FILL_INT;
80 
81     /* check if the contents of buf are expected */
82     for (i=0; i<LEN; i++) {
83         if (nprocs == 1) { if (i == 4) break; }
84         else if (nprocs == 2) {
85             if (3 < i && i < 7) continue;
86             if (i == 12) break;
87         }
88         else if (nprocs == 3) { if (i == 12) break; }
89 
90         if (buffer[i] != expected[i]) {
91             printf("Expected read buf[%d]=%d, but got %d\n",
92                    i,expected[i],buffer[i]);
93             return 1;
94         }
95     }
96     return 0;
97 }
98 
99 static
check_flt_buf(float * buffer,float extra)100 int check_flt_buf(float *buffer, float extra)
101 {
102     int i, nprocs;
103     float expected[LEN];
104 
105     MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
106 
107     for (i=0; i<LEN; i++) expected[i] = extra+i+1;
108     expected[1] = expected[7] = expected[14] = NC_FILL_FLOAT;
109 
110     /* check if the contents of buf are expected */
111     for (i=0; i<LEN; i++) {
112         if (nprocs == 1) { if (i == 4) break; }
113         else if (nprocs == 2) {
114             if (3 < i && i < 7) continue;
115             if (i == 12) break;
116         }
117         else if (nprocs == 3) { if (i == 12) break; }
118 
119         if (buffer[i] != expected[i]) {
120             printf("Expected read buf[%d]=%.1f, but got %.1f\n",
121                    i,expected[i],buffer[i]);
122             return 1;
123         }
124     }
125     return 0;
126 }
127 
128 static
check_dbl_buf(double * buffer,double extra)129 int check_dbl_buf(double *buffer, double extra)
130 {
131     int i, nprocs;
132     double expected[LEN];
133 
134     MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
135 
136     for (i=0; i<LEN; i++) expected[i] = extra+i+1;
137     expected[1] = expected[7] = expected[14] = NC_FILL_DOUBLE;
138 
139     /* check if the contents of buf are expected */
140     for (i=0; i<LEN; i++) {
141         if (nprocs == 1) { if (i == 4) break; }
142         else if (nprocs == 2) {
143             if (3 < i && i < 7) continue;
144             if (i == 12) break;
145         }
146         else if (nprocs == 3) { if (i == 12) break; }
147 
148         if (buffer[i] != expected[i]) {
149             printf("Expected read buf[%d]=%.1f, but got %.1f\n",
150                    i,expected[i],buffer[i]);
151             return 1;
152         }
153     }
154     return 0;
155 }
156 
main(int argc,char ** argv)157 int main(int argc, char** argv)
158 {
159     char filename[256];
160     int i, rank, nprocs, err, nerrs=0;
161     int ncid, cmode, dimid[2];
162     int vari0001, vari0002, varr0001, varr0002, vard0001, vard0002;
163     MPI_Offset **starts, **counts;
164     int req[LEN], st[LEN], num_reqs=0;
165     int ibuf1[LEN], ibuf2[LEN];
166     float rbuf1[LEN], rbuf2[LEN];
167     double dbuf1[LEN], dbuf2[LEN];
168 
169     MPI_Init(&argc, &argv);
170     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
171     MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
172 
173     if (argc > 2) {
174         if (!rank) printf("Usage: %s [filename]\n",argv[0]);
175         MPI_Finalize();
176         return 0;
177     }
178     if (argc == 2) snprintf(filename, 256, "%s", argv[1]);
179     else           strcpy(filename, "testfile.nc");
180     MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD);
181 
182     if (rank == 0) {
183         char *cmd_str = (char*)malloc(strlen(argv[0]) + 256);
184         sprintf(cmd_str, "*** TESTING C   %s for ncmpi_iput_varn_<type>() ", basename(argv[0]));
185         printf("%-66s ------ ", cmd_str); fflush(stdout);
186         free(cmd_str);
187     }
188 
189 #ifdef DEBUG
190     if (nprocs != 4 && rank == 0)
191         printf("Warning: %s is intended to run on 4 processes\n",argv[0]);
192 #endif
193 
194     /* create a new file for writing ----------------------------------------*/
195     cmode = NC_CLOBBER | NC_64BIT_DATA;
196     err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); ERR
197 
198     err = ncmpi_def_dim(ncid, "dim000001", LEN, &dimid[1]); ERR
199     err = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &dimid[0]); ERR
200     err = ncmpi_def_var(ncid, "vari0001", NC_INT, 2, dimid, &vari0001); ERR
201     err = ncmpi_def_var(ncid, "varr0001", NC_FLOAT, 2, dimid, &varr0001); ERR
202     err = ncmpi_def_var(ncid, "vard0001", NC_DOUBLE, 2, dimid, &vard0001); ERR
203     err = ncmpi_def_var(ncid, "vari0002", NC_INT, 2, dimid, &vari0002); ERR
204     err = ncmpi_def_var(ncid, "varr0002", NC_FLOAT, 2, dimid, &varr0002); ERR
205     err = ncmpi_def_var(ncid, "vard0002", NC_DOUBLE, 2, dimid, &vard0002); ERR
206     err = ncmpi_enddef(ncid); ERR
207 
208     starts    = (MPI_Offset**) malloc(2 *    sizeof(MPI_Offset*));
209     counts    = (MPI_Offset**) malloc(2 *    sizeof(MPI_Offset*));
210     starts[0] = (MPI_Offset*)  calloc(2 * 2, sizeof(MPI_Offset));
211     counts[0] = (MPI_Offset*)  calloc(2 * 2, sizeof(MPI_Offset));
212     for (i=1; i<2; i++) {
213         starts[i] = starts[i-1] + 2;
214         counts[i] = counts[i-1] + 2;
215     }
216 
217     /* assign arbitrary starts and counts */
218     if (rank == 0) {
219         /* vari0001 and vari0002 */
220         starts[0][0] = 0; starts[0][1] = 1; counts[0][0] = 1; counts[0][1] = 1;
221         ibuf1[0] = NC_FILL_INT;
222         err = ncmpi_iput_varn_int(ncid, vari0001, 1, starts, counts, ibuf1, &req[1]); ERR
223         ibuf2[0] = NC_FILL_INT;
224         err = ncmpi_iput_varn_int(ncid, vari0002, 1, starts, counts, ibuf2, &req[7]); ERR
225 
226         starts[0][0] = 0; starts[0][1] = 0; counts[0][0] = 1; counts[0][1] = 1;
227         starts[1][0] = 0; starts[1][1] = 2; counts[1][0] = 1; counts[1][1] = 2;
228         ibuf1[1] = 1; ibuf1[2] = 3; ibuf1[3] = 4;
229         err = ncmpi_iput_varn_int(ncid, vari0001, 2, starts, counts, ibuf1+1, &req[0]); ERR
230         ibuf2[1] = 1; ibuf2[2] = 3; ibuf2[3] = 4;
231         err = ncmpi_iput_varn_int(ncid, vari0002, 2, starts, counts, ibuf2+1, &req[6]); ERR
232 
233         /* varr0001 and varr0002 */
234         starts[0][0] = 0; starts[0][1] = 1; counts[0][0] = 1; counts[0][1] = 1;
235         rbuf1[0] = NC_FILL_FLOAT;
236         err = ncmpi_iput_varn_float(ncid, varr0001, 1, starts, counts, rbuf1, &req[3]); ERR
237         rbuf2[0] = NC_FILL_FLOAT;
238         err = ncmpi_iput_varn_float(ncid, varr0002, 1, starts, counts, rbuf2, &req[9]); ERR
239 
240         starts[0][0] = 0; starts[0][1] = 0; counts[0][0] = 1; counts[0][1] = 1;
241         starts[1][0] = 0; starts[1][1] = 2; counts[1][0] = 1; counts[1][1] = 2;
242         rbuf1[1] = 1.1; rbuf1[2] = 3.1; rbuf1[3] = 4.1;
243         err = ncmpi_iput_varn_float(ncid, varr0001, 2, starts, counts, rbuf1+1, &req[2]); ERR
244         rbuf2[1] = 1.2; rbuf2[2] = 3.2; rbuf2[3] = 4.2;
245         err = ncmpi_iput_varn_float(ncid, varr0002, 2, starts, counts, rbuf2+1, &req[8]); ERR
246 
247         /* vard0001 and vard0002 */
248         starts[0][0] = 0; starts[0][1] = 1; counts[0][0] = 1; counts[0][1] = 1;
249         dbuf1[0] = NC_FILL_DOUBLE;
250         err = ncmpi_iput_varn_double(ncid, vard0001, 1, starts, counts, dbuf1, &req[5]); ERR
251         dbuf2[0] = NC_FILL_DOUBLE;
252         err = ncmpi_iput_varn_double(ncid, vard0002, 1, starts, counts, dbuf2, &req[11]); ERR
253 
254         starts[0][0] = 0; starts[0][1] = 0; counts[0][0] = 1; counts[0][1] = 1;
255         starts[1][0] = 0; starts[1][1] = 2; counts[1][0] = 1; counts[1][1] = 2;
256         dbuf1[1] = 1.3; dbuf1[2] = 3.3; dbuf1[3] = 4.3;
257         err = ncmpi_iput_varn_double(ncid, vard0001, 2, starts, counts, dbuf1+1, &req[4]); ERR
258         dbuf2[1] = 1.4; dbuf2[2] = 3.4; dbuf2[3] = 4.4;
259         err = ncmpi_iput_varn_double(ncid, vard0002, 2, starts, counts, dbuf2+1, &req[10]); ERR
260 
261         num_reqs = 12;
262         /* rank 0 is writing the followings: ("x" means skip, "-" means fill value)
263                   1  -  3  4  x  x  x  x  x  x  x  x  x  x  x  x
264          */
265     } else if (rank ==1) {
266         /* vari0001 and vari0002 */
267         starts[0][0] = 0; starts[0][1] = 7; counts[0][0] = 1; counts[0][1] = 1;
268         ibuf1[0] = NC_FILL_INT;
269         err = ncmpi_iput_varn_int(ncid, vari0001, 1, starts, counts, ibuf1, &req[1]); ERR
270         ibuf2[0] = NC_FILL_INT;
271         err = ncmpi_iput_varn_int(ncid, vari0002, 1, starts, counts, ibuf2, &req[7]); ERR
272 
273         starts[0][0] = 0; starts[0][1] = 8; counts[0][0] = 1; counts[0][1] = 4;
274         ibuf1[1] = 9; ibuf1[2] = 10; ibuf1[3] = 11; ibuf1[4] = 12;
275         err = ncmpi_iput_varn_int(ncid, vari0001, 1, starts, counts, ibuf1+1, &req[0]); ERR
276         ibuf2[1] = 9; ibuf2[2] = 10; ibuf2[3] = 11; ibuf2[4] = 12;
277         err = ncmpi_iput_varn_int(ncid, vari0002, 1, starts, counts, ibuf2+1, &req[6]); ERR
278 
279         /* varr0001 and varr0002 */
280         starts[0][0] = 0; starts[0][1] = 7; counts[0][0] = 1; counts[0][1] = 1;
281         rbuf1[0] = NC_FILL_FLOAT;
282         err = ncmpi_iput_varn_float(ncid, varr0001, 1, starts, counts, rbuf1, &req[3]); ERR
283         rbuf2[0] = NC_FILL_FLOAT;
284         err = ncmpi_iput_varn_float(ncid, varr0002, 1, starts, counts, rbuf2, &req[9]); ERR
285 
286         starts[0][0] = 0; starts[0][1] = 8; counts[0][0] = 1; counts[0][1] = 4;
287         rbuf1[1] = 9.1; rbuf1[2] = 10.1; rbuf1[3] = 11.1; rbuf1[4] = 12.1;
288         err = ncmpi_iput_varn_float(ncid, varr0001, 1, starts, counts, rbuf1+1, &req[2]); ERR
289         rbuf2[1] = 9.2; rbuf2[2] = 10.2; rbuf2[3] = 11.2; rbuf2[4] = 12.2;
290         err = ncmpi_iput_varn_float(ncid, varr0002, 1, starts, counts, rbuf2+1, &req[8]); ERR
291 
292         /* vard0001 and vard0002 */
293         starts[0][0] = 0; starts[0][1] = 7; counts[0][0] = 1; counts[0][1] = 1;
294         dbuf1[0] = NC_FILL_DOUBLE;
295         err = ncmpi_iput_varn_double(ncid, vard0001, 1, starts, counts, dbuf1, &req[5]); ERR
296         dbuf2[0] = NC_FILL_DOUBLE;
297         err = ncmpi_iput_varn_double(ncid, vard0002, 1, starts, counts, dbuf2, &req[11]); ERR
298 
299         starts[0][0] = 0; starts[0][1] = 8; counts[0][0] = 1; counts[0][1] = 4;
300         dbuf1[1] = 9.3; dbuf1[2] = 10.3; dbuf1[3] = 11.3; dbuf1[4] = 12.3;
301         err = ncmpi_iput_varn_double(ncid, vard0001, 1, starts, counts, dbuf1+1, &req[4]); ERR
302         dbuf2[1] = 9.4; dbuf2[2] = 10.4; dbuf2[3] = 11.4; dbuf2[4] = 12.4;
303         err = ncmpi_iput_varn_double(ncid, vard0002, 1, starts, counts, dbuf2+1, &req[10]); ERR
304 
305         num_reqs = 12;
306         /* rank 1 is writing the followings: ("x" means skip, "-" means fill value)
307                   vari0001:
308                   x  x  x  x  x  x  x  x  -  9  10 11 12 x  x  x  x
309          */
310     } else if (rank ==2) {
311         /* vari0001 and vari0002 */
312         starts[0][0] = 0; starts[0][1] = 4; counts[0][0] = 1; counts[0][1] = 3;
313         ibuf1[0] = 5; ibuf1[1] = 6; ibuf1[2] = 7;
314         err = ncmpi_iput_varn_int(ncid, vari0001, 1, starts, counts, ibuf1, &req[0]); ERR
315         ibuf2[0] = 5; ibuf2[1] = 6; ibuf2[2] = 7;
316         err = ncmpi_iput_varn_int(ncid, vari0002, 1, starts, counts, ibuf2, &req[1]); ERR
317 
318         /* varr0001 and varr0002 */
319         rbuf1[0] = 5.1; rbuf1[1] = 6.1; rbuf1[2] = 7.1;
320         err = ncmpi_iput_varn_float(ncid, varr0001, 1, starts, counts, rbuf1, &req[2]); ERR
321         rbuf2[0] = 5.2; rbuf2[1] = 6.2; rbuf2[2] = 7.2;
322         err = ncmpi_iput_varn_float(ncid, varr0002, 1, starts, counts, rbuf2, &req[3]); ERR
323 
324         /* vard0001 and vard0002 */
325         dbuf1[0] = 5.3; dbuf1[1] = 6.3; dbuf1[2] = 7.3;
326         err = ncmpi_iput_varn_double(ncid, vard0001, 1, starts, counts, dbuf1, &req[4]); ERR
327         dbuf2[0] = 5.4; dbuf2[1] = 6.4; dbuf2[2] = 7.4;
328         err = ncmpi_iput_varn_double(ncid, vard0002, 1, starts, counts, dbuf2, &req[5]); ERR
329 
330         num_reqs = 6;
331         /* rank 2 is writing the followings: ("x" means skip, "-" means fill value)
332                   vari0001:
333                   x  x  x  x  x  5  6  7  x  x  x  x  x  x  x  x  x
334          */
335     } else if (rank ==3) {
336         /* vari0001 and vari0002 */
337         starts[0][0] = 0; starts[0][1] = 14; counts[0][0] = 1; counts[0][1] = 1;
338         ibuf1[0] = NC_FILL_INT;
339         err = ncmpi_iput_varn_int(ncid, vari0001, 1, starts, counts, ibuf1, &req[0]); ERR
340         ibuf2[0] = NC_FILL_INT;
341         err = ncmpi_iput_varn_int(ncid, vari0002, 1, starts, counts, ibuf2, &req[1]); ERR
342 
343         starts[0][0] = 0; starts[0][1] = 12; counts[0][0] = 1; counts[0][1] = 2;
344         starts[1][0] = 0; starts[1][1] = 15; counts[1][0] = 1; counts[1][1] = 1;
345         ibuf1[1] = 13; ibuf1[2] = 14; ibuf1[3] = 16;
346         err = ncmpi_iput_varn_int(ncid, vari0001, 2, starts, counts, ibuf1+1, &req[2]); ERR
347         ibuf2[1] = 13; ibuf2[2] = 14; ibuf2[3] = 16;
348         err = ncmpi_iput_varn_int(ncid, vari0002, 2, starts, counts, ibuf2+1, &req[3]); ERR
349 
350         /* varr0001 and varr0002 */
351         starts[0][0] = 0; starts[0][1] = 14; counts[0][0] = 1; counts[0][1] = 1;
352         rbuf1[0] = NC_FILL_FLOAT;
353         err = ncmpi_iput_varn_float(ncid, varr0001, 1, starts, counts, rbuf1, &req[4]); ERR
354         rbuf2[0] = NC_FILL_FLOAT;
355         err = ncmpi_iput_varn_float(ncid, varr0002, 1, starts, counts, rbuf2, &req[5]); ERR
356 
357         starts[0][0] = 0; starts[0][1] = 12; counts[0][0] = 1; counts[0][1] = 2;
358         starts[1][0] = 0; starts[1][1] = 15; counts[1][0] = 1; counts[1][1] = 1;
359         rbuf1[1] = 13.1; rbuf1[2] = 14.1; rbuf1[3] = 16.1;
360         err = ncmpi_iput_varn_float(ncid, varr0001, 2, starts, counts, rbuf1+1, &req[6]); ERR
361         rbuf2[1] = 13.2; rbuf2[2] = 14.2; rbuf2[3] = 16.2;
362         err = ncmpi_iput_varn_float(ncid, varr0002, 2, starts, counts, rbuf2+1, &req[7]); ERR
363 
364         /* vard0001 and vard0002 */
365         starts[0][0] = 0; starts[0][1] = 14; counts[0][0] = 1; counts[0][1] = 1;
366         dbuf1[0] = NC_FILL_DOUBLE;
367         err = ncmpi_iput_varn_double(ncid, vard0001, 1, starts, counts, dbuf1, &req[8]); ERR
368         dbuf2[0] = NC_FILL_DOUBLE;
369         err = ncmpi_iput_varn_double(ncid, vard0002, 1, starts, counts, dbuf2, &req[9]); ERR
370 
371         starts[0][0] = 0; starts[0][1] = 12; counts[0][0] = 1; counts[0][1] = 2;
372         starts[1][0] = 0; starts[1][1] = 15; counts[1][0] = 1; counts[1][1] = 1;
373         dbuf1[1] = 13.3; dbuf1[2] = 14.3; dbuf1[3] = 16.3;
374         err = ncmpi_iput_varn_double(ncid, vard0001, 2, starts, counts, dbuf1+1, &req[10]); ERR
375         dbuf2[1] = 13.4; dbuf2[2] = 14.4; dbuf2[3] = 16.4;
376         err = ncmpi_iput_varn_double(ncid, vard0002, 2, starts, counts, dbuf2+1, &req[11]); ERR
377 
378         num_reqs = 12;
379         /* rank 3 is writing the followings: ("x" means skip, "-" means fill value)
380                   vari0001:
381                   x  x  x  x  x  x  x  x  x  x  x  x 13 14  - 16
382          */
383     }
384 
385     err = ncmpi_wait_all(ncid, num_reqs, req, st); ERR
386     for (i=0; i<num_reqs; i++) {
387         err = st[i]; ERR
388     }
389 
390     err = ncmpi_close(ncid); ERR
391 
392     free(starts[0]);
393     free(counts[0]);
394     free(starts);
395     free(counts);
396 
397     err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR
398 
399     err = ncmpi_inq_varid(ncid, "vari0001", &vari0001); ERR
400     err = ncmpi_inq_varid(ncid, "varr0001", &varr0001); ERR
401     err = ncmpi_inq_varid(ncid, "vard0001", &vard0001); ERR
402     err = ncmpi_inq_varid(ncid, "vari0002", &vari0002); ERR
403     err = ncmpi_inq_varid(ncid, "varr0002", &varr0002); ERR
404     err = ncmpi_inq_varid(ncid, "vard0002", &vard0002); ERR
405 
406     for (i=0; i<LEN; i++) ibuf1[i] = -1;
407     err = ncmpi_get_var_int_all(ncid, vari0001, ibuf1); ERR
408     nerrs += check_int_buf(ibuf1);
409 
410     for (i=0; i<LEN; i++) ibuf2[i] = -1;
411     err = ncmpi_get_var_int_all(ncid, vari0002, ibuf2); ERR
412     nerrs += check_int_buf(ibuf2);
413 
414     for (i=0; i<LEN; i++) rbuf1[i] = -1;
415     err = ncmpi_get_var_float_all(ncid, varr0001, rbuf1); ERR
416     nerrs += check_flt_buf(rbuf1, 0.1);
417 
418     for (i=0; i<LEN; i++) rbuf2[i] = -1;
419     err = ncmpi_get_var_float_all(ncid, varr0002, rbuf2); ERR
420     nerrs += check_flt_buf(rbuf2, 0.2);
421 
422     for (i=0; i<LEN; i++) dbuf1[i] = -1;
423     err = ncmpi_get_var_double_all(ncid, vard0001, dbuf1); ERR
424     nerrs += check_dbl_buf(dbuf1, 0.3);
425 
426     for (i=0; i<LEN; i++) dbuf2[i] = -1;
427     err = ncmpi_get_var_double_all(ncid, vard0002, dbuf2); ERR
428     nerrs += check_dbl_buf(dbuf2, 0.4);
429 
430     err = ncmpi_close(ncid); ERR
431 
432     /* check if PnetCDF freed all internal malloc */
433     MPI_Offset malloc_size, sum_size;
434     err = ncmpi_inq_malloc_size(&malloc_size);
435     if (err == NC_NOERR) {
436         MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
437         if (rank == 0 && sum_size > 0)
438             printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
439                    sum_size);
440     }
441 
442     MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
443     if (rank == 0) {
444         if (nerrs) printf(FAIL_STR,nerrs);
445         else       printf(PASS_STR);
446     }
447 
448     MPI_Finalize();
449     return 0;
450 }
451 
452