1 /*********************************************************************
2 *
3 * Copyright (C) 2014, Northwestern University and Argonne National Laboratory
4 * See COPYRIGHT notice in top-level directory.
5 *
6 *********************************************************************/
7 /* $Id: bput_varn_uint.c 2717 2016-12-18 01:20:47Z wkliao $ */
8
9 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
10 * This example tests nonblocking buffered write varn APIs, including
11 * ncmpi_bput_varn_uint() and ncmpi_bput_varn(),
12 * It first writes a sequence of requests with arbitrary array indices and
13 * lengths to four variables of type NC_UINT, and reads back.
14 *
15 * The compile and run commands are given below, together with an ncmpidump of
16 * the output file.
17 *
18 * % mpicc -O2 -o bput_varn_uint bput_varn_uint.c -lpnetcdf
19 * % mpiexec -n 4 ./bput_varn_uint /pvfs2/wkliao/testfile.nc
20 * % ncmpidump /pvfs2/wkliao/testfile.nc
21 * netcdf testfile {
22 * // file format: CDF-5 (big variables)
23 * dimensions:
24 * Y = 4 ;
25 * X = 10 ;
26 * variables:
27 * uint var0(Y, X) ;
28 * uint var1(Y, X) ;
29 * uint var2(Y, X) ;
30 * uint var3(Y, X) ;
31 * data:
32 *
33 * var0 =
34 * 1, 1, 1, 3, 3, 0, 0, 2, 3, 3,
35 * 0, 2, 2, 2, 1, 3, 3, 2, 2, 2,
36 * 3, 3, 2, 1, 1, 1, 0, 0, 3, 3,
37 * 0, 0, 0, 2, 3, 3, 3, 1, 1, 1 ;
38 *
39 * var1 =
40 * 2, 2, 2, 0, 0, 1, 1, 3, 0, 0,
41 * 1, 3, 3, 3, 2, 0, 0, 3, 3, 3,
42 * 0, 0, 3, 2, 2, 2, 1, 1, 0, 0,
43 * 1, 1, 1, 3, 0, 0, 0, 2, 2, 2 ;
44 *
45 * var2 =
46 * 3, 3, 3, 1, 1, 2, 2, 0, 1, 1,
47 * 2, 0, 0, 0, 3, 1, 1, 0, 0, 0,
48 * 1, 1, 0, 3, 3, 3, 2, 2, 1, 1,
49 * 2, 2, 2, 0, 1, 1, 1, 3, 3, 3 ;
50 *
51 * var3 =
52 * 0, 0, 0, 2, 2, 3, 3, 1, 2, 2,
53 * 3, 1, 1, 1, 0, 2, 2, 1, 1, 1,
54 * 2, 2, 1, 0, 0, 0, 3, 3, 2, 2,
55 * 3, 3, 3, 1, 2, 2, 2, 0, 0, 0 ;
56 * }
57 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
58
59 #include <stdio.h>
60 #include <stdlib.h>
61 #include <string.h> /* strcpy(), strncpy() */
62 #include <unistd.h> /* getopt() */
63 #include <mpi.h>
64 #include <pnetcdf.h>
65
66 #ifndef MPI_OFFSET
67 #define MPI_OFFSET MPI_LONG_LONG_INT
68 #endif
69
70 #define NY 4
71 #define NX 10
72 #define NDIMS 2
73
74 #define ERR {if(err!=NC_NOERR){printf("Error at line=%d: %s\n", __LINE__, ncmpi_strerror(err));nerrs++;}}
75
76 #define ERRS(n,a) { \
77 int _i; \
78 for (_i=0; _i<(n); _i++) { \
79 if ((a)[_i] != NC_NOERR) { \
80 printf("Error at line=%d: err[%d] %s\n", __LINE__, _i, \
81 ncmpi_strerror((a)[_i])); \
82 nerrs++; \
83 } \
84 } \
85 }
86
calloc_3D(size_t z,size_t y,size_t x)87 static MPI_Offset *** calloc_3D(size_t z, size_t y, size_t x)
88 {
89 if (z*y*x == 0) return NULL;
90 int _j, _k;
91 MPI_Offset ***buf = (MPI_Offset***) malloc(z * sizeof(MPI_Offset**));
92 MPI_Offset **bufy = (MPI_Offset**) malloc(z*y * sizeof(MPI_Offset*));
93 MPI_Offset *bufx = (MPI_Offset*) calloc(z*y*x, sizeof(MPI_Offset));
94 for (_k=0; _k<z; _k++, bufy+=y) {
95 buf[_k] = bufy;
96 for (_j=0; _j<y; _j++, bufx+=x)
97 buf[_k][_j] = bufx;
98 }
99 return buf;
100 }
101
free_3D(MPI_Offset *** buf)102 static void free_3D(MPI_Offset ***buf)
103 {
104 free(buf[0][0]);
105 free(buf[0]);
106 free(buf);
107 }
108
109 static void
usage(char * argv0)110 usage(char *argv0)
111 {
112 char *help =
113 "Usage: %s [-h] | [-q] [file_name]\n"
114 " [-h] Print help\n"
115 " [-q] Quiet mode (reports when fail)\n"
116 " [filename] output netCDF file name\n";
117 fprintf(stderr, help, argv0);
118 }
119
check_contents(int ncid,int * varid)120 static int check_contents(int ncid, int *varid)
121 {
122 int i, j, err, nerrs=0;
123 unsigned int expected[4][NY*NX] = {{1, 1, 1, 3, 3, 0, 0, 2, 3, 3,
124 0, 2, 2, 2, 1, 3, 3, 2, 2, 2,
125 3, 3, 2, 1, 1, 1, 0, 0, 3, 3,
126 0, 0, 0, 2, 3, 3, 3, 1, 1, 1},
127 {2, 2, 2, 0, 0, 1, 1, 3, 0, 0,
128 1, 3, 3, 3, 2, 0, 0, 3, 3, 3,
129 0, 0, 3, 2, 2, 2, 1, 1, 0, 0,
130 1, 1, 1, 3, 0, 0, 0, 2, 2, 2},
131 {3, 3, 3, 1, 1, 2, 2, 0, 1, 1,
132 2, 0, 0, 0, 3, 1, 1, 0, 0, 0,
133 1, 1, 0, 3, 3, 3, 2, 2, 1, 1,
134 2, 2, 2, 0, 1, 1, 1, 3, 3, 3},
135 {0, 0, 0, 2, 2, 3, 3, 1, 2, 2,
136 3, 1, 1, 1, 0, 2, 2, 1, 1, 1,
137 2, 2, 1, 0, 0, 0, 3, 3, 2, 2,
138 3, 3, 3, 1, 2, 2, 2, 0, 0, 0}};
139
140 unsigned int *r_buffer = (unsigned int*) malloc(NY*NX*sizeof(unsigned int));
141
142 for (i=0; i<4; i++) {
143 for (j=0; j<NY*NX; j++) r_buffer[j] = 99999;
144 err = ncmpi_get_var_uint_all(ncid, varid[i], r_buffer);
145 ERR
146
147 /* check if the contents of buf are expected */
148 for (j=0; j<NY*NX; j++)
149 if (r_buffer[j] != expected[i][j]) {
150 printf("Expected file contents [%d][%d]=%u, but got %u\n",
151 i,j,expected[i][j],r_buffer[j]);
152 nerrs++;
153 }
154 }
155 free(r_buffer);
156 return nerrs;
157 }
158
main(int argc,char ** argv)159 int main(int argc, char** argv)
160 {
161 extern int optind;
162 char filename[256], *exec;
163 int i, j, k, n, rank, nprocs, verbose=1, err, nerrs=0;
164 int ncid, cmode, varid[4], dimid[2], nreqs, reqs[4], sts[4];
165 unsigned int *buffer[4];
166 int num_segs[4], req_lens[4];
167 MPI_Offset ***starts, ***counts;
168 MPI_Info info;
169
170 MPI_Init(&argc, &argv);
171 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
172 MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
173 exec = argv[0];
174
175 /* get command-line arguments */
176 while ((i = getopt(argc, argv, "hq")) != EOF)
177 switch(i) {
178 case 'q': verbose = 0;
179 break;
180 case 'h':
181 default: if (rank==0) usage(argv[0]);
182 MPI_Finalize();
183 return 0;
184 }
185 argc -= optind;
186 argv += optind;
187 if (argc == 1) snprintf(filename, 256, "%s", argv[0]);
188 else strcpy(filename, "testfile.nc");
189
190 if (nprocs != 4 && rank == 0 && verbose)
191 printf("Warning: %s is intended to run on 4 processes\n",exec);
192
193 /* set an MPI-IO hint to disable file offset alignment for fixed-size
194 * variables */
195 MPI_Info_create(&info);
196 MPI_Info_set(info, "nc_var_align_size", "1");
197
198 /* create a new file for writing ----------------------------------------*/
199 cmode = NC_CLOBBER | NC_64BIT_DATA;
200 err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid);
201 ERR
202
203 MPI_Info_free(&info);
204
205 /* create a global array of size NY * NX */
206 err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]); ERR
207 err = ncmpi_def_dim(ncid, "X", NX, &dimid[1]); ERR
208 err = ncmpi_def_var(ncid, "var0", NC_UINT, NDIMS, dimid, &varid[0]); ERR
209 err = ncmpi_def_var(ncid, "var1", NC_UINT, NDIMS, dimid, &varid[1]); ERR
210 err = ncmpi_def_var(ncid, "var2", NC_UINT, NDIMS, dimid, &varid[2]); ERR
211 err = ncmpi_def_var(ncid, "var3", NC_UINT, NDIMS, dimid, &varid[3]); ERR
212 err = ncmpi_enddef(ncid); ERR
213
214 /* allocate space for starts and counts */
215 starts = calloc_3D(4, 6, NDIMS);
216 counts = calloc_3D(4, 6, NDIMS);
217
218 n = rank % 4;
219 num_segs[n] = 4; /* number of segments for this request */
220 starts[n][0][0]=0; starts[n][0][1]=5; counts[n][0][0]=1; counts[n][0][1]=2;
221 starts[n][1][0]=1; starts[n][1][1]=0; counts[n][1][0]=1; counts[n][1][1]=1;
222 starts[n][2][0]=2; starts[n][2][1]=6; counts[n][2][0]=1; counts[n][2][1]=2;
223 starts[n][3][0]=3; starts[n][3][1]=0; counts[n][3][0]=1; counts[n][3][1]=3;
224 /* starts[n][][] n_counts[n][][] indicate the following: ("-" means skip)
225 - - - - - X X - - -
226 X - - - - - - - - -
227 - - - - - - X X - -
228 X X X - - - - - - -
229 */
230 n = (rank+1) % 4;
231 num_segs[n] = 6; /* number of segments for this request */
232 starts[n][0][0]=0; starts[n][0][1]=3; counts[n][0][0]=1; counts[n][0][1]=2;
233 starts[n][1][0]=0; starts[n][1][1]=8; counts[n][1][0]=1; counts[n][1][1]=2;
234 starts[n][2][0]=1; starts[n][2][1]=5; counts[n][2][0]=1; counts[n][2][1]=2;
235 starts[n][3][0]=2; starts[n][3][1]=0; counts[n][3][0]=1; counts[n][3][1]=2;
236 starts[n][4][0]=2; starts[n][4][1]=8; counts[n][4][0]=1; counts[n][4][1]=2;
237 starts[n][5][0]=3; starts[n][5][1]=4; counts[n][5][0]=1; counts[n][5][1]=3;
238 /* starts[n][][] counts[n][][] indicate the following pattern.
239 - - - X X - - - X X
240 - - - - - X X - - -
241 X X - - - - - - X X
242 - - - - X X X - - -
243 */
244 n = (rank+2) % 4;
245 num_segs[n] = 5; /* number of segments for this request */
246 starts[n][0][0]=0; starts[n][0][1]=7; counts[n][0][0]=1; counts[n][0][1]=1;
247 starts[n][1][0]=1; starts[n][1][1]=1; counts[n][1][0]=1; counts[n][1][1]=3;
248 starts[n][2][0]=1; starts[n][2][1]=7; counts[n][2][0]=1; counts[n][2][1]=3;
249 starts[n][3][0]=2; starts[n][3][1]=2; counts[n][3][0]=1; counts[n][3][1]=1;
250 starts[n][4][0]=3; starts[n][4][1]=3; counts[n][4][0]=1; counts[n][4][1]=1;
251 /* starts[n][][] counts[n][][] indicate the following pattern.
252 - - - - - - - X - -
253 - X X X - - - X X X
254 - - X - - - - - - -
255 - - - X - - - - - -
256 */
257 n = (rank+3) % 4;
258 num_segs[n] = 4; /* number of segments for this request */
259 starts[n][0][0]=0; starts[n][0][1]=0; counts[n][0][0]=1; counts[n][0][1]=3;
260 starts[n][1][0]=1; starts[n][1][1]=4; counts[n][1][0]=1; counts[n][1][1]=1;
261 starts[n][2][0]=2; starts[n][2][1]=3; counts[n][2][0]=1; counts[n][2][1]=3;
262 starts[n][3][0]=3; starts[n][3][1]=7; counts[n][3][0]=1; counts[n][3][1]=3;
263 /*starts[n][][] counts[n][][] indicate the following pattern.
264 X X X - - - - - - -
265 - - - - X - - - - -
266 - - - X X X - - - -
267 - - - - - - - X X X
268 */
269
270 /* only rank 0, 1, 2, and 3 do I/O:
271 * each of ranks 0 to 3 write 4 nonblocking requests */
272 nreqs = 4;
273 if (rank >= 4) nreqs = 0;
274
275 /* bufsize must be max of data type converted before and after */
276 MPI_Offset bufsize = 0;
277
278 /* calculate length of each varn request, number of segments in each
279 * varn request, and allocate write buffer */
280 for (i=0; i<nreqs; i++) {
281 req_lens[i] = 0; /* total length this request */
282 for (j=0; j<num_segs[i]; j++) {
283 MPI_Offset req_len=1;
284 for (k=0; k<NDIMS; k++)
285 req_len *= counts[i][j][k];
286 req_lens[i] += req_len;
287 }
288 if (verbose) printf("req_lens[%d]=%d\n",i,req_lens[i]);
289
290 /* allocate I/O buffer and initialize its contents */
291 buffer[i] = (unsigned int*) malloc(req_lens[i] * sizeof(unsigned int));
292 for (j=0; j<req_lens[i]; j++) buffer[i][j] = rank;
293
294 bufsize += req_lens[i];
295 }
296 bufsize *= sizeof(unsigned int);
297 if (verbose) printf("%d: Attach buffer size %lld\n", rank, bufsize);
298
299 /* give PnetCDF a space to buffer the nonblocking requests */
300 if (bufsize > 0) {
301 err = ncmpi_buffer_attach(ncid, bufsize); ERR
302 }
303
304 /* write using varn API */
305 for (i=0; i<nreqs; i++) {
306 err = ncmpi_bput_varn_uint(ncid, varid[i], num_segs[i], starts[i],
307 counts[i], buffer[i], &reqs[i]);
308 ERR
309 }
310 err = ncmpi_wait_all(ncid, nreqs, reqs, sts);
311 ERRS(nreqs, sts)
312
313 /* check file contents */
314 if (nprocs >= 4) check_contents(ncid, varid);
315
316 for (i=0; i<nreqs; i++) free(buffer[i]);
317
318 /* test flexible put API, using a noncontiguous buftype */
319 for (i=0; i<nreqs; i++) {
320 MPI_Datatype buftype;
321 MPI_Type_vector(req_lens[i], 1, 2, MPI_UNSIGNED, &buftype);
322 MPI_Type_commit(&buftype);
323 buffer[i] = (unsigned int*) malloc(req_lens[i] * 2 * sizeof(unsigned int));
324 for (j=0; j<req_lens[i]*2; j++) buffer[i][j] = rank;
325
326 err = ncmpi_bput_varn(ncid, varid[i], num_segs[i], starts[i],
327 counts[i], buffer[i], 1, buftype, &reqs[i]);
328 ERR
329 MPI_Type_free(&buftype);
330 }
331 err = ncmpi_wait_all(ncid, nreqs, reqs, sts);
332 ERRS(nreqs, sts)
333
334 /* check file contents */
335 if (nprocs >= 4) check_contents(ncid, varid);
336
337 /* free the buffer space for bput */
338 if (bufsize > 0) {
339 err = ncmpi_buffer_detach(ncid); ERR
340 }
341
342 err = ncmpi_close(ncid);
343 ERR
344
345 for (i=0; i<nreqs; i++) free(buffer[i]);
346 free_3D(starts);
347 free_3D(counts);
348
349 /* check if there is any PnetCDF internal malloc residue */
350 MPI_Offset malloc_size, sum_size;
351 err = ncmpi_inq_malloc_size(&malloc_size);
352 if (err == NC_NOERR) {
353 MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
354 if (rank == 0 && sum_size > 0)
355 printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
356 sum_size);
357 }
358
359 MPI_Finalize();
360 return nerrs;
361 }
362
363