1 /*********************************************************************
2 *
3 * Copyright (C) 2013, Northwestern University and Argonne National Laboratory
4 * See COPYRIGHT notice in top-level directory.
5 *
6 *********************************************************************/
7 /* $Id: column_wise.c 2717 2016-12-18 01:20:47Z wkliao $ */
8
9 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
10 * This example makes a number of nonblocking API calls, each writes a single
11 * column of a 2D integer array. Each process writes NX columns and any two
12 * consecutive columns are of nprocs columns distance apart from each other. In
13 * this case, the fileview of each process interleaves with all other processes.
14 * If simply concatenating fileviews of all the nonblocking calls will result
15 * in a fileview that violates the MPI-IO requirement on the fileview of which
16 * flattened file offsets must be monotonically non-decreasing. PnetCDF handles
17 * this case by breaking down each nonblocking call into a list of offset-length
18 * pairs, merging the pairs across multiple nonblocking calls, and sorting
19 * them into an increasing order. The sorted pairs are used to construct a
20 * fileview that meets the monotonically non-decreasing offset requirement,
21 * and thus the nonblocking requests can be serviced by a single MPI-IO call.
22 *
23 * The compile and run commands are given below, together with an ncmpidump of
24 * the output file.
25 *
26 * % mpicc -O2 -o column_wise column_wise.c -lpnetcdf
27 * % mpiexec -l -n 4 ./column_wise /pvfs2/wkliao/testfile.nc
28 * 0: 0: myOff= 0 myNX= 4
29 * 1: 1: myOff= 4 myNX= 4
30 * 2: 2: myOff= 8 myNX= 4
31 * 3: 3: myOff= 12 myNX= 4
32 * 0: 0: start= 0 0 count= 10 1
33 * 1: 1: start= 0 1 count= 10 1
34 * 2: 2: start= 0 2 count= 10 1
35 * 3: 3: start= 0 3 count= 10 1
36 *
37 * % ncmpidump /pvfs2/wkliao/testfile.nc
38 * netcdf testfile {
39 * // file format: CDF-5 (big variables)
40 * dimensions:
41 * Y = 10 ;
42 * X = 16 ;
43 * variables:
44 * int var(Y, X) ;
45 * data:
46 *
47 * var =
48 * 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3,
49 * 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3,
50 * 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3,
51 * 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3,
52 * 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3,
53 * 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3,
54 * 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3,
55 * 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3,
56 * 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3,
57 * 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 ;
58 * }
59 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
60
61 #include <stdio.h>
62 #include <stdlib.h>
63 #include <string.h> /* strcpy(), strncpy() */
64 #include <unistd.h> /* getopt() */
65 #include <mpi.h>
66 #include <pnetcdf.h>
67
68 #ifndef MPI_OFFSET
69 #define MPI_OFFSET MPI_LONG_LONG_INT
70 #endif
71
72 #define NY 10
73 #define NX 4
74
75 #define ERR {if(err!=NC_NOERR){printf("Error at line=%d: %s\n", __LINE__, ncmpi_strerror(err));nerrs++;}}
76
77 static void
usage(char * argv0)78 usage(char *argv0)
79 {
80 char *help =
81 "Usage: %s [-h] | [-q] [file_name]\n"
82 " [-h] Print help\n"
83 " [-q] Quiet mode (reports when fail)\n"
84 " [filename] output netCDF file name\n";
85 fprintf(stderr, help, argv0);
86 }
87
main(int argc,char ** argv)88 int main(int argc, char** argv)
89 {
90 extern int optind;
91 char filename[256];
92 int i, j, verbose=1, rank, nprocs, err, nerrs=0, myNX, G_NX, myOff, num_reqs;
93 int ncid, cmode, varid, dimid[2], *reqs, *sts, **buf;
94 MPI_Offset start[2], count[2];
95 MPI_Info info;
96
97 MPI_Init(&argc, &argv);
98 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
99 MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
100
101 /* get command-line arguments */
102 while ((i = getopt(argc, argv, "hq")) != EOF)
103 switch(i) {
104 case 'q': verbose = 0;
105 break;
106 case 'h':
107 default: if (rank==0) usage(argv[0]);
108 MPI_Finalize();
109 return 0;
110 }
111 argc -= optind;
112 argv += optind;
113 if (argc == 1) snprintf(filename, 256, "%s", argv[0]);
114 else strcpy(filename, "testfile.nc");
115
116 /* set an MPI-IO hint to disable file offset alignment for fixed-size
117 * variables */
118 MPI_Info_create(&info);
119 MPI_Info_set(info, "nc_var_align_size", "1");
120
121 cmode = NC_CLOBBER | NC_64BIT_DATA;
122 err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid);
123 ERR
124
125 MPI_Info_free(&info);
126
127 /* the global array is NY * (NX * nprocs) */
128 G_NX = NX * nprocs;
129 myOff = NX * rank;
130 myNX = NX;
131 if (verbose) printf("%2d: myOff=%3d myNX=%3d\n",rank,myOff,myNX);
132
133 err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]);
134 ERR
135 err = ncmpi_def_dim(ncid, "X", G_NX, &dimid[1]);
136 ERR
137 err = ncmpi_def_var(ncid, "var", NC_INT, 2, dimid, &varid);
138 ERR
139 err = ncmpi_enddef(ncid);
140 ERR
141
142 /* First, fill the entire array with zeros, using a blocking I/O.
143 Every process writes a subarray of size NY * myNX */
144 buf = (int**) malloc(myNX * sizeof(int*));
145 buf[0] = (int*) calloc(NY * myNX, sizeof(int));
146 start[0] = 0; start[1] = myOff;
147 count[0] = NY; count[1] = myNX;
148 err = ncmpi_put_vara_int_all(ncid, varid, start, count, buf[0]);
149 free(buf[0]);
150
151 /* initialize the buffer with rank ID. Also make the case interesting,
152 by allocating buffers separately */
153 for (i=0; i<myNX; i++) {
154 buf[i] = (int*) malloc(NY * sizeof(int));
155 for (j=0; j<NY; j++) buf[i][j] = rank;
156 }
157
158 reqs = (int*) malloc(myNX * sizeof(int));
159 sts = (int*) malloc(myNX * sizeof(int));
160
161 /* each proc writes myNX single columns of the 2D array */
162 start[0] = 0; start[1] = rank;
163 count[0] = NY; count[1] = 1;
164 if (verbose)
165 printf("%2d: start=%3lld %3lld count=%3lld %3lld\n",
166 rank, start[0],start[1], count[0],count[1]);
167
168 num_reqs = 0;
169 for (i=0; i<myNX; i++) {
170 err = ncmpi_iput_vara_int(ncid, varid, start, count, buf[i],
171 &reqs[num_reqs++]);
172 ERR
173 start[1] += nprocs;
174 }
175 err = ncmpi_wait_all(ncid, num_reqs, reqs, sts);
176 ERR
177
178 /* check status of all requests */
179 for (i=0; i<num_reqs; i++)
180 if (sts[i] != NC_NOERR)
181 printf("Error: nonblocking write fails on request %d (%s)\n",
182 i, ncmpi_strerror(sts[i]));
183
184 /* read back using the same access pattern */
185 for (i=0; i<myNX; i++)
186 for (j=0; j<NY; j++) buf[i][j] = -1;
187
188 /* each proc reads myNX single columns of the 2D array */
189 start[0] = 0; start[1] = rank;
190 count[0] = NY; count[1] = 1;
191
192 num_reqs = 0;
193 for (i=0; i<myNX; i++) {
194 err = ncmpi_iget_vara_int(ncid, varid, start, count, buf[i],
195 &reqs[num_reqs++]);
196 ERR
197 start[1] += nprocs;
198 }
199 err = ncmpi_wait_all(ncid, num_reqs, reqs, sts);
200 ERR
201
202 /* check status of all requests */
203 for (i=0; i<num_reqs; i++)
204 if (sts[i] != NC_NOERR)
205 printf("Error: nonblocking write fails on request %d (%s)\n",
206 i, ncmpi_strerror(sts[i]));
207
208 for (i=0; i<myNX; i++) {
209 for (j=0; j<NY; j++)
210 if (buf[i][j] != rank)
211 printf("Error: expect buf[%d][%d]=%d but got %d\n",i,j,rank,buf[i][j]);
212 }
213
214 err = ncmpi_close(ncid);
215 ERR
216
217 free(sts);
218 free(reqs);
219 for (i=0; i<myNX; i++) free(buf[i]);
220 free(buf);
221
222 /* check if there is any PnetCDF internal malloc residue */
223 MPI_Offset malloc_size, sum_size;
224 err = ncmpi_inq_malloc_size(&malloc_size);
225 if (err == NC_NOERR) {
226 MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
227 if (rank == 0 && sum_size > 0)
228 printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
229 sum_size);
230 }
231
232 MPI_Finalize();
233 return nerrs;
234 }
235
236