1 /*********************************************************************
2 *
3 * Copyright (C) 2013, Northwestern University and Argonne National Laboratory
4 * See COPYRIGHT notice in top-level directory.
5 *
6 *********************************************************************/
7 /* $Id: mput.c 2717 2016-12-18 01:20:47Z wkliao $ */
8
9 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
10 * This example shows how to use a single call of ncmpi_mput_vara_all() to
11 * write a sequence of requests with arbitrary array indices and lengths.
12 * Using ncmpi_mput_vara_all() can achieve the same effect of HDF5 writing
13 * a sequence of selected file locations through the following 2 APIs.
14 *
15 * H5Sselect_elements(fid, H5S_SELECT_SET, NUMP, (const hssize_t **)coord);
16 * H5Dwrite(dataset, H5T_NATIVE_INT, mid, fid, H5P_DEFAULT, val);
17 *
18 * Note that in ncmpi_mput_vara_all(), users can write more than one element
19 * starting at each selected location.
20 *
21 * The compile and run commands are given below, together with an ncmpidump of
22 * the output file.
23 *
24 * % mpicc -O2 -o mput mput.c -lpnetcdf
25 * % mpiexec -l -n 4 ./mput /pvfs2/wkliao/testfile.nc
26 * % ncmpidump /pvfs2/wkliao/testfile.nc
27 * netcdf testfile {
28 * // file format: CDF-5 (big variables)
29 * dimensions:
30 * Y = 4 ;
31 * X = 10 ;
32 * variables:
33 * int var(Y, X) ;
34 * data:
35 *
36 * var =
37 * 3, 3, 3, 1, 1, 0, 0, 2, 1, 1,
38 * 0, 2, 2, 2, 3, 1, 1, 2, 2, 2,
39 * 1, 1, 2, 3, 3, 3, 0, 0, 1, 1,
40 * 0, 0, 0, 2, 1, 1, 1, 3, 3, 3 ;
41 * }
42 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
43
44 #include <stdio.h>
45 #include <stdlib.h>
46 #include <string.h> /* strcpy(), strncpy() */
47 #include <unistd.h> /* getopt() */
48 #include <mpi.h>
49 #include <pnetcdf.h>
50
51 #ifndef MPI_OFFSET
52 #define MPI_OFFSET MPI_LONG_LONG_INT
53 #endif
54
55 #define NY 4
56 #define NX 10
57 #define NDIMS 2
58
59
60 #define ERR {if(err!=NC_NOERR){printf("Error at line=%d: %s\n", __LINE__, ncmpi_strerror(err));nerrs++;}}
61
62 static void
usage(char * argv0)63 usage(char *argv0)
64 {
65 char *help =
66 "Usage: %s [-h] | [-q] [file_name]\n"
67 " [-h] Print help\n"
68 " [-q] Quiet mode (reports when fail)\n"
69 " [filename] output netCDF file name\n";
70 fprintf(stderr, help, argv0);
71 }
72
main(int argc,char ** argv)73 int main(int argc, char** argv)
74 {
75 extern int optind;
76 char filename[256];
77 int i, rank, nprocs, verbose=1, err, nerrs=0;
78 int ncid, cmode, varid, dimid[2], num_reqs, *buffer, **bufs, *nvarids;
79 MPI_Offset w_len, **starts=NULL, **counts=NULL, *bufcounts;
80 MPI_Datatype *datatypes;
81 MPI_Info info;
82
83 MPI_Init(&argc, &argv);
84 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
85 MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
86
87 /* get command-line arguments */
88 while ((i = getopt(argc, argv, "hq")) != EOF)
89 switch(i) {
90 case 'q': verbose = 0;
91 break;
92 case 'h':
93 default: if (rank==0) usage(argv[0]);
94 MPI_Finalize();
95 return 0;
96 }
97 argc -= optind;
98 argv += optind;
99 if (argc == 1) snprintf(filename, 256, "%s", argv[0]);
100 else strcpy(filename, "testfile.nc");
101
102 if (nprocs != 4 && rank == 0 && verbose)
103 printf("Warning: this program is intended to run on 4 processes\n");
104
105 /* set an MPI-IO hint to disable file offset alignment for fixed-size
106 * variables */
107 MPI_Info_create(&info);
108 MPI_Info_set(info, "nc_var_align_size", "1");
109
110 /* create a new file for writing ----------------------------------------*/
111 cmode = NC_CLOBBER | NC_64BIT_DATA;
112 err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid);
113 ERR
114
115 MPI_Info_free(&info);
116
117 /* create a global array of size NY * NX */
118 err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]);
119 ERR
120 err = ncmpi_def_dim(ncid, "X", NX, &dimid[1]);
121 ERR
122 err = ncmpi_def_var(ncid, "var", NC_INT, NDIMS, dimid, &varid);
123 ERR
124 err = ncmpi_enddef(ncid);
125 ERR
126
127 /* pick arbitrary numbers of requests for 4 processes */
128 num_reqs = 0;
129 if (rank == 0) num_reqs = 4;
130 else if (rank == 1) num_reqs = 6;
131 else if (rank == 2) num_reqs = 5;
132 else if (rank == 3) num_reqs = 4;
133
134 if (num_reqs > 0) {
135 starts = (MPI_Offset**) malloc(num_reqs* sizeof(MPI_Offset*));
136 counts = (MPI_Offset**) malloc(num_reqs* sizeof(MPI_Offset*));
137 starts[0] = (MPI_Offset*) calloc(num_reqs*NDIMS, sizeof(MPI_Offset));
138 counts[0] = (MPI_Offset*) calloc(num_reqs*NDIMS, sizeof(MPI_Offset));
139 for (i=1; i<num_reqs; i++) {
140 starts[i] = starts[i-1] + NDIMS;
141 counts[i] = counts[i-1] + NDIMS;
142 }
143 }
144
145 /* assign arbitrary starts and counts */
146 const int y=0, x=1;
147 if (rank == 0) {
148 starts[0][y] = 0; starts[0][x] = 5; counts[0][y] = 1; counts[0][x] = 2;
149 starts[1][y] = 1; starts[1][x] = 0; counts[1][y] = 1; counts[1][x] = 1;
150 starts[2][y] = 2; starts[2][x] = 6; counts[2][y] = 1; counts[2][x] = 2;
151 starts[3][y] = 3; starts[3][x] = 0; counts[3][y] = 1; counts[3][x] = 3;
152 /* rank 0 is writing the followings: ("-" means skip)
153 - - - - - 0 0 - - -
154 0 - - - - - - - - -
155 - - - - - - 0 0 - -
156 0 0 0 - - - - - - -
157 */
158 } else if (rank ==1) {
159 starts[0][y] = 0; starts[0][x] = 3; counts[0][y] = 1; counts[0][x] = 2;
160 starts[1][y] = 0; starts[1][x] = 8; counts[1][y] = 1; counts[1][x] = 2;
161 starts[2][y] = 1; starts[2][x] = 5; counts[2][y] = 1; counts[2][x] = 2;
162 starts[3][y] = 2; starts[3][x] = 0; counts[3][y] = 1; counts[3][x] = 2;
163 starts[4][y] = 2; starts[4][x] = 8; counts[4][y] = 1; counts[4][x] = 2;
164 starts[5][y] = 3; starts[5][x] = 4; counts[5][y] = 1; counts[5][x] = 3;
165 /* rank 1 is writing the followings: ("-" means skip)
166 - - - 1 1 - - - 1 1
167 - - - - - 1 1 - - -
168 1 1 - - - - - - 1 1
169 - - - - 1 1 1 - - -
170 */
171 } else if (rank ==2) {
172 starts[0][y] = 0; starts[0][x] = 7; counts[0][y] = 1; counts[0][x] = 1;
173 starts[1][y] = 1; starts[1][x] = 1; counts[1][y] = 1; counts[1][x] = 3;
174 starts[2][y] = 1; starts[2][x] = 7; counts[2][y] = 1; counts[2][x] = 3;
175 starts[3][y] = 2; starts[3][x] = 2; counts[3][y] = 1; counts[3][x] = 1;
176 starts[4][y] = 3; starts[4][x] = 3; counts[4][y] = 1; counts[4][x] = 1;
177 /* rank 2 is writing the followings: ("-" means skip)
178 - - - - - - - 2 - -
179 - 2 2 2 - - - 2 2 2
180 - - 2 - - - - - - -
181 - - - 2 - - - - - -
182 */
183 } else if (rank ==3) {
184 starts[0][y] = 0; starts[0][x] = 0; counts[0][y] = 1; counts[0][x] = 3;
185 starts[1][y] = 1; starts[1][x] = 4; counts[1][y] = 1; counts[1][x] = 1;
186 starts[2][y] = 2; starts[2][x] = 3; counts[2][y] = 1; counts[2][x] = 3;
187 starts[3][y] = 3; starts[3][x] = 7; counts[3][y] = 1; counts[3][x] = 3;
188 /* rank 3 is writing the followings: ("-" means skip)
189 3 3 3 - - - - - - -
190 - - - - 3 - - - - -
191 - - - 3 3 3 - - - -
192 - - - - - - - 3 3 3
193 */
194 }
195
196 nvarids = (int*) malloc(num_reqs * sizeof(int));
197 bufcounts = (MPI_Offset*) malloc(num_reqs * sizeof(MPI_Offset));
198 datatypes = (MPI_Datatype*) malloc(num_reqs * sizeof(MPI_Datatype));
199 w_len = 0;
200 for (i=0; i<num_reqs; i++) {
201 nvarids[i] = varid;
202 bufcounts[i] = counts[i][x];
203 datatypes[i] = MPI_INT;
204 w_len += bufcounts[i];
205 }
206
207 /* allocate I/O buffer and initialize its contents */
208 buffer = (int*) malloc(w_len * sizeof(int));
209 for (i=0; i<w_len; i++) buffer[i] = rank;
210
211 /* set the buffer pointers to different offsets to the I/O buffer */
212 bufs = (int**) malloc(num_reqs * sizeof(int*));
213 if (num_reqs > 0) bufs[0] = buffer;
214 for (i=1; i<num_reqs; i++) bufs[i] = bufs[i-1] + bufcounts[i-1];
215
216 err = ncmpi_mput_vara_all(ncid, num_reqs, nvarids, starts, counts,
217 (void**)bufs, bufcounts, datatypes);
218 ERR
219
220 err = ncmpi_close(ncid);
221 ERR
222
223 free(buffer);
224 free(bufs);
225 free(nvarids);
226 free(bufcounts);
227 free(datatypes);
228 if (num_reqs > 0) {
229 free(starts[0]);
230 free(counts[0]);
231 free(starts);
232 free(counts);
233 }
234
235 /* check if there is any PnetCDF internal malloc residue */
236 MPI_Offset malloc_size, sum_size;
237 err = ncmpi_inq_malloc_size(&malloc_size);
238 if (err == NC_NOERR) {
239 MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
240 if (rank == 0 && sum_size > 0)
241 printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
242 sum_size);
243 }
244
245 MPI_Finalize();
246 return nerrs;
247 }
248
249