1 /* Copyright 2007-2018, UCAR/Unidata. See COPYRIGHT file for copying
2 * and redistribution conditions.
3 *
4 * This is part of the netCDF package.
5 *
6 * This test is for parallel IO and the collective access of metadata
7 * with HDF5.
8 *
9 * Ward Fisher, Ed Hartnett
10 */
11
12 #include "config.h"
13 #include "nc_tests.h"
14 #include "err_macros.h"
15
16 #define TEST_NAME "tst_parallel4_simplerw_coll"
17 #define NDIMS 3
18 #define DIMSIZE 16
19 #define NUM_SLABS 16
20 #define DIM1_NAME "slab"
21 #define DIM2_NAME "x"
22 #define DIM3_NAME "y"
23 #define VAR_NAME "Bond_James_Bond"
24 #define NUM_FILL_TEST_RUNS 3
25
26 int
main(int argc,char ** argv)27 main(int argc, char **argv)
28 {
29 int mpi_namelen;
30 char mpi_name[MPI_MAX_PROCESSOR_NAME];
31 int mpi_size, mpi_rank;
32 MPI_Comm comm = MPI_COMM_WORLD;
33 MPI_Info info = MPI_INFO_NULL;
34 double start_time = 0, total_time;
35 int mpi_size_in;
36 #define NUM_TEST_TYPES 11
37 nc_type test_type[NUM_TEST_TYPES] = {NC_BYTE, NC_CHAR, NC_SHORT, NC_INT, NC_FLOAT, NC_DOUBLE,
38 NC_UBYTE, NC_USHORT, NC_UINT, NC_INT64, NC_UINT64};
39 int tt, fv;
40 int j, i, k, ret;
41
42 /* Initialize MPI. */
43 MPI_Init(&argc,&argv);
44 MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
45 MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
46 MPI_Get_processor_name(mpi_name, &mpi_namelen);
47
48 /* Must be able to evenly divide my slabs between processors. */
49 if (NUM_SLABS % mpi_size)
50 {
51 if (!mpi_rank)
52 printf("NUM_SLABS (%d) is not evenly divisible by mpi_size(%d)\n",
53 NUM_SLABS, mpi_size);
54 ERR;
55 }
56
57 if (!mpi_rank)
58 printf("\n*** Testing parallel I/O some more.\n");
59
60 /* Test for different fill value settings. */
61 for (fv = 0; fv < NUM_FILL_TEST_RUNS; fv++)
62 {
63 /* Test for different netCDF types. */
64 for (tt = 0; tt < NUM_TEST_TYPES; tt++)
65 {
66 char file_name[NC_MAX_NAME + 1];
67 int fill_mode_in;
68 void *data, *data_in;
69 void *fill_value, *fill_value_in;
70 size_t type_size;
71 size_t write_start[NDIMS] = {0, 0, 1};
72 size_t write_count[NDIMS] = {1, DIMSIZE, DIMSIZE - 1};
73 size_t read_start[NDIMS] = {0, 0, 0};
74 size_t read_count[NDIMS] = {1, DIMSIZE, DIMSIZE};
75 int ncid, varid, dimids[NDIMS];
76 int ndims_in, nvars_in, natts_in, unlimdimid_in;
77
78 /* Fill values to be expected. */
79 signed char byte_expected_fill_value;
80 unsigned char char_expected_fill_value;
81 short short_expected_fill_value;
82 int int_expected_fill_value;
83 float float_expected_fill_value;
84 double double_expected_fill_value;
85 unsigned char ubyte_expected_fill_value;
86 unsigned short ushort_expected_fill_value;
87 unsigned int uint_expected_fill_value;
88 long long int int64_expected_fill_value;
89 unsigned long long int uint64_expected_fill_value;
90
91 /* Fill values used when writing. */
92 signed char byte_fill_value = -TEST_VAL_42;
93 unsigned char char_fill_value = 'x';
94 short short_fill_value = TEST_VAL_42 * 100;
95 int int_fill_value = TEST_VAL_42 * 1000;
96 float float_fill_value = TEST_VAL_42 * 1000;
97 double double_fill_value = TEST_VAL_42 * 1000;
98 unsigned char ubyte_fill_value = TEST_VAL_42;
99 unsigned short ushort_fill_value = TEST_VAL_42 * 100;
100 unsigned int uint_fill_value = TEST_VAL_42 * 1000;
101 long long int int64_fill_value = TEST_VAL_42 * 1000;
102 unsigned long long int uint64_fill_value = TEST_VAL_42 * 1000;
103
104 /* Fill values read in. */
105 signed char byte_fill_value_in;
106 unsigned char char_fill_value_in;
107 short short_fill_value_in;
108 int int_fill_value_in;
109 float float_fill_value_in;
110 double double_fill_value_in;
111 unsigned char ubyte_fill_value_in;
112 unsigned short ushort_fill_value_in;
113 unsigned int uint_fill_value_in;
114 long long int int64_fill_value_in;
115 unsigned long long int uint64_fill_value_in;
116
117 /* Data to write and read. */
118 signed char byte_data[DIMSIZE * DIMSIZE], byte_data_in[DIMSIZE * DIMSIZE];
119 unsigned char char_data[DIMSIZE * DIMSIZE], char_data_in[DIMSIZE * DIMSIZE];
120 short short_data[DIMSIZE * DIMSIZE], short_data_in[DIMSIZE * DIMSIZE];
121 int int_data[DIMSIZE * DIMSIZE], int_data_in[DIMSIZE * DIMSIZE];
122 float float_data[DIMSIZE * DIMSIZE], float_data_in[DIMSIZE * DIMSIZE];
123 double double_data[DIMSIZE * DIMSIZE], double_data_in[DIMSIZE * DIMSIZE];
124 unsigned char ubyte_data[DIMSIZE * DIMSIZE], ubyte_data_in[DIMSIZE * DIMSIZE];
125 unsigned short ushort_data[DIMSIZE * DIMSIZE], ushort_data_in[DIMSIZE * DIMSIZE];
126 unsigned int uint_data[DIMSIZE * DIMSIZE], uint_data_in[DIMSIZE * DIMSIZE];
127 long long int int64_data[DIMSIZE * DIMSIZE], int64_data_in[DIMSIZE * DIMSIZE];
128 unsigned long long int uint64_data[DIMSIZE * DIMSIZE], uint64_data_in[DIMSIZE * DIMSIZE];
129
130 if (!mpi_rank)
131 printf("*** writing a %d x %d x %d file from %d processors for fill value test %d type %d...\n",
132 NUM_SLABS, DIMSIZE, DIMSIZE, mpi_size, fv, test_type[tt]);
133
134 /* Initialize test data. */
135 switch(test_type[tt])
136 {
137 case NC_BYTE:
138 for (i = 0; i < DIMSIZE * DIMSIZE; i++)
139 byte_data[i] = mpi_rank;
140 data = byte_data;
141 data_in = byte_data_in;
142 byte_expected_fill_value = fv ? byte_fill_value : NC_FILL_BYTE;
143 fill_value = &byte_expected_fill_value;
144 fill_value_in = &byte_fill_value_in;
145 break;
146 case NC_CHAR:
147 for (i = 0; i < DIMSIZE * DIMSIZE; i++)
148 char_data[i] = mpi_rank;
149 data = char_data;
150 data_in = char_data_in;
151 char_expected_fill_value = fv ? char_fill_value : NC_FILL_CHAR;
152 fill_value = &char_expected_fill_value;
153 fill_value_in = &char_fill_value_in;
154 break;
155 case NC_SHORT:
156 for (i = 0; i < DIMSIZE * DIMSIZE; i++)
157 short_data[i] = mpi_rank;
158 data = short_data;
159 data_in = short_data_in;
160 short_expected_fill_value = fv ? short_fill_value : NC_FILL_SHORT;
161 fill_value = &short_expected_fill_value;
162 fill_value_in = &short_fill_value_in;
163 break;
164 case NC_INT:
165 for (i = 0; i < DIMSIZE * DIMSIZE; i++)
166 int_data[i] = mpi_rank;
167 data = int_data;
168 data_in = int_data_in;
169 int_expected_fill_value = fv ? int_fill_value : NC_FILL_INT;
170 fill_value = &int_expected_fill_value;
171 fill_value_in = &int_fill_value_in;
172 break;
173 case NC_FLOAT:
174 for (i = 0; i < DIMSIZE * DIMSIZE; i++)
175 float_data[i] = mpi_rank;
176 data = float_data;
177 data_in = float_data_in;
178 float_expected_fill_value = fv ? float_fill_value : NC_FILL_FLOAT;
179 fill_value = &float_expected_fill_value;
180 fill_value_in = &float_fill_value_in;
181 break;
182 case NC_DOUBLE:
183 for (i = 0; i < DIMSIZE * DIMSIZE; i++)
184 double_data[i] = mpi_rank;
185 data = double_data;
186 data_in = double_data_in;
187 double_expected_fill_value = fv ? double_fill_value : NC_FILL_DOUBLE;
188 fill_value = &double_expected_fill_value;
189 fill_value_in = &double_fill_value_in;
190 break;
191 case NC_UBYTE:
192 for (i = 0; i < DIMSIZE * DIMSIZE; i++)
193 ubyte_data[i] = mpi_rank;
194 data = ubyte_data;
195 data_in = ubyte_data_in;
196 ubyte_expected_fill_value = fv ? ubyte_fill_value : NC_FILL_UBYTE;
197 fill_value = &ubyte_expected_fill_value;
198 fill_value_in = &ubyte_fill_value_in;
199 break;
200 case NC_USHORT:
201 for (i = 0; i < DIMSIZE * DIMSIZE; i++)
202 ushort_data[i] = mpi_rank;
203 data = ushort_data;
204 data_in = ushort_data_in;
205 ushort_expected_fill_value = fv ? ushort_fill_value : NC_FILL_USHORT;
206 fill_value = &ushort_expected_fill_value;
207 fill_value_in = &ushort_fill_value_in;
208 break;
209 case NC_UINT:
210 for (i = 0; i < DIMSIZE * DIMSIZE; i++)
211 uint_data[i] = mpi_rank;
212 data = uint_data;
213 data_in = uint_data_in;
214 uint_expected_fill_value = fv ? uint_fill_value : NC_FILL_UINT;
215 fill_value = &uint_expected_fill_value;
216 fill_value_in = &uint_fill_value_in;
217 break;
218 case NC_INT64:
219 for (i = 0; i < DIMSIZE * DIMSIZE; i++)
220 int64_data[i] = mpi_rank;
221 data = int64_data;
222 data_in = int64_data_in;
223 int64_expected_fill_value = fv ? int64_fill_value : NC_FILL_INT64;
224 fill_value = &int64_expected_fill_value;
225 fill_value_in = &int64_fill_value_in;
226 break;
227 case NC_UINT64:
228 for (i = 0; i < DIMSIZE * DIMSIZE; i++)
229 uint64_data[i] = mpi_rank;
230 data = uint64_data;
231 data_in = uint64_data_in;
232 uint64_expected_fill_value = fv ? uint64_fill_value : NC_FILL_UINT64;
233 fill_value = &uint64_expected_fill_value;
234 fill_value_in = &uint64_fill_value_in;
235 break;
236 }
237
238 /* Create a file name. */
239 sprintf(file_name, "%s_type_%d_fv_%d.nc", TEST_NAME, test_type[tt], fv);
240
241 /* Create a parallel netcdf-4 file. */
242 if (nc_create_par(file_name, NC_NETCDF4, comm, info, &ncid)) ERR;
243
244 /* Get the type len. */
245 if (nc_inq_type(ncid, test_type[tt], NULL, &type_size)) ERR;
246
247 /* A global attribute holds the number of processors that created
248 * the file. */
249 if (nc_put_att_int(ncid, NC_GLOBAL, "num_processors", NC_INT, 1, &mpi_size)) ERR;
250
251 /* Create three dimensions. */
252 if (nc_def_dim(ncid, DIM1_NAME, NUM_SLABS, dimids)) ERR;
253 if (nc_def_dim(ncid, DIM2_NAME, DIMSIZE, &dimids[1])) ERR;
254 if (nc_def_dim(ncid, DIM3_NAME, DIMSIZE, &dimids[2])) ERR;
255
256 /* Create one var. */
257 if (nc_def_var(ncid, VAR_NAME, test_type[tt], NDIMS, dimids, &varid)) ERR;
258 if (nc_put_att_int(ncid, varid, "var_num_processors", NC_INT, 1, &mpi_size)) ERR;
259 if (fv == 1)
260 {
261 if (nc_def_var_fill(ncid, varid, NC_FILL, fill_value)) ERR;
262 if (nc_inq_var_fill(ncid, varid, &fill_mode_in, fill_value_in)) ERR;
263 if (fill_mode_in != NC_FILL) ERR;
264 if (memcmp(fill_value_in, fill_value, type_size)) ERR;
265 }
266 else if (fv == 2)
267 {
268 if (nc_def_var_fill(ncid, varid, NC_NOFILL, NULL)) ERR;
269 if (nc_inq_var_fill(ncid, varid, &fill_mode_in, NULL)) ERR;
270 if (!fill_mode_in) ERR; /* nofill will be true */
271 }
272
273 /* Write metadata to file. */
274 if (nc_enddef(ncid)) ERR;
275
276 /* Change access mode to collective, then back to independent. */
277 if (nc_var_par_access(ncid, varid, NC_COLLECTIVE)) ERR;
278 if (nc_var_par_access(ncid, varid, NC_INDEPENDENT)) ERR;
279
280 if (!mpi_rank)
281 start_time = MPI_Wtime();
282
283 /* Write all the slabs this process is responsible for. */
284 for (i = 0; i < NUM_SLABS / mpi_size; i++)
285 {
286 write_start[0] = NUM_SLABS / mpi_size * mpi_rank + i;
287
288 /* Write one slab of data. Due to start/count settings,
289 * every 16th value will be a fill value. */
290 if (nc_put_vara(ncid, varid, write_start, write_count, data)) ERR;
291 }
292
293 /* On rank 0, keep track of time. */
294 if (!mpi_rank)
295 {
296 total_time = MPI_Wtime() - start_time;
297 printf("%d\t%g\t%g\n", mpi_size, total_time, DIMSIZE * DIMSIZE * NUM_SLABS *
298 sizeof(int) / total_time);
299 }
300
301 /* Close the netcdf file. */
302 if (nc_close(ncid)) ERR;
303
304 /* Reopen the file and check it. */
305 if ((ret = nc_open_par(file_name, NC_NOWRITE, comm, info, &ncid))) ERR;
306 if (nc_inq(ncid, &ndims_in, &nvars_in, &natts_in, &unlimdimid_in)) ERR;
307 if (ndims_in != NDIMS || nvars_in != 1 || natts_in != 1 ||
308 unlimdimid_in != -1) ERR;
309
310 /* Check the attributes. */
311 if (nc_get_att_int(ncid, NC_GLOBAL, "num_processors", &mpi_size_in)) ERR;
312 if (mpi_size_in != mpi_size) ERR;
313 if (nc_get_att_int(ncid, 0, "var_num_processors", &mpi_size_in)) ERR;
314 if (mpi_size_in != mpi_size) ERR;
315 if (fv == 1)
316 {
317 if (nc_inq_var_fill(ncid, varid, &fill_mode_in, fill_value_in)) ERR;
318 if (fill_mode_in != NC_FILL) ERR;
319 if (memcmp(fill_value_in, fill_value, type_size)) ERR;
320 }
321
322 /* Read all the slabs this process is responsible for. */
323 for (i = 0; i < NUM_SLABS / mpi_size; i++)
324 {
325 read_start[0] = NUM_SLABS / mpi_size * mpi_rank + i;
326 /* printf("mpi_rank %d i %d read_start[0] %ld\n", mpi_rank, i, read_start[0]); */
327
328 /* Read one slab of data. */
329 if (nc_get_vara(ncid, varid, read_start, read_count, data_in)) ERR;
330
331 /* Check data. For the third fill value test, fill is
332 * turned off. So don't bother testing the values where k
333 * is zero. */
334 /* printf("mpi_rank %d fv %d i %d j %d k %d int_data_in[j * k] %d int_expected_fill_value %d " */
335 /* "expected_value %d\n", mpi_rank, fv, i, j, k, int_data_in[j * k], */
336 /* int_expected_fill_value, expected_value); */
337 switch (test_type[tt])
338 {
339 case NC_BYTE:
340 for (j = 0; j < DIMSIZE; j++)
341 for (k = 0; k < DIMSIZE; k++)
342 if (fv < 2 || k)
343 if (byte_data_in[j * DIMSIZE + k] != (signed char)(k ? mpi_rank : byte_expected_fill_value)) ERR;
344 break;
345 case NC_SHORT:
346 for (j = 0; j < DIMSIZE; j++)
347 for (k = 0; k < DIMSIZE; k++)
348 if (fv < 2 || k)
349 if (short_data_in[j * DIMSIZE + k] != (short)(k ? mpi_rank : short_expected_fill_value)) ERR;
350 break;
351 case NC_INT:
352 for (j = 0; j < DIMSIZE; j++)
353 for (k = 0; k < DIMSIZE; k++)
354 if (fv < 2 || k)
355 if (int_data_in[j * DIMSIZE + k] != (int)(k ? mpi_rank : int_expected_fill_value)) ERR;
356 break;
357 case NC_FLOAT:
358 for (j = 0; j < DIMSIZE; j++)
359 for (k = 0; k < DIMSIZE; k++)
360 if (fv < 2 || k)
361 if (float_data_in[j * DIMSIZE + k] != (float)(k ? mpi_rank : float_expected_fill_value)) ERR;
362 break;
363 case NC_DOUBLE:
364 for (j = 0; j < DIMSIZE; j++)
365 for (k = 0; k < DIMSIZE; k++)
366 if (fv < 2 || k)
367 if (double_data_in[j * DIMSIZE + k] != (double)(k ? mpi_rank : double_expected_fill_value)) ERR;
368 break;
369 case NC_UBYTE:
370 for (j = 0; j < DIMSIZE; j++)
371 for (k = 0; k < DIMSIZE; k++)
372 if (fv < 2 || k)
373 if (ubyte_data_in[j * DIMSIZE + k] != (unsigned char)(k ? mpi_rank : ubyte_expected_fill_value)) ERR;
374 break;
375 case NC_USHORT:
376 for (j = 0; j < DIMSIZE; j++)
377 for (k = 0; k < DIMSIZE; k++)
378 if (fv < 2 || k)
379 if (ushort_data_in[j * DIMSIZE + k] != (unsigned short)(k ? mpi_rank : ushort_expected_fill_value)) ERR;
380 break;
381 case NC_UINT:
382 for (j = 0; j < DIMSIZE; j++)
383 for (k = 0; k < DIMSIZE; k++)
384 if (fv < 2 || k)
385 if (uint_data_in[j * DIMSIZE + k] != (unsigned int)(k ? mpi_rank : uint_expected_fill_value)) ERR;
386 break;
387 case NC_INT64:
388 for (j = 0; j < DIMSIZE; j++)
389 for (k = 0; k < DIMSIZE; k++)
390 if (fv < 2 || k)
391 if (int64_data_in[j * DIMSIZE + k] != (long long int)(k ? mpi_rank : int64_expected_fill_value)) ERR;
392 break;
393 case NC_UINT64:
394 for (j = 0; j < DIMSIZE; j++)
395 for (k = 0; k < DIMSIZE; k++)
396 if (fv < 2 || k)
397 if (uint64_data_in[j * DIMSIZE + k] != (unsigned long long int)(k ? mpi_rank : uint64_expected_fill_value)) ERR;
398 break;
399 }
400 } /* next slab */
401
402 /* Close the netcdf file. */
403 if (nc_close(ncid)) ERR;
404
405 if (!mpi_rank)
406 SUMMARIZE_ERR;
407 } /* next test type */
408 } /* next fill value test run */
409
410 /* Shut down MPI. */
411 MPI_Finalize();
412
413 if (!mpi_rank)
414 FINAL_RESULTS;
415 return 0;
416 }
417