1 /*! \file
2
3 Copyright 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
4 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014,
5 2015, 2016, 2017, 2018
6 University Corporation for Atmospheric Research/Unidata.
7
8 See \ref copyright file for more info.
9
10 */
11
12 /* This is a benchmarking program for netCDF-4 parallel I/O. */
13
14 /* Defining USE_MPE causes the MPE trace library to be used (and you
15 * must also relink with -llmpe -lmpe). This causes clog2 output to be
16 * written, which can be converted to slog2 (by the program
17 * clog2TOslog2) and then used in the analysis program jumpshot. */
18 /*#define USE_MPE 1*/
19
20 #include <nc_tests.h>
21 #include "err_macros.h"
22 #include <mpi.h>
23 #include <pnetcdf.h>
24
25 #ifdef USE_MPE
26 #include <mpe.h>
27 #endif /* USE_MPE */
28
29 #undef DEBUG
30
31 #define FILE_NAME "tst_parallel2.nc"
32 #define NDIMS 3
33 #define DIMSIZE 8
34 #define NUM_SLABS 8
35 #define DIM1_NAME "slab"
36 #define DIM2_NAME "x"
37 #define DIM3_NAME "y"
38 #define VAR_NAME "Bond_James_Bond"
39
40 int
main(int argc,char ** argv)41 main(int argc, char **argv)
42 {
43 /* MPI stuff. */
44 int mpi_namelen;
45 char mpi_name[MPI_MAX_PROCESSOR_NAME];
46 int mpi_size, mpi_rank;
47 MPI_Comm comm = MPI_COMM_WORLD;
48 MPI_Info info = MPI_INFO_NULL;
49 double start_time = 0, total_time;
50
51 /* Netcdf-4 stuff. */
52 int ncid, varid, dimids[NDIMS];
53 size_t start[NDIMS] = {0, 0, 0};
54 size_t count[NDIMS] = {1, DIMSIZE, DIMSIZE};
55 int data[DIMSIZE * DIMSIZE], data_in[DIMSIZE * DIMSIZE];
56 int j, i;
57
58 char file_name[NC_MAX_NAME + 1];
59 int ndims_in, nvars_in, natts_in, unlimdimid_in;
60
61 #ifdef USE_MPE
62 int s_init, e_init, s_define, e_define, s_write, e_write, s_close, e_close;
63 #endif /* USE_MPE */
64
65 /* Initialize MPI. */
66 MPI_Init(&argc,&argv);
67 MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
68 MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
69 MPI_Get_processor_name(mpi_name, &mpi_namelen);
70 #ifdef DEBUG
71 printf("mpi_name: %s size: %d rank: %d\n", mpi_name, mpi_size, mpi_rank);
72 #endif
73
74 /* Must be able to evenly divide my slabs between processors. */
75 if (NUM_SLABS % mpi_size != 0)
76 {
77 if (!mpi_rank) printf("NUM_SLABS (%d) is not evenly divisible by mpi_size(%d)\n",
78 NUM_SLABS, mpi_size);
79 ERR;
80 }
81
82 #ifdef USE_MPE
83 MPE_Init_log();
84 s_init = MPE_Log_get_event_number();
85 e_init = MPE_Log_get_event_number();
86 s_define = MPE_Log_get_event_number();
87 e_define = MPE_Log_get_event_number();
88 s_write = MPE_Log_get_event_number();
89 e_write = MPE_Log_get_event_number();
90 s_close = MPE_Log_get_event_number();
91 e_close = MPE_Log_get_event_number();
92 s_open = MPE_Log_get_event_number();
93 e_open = MPE_Log_get_event_number();
94 MPE_Describe_state(s_init, e_init, "Init", "red");
95 MPE_Describe_state(s_define, e_define, "Define", "yellow");
96 MPE_Describe_state(s_write, e_write, "Write", "green");
97 MPE_Describe_state(s_close, e_close, "Close", "purple");
98 MPE_Describe_state(s_open, e_open, "Open", "blue");
99 MPE_Start_log();
100 MPE_Log_event(s_init, 0, "start init");
101 #endif /* USE_MPE */
102
103 #ifdef DEBUG
104 if (!mpi_rank)
105 {
106 printf("\n*** Testing parallel I/O some more.\n");
107 printf("*** writing a %d x %d x %d file from %d processors...\n",
108 NUM_SLABS, DIMSIZE, DIMSIZE, mpi_size);
109 }
110 #endif
111
112 /* We will write the same slab over and over. */
113 for (i = 0; i < DIMSIZE * DIMSIZE; i++)
114 data[i] = mpi_rank;
115
116 #ifdef USE_MPE
117 MPE_Log_event(e_init, 0, "end init");
118 MPE_Log_event(s_define, 0, "start define file");
119 #endif /* USE_MPE */
120
121 /* Create a parallel netcdf-4 file. */
122 sprintf(file_name, "%s/%s", TEMP_LARGE, FILE_NAME);
123 #ifdef DEBUG
124 fprintf(stderr,"create: file_name=%s\n",file_name);
125 #endif
126 if (nc_create_par(file_name, 0, comm, info, &ncid)) ERR;
127
128 /* A global attribute holds the number of processors that created
129 * the file. */
130 if (nc_put_att_int(ncid, NC_GLOBAL, "num_processors", NC_INT, 1, &mpi_size)) ERR;
131
132 /* Create three dimensions. */
133 if (nc_def_dim(ncid, DIM1_NAME, NUM_SLABS, dimids)) ERR;
134 if (nc_def_dim(ncid, DIM2_NAME, DIMSIZE, &dimids[1])) ERR;
135 if (nc_def_dim(ncid, DIM3_NAME, DIMSIZE, &dimids[2])) ERR;
136
137 /* Create one var. */
138 if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIMS, dimids, &varid)) ERR;
139
140 /* Write metadata to file. */
141 if (nc_enddef(ncid)) ERR;
142
143 #ifdef USE_MPE
144 MPE_Log_event(e_define, 0, "end define file");
145 if (mpi_rank)
146 sleep(mpi_rank);
147 #endif /* USE_MPE */
148
149 #ifdef USE_PNETCDF
150 /* if (nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE)) ERR;*/
151 if (nc_var_par_access(ncid, NC_GLOBAL, NC_INDEPENDENT)) ERR;
152 #else
153 /* if (nc_var_par_access(ncid, varid, NC_COLLECTIVE)) ERR;*/
154 if (nc_var_par_access(ncid, varid, NC_INDEPENDENT)) ERR;
155 #endif
156
157 if (!mpi_rank)
158 start_time = MPI_Wtime();
159
160 /* Write all the slabs this process is responsible for. */
161 for (i = 0; i < NUM_SLABS / mpi_size; i++)
162 {
163 start[0] = NUM_SLABS / mpi_size * mpi_rank + i;
164
165 #ifdef USE_MPE
166 MPE_Log_event(s_write, 0, "start write slab");
167 #endif /* USE_MPE */
168
169 /* Write one slab of data. */
170 if (nc_put_vara_int(ncid, varid, start, count, data)) ERR;
171
172 #ifdef USE_MPE
173 MPE_Log_event(e_write, 0, "end write file");
174 #endif /* USE_MPE */
175 }
176
177 if (!mpi_rank)
178 {
179 total_time = MPI_Wtime() - start_time;
180 /* printf("num_proc\ttime(s)\n");*/
181 printf("%d\t%g\t%g\n", mpi_size, total_time, DIMSIZE * DIMSIZE * NUM_SLABS * sizeof(int) / total_time);
182 }
183
184 #ifdef USE_MPE
185 MPE_Log_event(s_close, 0, "start close file");
186 #endif /* USE_MPE */
187
188 /* Close the netcdf file. */
189 if (nc_close(ncid)) ERR;
190
191 #ifdef USE_MPE
192 MPE_Log_event(e_close, 0, "end close file");
193 #endif /* USE_MPE */
194
195 /* Reopen the file and check it. */
196 #ifdef DEBUG
197 fprintf(stderr,"open: file_name=%s\n",file_name);
198 #endif
199 if (nc_open_par(file_name, NC_NOWRITE, comm, info, &ncid)) ERR;
200 if (nc_inq(ncid, &ndims_in, &nvars_in, &natts_in, &unlimdimid_in)) ERR;
201 if (ndims_in != NDIMS || nvars_in != 1 || natts_in != 1 ||
202 unlimdimid_in != -1) ERR;
203
204 /* Read all the slabs this process is responsible for. */
205 for (i = 0; i < NUM_SLABS / mpi_size; i++)
206 {
207 start[0] = NUM_SLABS / mpi_size * mpi_rank + i;
208
209 #ifdef USE_MPE
210 MPE_Log_event(s_read, 0, "start read slab");
211 #endif /* USE_MPE */
212
213 /* Read one slab of data. */
214 if (nc_get_vara_int(ncid, varid, start, count, data_in)) ERR;
215
216 /* Check data. */
217 for (j = 0; j < DIMSIZE * DIMSIZE; j++)
218 if (data_in[j] != mpi_rank)
219 {
220 ERR;
221 break;
222 }
223
224 #ifdef USE_MPE
225 MPE_Log_event(e_read, 0, "end read file");
226 #endif /* USE_MPE */
227 }
228
229 #ifdef USE_MPE
230 MPE_Log_event(s_close, 0, "start close file");
231 #endif /* USE_MPE */
232
233 /* Close the netcdf file. */
234 if (nc_close(ncid)) ERR;
235
236 #ifdef USE_MPE
237 MPE_Log_event(e_close, 0, "end close file");
238 #endif /* USE_MPE */
239
240 MPI_Barrier(MPI_COMM_WORLD);
241 if (mpi_rank == 0)
242 remove(file_name);
243
244 /* Shut down MPI. */
245 MPI_Finalize();
246
247 #ifdef DEBUG
248 if (!mpi_rank)
249 {
250 SUMMARIZE_ERR;
251 FINAL_RESULTS;
252 }
253 #endif
254
255 return total_err;
256 }
257