1 /*
2 Copyright 2009, UCAR/Unidata
3 See COPYRIGHT file for copying and redistribution conditions.
4
5 This program tests netcdf-4 parallel I/O.
6
7 $Id: tst_parallel.c,v 1.7 2009/08/19 15:58:57 ed Exp $
8 */
9
10 /* Defining USE_MPE causes the MPE trace library to be used (and you
11 * must also relink with -llmpe -lmpe). This causes clog2 output to be
12 * written, which can be converted to slog2 (by the program
13 * clog2TOslog2) and then used in the analysis program jumpshot. */
14 /*#define USE_MPE 1*/
15
16 #include <nc_tests.h>
17 #include "err_macros.h"
18 #include <mpi.h>
19 #ifdef USE_MPE
20 #include <mpe.h>
21 #endif /* USE_MPE */
22 #define FILE "tst_parallel.nc"
23 #define NDIMS 3
24 #define DIMSIZE 24
25 #define QTR_DATA (DIMSIZE * DIMSIZE / 4)
26 #define NUM_PROC 4
27 #define NUM_SLABS 10
28
29 int
main(int argc,char ** argv)30 main(int argc, char **argv)
31 {
32 /* MPI stuff. */
33 int mpi_namelen;
34 char mpi_name[MPI_MAX_PROCESSOR_NAME];
35 int mpi_size, mpi_rank;
36 MPI_Comm comm = MPI_COMM_WORLD;
37 MPI_Info info = MPI_INFO_NULL;
38
39 /* Netcdf-4 stuff. */
40 int ncid, v1id, dimids[NDIMS];
41 size_t start[NDIMS], count[NDIMS];
42
43 int i, res;
44 int slab_data[DIMSIZE * DIMSIZE / 4]; /* one slab */
45 char file_name[NC_MAX_NAME + 1];
46
47 #ifdef USE_MPE
48 int s_init, e_init, s_define, e_define, s_write, e_write, s_close, e_close;
49 #endif /* USE_MPE */
50
51 /* Initialize MPI. */
52 MPI_Init(&argc,&argv);
53 MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
54 MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
55 MPI_Get_processor_name(mpi_name, &mpi_namelen);
56 /*printf("mpi_name: %s size: %d rank: %d\n", mpi_name,
57 mpi_size, mpi_rank);*/
58
59 #ifdef USE_MPE
60 MPE_Init_log();
61 s_init = MPE_Log_get_event_number();
62 e_init = MPE_Log_get_event_number();
63 s_define = MPE_Log_get_event_number();
64 e_define = MPE_Log_get_event_number();
65 s_write = MPE_Log_get_event_number();
66 e_write = MPE_Log_get_event_number();
67 s_close = MPE_Log_get_event_number();
68 e_close = MPE_Log_get_event_number();
69 MPE_Describe_state(s_init, e_init, "Init", "red");
70 MPE_Describe_state(s_define, e_define, "Define", "yellow");
71 MPE_Describe_state(s_write, e_write, "Write", "green");
72 MPE_Describe_state(s_close, e_close, "Close", "purple");
73 MPE_Start_log();
74 MPE_Log_event(s_init, 0, "start init");
75 #endif /* USE_MPE */
76
77 if (mpi_rank == 1)
78 {
79 printf("\n*** tst_parallel testing very basic parallel access.\n");
80 printf("*** tst_parallel testing whether we can create file for parallel access and write to it...");
81 }
82
83 /* Create phony data. We're going to write a 24x24 array of ints,
84 in 4 sets of 144. */
85 for (i = 0; i < DIMSIZE * DIMSIZE / 4; i++)
86 slab_data[i] = mpi_rank;
87
88 #ifdef USE_MPE
89 MPE_Log_event(e_init, 0, "end init");
90 MPE_Log_event(s_define, 0, "start define file");
91 #endif /* USE_MPE */
92
93 /* Create a parallel netcdf-4 file. */
94 /*nc_set_log_level(3);*/
95 sprintf(file_name, "%s/%s", TEMP_LARGE, FILE);
96 if ((res = nc_create_par(file_name, NC_NETCDF4, comm,
97 info, &ncid))) ERR;
98
99 /* Create three dimensions. */
100 if (nc_def_dim(ncid, "d1", DIMSIZE, dimids)) ERR;
101 if (nc_def_dim(ncid, "d2", DIMSIZE, &dimids[1])) ERR;
102 if (nc_def_dim(ncid, "d3", NUM_SLABS, &dimids[2])) ERR;
103
104 /* Create one var. */
105 if ((res = nc_def_var(ncid, "v1", NC_INT, NDIMS, dimids, &v1id))) ERR;
106
107 /* Write metadata to file. */
108 if ((res = nc_enddef(ncid))) ERR;
109
110 #ifdef USE_MPE
111 MPE_Log_event(e_define, 0, "end define file");
112 if (mpi_rank)
113 sleep(mpi_rank);
114 #endif /* USE_MPE */
115
116 /* Set up slab for this process. */
117 start[0] = mpi_rank * DIMSIZE/mpi_size;
118 start[1] = 0;
119 count[0] = DIMSIZE/mpi_size;
120 count[1] = DIMSIZE;
121 count[2] = 1;
122 /*printf("mpi_rank=%d start[0]=%d start[1]=%d count[0]=%d count[1]=%d\n",
123 mpi_rank, start[0], start[1], count[0], count[1]);*/
124
125 if (nc_var_par_access(ncid, v1id, NC_COLLECTIVE)) ERR;
126 /* if (nc_var_par_access(ncid, v1id, NC_INDEPENDENT)) ERR;*/
127
128 for (start[2] = 0; start[2] < NUM_SLABS; start[2]++)
129 {
130 #ifdef USE_MPE
131 MPE_Log_event(s_write, 0, "start write slab");
132 #endif /* USE_MPE */
133
134 /* Write slabs of phoney data. */
135 if (nc_put_vara_int(ncid, v1id, start, count, slab_data)) ERR;
136 #ifdef USE_MPE
137 MPE_Log_event(e_write, 0, "end write file");
138 #endif /* USE_MPE */
139 }
140
141 #ifdef USE_MPE
142 MPE_Log_event(s_close, 0, "start close file");
143 #endif /* USE_MPE */
144
145 /* Close the netcdf file. */
146 if ((res = nc_close(ncid))) ERR;
147
148 #ifdef USE_MPE
149 MPE_Log_event(e_close, 0, "end close file");
150 #endif /* USE_MPE */
151
152 /* Delete this large file. */
153 remove(file_name);
154
155 /* Shut down MPI. */
156 MPI_Finalize();
157
158 if (mpi_rank == 1)
159 {
160 SUMMARIZE_ERR;
161 FINAL_RESULTS;
162 }
163 return 0;
164 }
165