1 /*
2  * Copyright (c) 1998 Sandia Corporation. Under the terms of Contract
3  * DE-AC04-94AL85000 with Sandia Corporation, the U.S. Governement
4  * retains certain rights in this software.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions are
8  * met:
9  *
10  *     * Redistributions of source code must retain the above copyright
11  *       notice, this list of conditions and the following disclaimer.
12  *
13  *     * Redistributions in binary form must reproduce the above
14  *       copyright notice, this list of conditions and the following
15  *       disclaimer in the documentation and/or other materials provided
16  *       with the distribution.
17  *
18  *     * Neither the name of Sandia Corporation nor the names of its
19  *       contributors may be used to endorse or promote products derived
20  *       from this software without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33  *
34  */
35 
36 /*****************************************************************************/
37 /*****************************************************************************/
38 /*****************************************************************************/
39 /* Function(s) contained in this file:
40  *      ex_put_loadbal_param_cc()
41  *****************************************************************************
42  * This function outputs the concatenated list of load-balance parameters.
43  *****************************************************************************
44  *  Variable Index:
45  *      exoid             - The NetCDF ID of an already open NemesisI file.
46  *      num_int_nodes    - Vector of number of internal FEM nodes for
47  *			   "num_proc_in_f" processors.
48  *      num_bor_nodes    - Vector of number of border FEM nodes for
49  *			   "num_proc_in_f" processors.
50  *      num_ext_nodes    - Vector of number of external FEM nodes for
51  *			   "num_proc_in_f" processors.
52  *      num_int_elems    - Vector of number of internal FEM elems for
53  *			   "num_proc_in_f" processors.
54  *      num_bor_elems    - Vector of number of border FEM elems for
55  *			   "num_proc_in_f" processors.
56  *      num_node_cmaps   - Vector of number of nodal communication maps
57  *                         for "num_proc_in_f" processors.
58  *      num_elem_cmaps   - Vector of number of elemental communication maps
59  *                         for "num_proc_in_f" processors.
60  */
61 /*****************************************************************************/
62 /*****************************************************************************/
63 /*****************************************************************************/
64 
65 #include <stdio.h>
66 
67 #include <netcdf.h>
68 
69 #include <exodusII.h>
70 #include <exodusII_int.h>
71 #ifndef NC_INT64
72 #define NC_INT64 NC_INT
73 #endif
74 
ex_put_loadbal_param_cc(int exoid,void_int * num_int_nodes,void_int * num_bor_nodes,void_int * num_ext_nodes,void_int * num_int_elems,void_int * num_bor_elems,void_int * num_node_cmaps,void_int * num_elem_cmaps)75 int ex_put_loadbal_param_cc(int   exoid,
76                             void_int  *num_int_nodes,
77                             void_int  *num_bor_nodes,
78                             void_int  *num_ext_nodes,
79                             void_int  *num_int_elems,
80                             void_int  *num_bor_elems,
81                             void_int  *num_node_cmaps,
82                             void_int  *num_elem_cmaps
83                             )
84 {
85   const char  *func_name="ex_put_loadbal_param_cc";
86 
87   int     status;
88   int     iproc, varid, dimid_npf, dimid[3];
89   int     num_proc, num_proc_in_f;
90   int     varid_nm[3], varid_em[2];
91   int     varid_idx[7] = {0, 0, 0, 0, 0, 0, 0};
92   size_t  start[1];
93   char    ftype[2];
94   int  oldfill;
95 
96 #if defined(NC_NETCDF4)
97   int64_t num_int_elem = 0, num_int_node = 0, num_bor_elem = 0;
98   int64_t num_bor_node = 0, num_ext_node = 0;
99   int64_t num_n_cmaps = 0, num_e_cmaps = 0;
100 #else
101   int num_int_elem = 0, num_int_node = 0, num_bor_elem = 0;
102   int num_bor_node = 0, num_ext_node = 0;
103   int num_n_cmaps = 0, num_e_cmaps = 0;
104 #endif
105 
106   int  nmstat;
107 
108   char   errmsg[MAX_ERR_LENGTH];
109 
110   int     format;
111 
112   int index_type = NC_INT;
113   int map_type = NC_INT;
114   int id_type  = NC_INT;
115 
116   /*-----------------------------Execution begins-----------------------------*/
117   exerrval = 0; /* clear error code */
118 
119   if (ex_int64_status(exoid) & EX_MAPS_INT64_DB) {
120     map_type = NC_INT64;
121   }
122   if (ex_int64_status(exoid) & EX_IDS_INT64_DB) {
123     id_type = NC_INT64;
124   }
125 
126   /* See if using NC_FORMAT_NETCDF4 format... */
127   nc_inq_format(exoid, &format);
128   if ((ex_int64_status(exoid) & EX_BULK_INT64_DB) || (format == NC_FORMAT_NETCDF4)) {
129     index_type = NC_INT64;
130   }
131 
132   /* Get the processor information from the file */
133   if (ex_get_init_info(exoid, &num_proc, &num_proc_in_f, ftype) != EX_NOERR) {
134     exerrval = EX_MSG;
135     sprintf(errmsg,
136             "Error: Unable to get processor info from file ID %d",
137             exoid);
138     ex_err(func_name, errmsg, exerrval);
139 
140     return (EX_FATAL);
141   }
142 
143   /*
144    * Get the dimension ID for the number of processors storing
145    * information in this file.
146    */
147   if ((status = nc_inq_dimid(exoid, DIM_NUM_PROCS_F, &dimid_npf)) != NC_NOERR) {
148     exerrval = status;
149     sprintf(errmsg,
150             "Error: failed to find dimension ID for \"%s\" in file ID %d",
151             DIM_NUM_PROCS_F, exoid);
152     ex_err(func_name, errmsg, exerrval);
153 
154     return (EX_FATAL);
155   }
156 
157   /* Put NetCDF file into define mode */
158   if ((status = nc_redef(exoid)) != NC_NOERR) {
159     exerrval = status;
160     sprintf(errmsg,
161             "Error: failed to put file id %d into define mode", exoid);
162     ex_err(func_name, errmsg, exerrval);
163     return (EX_FATAL);
164   }
165 
166   /* Set the fill mode */
167   if ((status = nc_set_fill(exoid, NC_NOFILL, &oldfill)) != NC_NOERR) {
168     exerrval = status;
169     sprintf(errmsg,
170             "Error: failed to put file ID %d into no-fill mode",
171             exoid);
172     ex_err(func_name, errmsg, exerrval);
173     return (EX_FATAL);
174   }
175 
176   /* Output the file version */
177   if ((status=ex_put_nemesis_version(exoid)) < 0) return (status);
178 
179   /* Output the file type */
180   if (nc_inq_varid(exoid, VAR_FILE_TYPE, &varid) != NC_NOERR) {
181     if ((status = nc_def_var(exoid, VAR_FILE_TYPE, NC_INT, 0, NULL, &varid)) != NC_NOERR) {
182       exerrval = status;
183       sprintf(errmsg,
184               "Error: failed to define file type in file ID %d",
185               exoid);
186       ex_err(func_name, errmsg, exerrval);
187 
188       /* Leave define mode before returning */
189       ex_leavedef(exoid, func_name);
190 
191       return (EX_FATAL);
192     }
193   }
194 
195   /* Define the status variables for the nodal vectors */
196   if (nc_inq_varid(exoid, VAR_INT_N_STAT, &varid) != NC_NOERR) {
197     if ((status = nc_def_var(exoid, VAR_INT_N_STAT, NC_INT, 1, &dimid_npf, &varid)) != NC_NOERR) {
198       exerrval = status;
199       sprintf(errmsg,
200               "Error: failed to define variable \"%s\" in file ID %d",
201               VAR_INT_N_STAT, exoid);
202       ex_err(func_name, errmsg, exerrval);
203       /* Leave define mode before returning */
204       ex_leavedef(exoid, func_name);
205 
206       return (EX_FATAL);
207     }
208   }
209 
210   /* Set the dimension for status vectors */
211   if (nc_inq_varid(exoid, VAR_BOR_N_STAT, &varid) != NC_NOERR) {
212     if ((status = nc_def_var(exoid, VAR_BOR_N_STAT, NC_INT, 1, &dimid_npf, &varid)) != NC_NOERR) {
213       exerrval = status;
214       sprintf(errmsg,
215               "Error: failed to define variable \"%s\" in file ID %d",
216               VAR_BOR_N_STAT, exoid);
217       ex_err(func_name, errmsg, exerrval);
218       /* Leave define mode before returning */
219       ex_leavedef(exoid, func_name);
220 
221       return (EX_FATAL);
222     }
223   }
224 
225   if (nc_inq_varid(exoid, VAR_EXT_N_STAT, &varid) != NC_NOERR) {
226     if ((status = nc_def_var(exoid, VAR_EXT_N_STAT, NC_INT, 1, &dimid_npf, &varid)) != NC_NOERR) {
227       exerrval = status;
228       sprintf(errmsg,
229               "Error: failed to define variable \"%s\" in file ID %d",
230               VAR_EXT_N_STAT, exoid);
231       ex_err(func_name, errmsg, exerrval);
232       /* Leave define mode before returning */
233       ex_leavedef(exoid, func_name);
234 
235       return (EX_FATAL);
236     }
237   }
238 
239   /* Define the variable IDs for the elemental status vectors */
240   if (nc_inq_varid(exoid, VAR_INT_E_STAT, &varid) != NC_NOERR) {
241     if ((status = nc_def_var(exoid, VAR_INT_E_STAT, NC_INT, 1, &dimid_npf, &varid)) != NC_NOERR) {
242       exerrval = status;
243       sprintf(errmsg,
244               "Error: failed to define variable \"%s\" in file ID %d",
245               VAR_INT_E_STAT, exoid);
246       ex_err(func_name, errmsg, exerrval);
247       /* Leave define mode before returning */
248       ex_leavedef(exoid, func_name);
249 
250       return (EX_FATAL);
251     }
252   }
253 
254   if (nc_inq_varid(exoid, VAR_BOR_E_STAT, &varid) != NC_NOERR) {
255     if ((status = nc_def_var(exoid, VAR_BOR_E_STAT, NC_INT, 1, &dimid_npf, &varid)) != NC_NOERR) {
256       exerrval = status;
257       sprintf(errmsg,
258               "Error: Failed to define variable \"%s\" in file ID %d",
259               VAR_BOR_E_STAT, exoid);
260       /* Leave define mode before returning */
261       ex_leavedef(exoid, func_name);
262 
263       return (EX_FATAL);
264     }
265   }
266 
267   /* Get the variable ID for the nodal status vectors */
268   if ((status = nc_inq_varid(exoid, VAR_INT_N_STAT, &varid_nm[0])) != NC_NOERR) {
269     exerrval = status;
270     sprintf(errmsg,
271 	    "Error: failed to find variable ID for \"%s\" in file ID %d",
272 	    VAR_INT_N_STAT, exoid);
273     ex_err(func_name, errmsg, exerrval);
274     /* Leave define mode before returning */
275     ex_leavedef(exoid, func_name);
276 
277     return (EX_FATAL);
278   }
279 
280   if ((status = nc_inq_varid(exoid, VAR_BOR_N_STAT, &varid_nm[1])) != NC_NOERR) {
281     exerrval = status;
282     sprintf(errmsg,
283             "Error: failed to find variable ID for \"%s\" in file ID %d",
284             VAR_BOR_N_STAT, exoid);
285     ex_err(func_name, errmsg, exerrval);
286     /* Leave define mode before returning */
287     ex_leavedef(exoid, func_name);
288 
289     return (EX_FATAL);
290   }
291 
292   if ((status = nc_inq_varid(exoid, VAR_EXT_N_STAT, &varid_nm[2])) != NC_NOERR) {
293     exerrval = status;
294     sprintf(errmsg,
295             "Error: failed to find variable ID for \"%s\" in file ID %d",
296             VAR_EXT_N_STAT, exoid);
297     ex_err(func_name, errmsg, exerrval);
298     /* Leave define mode before returning */
299     ex_leavedef(exoid, func_name);
300 
301     return (EX_FATAL);
302   }
303 
304   /* Get the variable IDs for the elemental status vectors */
305   if ((status = nc_inq_varid(exoid, VAR_INT_E_STAT, &varid_em[0])) != NC_NOERR) {
306     exerrval = status;
307     sprintf(errmsg,
308             "Error: failed to find variable ID for \"%s\" in file ID %d",
309             VAR_INT_E_STAT, exoid);
310     ex_err(func_name, errmsg, exerrval);
311     /* Leave define mode before returning */
312     ex_leavedef(exoid, func_name);
313 
314     return (EX_FATAL);
315   }
316 
317   if ((status = nc_inq_varid(exoid, VAR_BOR_E_STAT, &varid_em[1])) != NC_NOERR) {
318     exerrval = status;
319     sprintf(errmsg,
320             "Error: failed to find variable ID for \"%s\" in file ID %d",
321             VAR_BOR_E_STAT, exoid);
322     ex_err(func_name, errmsg, exerrval);
323     /* Leave define mode before returning */
324     ex_leavedef(exoid, func_name);
325 
326     return (EX_FATAL);
327   }
328 
329   /* mms: NEW STUFF HERE */
330   /*
331   ** first need to loop through the processors in this
332   ** file and get the counts of the element and cmap lists
333   */
334   if (ex_int64_status(exoid) & EX_BULK_INT64_API) {
335     for(iproc=0; iproc < num_proc_in_f; iproc++) {
336       num_int_elem += ((int64_t*)num_int_elems)[iproc];
337       num_int_node += ((int64_t*)num_int_nodes)[iproc];
338       num_bor_elem += ((int64_t*)num_bor_elems)[iproc];
339       num_bor_node += ((int64_t*)num_bor_nodes)[iproc];
340       num_ext_node += ((int64_t*)num_ext_nodes)[iproc];
341       num_e_cmaps += ((int64_t*)num_elem_cmaps)[iproc];
342       num_n_cmaps += ((int64_t*)num_node_cmaps)[iproc];
343     }
344   } else {
345     for(iproc=0; iproc < num_proc_in_f; iproc++) {
346       num_int_elem += ((int*)num_int_elems)[iproc];
347       num_int_node += ((int*)num_int_nodes)[iproc];
348       num_bor_elem += ((int*)num_bor_elems)[iproc];
349       num_bor_node += ((int*)num_bor_nodes)[iproc];
350       num_ext_node += ((int*)num_ext_nodes)[iproc];
351       num_e_cmaps += ((int*)num_elem_cmaps)[iproc];
352       num_n_cmaps += ((int*)num_node_cmaps)[iproc];
353     }
354   }
355 
356   /* Define variable for the internal element information */
357   if (num_int_elem > 0) {
358     if ((status = nc_def_dim(exoid, DIM_NUM_INT_ELEMS, num_int_elem, &dimid[0])) != NC_NOERR) {
359       exerrval = status;
360       sprintf(errmsg,
361               "Error: failed to dimension \"%s\" in file id %d",
362               DIM_NUM_INT_ELEMS, exoid);
363       ex_err(func_name, errmsg, exerrval);
364       /* Leave define mode before returning */
365       ex_leavedef(exoid, func_name);
366 
367       return (EX_FATAL);
368     }
369 
370     if ((status = nc_def_var(exoid, VAR_ELEM_MAP_INT, map_type, 1, dimid, &varid)) != NC_NOERR) {
371       exerrval = status;
372       sprintf(errmsg,
373               "Error: failed to define variable \"%s\" in file ID %d",
374               VAR_ELEM_MAP_INT, exoid);
375       ex_err(func_name, errmsg, exerrval);
376       /* Leave define mode before returning */
377       ex_leavedef(exoid, func_name);
378 
379       return (EX_FATAL);
380     }
381     ex_compress_variable(exoid, varid, 1);
382 
383     /* and the index variable */
384     if ((status = nc_def_var(exoid, VAR_ELEM_MAP_INT_IDX, index_type, 1,
385 			     &dimid_npf, &varid_idx[0])) != NC_NOERR) {
386       exerrval = status;
387       sprintf(errmsg,
388               "Error: failed to define variable \"%s\" in file ID %d",
389               VAR_ELEM_MAP_INT_IDX, exoid);
390       ex_err(func_name, errmsg, exerrval);
391       /* Leave define mode before returning */
392       ex_leavedef(exoid, func_name);
393 
394       return (EX_FATAL);
395     }
396   } /* End "if (num_int_elem > 0)" */
397 
398   /* Define variable for the border element information */
399   if (num_bor_elem > 0) {
400     if ((status = nc_def_dim(exoid, DIM_NUM_BOR_ELEMS, num_bor_elem, &dimid[0])) != NC_NOERR) {
401       exerrval = status;
402       sprintf(errmsg,
403               "Error: failed to dimension \"%s\" in file id %d",
404               DIM_NUM_BOR_ELEMS, exoid);
405       ex_err(func_name, errmsg, exerrval);
406       /* Leave define mode before returning */
407       ex_leavedef(exoid, func_name);
408 
409       return (EX_FATAL);
410     }
411 
412     if ((status = nc_def_var(exoid, VAR_ELEM_MAP_BOR, map_type, 1, dimid, &varid)) != NC_NOERR) {
413       exerrval = status;
414       sprintf(errmsg,
415               "Error: failed to define variable \"%s\" in file ID %d",
416               VAR_ELEM_MAP_BOR, exoid);
417       ex_err(func_name, errmsg, exerrval);
418       /* Leave define mode before returning */
419       ex_leavedef(exoid, func_name);
420 
421       return (EX_FATAL);
422     }
423     ex_compress_variable(exoid, varid, 1);
424 
425     /* and the index variable */
426     if ((status = nc_def_var(exoid, VAR_ELEM_MAP_BOR_IDX, index_type, 1,
427 			     &dimid_npf, &varid_idx[1])) != NC_NOERR) {
428       exerrval = status;
429       sprintf(errmsg,
430               "Error: failed to define variable \"%s\" in file ID %d",
431               VAR_ELEM_MAP_BOR_IDX, exoid);
432       ex_err(func_name, errmsg, exerrval);
433       /* Leave define mode before returning */
434       ex_leavedef(exoid, func_name);
435 
436       return (EX_FATAL);
437     }
438 
439   } /* End "if (num_bor_elem > 0)" */
440 
441   if (num_int_node > 0) {
442     /* Define variable for vector of internal FEM node IDs */
443     if ((status = nc_def_dim(exoid, DIM_NUM_INT_NODES, num_int_node, &dimid[0])) != NC_NOERR) {
444       exerrval = status;
445       sprintf(errmsg,
446               "Error: failed to dimension \"%s\" in file id %d",
447               DIM_NUM_INT_NODES, exoid);
448       ex_err(func_name, errmsg, exerrval);
449       /* Leave define mode before returning */
450       ex_leavedef(exoid, func_name);
451 
452       return (EX_FATAL);
453     }
454 
455     if ((status = nc_def_var(exoid, VAR_NODE_MAP_INT, map_type, 1, &dimid[0], &varid)) != NC_NOERR) {
456       exerrval = status;
457       sprintf(errmsg,
458               "Error: failed to define variable \"%s\" in file ID %d",
459               VAR_NODE_MAP_INT, exoid);
460       ex_err(func_name, errmsg, exerrval);
461       /* Leave define mode before returning */
462       ex_leavedef(exoid, func_name);
463 
464       return (EX_FATAL);
465     }
466     ex_compress_variable(exoid, varid, 1);
467 
468     /* and the index variable */
469     if ((status = nc_def_var(exoid, VAR_NODE_MAP_INT_IDX, index_type, 1,
470 			     &dimid_npf, &varid_idx[2])) != NC_NOERR) {
471       exerrval = status;
472       sprintf(errmsg,
473               "Error: failed to define variable \"%s\" in file ID %d",
474               VAR_NODE_MAP_INT_IDX, exoid);
475       ex_err(func_name, errmsg, exerrval);
476       /* Leave define mode before returning */
477       ex_leavedef(exoid, func_name);
478 
479       return (EX_FATAL);
480     }
481 
482   } /* End "if (num_int_node > 0)" */
483 
484   if (num_bor_node > 0) {
485     /* Define variable for vector of border FEM node IDs */
486     if ((status = nc_def_dim(exoid, DIM_NUM_BOR_NODES, num_bor_node, &dimid[1])) != NC_NOERR) {
487       exerrval = status;
488       sprintf(errmsg,
489               "Error: failed to dimension \"%s\" in file id %d",
490               DIM_NUM_BOR_NODES, exoid);
491       ex_err(func_name, errmsg, exerrval);
492       /* Leave define mode before returning */
493       ex_leavedef(exoid, func_name);
494 
495       return (EX_FATAL);
496     }
497 
498     if ((status = nc_def_var(exoid, VAR_NODE_MAP_BOR, map_type, 1, &dimid[1], &varid)) != NC_NOERR) {
499       exerrval = status;
500       sprintf(errmsg,
501               "Error: failed to define variable \"%s\" in file ID %d",
502               VAR_NODE_MAP_BOR, exoid);
503       ex_err(func_name, errmsg, exerrval);
504       /* Leave define mode before returning */
505       ex_leavedef(exoid, func_name);
506 
507       return (EX_FATAL);
508     }
509     ex_compress_variable(exoid, varid, 1);
510 
511     /* and the index variable */
512     if ((status = nc_def_var(exoid, VAR_NODE_MAP_BOR_IDX, index_type, 1,
513 			     &dimid_npf, &varid_idx[3])) != NC_NOERR) {
514       exerrval = status;
515       sprintf(errmsg,
516               "Error: failed to define variable \"%s\" in file ID %d",
517               VAR_NODE_MAP_BOR_IDX, exoid);
518       ex_err(func_name, errmsg, exerrval);
519       /* Leave define mode before returning */
520       ex_leavedef(exoid, func_name);
521 
522       return (EX_FATAL);
523     }
524 
525   } /* End "if (num_bor_node > 0)" */
526 
527   if (num_ext_node > 0) {
528     /* Define dimension for vector of external FEM node IDs */
529     if ((status = nc_def_dim(exoid, DIM_NUM_EXT_NODES, num_ext_node, &dimid[2])) != NC_NOERR) {
530       exerrval = status;
531       sprintf(errmsg,
532               "Error: failed to dimension \"%s\" in file id %d",
533               DIM_NUM_EXT_NODES, exoid);
534       ex_err(func_name, errmsg, exerrval);
535       /* Leave define mode before returning */
536       ex_leavedef(exoid, func_name);
537 
538       return (EX_FATAL);
539     }
540 
541     if ((status = nc_def_var(exoid, VAR_NODE_MAP_EXT, map_type, 1, &dimid[2], &varid)) != NC_NOERR) {
542       exerrval = status;
543       sprintf(errmsg,
544               "Error: failed to define variable \"%s\" in file ID %d",
545               VAR_NODE_MAP_EXT, exoid);
546       ex_err(func_name, errmsg, exerrval);
547       /* Leave define mode before returning */
548       ex_leavedef(exoid, func_name);
549 
550       return (EX_FATAL);
551     }
552     ex_compress_variable(exoid, varid, 1);
553 
554     /* and the index variable */
555     if ((status = nc_def_var(exoid, VAR_NODE_MAP_EXT_IDX, index_type, 1,
556 			     &dimid_npf, &varid_idx[4])) != NC_NOERR) {
557       exerrval = status;
558       sprintf(errmsg,
559               "Error: failed to define variable \"%s\" in file ID %d",
560               VAR_NODE_MAP_EXT_IDX, exoid);
561       ex_err(func_name, errmsg, exerrval);
562       /* Leave define mode before returning */
563       ex_leavedef(exoid, func_name);
564 
565       return (EX_FATAL);
566     }
567 
568   } /* End "if (num_ext_node > 0)" */
569 
570   /* Output the communication map dimensions */
571   if (num_n_cmaps > 0) {
572     if ((status = nc_def_dim(exoid, DIM_NUM_N_CMAPS, num_n_cmaps, &dimid[0])) != NC_NOERR) {
573       exerrval = status;
574       sprintf(errmsg,
575               "Error: failed to add dimension \"%s\" in file ID %d",
576               DIM_NUM_N_CMAPS, exoid);
577       ex_err(func_name, errmsg, exerrval);
578       /* Leave define mode before returning */
579       ex_leavedef(exoid, func_name);
580 
581       return (EX_FATAL);
582     }
583 
584     /* Add variables for communication maps */
585     if ((status = nc_def_var(exoid, VAR_N_COMM_IDS, id_type, 1, dimid, &varid)) != NC_NOERR) {
586       exerrval = status;
587       sprintf(errmsg,
588               "Error: failed to define variable \"%s\" in file ID %d",
589               VAR_N_COMM_IDS, exoid);
590       ex_err(func_name, errmsg, exerrval);
591       /* Leave define mode before returning */
592       ex_leavedef(exoid, func_name);
593 
594       return (EX_FATAL);
595     }
596 
597     if ((status = nc_def_var(exoid, VAR_N_COMM_STAT, NC_INT, 1, dimid, &varid)) != NC_NOERR) {
598       exerrval = status;
599       sprintf(errmsg,
600               "Error: failed to define variable \"%s\" in file ID %d",
601               VAR_N_COMM_STAT, exoid);
602       ex_err(func_name, errmsg, exerrval);
603       /* Leave define mode before returning */
604       ex_leavedef(exoid, func_name);
605 
606       return (EX_FATAL);
607     }
608 
609     /* and the index variable */
610     if ((status = nc_def_var(exoid, VAR_N_COMM_INFO_IDX, index_type, 1,
611 			     &dimid_npf, &varid_idx[5])) != NC_NOERR) {
612       exerrval = status;
613       sprintf(errmsg,
614               "Error: failed to define variable \"%s\" in file ID %d",
615               VAR_N_COMM_INFO_IDX, exoid);
616       ex_err(func_name, errmsg, exerrval);
617       /* Leave define mode before returning */
618       ex_leavedef(exoid, func_name);
619 
620       return (EX_FATAL);
621     }
622 
623   } /* End "if (num_n_cmaps > 0)" */
624 
625   if (num_e_cmaps > 0) {
626     if ((status = nc_def_dim(exoid, DIM_NUM_E_CMAPS, num_e_cmaps, &dimid[0])) != NC_NOERR) {
627       exerrval = status;
628       sprintf(errmsg,
629               "Error: failed to add dimension \"%s\" in file ID %d",
630               DIM_NUM_E_CMAPS, exoid);
631       ex_err(func_name, errmsg, exerrval);
632       /* Leave define mode before returning */
633       ex_leavedef(exoid, func_name);
634 
635       return (EX_FATAL);
636     }
637 
638     /* Add variables for elemental communication maps */
639     if ((status = nc_def_var(exoid, VAR_E_COMM_IDS, id_type, 1, dimid, &varid)) != NC_NOERR) {
640       exerrval = status;
641       sprintf(errmsg,
642               "Error: failed to define variable \"%s\" in file ID %d",
643               VAR_E_COMM_IDS, exoid);
644       ex_err(func_name, errmsg, exerrval);
645       /* Leave define mode before returning */
646       ex_leavedef(exoid, func_name);
647 
648       return (EX_FATAL);
649     }
650 
651     if ((status = nc_def_var(exoid, VAR_E_COMM_STAT, NC_INT, 1, dimid, &varid)) != NC_NOERR) {
652       exerrval = status;
653       sprintf(errmsg,
654               "Error: failed to define variable \"%s\" in file ID %d",
655               VAR_E_COMM_STAT, exoid);
656       ex_err(func_name, errmsg, exerrval);
657       /* Leave define mode before returning */
658       ex_leavedef(exoid, func_name);
659 
660       return (EX_FATAL);
661     }
662 
663     /* and the index variable */
664     if ((status = nc_def_var(exoid, VAR_E_COMM_INFO_IDX, index_type, 1,
665 			     &dimid_npf, &varid_idx[6])) != NC_NOERR) {
666       exerrval = status;
667       sprintf(errmsg,
668               "Error: failed to define variable \"%s\" in file ID %d",
669               VAR_E_COMM_INFO_IDX, exoid);
670       ex_err(func_name, errmsg, exerrval);
671       /* Leave define mode before returning */
672       ex_leavedef(exoid, func_name);
673 
674       return (EX_FATAL);
675     }
676 
677   } /* End "if (num_e_cmaps > 0)" */
678 
679   /* Leave define mode */
680   if (ex_leavedef(exoid, func_name) != EX_NOERR)
681     return (EX_FATAL);
682 
683   /* need to reset these counters */
684   num_int_elem = 0;
685   num_int_node = 0;
686   num_bor_elem = 0;
687   num_bor_node = 0;
688   num_ext_node = 0;
689   num_n_cmaps = 0;
690   num_e_cmaps = 0;
691 
692   /* Update the status vectors */
693   for(iproc=0; iproc < num_proc_in_f; iproc++) {
694     size_t nin;
695     size_t nbn;
696     size_t nen;
697     size_t nie;
698     size_t nbe;
699     size_t nnc;
700     size_t nec;
701     if (ex_int64_status(exoid) & EX_BULK_INT64_API) {
702       nie = ((int64_t*)num_int_elems)[iproc];
703       nin = ((int64_t*)num_int_nodes)[iproc];
704       nbe = ((int64_t*)num_bor_elems)[iproc];
705       nbn = ((int64_t*)num_bor_nodes)[iproc];
706       nen = ((int64_t*)num_ext_nodes)[iproc];
707       nec = ((int64_t*)num_elem_cmaps)[iproc];
708       nnc = ((int64_t*)num_node_cmaps)[iproc];
709     } else {
710       nie = ((int*)num_int_elems)[iproc];
711       nin = ((int*)num_int_nodes)[iproc];
712       nbe = ((int*)num_bor_elems)[iproc];
713       nbn = ((int*)num_bor_nodes)[iproc];
714       nen = ((int*)num_ext_nodes)[iproc];
715       nec = ((int*)num_elem_cmaps)[iproc];
716       nnc = ((int*)num_node_cmaps)[iproc];
717     }
718 
719     start[0] = iproc;
720 
721     if (nin > 0)
722       nmstat = 1;
723     else
724       nmstat = 0;
725 
726     if ((status = nc_put_var1_int(exoid, varid_nm[0], start, &nmstat)) != NC_NOERR) {
727       exerrval = status;
728       sprintf(errmsg,
729               "Error: failed to output status int node map in file ID %d",
730               exoid);
731       ex_err(func_name, errmsg, exerrval);
732       return (EX_FATAL);
733     }
734 
735     if (nbn > 0)
736       nmstat = 1;
737     else
738       nmstat = 0;
739 
740     if ((status = nc_put_var1_int(exoid, varid_nm[1], start, &nmstat)) != NC_NOERR) {
741       exerrval = status;
742       sprintf(errmsg,
743               "Error: failed to output status bor node map in file ID %d",
744               exoid);
745       ex_err(func_name, errmsg, exerrval);
746       return (EX_FATAL);
747     }
748 
749     if (nen > 0)
750       nmstat = 1;
751     else
752       nmstat = 0;
753 
754     if ((status = nc_put_var1_int(exoid, varid_nm[2], start, &nmstat)) != NC_NOERR) {
755       exerrval = status;
756       sprintf(errmsg,
757               "Error: failed to output status ext node map in file ID %d",
758               exoid);
759       ex_err(func_name, errmsg, exerrval);
760       return (EX_FATAL);
761     }
762 
763     if (nie > 0)
764       nmstat = 1;
765     else
766       nmstat = 0;
767 
768     if ((status = nc_put_var1_int(exoid, varid_em[0], start, &nmstat)) != NC_NOERR) {
769       exerrval = status;
770       sprintf(errmsg,
771               "Error: failed to output status int elem map in file ID %d",
772               exoid);
773       ex_err(func_name, errmsg, exerrval);
774       return (EX_FATAL);
775     }
776 
777     if (nbe > 0)
778       nmstat = 1;
779     else
780       nmstat = 0;
781 
782     if ((status = nc_put_var1_int(exoid, varid_em[1], start, &nmstat)) != NC_NOERR) {
783       exerrval = status;
784       sprintf(errmsg,
785               "Error: failed to output status bor elem map in file ID %d",
786               exoid);
787       ex_err(func_name, errmsg, exerrval);
788       return (EX_FATAL);
789     }
790 
791     /* now fill the index variables */
792     if (varid_idx[0] > 0) {
793       /* increment to the next starting position */
794       num_int_elem += nie;
795 #if defined(NC_NETCDF4)
796       status = nc_put_var1_longlong(exoid, varid_idx[0], start, (long long*)&num_int_elem);
797 #else
798       status = nc_put_var1_int(exoid, varid_idx[0], start, &num_int_elem);
799 #endif
800       if (status != NC_NOERR) {
801         exerrval = status;
802         sprintf(errmsg,
803                 "Error: failed to output internal element map index in file ID %d",
804                 exoid);
805         ex_err(func_name, errmsg, exerrval);
806         return (EX_FATAL);
807       }
808     }
809 
810     if (varid_idx[1] > 0) {
811       /* increment to the next starting position */
812       num_bor_elem += nbe;
813 #if defined(NC_NETCDF4)
814       status = nc_put_var1_longlong(exoid, varid_idx[1], start, (long long*)&num_bor_elem);
815 #else
816       status = nc_put_var1_int(exoid, varid_idx[1], start, &num_bor_elem);
817 #endif
818       if (status != NC_NOERR) {
819         exerrval = status;
820         sprintf(errmsg,
821                 "Error: failed to output border element map index in file ID %d",
822                 exoid);
823         ex_err(func_name, errmsg, exerrval);
824         return (EX_FATAL);
825       }
826     }
827 
828     if (varid_idx[2] > 0) {
829       /* increment to the next starting position */
830       num_int_node += nin;
831 #if defined(NC_NETCDF4)
832       status = nc_put_var1_longlong(exoid, varid_idx[2], start, (long long*)&num_int_node);
833 #else
834       status = nc_put_var1_int(exoid, varid_idx[2], start, &num_int_node);
835 #endif
836       if (status != NC_NOERR) {
837         exerrval = status;
838         sprintf(errmsg,
839                 "Error: failed to output internal node map index in file ID %d",
840                 exoid);
841         ex_err(func_name, errmsg, exerrval);
842         return (EX_FATAL);
843       }
844     }
845 
846     if (varid_idx[3] > 0) {
847       /* increment to the next starting position */
848       num_bor_node += nbn;
849 #if defined(NC_NETCDF4)
850       status = nc_put_var1_longlong(exoid, varid_idx[3], start, (long long*)&num_bor_node);
851 #else
852       status = nc_put_var1_int(exoid, varid_idx[3], start, &num_bor_node);
853 #endif
854       if (status != NC_NOERR) {
855         exerrval = status;
856         sprintf(errmsg,
857                 "Error: failed to output border node map index in file ID %d",
858                 exoid);
859         ex_err(func_name, errmsg, exerrval);
860         return (EX_FATAL);
861       }
862     }
863 
864     if (varid_idx[4] > 0) {
865       /* increment to the next starting position */
866       num_ext_node += nen;
867 #if defined(NC_NETCDF4)
868       status = nc_put_var1_longlong(exoid, varid_idx[4], start, (long long*)&num_ext_node);
869 #else
870       status = nc_put_var1_int(exoid, varid_idx[4], start, &num_ext_node);
871 #endif
872       if (status != NC_NOERR) {
873         exerrval = status;
874         sprintf(errmsg,
875                 "Error: failed to output external node map index in file ID %d",
876                 exoid);
877         ex_err(func_name, errmsg, exerrval);
878         return (EX_FATAL);
879       }
880     }
881 
882     if (varid_idx[5] > 0) {
883       /* increment to the next starting position */
884       num_n_cmaps += nnc;
885 #if defined(NC_NETCDF4)
886       status = nc_put_var1_longlong(exoid, varid_idx[5], start, (long long*)&num_n_cmaps);
887 #else
888       status = nc_put_var1_int(exoid, varid_idx[5], start, &num_n_cmaps);
889 #endif
890       if (status != NC_NOERR) {
891         exerrval = status;
892         sprintf(errmsg,
893                 "Error: failed to output node communication map index in file ID %d",
894                 exoid);
895         ex_err(func_name, errmsg, exerrval);
896         return (EX_FATAL);
897       }
898     }
899 
900     if (varid_idx[6] > 0) {
901       /* increment to the next starting position */
902       num_e_cmaps += nec;
903 #if defined(NC_NETCDF4)
904       status = nc_put_var1_longlong(exoid, varid_idx[6], start, (long long*)&num_e_cmaps);
905 #else
906       status = nc_put_var1_int(exoid, varid_idx[6], start, &num_e_cmaps);
907 #endif
908       if (status != NC_NOERR) {
909         exerrval = status;
910         sprintf(errmsg,
911                 "Error: failed to output elem communication map index in file ID %d",
912                 exoid);
913         ex_err(func_name, errmsg, exerrval);
914         return (EX_FATAL);
915       }
916     }
917 
918   } /* End "for(iproc=0; iproc < num_proc_in_f; iproc++)" */
919   return (EX_NOERR);
920 }
921