1 #include "TestUtil.hpp"
2 #include "moab/Core.hpp"
3 #include "moab/ParallelComm.hpp"
4 #include "moab/ProgOptions.hpp"
5 #include "MBParallelConventions.h"
6 #include "moab/ReadUtilIface.hpp"
7 #include "MBTagConventions.hpp"
8
9 #include <sstream>
10
11 using namespace moab;
12
13 std::string example = TestDir + "/io/mpasx1.642.t.2.nc";
14
15 void test_read_onevar_trivial();
16 void test_read_onevar_trivial_no_mixed_elements();
17 #if defined(MOAB_HAVE_MPI) && defined(MOAB_HAVE_ZOLTAN)
18 void test_read_onevar_rcbzoltan();
19 void test_read_onevar_rcbzoltan_no_mixed_elements();
20 #endif
21
22 void test_read_mesh_parallel_trivial();
23 void test_read_mesh_parallel_trivial_no_mixed_elements();
24 #if defined(MOAB_HAVE_MPI) && defined(MOAB_HAVE_ZOLTAN)
25 void test_read_mesh_parallel_rcbzoltan();
26 void test_read_mesh_parallel_rcbzoltan_no_mixed_elements();
27 #endif
28
29 void test_gather_onevar_on_rank0();
30 void test_gather_onevar_on_rank1();
31
32 void test_multiple_loads_of_same_file();
33 void test_multiple_loads_of_same_file_no_mixed_elements();
34
35 // Helper functions
36 void read_one_cell_var(bool rcbzoltan, bool no_mixed_elements);
37 void read_mesh_parallel(bool rcbzoltan, bool no_mixed_elements);
38 void gather_one_cell_var(int gather_set_rank);
39 void multiple_loads_of_same_file(bool no_mixed_elements);
40
41 std::string read_options;
42 const double eps = 1e-20;
43
main(int argc,char * argv[])44 int main(int argc, char* argv[])
45 {
46 MPI_Init(&argc, &argv);
47 int result = 0;
48
49 result += RUN_TEST(test_read_onevar_trivial);
50 result += RUN_TEST(test_read_onevar_trivial_no_mixed_elements);
51 #if defined(MOAB_HAVE_MPI) && defined(MOAB_HAVE_ZOLTAN)
52 result += RUN_TEST(test_read_onevar_rcbzoltan);
53 result += RUN_TEST(test_read_onevar_rcbzoltan_no_mixed_elements);
54 #endif
55
56 result += RUN_TEST(test_read_mesh_parallel_trivial);
57 result += RUN_TEST(test_read_mesh_parallel_trivial_no_mixed_elements);
58 #if defined(MOAB_HAVE_MPI) && defined(MOAB_HAVE_ZOLTAN)
59 result += RUN_TEST(test_read_mesh_parallel_rcbzoltan);
60 result += RUN_TEST(test_read_mesh_parallel_rcbzoltan_no_mixed_elements);
61 #endif
62
63 result += RUN_TEST(test_gather_onevar_on_rank0);
64 result += RUN_TEST(test_gather_onevar_on_rank1);
65
66 result += RUN_TEST(test_multiple_loads_of_same_file);
67 result += RUN_TEST(test_multiple_loads_of_same_file_no_mixed_elements);
68
69 MPI_Finalize();
70 return result;
71 }
72
test_read_onevar_trivial()73 void test_read_onevar_trivial()
74 {
75 read_one_cell_var(false, false);
76 }
77
test_read_onevar_trivial_no_mixed_elements()78 void test_read_onevar_trivial_no_mixed_elements()
79 {
80 read_one_cell_var(false, true);
81 }
82
test_read_onevar_rcbzoltan()83 void test_read_onevar_rcbzoltan()
84 {
85 read_one_cell_var(true, false);
86 }
87
test_read_onevar_rcbzoltan_no_mixed_elements()88 void test_read_onevar_rcbzoltan_no_mixed_elements()
89 {
90 read_one_cell_var(true, true);
91 }
92
test_read_mesh_parallel_trivial()93 void test_read_mesh_parallel_trivial()
94 {
95 read_mesh_parallel(false, false);
96 }
97
test_read_mesh_parallel_trivial_no_mixed_elements()98 void test_read_mesh_parallel_trivial_no_mixed_elements()
99 {
100 read_mesh_parallel(false, true);
101 }
102
test_read_mesh_parallel_rcbzoltan()103 void test_read_mesh_parallel_rcbzoltan()
104 {
105 read_mesh_parallel(true, false);
106 }
107
test_read_mesh_parallel_rcbzoltan_no_mixed_elements()108 void test_read_mesh_parallel_rcbzoltan_no_mixed_elements()
109 {
110 read_mesh_parallel(true, true);
111 }
112
test_gather_onevar_on_rank0()113 void test_gather_onevar_on_rank0()
114 {
115 gather_one_cell_var(0);
116 }
117
test_gather_onevar_on_rank1()118 void test_gather_onevar_on_rank1()
119 {
120 gather_one_cell_var(1);
121 }
122
test_multiple_loads_of_same_file()123 void test_multiple_loads_of_same_file()
124 {
125 multiple_loads_of_same_file(false);
126 }
127
test_multiple_loads_of_same_file_no_mixed_elements()128 void test_multiple_loads_of_same_file_no_mixed_elements()
129 {
130 multiple_loads_of_same_file(true);
131 }
132
133 // Helper functions
read_one_cell_var(bool rcbzoltan,bool no_mixed_elements)134 void read_one_cell_var(bool rcbzoltan, bool no_mixed_elements)
135 {
136 Core moab;
137 Interface& mb = moab;
138
139 read_options = "PARALLEL=READ_PART;PARTITION_METHOD=TRIVIAL;NO_EDGES;VARIABLE=ke";
140 if (rcbzoltan)
141 read_options = "PARALLEL=READ_PART;PARTITION_METHOD=RCBZOLTAN;NO_EDGES;VARIABLE=ke";
142
143 if (no_mixed_elements)
144 read_options += ";NO_MIXED_ELEMENTS";
145
146 ErrorCode rval = mb.load_file(example.c_str(), NULL, read_options.c_str());
147 CHECK_ERR(rval);
148
149 // Get local edges
150 Range local_edges;
151 rval = mb.get_entities_by_type(0, MBEDGE, local_edges);
152 CHECK_ERR(rval);
153 CHECK_EQUAL((size_t)0, local_edges.size());
154
155 // Get local cells
156 Range local_cells;
157 rval = mb.get_entities_by_type(0, MBPOLYGON, local_cells);
158 CHECK_ERR(rval);
159
160 Tag gid_tag;
161 rval = mb.tag_get_handle(GLOBAL_ID_TAG_NAME, 1, MB_TYPE_INTEGER, gid_tag, MB_TAG_DENSE);
162 CHECK_ERR(rval);
163
164 std::vector<int> gids(local_cells.size());
165 rval = mb.tag_get_data(gid_tag, local_cells, &gids[0]);
166 Range local_cell_gids;
167 std::copy(gids.rbegin(), gids.rend(), range_inserter(local_cell_gids));
168
169 ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
170 int procs = pcomm->proc_config().proc_size();
171 int rank = pcomm->proc_config().proc_rank();
172
173 // Make check runs this test on two processors
174 if (2 == procs) {
175 CHECK_EQUAL((size_t)321, local_cells.size());
176 CHECK_EQUAL((size_t)321, local_cell_gids.size());
177
178 // Check tag for cell variable ke at timestep 0
179 Tag ke_tag0;
180 rval = mb.tag_get_handle("ke0", 1, MB_TYPE_DOUBLE, ke_tag0);
181 CHECK_ERR(rval);
182
183 // Check tag for cell variable ke at timestep 1
184 Tag ke_tag1;
185 rval = mb.tag_get_handle("ke1", 1, MB_TYPE_DOUBLE, ke_tag1);
186 CHECK_ERR(rval);
187
188 // Get ke0 and ke1 tag values on 3 local cells
189 EntityHandle cell_ents[] = {local_cells[0], local_cells[160], local_cells[320]};
190 double ke0_val[3];
191 rval = mb.tag_get_data(ke_tag0, cell_ents, 3, ke0_val);
192 CHECK_ERR(rval);
193 double ke1_val[3];
194 rval = mb.tag_get_data(ke_tag1, cell_ents, 3, ke1_val);
195 CHECK_ERR(rval);
196
197 if (rcbzoltan) {
198
199 CHECK_EQUAL((size_t)1, local_cells.psize());
200 CHECK_EQUAL((size_t)37, local_cell_gids.psize());
201
202 if (0 == rank) {
203 CHECK_EQUAL(1, (int)local_cell_gids[0]);
204 CHECK_EQUAL(284, (int)local_cell_gids[160]);
205 CHECK_EQUAL(630, (int)local_cell_gids[320]);
206
207 CHECK_REAL_EQUAL(15.001, ke0_val[0], eps);
208 CHECK_REAL_EQUAL(16.284, ke0_val[1], eps);
209 CHECK_REAL_EQUAL(16.630, ke0_val[2], eps);
210 CHECK_REAL_EQUAL(25.001, ke1_val[0], eps);
211 CHECK_REAL_EQUAL(26.284, ke1_val[1], eps);
212 CHECK_REAL_EQUAL(26.630, ke1_val[2], eps);
213 }
214 else if (1 == rank) {
215 CHECK_EQUAL(4, (int)local_cell_gids[0]);
216 CHECK_EQUAL(341, (int)local_cell_gids[160]);
217 CHECK_EQUAL(642, (int)local_cell_gids[320]);
218
219 CHECK_REAL_EQUAL(15.004, ke0_val[0], eps);
220 CHECK_REAL_EQUAL(16.341, ke0_val[1], eps);
221 CHECK_REAL_EQUAL(16.642, ke0_val[2], eps);
222 CHECK_REAL_EQUAL(25.004, ke1_val[0], eps);
223 CHECK_REAL_EQUAL(26.341, ke1_val[1], eps);
224 CHECK_REAL_EQUAL(26.642, ke1_val[2], eps);
225 }
226 }
227 else {
228 CHECK_EQUAL((size_t)1, local_cell_gids.psize());
229
230 if (0 == rank) {
231 CHECK_EQUAL((size_t)1, local_cells.psize());
232 CHECK_EQUAL(1, (int)local_cell_gids[0]);
233 CHECK_EQUAL(161, (int)local_cell_gids[160]);
234 CHECK_EQUAL(321, (int)local_cell_gids[320]);
235
236 CHECK_REAL_EQUAL(15.001, ke0_val[0], eps);
237 CHECK_REAL_EQUAL(16.161, ke0_val[1], eps);
238 CHECK_REAL_EQUAL(16.321, ke0_val[2], eps);
239 CHECK_REAL_EQUAL(25.001, ke1_val[0], eps);
240 CHECK_REAL_EQUAL(26.161, ke1_val[1], eps);
241 CHECK_REAL_EQUAL(26.321, ke1_val[2], eps);
242 }
243 else if (1 == rank) {
244 CHECK_EQUAL((size_t)1, local_cells.psize());
245 CHECK_EQUAL(322, (int)local_cell_gids[0]);
246 CHECK_EQUAL(482, (int)local_cell_gids[160]);
247 CHECK_EQUAL(642, (int)local_cell_gids[320]);
248
249 CHECK_REAL_EQUAL(16.322, ke0_val[0], eps);
250 CHECK_REAL_EQUAL(16.482, ke0_val[1], eps);
251 CHECK_REAL_EQUAL(16.642, ke0_val[2], eps);
252 CHECK_REAL_EQUAL(26.322, ke1_val[0], eps);
253 CHECK_REAL_EQUAL(26.482, ke1_val[1], eps);
254 CHECK_REAL_EQUAL(26.642, ke1_val[2], eps);
255 }
256 }
257 }
258 }
259
read_mesh_parallel(bool rcbzoltan,bool no_mixed_elements)260 void read_mesh_parallel(bool rcbzoltan, bool no_mixed_elements)
261 {
262 Core moab;
263 Interface& mb = moab;
264
265 read_options = "PARALLEL=READ_PART;PARTITION_METHOD=TRIVIAL;PARALLEL_RESOLVE_SHARED_ENTS;VARIABLE=";
266 if (rcbzoltan)
267 read_options = "PARALLEL=READ_PART;PARTITION_METHOD=RCBZOLTAN;PARALLEL_RESOLVE_SHARED_ENTS;VARIABLE=";
268
269 if (no_mixed_elements)
270 read_options += ";NO_MIXED_ELEMENTS";
271
272 ErrorCode rval = mb.load_file(example.c_str(), NULL, read_options.c_str());
273 CHECK_ERR(rval);
274
275 ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
276 int procs = pcomm->proc_config().proc_size();
277 int rank = pcomm->proc_config().proc_rank();
278
279 rval = pcomm->check_all_shared_handles();
280 CHECK_ERR(rval);
281
282 // Get local vertices
283 Range local_verts;
284 rval = mb.get_entities_by_type(0, MBVERTEX, local_verts);
285 CHECK_ERR(rval);
286
287 int verts_num = local_verts.size();
288 if (2 == procs) {
289 if (rcbzoltan) {
290 if (0 == rank)
291 CHECK_EQUAL(685, verts_num);
292 else if (1 == rank)
293 CHECK_EQUAL(685, verts_num); // Not owned vertices included
294 }
295 else {
296 if (0 == rank)
297 CHECK_EQUAL(1120, verts_num);
298 else if (1 == rank)
299 CHECK_EQUAL(1122, verts_num); // Not owned vertices included
300 }
301 }
302
303 rval = pcomm->filter_pstatus(local_verts, PSTATUS_NOT_OWNED, PSTATUS_NOT);
304 CHECK_ERR(rval);
305
306 verts_num = local_verts.size();
307 if (2 == procs) {
308 if (rcbzoltan) {
309 if (0 == rank)
310 CHECK_EQUAL(685, verts_num);
311 else if (1 == rank)
312 CHECK_EQUAL(595, verts_num); // Not owned vertices excluded
313 }
314 else {
315 if (0 == rank)
316 CHECK_EQUAL(1120, verts_num);
317 else if (1 == rank)
318 CHECK_EQUAL(160, verts_num); // Not owned vertices excluded
319 }
320 }
321
322 // Get local edges
323 Range local_edges;
324 rval = mb.get_entities_by_type(0, MBEDGE, local_edges);
325 CHECK_ERR(rval);
326
327 int edges_num = local_edges.size();
328 if (2 == procs) {
329 if (rcbzoltan) {
330 if (0 == rank)
331 CHECK_EQUAL(1005, edges_num);
332 else if (1 == rank)
333 CHECK_EQUAL(1005, edges_num); // Not owned edges included
334 }
335 else {
336 if (0 == rank)
337 CHECK_EQUAL(1438, edges_num);
338 else if (1 == rank)
339 CHECK_EQUAL(1444, edges_num); // Not owned edges included
340 }
341 }
342
343 rval = pcomm->filter_pstatus(local_edges, PSTATUS_NOT_OWNED, PSTATUS_NOT);
344 CHECK_ERR(rval);
345
346 edges_num = local_edges.size();
347 if (2 == procs) {
348 if (rcbzoltan) {
349 if (0 == rank)
350 CHECK_EQUAL(1005, edges_num);
351 else if (1 == rank)
352 CHECK_EQUAL(915, edges_num); // Not owned edges excluded
353 }
354 else {
355 if (0 == rank)
356 CHECK_EQUAL(1438, edges_num);
357 else if (1 == rank)
358 CHECK_EQUAL(482, edges_num); // Not owned edges excluded
359 }
360 }
361
362 // Get local cells
363 Range local_cells;
364 rval = mb.get_entities_by_type(0, MBPOLYGON, local_cells);
365 CHECK_ERR(rval);
366
367 int cells_num = local_cells.size();
368 if (2 == procs) {
369 CHECK_EQUAL(321, cells_num);
370 CHECK_EQUAL((size_t)1, local_cells.psize());
371 }
372
373 rval = pcomm->filter_pstatus(local_cells, PSTATUS_NOT_OWNED, PSTATUS_NOT);
374 CHECK_ERR(rval);
375
376 cells_num = local_cells.size();
377 if (2 == procs) {
378 CHECK_EQUAL(321, cells_num);
379 CHECK_EQUAL((size_t)1, local_cells.psize());
380 }
381
382 std::cout << "proc: " << rank << " verts:" << verts_num << "\n";
383
384 int total_verts_num;
385 MPI_Reduce(&verts_num, &total_verts_num, 1, MPI_INT, MPI_SUM, 0, pcomm->proc_config().proc_comm());
386 if (0 == rank) {
387 std::cout << "total vertices: " << total_verts_num << "\n";
388 CHECK_EQUAL(1280, total_verts_num);
389 }
390
391 std::cout << "proc: " << rank << " edges:" << edges_num << "\n";
392
393 int total_edges_num;
394 MPI_Reduce(&edges_num, &total_edges_num, 1, MPI_INT, MPI_SUM, 0, pcomm->proc_config().proc_comm());
395 if (0 == rank) {
396 std::cout << "total edges: " << total_edges_num << "\n";
397 CHECK_EQUAL(1920, total_edges_num);
398 }
399
400 std::cout << "proc: " << rank << " cells:" << cells_num << "\n";
401
402 int total_cells_num;
403 MPI_Reduce(&cells_num, &total_cells_num, 1, MPI_INT, MPI_SUM, 0, pcomm->proc_config().proc_comm());
404 if (0 == rank) {
405 std::cout << "total cells: " << total_cells_num << "\n";
406 CHECK_EQUAL(642, total_cells_num);
407 }
408
409 #ifdef MOAB_HAVE_HDF5_PARALLEL
410 std::string write_options("PARALLEL=WRITE_PART;");
411
412 std::string output_file = "test_mpas";
413 if (rcbzoltan)
414 output_file += "_rcbzoltan";
415 if (no_mixed_elements)
416 output_file += "_no_mixed_elements";
417 output_file += ".h5m";
418
419 mb.write_file(output_file.c_str(), NULL, write_options.c_str());
420 #endif
421 }
422
gather_one_cell_var(int gather_set_rank)423 void gather_one_cell_var(int gather_set_rank)
424 {
425 Core moab;
426 Interface& mb = moab;
427
428 EntityHandle file_set;
429 ErrorCode rval = mb.create_meshset(MESHSET_SET, file_set);
430 CHECK_ERR(rval);
431
432 read_options = "PARALLEL=READ_PART;PARTITION_METHOD=TRIVIAL;PARALLEL_RESOLVE_SHARED_ENTS";
433 std::ostringstream gather_set_option;
434 gather_set_option << ";GATHER_SET=" << gather_set_rank;
435 read_options += gather_set_option.str();
436
437 rval = mb.load_file(example.c_str(), &file_set, read_options.c_str());
438 CHECK_ERR(rval);
439
440 ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
441 int procs = pcomm->proc_config().proc_size();
442 int rank = pcomm->proc_config().proc_rank();
443
444 // Make sure gather_set_rank is valid
445 if (gather_set_rank < 0 || gather_set_rank >= procs)
446 return;
447
448 Range cells, cells_owned;
449 rval = mb.get_entities_by_type(file_set, MBPOLYGON, cells);
450 CHECK_ERR(rval);
451
452 // Get local owned cells
453 rval = pcomm->filter_pstatus(cells, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, &cells_owned);
454 CHECK_ERR(rval);
455
456 EntityHandle gather_set = 0;
457 if (gather_set_rank == rank) {
458 // Get gather set
459 ReadUtilIface* readUtilIface;
460 mb.query_interface(readUtilIface);
461 rval = readUtilIface->get_gather_set(gather_set);
462 CHECK_ERR(rval);
463 assert(gather_set != 0);
464 }
465
466 Tag ke_tag0, gid_tag;
467 rval = mb.tag_get_handle("ke0", 1, MB_TYPE_DOUBLE, ke_tag0, MB_TAG_DENSE);
468 CHECK_ERR(rval);
469
470 rval = mb.tag_get_handle(GLOBAL_ID_TAG_NAME, 1, MB_TYPE_INTEGER, gid_tag, MB_TAG_DENSE);
471 CHECK_ERR(rval);
472
473 pcomm->gather_data(cells_owned, ke_tag0, gid_tag, gather_set, gather_set_rank);
474
475 if (gather_set_rank == rank) {
476 // Get gather set cells
477 Range gather_set_cells;
478 rval = mb.get_entities_by_type(gather_set, MBPOLYGON, gather_set_cells);
479 CHECK_ERR(rval);
480 CHECK_EQUAL((size_t)642, gather_set_cells.size());
481 CHECK_EQUAL((size_t)1, gather_set_cells.psize());
482
483 // Check ke0 tag values on 4 gather set cells: first pentagon, last pentagon,
484 // first hexagon and last hexagon
485 EntityHandle cell_ents[] = {gather_set_cells[0], gather_set_cells[11],
486 gather_set_cells[12], gather_set_cells[641]};
487 double ke0_val[4];
488 rval = mb.tag_get_data(ke_tag0, &cell_ents[0], 4, ke0_val);
489 CHECK_ERR(rval);
490
491 CHECK_REAL_EQUAL(15.001, ke0_val[0], eps);
492 CHECK_REAL_EQUAL(15.012, ke0_val[1], eps);
493 CHECK_REAL_EQUAL(16.013, ke0_val[2], eps);
494 CHECK_REAL_EQUAL(16.642, ke0_val[3], eps);
495 }
496 }
497
multiple_loads_of_same_file(bool no_mixed_elements)498 void multiple_loads_of_same_file(bool no_mixed_elements)
499 {
500 Core moab;
501 Interface& mb = moab;
502
503 // Need a file set for nomesh to work right
504 EntityHandle file_set;
505 ErrorCode rval;
506 rval = mb.create_meshset(MESHSET_SET, file_set);
507 CHECK_ERR(rval);
508
509 // Read first only header information, no mesh, no variable
510 read_options = "PARALLEL=READ_PART;PARTITION;NOMESH;VARIABLE=;PARTITION_METHOD=TRIVIAL";
511 if (no_mixed_elements)
512 read_options += ";NO_MIXED_ELEMENTS";
513
514 rval = mb.load_file(example.c_str(), &file_set, read_options.c_str());
515 CHECK_ERR(rval);
516
517 // Create mesh, no variable
518 read_options = "PARALLEL=READ_PART;PARTITION;PARALLEL_RESOLVE_SHARED_ENTS;PARTITION_METHOD=TRIVIAL;VARIABLE=";
519 if (no_mixed_elements)
520 read_options += ";NO_MIXED_ELEMENTS";
521
522 rval = mb.load_file(example.c_str(), &file_set, read_options.c_str());
523 CHECK_ERR(rval);
524
525 // Read variable ke at timestep 0, no mesh
526 read_options = "PARALLEL=READ_PART;PARTITION;PARTITION_METHOD=TRIVIAL;NOMESH;VARIABLE=ke;TIMESTEP=0";
527 if (no_mixed_elements)
528 read_options += ";NO_MIXED_ELEMENTS";
529
530 rval = mb.load_file(example.c_str(), &file_set, read_options.c_str());
531 CHECK_ERR(rval);
532
533 Range local_verts;
534 rval = mb.get_entities_by_type(file_set, MBVERTEX, local_verts);
535 CHECK_ERR(rval);
536
537 Range local_edges;
538 rval = mb.get_entities_by_type(file_set, MBEDGE, local_edges);
539 CHECK_ERR(rval);
540
541 Range local_cells;
542 rval = mb.get_entities_by_type(file_set, MBPOLYGON, local_cells);
543 CHECK_ERR(rval);
544
545 ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0);
546 int procs = pcomm->proc_config().proc_size();
547 int rank = pcomm->proc_config().proc_rank();
548
549 // Make check runs this test on two processors
550 if (2 == procs) {
551 CHECK_EQUAL((size_t)321, local_cells.size());
552
553 // Check tag for cell variable ke at timestep 0
554 Tag ke_tag0;
555 rval = mb.tag_get_handle("ke0", 1, MB_TYPE_DOUBLE, ke_tag0);
556 CHECK_ERR(rval);
557
558 // Get ke0 tag values on 3 local cells
559 EntityHandle cell_ents[] = {local_cells[0], local_cells[160], local_cells[320]};
560 double ke0_val[3];
561 rval = mb.tag_get_data(ke_tag0, cell_ents, 3, ke0_val);
562 CHECK_ERR(rval);
563
564 if (0 == rank) {
565 CHECK_EQUAL((size_t)1120, local_verts.size());
566 CHECK_EQUAL((size_t)1438, local_edges.size());
567 CHECK_EQUAL((size_t)1, local_cells.psize());
568 CHECK_REAL_EQUAL(15.001, ke0_val[0], eps);
569 CHECK_REAL_EQUAL(16.161, ke0_val[1], eps);
570 CHECK_REAL_EQUAL(16.321, ke0_val[2], eps);
571 }
572 else if (1 == rank) {
573 CHECK_EQUAL((size_t)1122, local_verts.size());
574 CHECK_EQUAL((size_t)1444, local_edges.size());
575 CHECK_EQUAL((size_t)1, local_cells.psize());
576
577 CHECK_REAL_EQUAL(16.322, ke0_val[0], eps);
578 CHECK_REAL_EQUAL(16.482, ke0_val[1], eps);
579 CHECK_REAL_EQUAL(16.642, ke0_val[2], eps);
580 }
581 }
582 }
583