1 #include "moab/Core.hpp"
2 #include "moab/ProgOptions.hpp"
3 #include "moab/ReorderTool.hpp"
4
5 #ifdef MOAB_HAVE_MPI
6 #include "moab/ParallelComm.hpp"
7 #endif
8
9 #ifdef MOAB_HAVE_ZOLTAN
10 #include "moab/ZoltanPartitioner.hpp"
11
12 #ifdef MOAB_HAVE_CGM
13 #include "InitCGMA.hpp"
14 #include "CubitCompat.hpp"
15 #endif
16
17 #endif
18
19 #ifdef MOAB_HAVE_METIS
20 #include "moab/MetisPartitioner.hpp"
21 typedef idx_t PartType;
22 #else
23 typedef int PartType;
24 #endif
25
26 #include <iostream>
27 #include <sstream>
28 #include <stdlib.h>
29 #include <list>
30 #include <time.h>
31
32 using namespace moab;
33
34 const char DEFAULT_TAGGEDSETS_TAG[] = "PARALLEL_PARTITION";
35
36 const char DEFAULT_ZOLTAN_METHOD[] = "RCB";
37 #ifdef MOAB_HAVE_ZOLTAN
38 const char ZOLTAN_PARMETIS_METHOD[] = "PARMETIS";
39 const char ZOLTAN_OCTPART_METHOD[] = "OCTPART";
40 #endif
41
42 const char METIS_DEFAULT_METHOD[] = "ML_KWAY";
43 /* const char METIS_ALTERNATIVE_METHOD[] = "ML_RB"; */
44
45 const char BRIEF_DESC[] = "Use Zoltan or Metis to partition MOAB meshes for use on parallel computers";
46 std::ostringstream LONG_DESC;
47
main(int argc,char * argv[])48 int main(int argc, char* argv[])
49 {
50 #ifdef MOAB_HAVE_MPI
51 int err = MPI_Init(&argc, &argv);
52 if (err)
53 {
54 std::cerr << "MPI_Init failed. Aborting." << std::endl;
55 return 3;
56 }
57 #endif
58 Core moab;
59 Interface& mb = moab;
60 std::vector<int> set_l;
61
62 #ifdef MOAB_HAVE_ZOLTAN
63 bool moab_use_zoltan=false;
64 #endif
65 #ifdef MOAB_HAVE_METIS
66 bool moab_use_metis=false;
67 #endif
68
69 LONG_DESC << "This utility invokes the ZoltanPartitioner or MetisPartitioner component of MOAB/CGM "
70 "to partition a mesh/geometry." << std::endl
71 << "If no partitioning method is specified, the defaults are: "
72 << "for Zoltan=\"" << DEFAULT_ZOLTAN_METHOD
73 << "\" and Metis=\"" << METIS_DEFAULT_METHOD
74 << " method" << std::endl;
75
76 ProgOptions opts(LONG_DESC.str(), BRIEF_DESC);
77
78 int part_dim = 3;
79 opts.addOpt<int>("dimension", "Specify dimension of entities to partition."
80 " Default is largest in file.", &part_dim, ProgOptions::int_flag);
81
82 std::string zoltan_method, parm_method, oct_method, metis_method;
83 #ifdef MOAB_HAVE_ZOLTAN
84 opts.addOpt<std::string>("zoltan,z", "(Zoltan) Specify Zoltan partition method. "
85 "One of RR, RCB, RIB, HFSC, PHG, or Hypergraph (PHG and Hypergraph "
86 "are synonymous).", &zoltan_method);
87 #ifdef MOAB_HAVE_PARMETIS
88 opts.addOpt<std::string>("parmetis,p", "(Zoltan+PARMetis) Specify PARMetis partition method.", &parm_method);
89 #endif // MOAB_HAVE_PARMETIS
90 opts.addOpt<std::string>("octpart,o", "(Zoltan) Specify OctPart partition method.", &oct_method);
91
92 bool incl_closure = false;
93 opts.addOpt<void>("include_closure,c", "Include element closure for part sets.", &incl_closure);
94 #endif // MOAB_HAVE_ZOLTAN
95
96 double imbal_tol = 1.03;
97 opts.addOpt<double>("imbalance,i", "Imbalance tolerance (used in PHG/Hypergraph method)", &imbal_tol);
98
99 #ifdef MOAB_HAVE_METIS
100 opts.addOpt<std::string>( "metis,m", "(Metis) Specify Metis partition method. One of ML_RB or ML_KWAY.", &metis_method);
101 #endif // MOAB_HAVE_METIS
102
103 bool write_sets = true, write_tags = false;
104 opts.addOpt<void>( "sets,s", "Write partition as tagged sets (Default)", &write_sets);
105 opts.addOpt<void>( "tags,t", "Write partition by tagging entities", &write_tags);
106
107 int power = -1;
108 opts.addOpt<int>("power,M", "Generate multiple partitions, in powers of 2, up to 2^(pow)", &power);
109
110 bool reorder = false;
111 opts.addOpt<void>("reorder,R", "Reorder mesh to group entities by partition", &reorder);
112
113 double part_geom_mesh_size = -1.0;
114 #ifdef MOAB_HAVE_ZOLTAN
115 bool part_surf = false;
116 #ifdef MOAB_HAVE_CGM
117 opts.addOpt<double>("geom,g", "(CGM) If partition geometry, specify mesh size.", &part_geom_mesh_size);
118 opts.addOpt<void>("surf,f", "(CGM) Specify if partition geometry surface.", &part_surf);
119 #endif // MOAB_HAVE_CGM
120
121 bool ghost = false;
122 opts.addOpt<void>("ghost,H", "Specify if partition ghost geometry body.");
123
124 int obj_weight = 0;
125 opts.addOpt<int>("vertex_w,v", "Number of weights associated with a graph vertex.");
126
127 int edge_weight = 0;
128 opts.addOpt<int>("edge_w,e", "Number of weights associated with an edge.");
129
130 #endif // MOAB_HAVE_ZOLTAN
131
132 long num_parts;
133 opts.addOpt<std::vector<int> >("set_l,l", "Load material set(s) with specified ids (comma seperated) for partition");
134
135 opts.addRequiredArg<int>("#parts", "Number of parts in partition");
136
137 std::string input_file, output_file;
138 opts.addRequiredArg<std::string>("input_file", "Mesh/geometry to partition", &input_file);
139 opts.addRequiredArg<std::string>("output_file", "File to which to write partitioned mesh/geometry", &output_file);
140
141 bool print_time = false;
142 opts.addOpt<void>(",T", "Print CPU time for each phase.", &print_time);
143
144 bool spherical_coords = false;
145 opts.addOpt<void>("project_on_sphere,s", "use spherical coordinates for partitioning ", &spherical_coords);
146
147 #ifdef MOAB_HAVE_METIS
148 bool partition_tagged_sets = false;
149 opts.addOpt<void>( "taggedsets,x", "(Metis) Partition tagged sets.", &partition_tagged_sets);
150
151 bool partition_tagged_ents = false;
152 opts.addOpt<void>( "taggedents,y", "(Metis) Partition tagged ents.", &partition_tagged_ents);
153
154 std::string aggregating_tag;
155 opts.addOpt<std::string>( "aggregatingtag,a", "(Metis) Specify aggregating tag to partion tagged sets or tagged entities.", &aggregating_tag);
156
157 std::string aggregating_bc_tag;
158 opts.addOpt<std::string>( "aggregatingBCtag,B", "(Metis) Specify boundary id tag name used to group cells with same boundary ids.", &aggregating_bc_tag);
159
160 std::string boundaryIds;
161 std::vector<int> BCids;
162 opts.addOpt<std::string>( "aggregatingBCids,I", " (Metis) Specify id or ids of boundaries to be aggregated before partitioning (all elements with same boundary id will be in the same partition). Comma separated e.g. -I 1,2,5 ", &boundaryIds);
163 #endif // MOAB_HAVE_METIS
164
165 opts.parseCommandLine(argc, argv);
166
167 #ifdef MOAB_HAVE_ZOLTAN
168 if (!zoltan_method.empty())
169 moab_use_zoltan=true;
170 else
171 #endif
172 #ifdef MOAB_HAVE_METIS
173 if (!metis_method.empty())
174 moab_use_metis=true;
175 else
176 #endif
177 MB_SET_ERR(MB_FAILURE, "Specify either Zoltan or Metis partitioner type");
178
179 #ifdef MOAB_HAVE_ZOLTAN
180 ZoltanPartitioner *zoltan_tool = NULL;
181 // check if partition geometry, if it is, should get mesh size for the geometry
182 if (part_geom_mesh_size != -1.0 && part_geom_mesh_size <= 0.0)
183 {
184 std::cerr << part_geom_mesh_size
185 << ": invalid geometry partition mesh size." << std::endl << std::endl;
186 opts.printHelp();
187 return EXIT_FAILURE;
188 }
189
190 if (moab_use_zoltan) {
191 if (part_geom_mesh_size < 0.)
192 {
193 // partition mesh
194 zoltan_tool = new ZoltanPartitioner(&mb, false, argc, argv);
195 }
196 else
197 {
198 // partition geometry
199 #ifdef MOAB_HAVE_CGM
200 CubitStatus status = InitCGMA::initialize_cgma();
201 if (CUBIT_SUCCESS != status)
202 {
203 std::cerr << "CGM couldn't be initialized." << std::endl << std::endl;
204 opts.printHelp();
205 return EXIT_FAILURE;
206 }
207 GeometryQueryTool *gti = GeometryQueryTool::instance();
208 zoltan_tool = new ZoltanPartitioner (&mb, false, argc, argv, gti);
209 #else
210 std::cerr << "CGM should be configured to partition geometry." << std::endl << std::endl;
211 opts.printHelp();
212 return EXIT_FAILURE;
213 #endif // MOAB_HAVE_CGM
214 }
215 }
216
217 if (zoltan_method.empty() && parm_method.empty() && oct_method.empty())
218 zoltan_method = DEFAULT_ZOLTAN_METHOD;
219 if (!parm_method.empty())
220 zoltan_method = ZOLTAN_PARMETIS_METHOD;
221 if (!oct_method.empty())
222 zoltan_method = ZOLTAN_OCTPART_METHOD;
223 #endif // MOAB_HAVE_ZOLTAN
224
225 #ifdef MOAB_HAVE_METIS
226 MetisPartitioner *metis_tool = NULL;
227 if (moab_use_metis && !metis_tool) {
228 metis_tool = new MetisPartitioner (&mb, false);
229 }
230
231 if ((aggregating_tag.empty() && partition_tagged_sets) || (aggregating_tag.empty() && partition_tagged_ents))
232 aggregating_tag = DEFAULT_TAGGEDSETS_TAG;
233 if (!write_sets && !write_tags)
234 write_sets = true;
235
236 if (!boundaryIds.empty())
237 {
238 std::vector<std::string> ids;
239 std::stringstream ss(boundaryIds);
240 std::string item;
241 while (std::getline(ss, item, ',')) {
242 ids.push_back(item);
243 }
244 for (unsigned int i = 0; i < ids.size(); i++)
245 BCids.push_back(std::atoi(ids[i].c_str()));
246 }
247
248 if (metis_method.empty()) {
249 metis_method = METIS_DEFAULT_METHOD;
250 }
251 #endif // MOAB_HAVE_METIS
252
253 if (!write_sets && !write_tags)
254 write_sets = true;
255
256 if (-1 == power)
257 {
258 num_parts = opts.getReqArg<int>("#parts");
259 power = 1;
260 }
261 else if (power < 1 || power > 18)
262 {
263 std::cerr << power
264 << ": invalid power for multiple partitions. Expected value in [1,18]"
265 << std::endl << std::endl;
266 opts.printHelp();
267 return EXIT_FAILURE;
268 }
269 else
270 {
271 num_parts = 2;
272 }
273
274 if (part_dim < 0 || part_dim > 3)
275 {
276 std::cerr << part_dim << " : invalid dimension" << std::endl << std::endl;
277 opts.printHelp();
278 return EXIT_FAILURE;
279 }
280
281 if (imbal_tol < 0.0)
282 {
283 std::cerr << imbal_tol << ": invalid imbalance tolerance" << std::endl << std::endl;
284 opts.printHelp();
285 return EXIT_FAILURE;
286 }
287
288 bool load_msets = false;
289 if (opts.getOpt("set_l,l", &set_l))
290 {
291 load_msets = true;
292 if (set_l.size() <= 0)
293 {
294 std::cerr << " No material set id's to load" << std::endl << std::endl;
295 opts.printHelp();
296 return EXIT_FAILURE;
297 }
298 }
299
300 if (num_parts <= 1) {
301 std::cerr << "** Please specify #parts = " << num_parts << " to be greater than 1." << std::endl << std::endl;
302 opts.printHelp();
303 return EXIT_FAILURE;
304 }
305
306 clock_t t = clock();
307
308 const char* options = NULL;
309 ErrorCode rval;
310 #ifdef MOAB_HAVE_ZOLTAN
311 if (part_geom_mesh_size > 0.)
312 options = "FACET_DISTANCE_TOLERANCE=0.1";
313 #endif // MOAB_HAVE_ZOLTAN
314
315 std::cout << "Loading file " << input_file << "..." << std::endl;
316 if (load_msets == false)
317 {
318 rval = mb.load_file(input_file.c_str(), 0, options);
319 if (MB_SUCCESS != rval)
320 {
321 std::cerr << input_file << " : failed to read file." << std::endl;
322 std::cerr << " Error code: " << mb.get_error_string(rval) << " (" << rval
323 << ")" << std::endl;
324 std::string errstr;
325 mb.get_last_error(errstr);
326 if (!errstr.empty())
327 std::cerr << " Error message: " << errstr << std::endl;
328 return 2;
329 }
330 }
331 // load the material set(s)
332 else
333 {
334 rval = mb.load_mesh(input_file.c_str(), &set_l[0], (int) set_l.size());
335 if (MB_SUCCESS != rval)
336 {
337 std::cerr << input_file << " : failed to read file." << std::endl;
338 std::cerr << " Error code: " << mb.get_error_string(rval) << " (" << rval
339 << ")" << std::endl;
340 std::string errstr;
341 mb.get_last_error(errstr);
342 if (!errstr.empty())
343 std::cerr << " Error message: " << errstr << std::endl;
344 return 2;
345 }
346 }
347 if (print_time)
348 std::cout << "Read input file in "
349 << (clock() - t) / (double) CLOCKS_PER_SEC << " seconds" << std::endl;
350
351 for (int dim = part_dim; dim >= 0; --dim)
352 {
353 int n;
354 rval = mb.get_number_entities_by_dimension(0, dim, n);
355 if (MB_SUCCESS == rval && 0 != n)
356 {
357 part_dim = dim;
358 break;
359 }
360 }
361 if (part_dim < 0)
362 {
363 std::cerr << input_file << " : file does not contain any mesh entities" << std::endl;
364 return 2;
365 }
366
367 ReorderTool reorder_tool(&moab);
368
369 for (int p = 0; p < power; p++)
370 {
371 t = clock();
372 #ifdef MOAB_HAVE_ZOLTAN
373 if (moab_use_zoltan) {
374 rval = zoltan_tool->partition_mesh_and_geometry(part_geom_mesh_size, num_parts,
375 zoltan_method.c_str(),
376 (!parm_method.empty() ? parm_method.c_str() : oct_method.c_str()),
377 imbal_tol, part_dim, write_sets, write_tags, obj_weight,
378 edge_weight, part_surf, ghost, spherical_coords, print_time);
379 }
380 #endif
381 #ifdef MOAB_HAVE_METIS
382 if (moab_use_metis) {
383 rval = metis_tool->partition_mesh( num_parts, metis_method.c_str(), part_dim,
384 write_sets, write_tags,
385 partition_tagged_sets, partition_tagged_ents,
386 aggregating_tag.c_str(), print_time);
387 }
388 #endif
389 if (MB_SUCCESS != rval)
390 {
391 std::cerr << "Partitioner failed!" << std::endl;
392 std::cerr << " Error code: " << mb.get_error_string(rval) << " (" << rval
393 << ")" << std::endl;
394 std::string errstr;
395 mb.get_last_error(errstr);
396 if (!errstr.empty())
397 std::cerr << " Error message: " << errstr << std::endl;
398 return 3;
399 }
400 if (print_time)
401 std::cout << "Generated " << num_parts << " part partitioning in "
402 << (clock() - t) / (double) CLOCKS_PER_SEC << " seconds" << std::endl;
403
404 if (reorder && part_geom_mesh_size < 0.)
405 {
406 std::cout << "Reordering mesh for partition..." << std::endl;
407
408 Tag tag, order;
409 rval = mb.tag_get_handle(DEFAULT_TAGGEDSETS_TAG, 1, MB_TYPE_INTEGER, tag);
410 if (MB_SUCCESS != rval)
411 {
412 std::cerr << "Partitioner did not create " << DEFAULT_TAGGEDSETS_TAG << " tag" << std::endl;
413 return 2;
414 }
415
416 t = clock();
417 if (write_sets)
418 {
419 Range sets;
420 mb.get_entities_by_type_and_tag(0, MBENTITYSET, &tag, 0, 1, sets);
421 rval = reorder_tool.handle_order_from_sets_and_adj(sets, order);
422 }
423 else
424 {
425 rval = reorder_tool.handle_order_from_int_tag(tag, -1, order);
426 }
427 if (MB_SUCCESS != rval)
428 {
429 std::cerr << "Failed to calculate reordering!" << std::endl;
430 return 2;
431 }
432
433 rval = reorder_tool.reorder_entities(order);
434 if (MB_SUCCESS != rval)
435 {
436 std::cerr << "Failed to perform reordering!" << std::endl;
437 return 2;
438 }
439
440 mb.tag_delete(order);
441 if (print_time)
442 std::cout << "Reordered mesh in "
443 << (clock() - t) / (double) CLOCKS_PER_SEC << " seconds"
444 << std::endl;
445 }
446
447 #ifdef MOAB_HAVE_ZOLTAN
448 if (incl_closure)
449 {
450 rval = zoltan_tool->include_closure();
451 if (MB_SUCCESS != rval)
452 {
453 std::cerr << "Closure inclusion failed." << std::endl;
454 return 1;
455 }
456 }
457 #endif
458
459 std::ostringstream tmp_output_file;
460
461 if (power > 1)
462 {
463 // append num_parts to output filename
464 std::string::size_type idx = output_file.find_last_of(".");
465 if (idx == std::string::npos)
466 {
467 tmp_output_file << output_file << "_" << num_parts;
468 if (part_geom_mesh_size < 0.)
469 tmp_output_file << ".h5m";
470 else
471 {
472 std::cerr << "output file type is not specified." << std::endl;
473 return 1;
474 }
475 }
476 else
477 {
478 tmp_output_file << output_file.substr(0, idx) << "_" << num_parts
479 << output_file.substr(idx);
480 }
481 }
482 else
483 tmp_output_file << output_file;
484
485 t = clock();
486 std::cout << "Saving file to " << output_file << "..." << std::endl;
487 if (part_geom_mesh_size < 0.)
488 {
489 rval = mb.write_file(tmp_output_file.str().c_str());
490 if (MB_SUCCESS != rval)
491 {
492 std::cerr << tmp_output_file.str() << " : failed to write file." << std::endl;
493 std::cerr << " Error code: " << mb.get_error_string(rval) << " ("
494 << rval << ")" << std::endl;
495 std::string errstr;
496 mb.get_last_error(errstr);
497 if (!errstr.empty())
498 std::cerr << " Error message: " << errstr << std::endl;
499 return 2;
500 }
501 }
502 #ifdef MOAB_HAVE_ZOLTAN
503 #ifdef MOAB_HAVE_CGM
504 else
505 {
506 std::string::size_type idx = output_file.find_last_of( "." );
507 int c_size = output_file.length() - idx;
508 const char* file_type = NULL;
509 if (output_file.compare(idx, c_size, ".occ") == 0
510 || output_file.compare(idx, c_size, ".OCC") == 0) file_type = "OCC";
511 else if (output_file.compare(idx, c_size, ".sab") == 0) file_type = "ACIS_SAB";
512 else if (output_file.compare(idx, c_size, ".sat") == 0) file_type = "ACIS_SAT";
513 else
514 {
515 std::cerr << "File type for " << output_file.c_str() << " not supported." << std::endl;
516 return 1;
517 }
518
519 int num_ents_exported=0;
520 DLIList<RefEntity*> ref_entity_list;
521 CubitStatus status = CubitCompat_export_solid_model(ref_entity_list,
522 tmp_output_file.str().c_str(),
523 file_type, num_ents_exported,
524 CubitString(__FILE__));
525 if (CUBIT_SUCCESS != status)
526 {
527 std::cerr << "CGM couldn't export models." << std::endl;
528 return 1;
529 }
530 }
531 #endif
532 #endif
533
534 if (print_time)
535 std::cout << "Wrote \"" << tmp_output_file.str() << "\" in "
536 << (clock() - t) / (double) CLOCKS_PER_SEC << " seconds" << std::endl;
537
538 num_parts *= 2;
539 }
540
541 #ifdef MOAB_HAVE_ZOLTAN
542 delete zoltan_tool;
543 #endif
544 #ifdef MOAB_HAVE_METIS
545 delete metis_tool;
546 #endif
547
548 #ifdef MOAB_HAVE_MPI
549 err = MPI_Finalize();
550 assert(MPI_SUCCESS == err);
551 #endif
552 return 0;
553 }
554