1 #include "moab/ScdInterface.hpp"
2 #include "moab/Core.hpp"
3 #include "SequenceManager.hpp"
4 #include "EntitySequence.hpp"
5 #include "StructuredElementSeq.hpp"
6 #include "VertexSequence.hpp"
7 #include "ScdVertexData.hpp"
8 #include "MBTagConventions.hpp"
9 #ifdef MOAB_HAVE_MPI
10 # include "moab/ParallelComm.hpp"
11 # include "moab/TupleList.hpp"
12 # include "moab/gs.hpp"
13 #endif
14 #include "assert.h"
15 #include <iostream>
16 #include <functional>
17
18 #define ERRORR(rval, str) {if (MB_SUCCESS != rval) \
19 {std::cerr << str; return rval; }}
20
21
22 namespace moab
23 {
24
25 const char *ScdParData::PartitionMethodNames[] = {"alljorkori", "alljkbal", "sqij", "sqjk", "sqijk", "trivial","rcbzoltan", "nopart"};
26
ScdInterface(Interface * imp,bool boxes)27 ScdInterface::ScdInterface(Interface *imp, bool boxes)
28 : mbImpl(imp),
29 searchedBoxes(false),
30 boxPeriodicTag(0),
31 boxDimsTag(0),
32 globalBoxDimsTag(0),
33 partMethodTag(0),
34 boxSetTag(0)
35 {
36 if (boxes) find_boxes(scdBoxes);
37 }
38
39 // Destructor
~ScdInterface()40 ScdInterface::~ScdInterface()
41 {
42 std::vector<ScdBox*> tmp_boxes;
43 tmp_boxes.swap(scdBoxes);
44
45 for (std::vector<ScdBox*>::iterator rit = tmp_boxes.begin(); rit != tmp_boxes.end(); ++rit)
46 delete *rit;
47
48 if (box_set_tag(false))
49 mbImpl->tag_delete(box_set_tag());
50
51 }
52
impl() const53 Interface *ScdInterface::impl() const
54 {
55 return mbImpl;
56 }
57
find_boxes(std::vector<ScdBox * > & scd_boxes)58 ErrorCode ScdInterface::find_boxes(std::vector<ScdBox*> &scd_boxes)
59 {
60 Range tmp_boxes;
61 ErrorCode rval = find_boxes(tmp_boxes);
62 if (MB_SUCCESS != rval) return rval;
63
64 for (Range::iterator rit = tmp_boxes.begin(); rit != tmp_boxes.end(); ++rit) {
65 ScdBox *tmp_box = get_scd_box(*rit);
66 if (tmp_box) scd_boxes.push_back(tmp_box);
67 else rval = MB_FAILURE;
68 }
69
70 return rval;
71 }
72
find_boxes(Range & scd_boxes)73 ErrorCode ScdInterface::find_boxes(Range &scd_boxes)
74 {
75 ErrorCode rval = MB_SUCCESS;
76 box_dims_tag();
77 Range boxes;
78 if (!searchedBoxes) {
79 rval = mbImpl->get_entities_by_type_and_tag(0, MBENTITYSET, &boxDimsTag, NULL, 1,
80 boxes, Interface::UNION);
81 searchedBoxes = true;
82 if (!boxes.empty()) {
83 scdBoxes.resize(boxes.size());
84 rval = mbImpl->tag_get_data(boxSetTag, boxes, &scdBoxes[0]);
85 ScdBox *dum = NULL;
86 std::remove_if(scdBoxes.begin(), scdBoxes.end(), std::bind2nd(std::equal_to<ScdBox*>(), dum) ) ;
87 }
88 }
89
90 for (std::vector<ScdBox*>::iterator vit = scdBoxes.begin(); vit != scdBoxes.end(); ++vit)
91 scd_boxes.insert((*vit)->box_set());
92
93 return rval;
94 }
95
get_scd_box(EntityHandle eh)96 ScdBox *ScdInterface::get_scd_box(EntityHandle eh)
97 {
98 ScdBox *scd_box = NULL;
99 if (!box_set_tag(false)) return scd_box;
100
101 mbImpl->tag_get_data(box_set_tag(), &eh, 1, &scd_box);
102 return scd_box;
103 }
104
construct_box(HomCoord low,HomCoord high,const double * const coords,unsigned int num_coords,ScdBox * & new_box,int * const lperiodic,ScdParData * par_data,bool assign_gids,int tag_shared_ents)105 ErrorCode ScdInterface::construct_box(HomCoord low, HomCoord high, const double * const coords, unsigned int num_coords,
106 ScdBox *& new_box, int * const lperiodic, ScdParData *par_data,
107 bool assign_gids, int tag_shared_ents)
108 {
109 // create a rectangular structured mesh block
110 ErrorCode rval;
111
112 int tmp_lper[3] = {0, 0, 0};
113 if (lperiodic) std::copy(lperiodic, lperiodic+3, tmp_lper);
114
115 #ifndef MOAB_HAVE_MPI
116 if (-1 != tag_shared_ents) ERRORR(MB_FAILURE, "Parallel capability requested but MOAB not compiled parallel.");
117 if (-1 == tag_shared_ents && !assign_gids) assign_gids = true; // need to assign gids in order to tag shared verts
118 #else
119 if (par_data && low == high && ScdParData::NOPART != par_data->partMethod) {
120 // requesting creation of parallel mesh, so need to compute partition
121 if (!par_data->pComm) {
122 // this is a really boneheaded way to have to create a PC
123 par_data->pComm = ParallelComm::get_pcomm(mbImpl, 0);
124 if (NULL == par_data->pComm) par_data->pComm = new ParallelComm(mbImpl, MPI_COMM_WORLD);
125 }
126 int ldims[6];
127 rval = compute_partition(par_data->pComm->size(), par_data->pComm->rank(), *par_data,
128 ldims, tmp_lper, par_data->pDims);
129 ERRORR(rval, "Error returned from compute_partition.");
130 low.set(ldims[0],ldims[1],ldims[2]);
131 high.set(ldims[3],ldims[4],ldims[5]);
132 if (par_data->pComm->get_debug_verbosity() > 0) {
133 std::cout << "Proc " << par_data->pComm->rank() << ": " << *par_data;
134 std::cout << "Proc " << par_data->pComm->rank() << " local dims: "
135 << low << "-" << high << std::endl;
136 }
137 }
138 #endif
139
140 HomCoord tmp_size = high - low + HomCoord(1, 1, 1, 0);
141 if ((tmp_size[1] && num_coords && (int)num_coords < tmp_size[0]) ||
142 (tmp_size[2] && num_coords && (int)num_coords < tmp_size[0]*tmp_size[1]))
143 return MB_FAILURE;
144
145 rval = create_scd_sequence(low, high, MBVERTEX, 0, new_box);
146 ERRORR(rval, "Trouble creating scd vertex sequence.");
147
148 // set the vertex coordinates
149 double *xc, *yc, *zc;
150 rval = new_box->get_coordinate_arrays(xc, yc, zc);
151 ERRORR(rval, "Couldn't get vertex coordinate arrays.");
152
153 if (coords && num_coords) {
154 unsigned int i = 0;
155 for (int kl = low[2]; kl <= high[2]; kl++) {
156 for (int jl = low[1]; jl <= high[1]; jl++) {
157 for (int il = low[0]; il <= high[0]; il++) {
158 xc[i] = coords[3*i];
159 if (new_box->box_size()[1])
160 yc[i] = coords[3*i+1];
161 if (new_box->box_size()[2])
162 zc[i] = coords[3*i+2];
163 i++;
164 }
165 }
166 }
167 }
168 else {
169 unsigned int i = 0;
170 for (int kl = low[2]; kl <= high[2]; kl++) {
171 for (int jl = low[1]; jl <= high[1]; jl++) {
172 for (int il = low[0]; il <= high[0]; il++) {
173 xc[i] = (double) il;
174 if (new_box->box_size()[1])
175 yc[i] = (double) jl;
176 else yc[i] = 0.0;
177 if (new_box->box_size()[2])
178 zc[i] = (double) kl;
179 else zc[i] = 0.0;
180 i++;
181 }
182 }
183 }
184 }
185
186 // create element sequence
187 Core *mbcore = dynamic_cast<Core*>(mbImpl);
188 SequenceManager *seq_mgr = mbcore->sequence_manager();
189
190 EntitySequence *tmp_seq;
191 EntityHandle start_ent;
192
193 // construct the sequence
194 EntityType this_tp = MBHEX;
195 if (1 >= tmp_size[2]) this_tp = MBQUAD;
196 if (1 >= tmp_size[2] && 1 >= tmp_size[1]) this_tp = MBEDGE;
197 rval = seq_mgr->create_scd_sequence(low, high, this_tp, 0, start_ent, tmp_seq,
198 tmp_lper);
199 ERRORR(rval, "Trouble creating scd element sequence.");
200
201 new_box->elem_seq(tmp_seq);
202 new_box->start_element(start_ent);
203
204 // add vertex seq to element seq, forward orientation, unity transform
205 rval = new_box->add_vbox(new_box,
206 // p1: imin,jmin
207 low, low,
208 // p2: imax,jmin
209 low + HomCoord(1, 0, 0),
210 low + HomCoord(1, 0, 0),
211 // p3: imin,jmax
212 low + HomCoord(0, 1, 0),
213 low + HomCoord(0, 1, 0));
214 ERRORR(rval, "Error constructing structured element sequence.");
215
216 // add the new hexes to the scd box set; vertices were added in call to create_scd_sequence
217 Range tmp_range(new_box->start_element(), new_box->start_element() + new_box->num_elements() - 1);
218 rval = mbImpl->add_entities(new_box->box_set(), tmp_range);
219 ERRORR(rval, "Couldn't add new hexes to box set.");
220
221 if (par_data) new_box->par_data(*par_data);
222
223
224 if (assign_gids) {
225 rval = assign_global_ids(new_box);
226 ERRORR(rval, "Trouble assigning global ids");
227 }
228
229 #ifdef MOAB_HAVE_MPI
230 if (par_data && -1 != tag_shared_ents) {
231 rval = tag_shared_vertices(par_data->pComm, new_box);
232 }
233 #endif
234
235 return MB_SUCCESS;
236 }
237
assign_global_ids(ScdBox * box)238 ErrorCode ScdInterface::assign_global_ids(ScdBox *box)
239 {
240 // Get a ptr to global id memory
241 void* data;
242 int count = 0;
243 Tag gid_tag;
244 ErrorCode rval = mbImpl->tag_get_handle(GLOBAL_ID_TAG_NAME, 1, MB_TYPE_INTEGER, gid_tag,
245 MB_TAG_CREAT & MB_TAG_DENSE, &count);
246 ERRORR(rval, "Trouble getting global id tag handle.");
247 Range tmp_range(box->start_vertex(), box->start_vertex() + box->num_vertices());
248 rval = mbImpl->tag_iterate(gid_tag, tmp_range.begin(), tmp_range.end(), count, data);
249 ERRORR(rval, "Failed to get tag iterator.");
250 assert(count == box->num_vertices());
251 int* gid_data = (int*) data;
252 int di = box->par_data().gDims[3] - box->par_data().gDims[0] + 1;
253 int dj = box->par_data().gDims[4] - box->par_data().gDims[1] + 1;
254
255 for (int kl = box->box_dims()[2]; kl <= box->box_dims()[5]; kl++) {
256 for (int jl = box->box_dims()[1]; jl <= box->box_dims()[4]; jl++) {
257 for (int il = box->box_dims()[0]; il <= box->box_dims()[3]; il++) {
258 int itmp = (!box->locally_periodic()[0] && box->par_data().gPeriodic[0] && il == box->par_data().gDims[3] ?
259 box->par_data().gDims[0] : il);
260 *gid_data = (-1 != kl ? kl * di * dj : 0) + jl * di + itmp + 1;
261 gid_data++;
262 }
263 }
264 }
265
266 return MB_SUCCESS;
267 }
268
create_scd_sequence(const HomCoord & low,const HomCoord & high,EntityType tp,int starting_id,ScdBox * & new_box,int * is_periodic)269 ErrorCode ScdInterface::create_scd_sequence(const HomCoord &low, const HomCoord &high, EntityType tp,
270 int starting_id, ScdBox *&new_box,
271 int *is_periodic)
272 {
273 HomCoord tmp_size = high - low + HomCoord(1, 1, 1, 0);
274 if ((tp == MBHEX && 1 >= tmp_size[2]) ||
275 (tp == MBQUAD && 1 >= tmp_size[1]) ||
276 (tp == MBEDGE && 1 >= tmp_size[0]))
277 return MB_TYPE_OUT_OF_RANGE;
278
279 Core *mbcore = dynamic_cast<Core*>(mbImpl);
280 assert(mbcore != NULL);
281 SequenceManager *seq_mgr = mbcore->sequence_manager();
282
283 EntitySequence *tmp_seq;
284 EntityHandle start_ent, scd_set;
285
286 // construct the sequence
287 ErrorCode rval = seq_mgr->create_scd_sequence(low, high, tp, starting_id, start_ent, tmp_seq,
288 is_periodic);
289 if (MB_SUCCESS != rval) return rval;
290
291 // create the set for this rectangle
292 rval = create_box_set(low, high, scd_set);
293 if (MB_SUCCESS != rval) return rval;
294
295 // make the ScdBox
296 new_box = new ScdBox(this, scd_set, tmp_seq);
297 if (!new_box) return MB_FAILURE;
298
299 // set the start vertex/element
300 Range new_range;
301 if (MBVERTEX == tp) {
302 new_range.insert(start_ent, start_ent+new_box->num_vertices()-1);
303 }
304 else {
305 new_range.insert(start_ent, start_ent+new_box->num_elements()-1);
306 }
307
308 // put the entities in the box set
309 rval = mbImpl->add_entities(scd_set, new_range);
310 if (MB_SUCCESS != rval) return rval;
311
312 // tag the set with the box
313 rval = mbImpl->tag_set_data(box_set_tag(), &scd_set, 1, &new_box);
314 if (MB_SUCCESS != rval) return rval;
315
316 return MB_SUCCESS;
317 }
318
create_box_set(const HomCoord & low,const HomCoord & high,EntityHandle & scd_set,int * is_periodic)319 ErrorCode ScdInterface::create_box_set(const HomCoord &low, const HomCoord &high,
320 EntityHandle &scd_set, int *is_periodic)
321 {
322 // create the set and put the entities in it
323 ErrorCode rval = mbImpl->create_meshset(MESHSET_SET, scd_set);
324 if (MB_SUCCESS != rval) return rval;
325
326 // tag the set with parameter extents
327 int boxdims[6];
328 for (int i = 0; i < 3; i++) boxdims[i] = low[i];
329 for (int i = 0; i < 3; i++) boxdims[3+i] = high[i];
330 rval = mbImpl->tag_set_data(box_dims_tag(), &scd_set, 1, boxdims);
331 if (MB_SUCCESS != rval) return rval;
332
333 if (is_periodic) {
334 rval = mbImpl->tag_set_data(box_periodic_tag(), &scd_set, 1, is_periodic);
335 if (MB_SUCCESS != rval) return rval;
336 }
337
338 return rval;
339 }
340
box_periodic_tag(bool create_if_missing)341 Tag ScdInterface::box_periodic_tag(bool create_if_missing)
342 {
343 // Reset boxPeriodicTag in case it has been deleted (e.g. by Core::clean_up_failed_read)
344 if (boxPeriodicTag) {
345 std::string tag_name;
346 if (MB_TAG_NOT_FOUND == mbImpl->tag_get_name(boxPeriodicTag, tag_name))
347 boxPeriodicTag = NULL;
348 }
349
350 if (boxPeriodicTag || !create_if_missing) return boxPeriodicTag;
351
352 ErrorCode rval = mbImpl->tag_get_handle("BOX_PERIODIC", 3, MB_TYPE_INTEGER,
353 boxPeriodicTag, MB_TAG_SPARSE|MB_TAG_CREAT);
354 if (MB_SUCCESS != rval) return 0;
355 return boxPeriodicTag;
356 }
357
box_dims_tag(bool create_if_missing)358 Tag ScdInterface::box_dims_tag(bool create_if_missing)
359 {
360 // Reset boxDimsTag in case it has been deleted (e.g. by clean_up_failed_read)
361 if (boxDimsTag) {
362 std::string tag_name;
363 if (MB_TAG_NOT_FOUND == mbImpl->tag_get_name(boxDimsTag, tag_name))
364 boxDimsTag = NULL;
365 }
366
367 if (boxDimsTag || !create_if_missing) return boxDimsTag;
368
369 ErrorCode rval = mbImpl->tag_get_handle("BOX_DIMS", 6, MB_TYPE_INTEGER,
370 boxDimsTag, MB_TAG_SPARSE|MB_TAG_CREAT);
371 if (MB_SUCCESS != rval) return 0;
372 return boxDimsTag;
373 }
374
global_box_dims_tag(bool create_if_missing)375 Tag ScdInterface::global_box_dims_tag(bool create_if_missing)
376 {
377 // Reset globalBoxDimsTag in case it has been deleted (e.g. by Core::clean_up_failed_read)
378 if (globalBoxDimsTag) {
379 std::string tag_name;
380 if (MB_TAG_NOT_FOUND == mbImpl->tag_get_name(globalBoxDimsTag, tag_name))
381 globalBoxDimsTag = NULL;
382 }
383
384 if (globalBoxDimsTag || !create_if_missing) return globalBoxDimsTag;
385
386 ErrorCode rval = mbImpl->tag_get_handle("GLOBAL_BOX_DIMS", 6, MB_TYPE_INTEGER,
387 globalBoxDimsTag, MB_TAG_SPARSE|MB_TAG_CREAT);
388 if (MB_SUCCESS != rval) return 0;
389 return globalBoxDimsTag;
390 }
391
part_method_tag(bool create_if_missing)392 Tag ScdInterface::part_method_tag(bool create_if_missing)
393 {
394 // Reset partMethodTag in case it has been deleted (e.g. by Core::clean_up_failed_read)
395 if (partMethodTag) {
396 std::string tag_name;
397 if (MB_TAG_NOT_FOUND == mbImpl->tag_get_name(partMethodTag, tag_name))
398 partMethodTag = NULL;
399 }
400
401 if (partMethodTag || !create_if_missing) return partMethodTag;
402
403 ErrorCode rval = mbImpl->tag_get_handle("PARTITION_METHOD", 1, MB_TYPE_INTEGER,
404 partMethodTag, MB_TAG_SPARSE|MB_TAG_CREAT);
405 if (MB_SUCCESS != rval) return 0;
406 return partMethodTag;
407 }
408
box_set_tag(bool create_if_missing)409 Tag ScdInterface::box_set_tag(bool create_if_missing)
410 {
411 // Reset boxSetTag in case it has been deleted (e.g. by Core::clean_up_failed_read)
412 if (boxSetTag) {
413 std::string tag_name;
414 if (MB_TAG_NOT_FOUND == mbImpl->tag_get_name(boxSetTag, tag_name))
415 boxSetTag = NULL;
416 }
417
418 if (boxSetTag || !create_if_missing) return boxSetTag;
419
420 ErrorCode rval = mbImpl->tag_get_handle("__BOX_SET", sizeof(ScdBox*), MB_TYPE_OPAQUE,
421 boxSetTag, MB_TAG_SPARSE|MB_TAG_CREAT);
422 if (MB_SUCCESS != rval) return 0;
423 return boxSetTag;
424 }
425
426 //! Remove the box from the list on ScdInterface
remove_box(ScdBox * box)427 ErrorCode ScdInterface::remove_box(ScdBox *box)
428 {
429 std::vector<ScdBox*>::iterator vit = std::find(scdBoxes.begin(), scdBoxes.end(), box);
430 if (vit != scdBoxes.end()) {
431 scdBoxes.erase(vit);
432 return MB_SUCCESS;
433 }
434 else return MB_FAILURE;
435 }
436
437 //! Add the box to the list on ScdInterface
add_box(ScdBox * box)438 ErrorCode ScdInterface::add_box(ScdBox *box)
439 {
440 scdBoxes.push_back(box);
441 return MB_SUCCESS;
442 }
443
get_boxes(std::vector<ScdBox * > & boxes)444 ErrorCode ScdInterface::get_boxes(std::vector<ScdBox*> &boxes)
445 {
446 std::copy(scdBoxes.begin(), scdBoxes.end(), std::back_inserter(boxes));
447 return MB_SUCCESS;
448 }
449
ScdBox(ScdInterface * impl,EntityHandle bset,EntitySequence * seq1,EntitySequence * seq2)450 ScdBox::ScdBox(ScdInterface *impl, EntityHandle bset,
451 EntitySequence *seq1, EntitySequence *seq2)
452 : scImpl(impl), boxSet(bset), vertDat(NULL), elemSeq(NULL), startVertex(0), startElem(0)
453 {
454 for (int i = 0; i < 6; i++) boxDims[i] = 0;
455 for (int i = 0; i < 3; i++) locallyPeriodic[i] = false;
456 VertexSequence *vseq = dynamic_cast<VertexSequence *>(seq1);
457 if (vseq) vertDat = dynamic_cast<ScdVertexData*>(vseq->data());
458 if (vertDat) {
459 // retrieve the parametric space
460 for (int i = 0; i < 3; i++) {
461 boxDims[i] = vertDat->min_params()[i];
462 boxDims[3+i] = vertDat->max_params()[i];
463 }
464 startVertex = vertDat->start_handle();
465 }
466 else if (impl->boxDimsTag) {
467 // look for parametric space info on set
468 ErrorCode rval = impl->mbImpl->tag_get_data(impl->boxDimsTag, &bset, 1, boxDims);
469 if (MB_SUCCESS == rval) {
470 Range verts;
471 impl->mbImpl->get_entities_by_dimension(bset, 0, verts);
472 if (!verts.empty()) startVertex = *verts.begin();
473 }
474 }
475
476 elemSeq = dynamic_cast<StructuredElementSeq *>(seq2);
477 if (!elemSeq)
478 elemSeq = dynamic_cast<StructuredElementSeq *>(seq1);
479
480 if (elemSeq) {
481 if (vertDat) {
482 // check the parametric space to make sure it's consistent
483 assert(elemSeq->sdata()->min_params() == HomCoord(boxDims, 3) &&
484 (elemSeq->sdata()->max_params() + HomCoord(1, 1, 1)) == HomCoord(boxDims, 3));
485
486 }
487 else {
488 // get the parametric space from the element sequence
489 for (int i = 0; i < 3; i++) {
490 boxDims[i] = elemSeq->sdata()->min_params()[i];
491 boxDims[3+i] = elemSeq->sdata()->max_params()[i];
492 }
493 }
494
495 startElem = elemSeq->start_handle();
496 }
497 else {
498 Range elems;
499 impl->mbImpl->get_entities_by_dimension(bset, (boxDims[2] == boxDims[5] ? (boxDims[1] == boxDims[4] ? 1 : 2) : 3), elems);
500 if (!elems.empty()) startElem = *elems.begin();
501 // call the following w/o looking at return value, since it doesn't really need to be there
502 if (impl->boxPeriodicTag)
503 impl->mbImpl->tag_get_data(impl->boxPeriodicTag, &bset, 1, locallyPeriodic);
504 }
505
506 assert(vertDat || elemSeq ||
507 boxDims[0] != boxDims[3]|| boxDims[1] != boxDims[4]|| boxDims[2] != boxDims[5]);
508
509 boxSize = HomCoord(boxDims+3, 3) - HomCoord(boxDims, 3) + HomCoord(1, 1, 1);
510 boxSizeIJ = (boxSize[1] ? boxSize[1] : 1) * boxSize[0];
511 boxSizeIM1 = boxSize[0]-(locallyPeriodic[0] ? 0 : 1);
512 boxSizeIJM1 = (boxSize[1] ? (boxSize[1]-(locallyPeriodic[1] ? 0 : 1)) : 1) * boxSizeIM1;
513
514 scImpl->add_box(this);
515 }
516
~ScdBox()517 ScdBox::~ScdBox()
518 {
519 // Reset the tag on the set
520 if (boxSet) {
521 // It is possible that the box set entity has been deleted (e.g. by Core::clean_up_failed_read)
522 Core* mbcore = dynamic_cast<Core*>(scImpl->mbImpl);
523 assert(mbcore != NULL);
524 if (mbcore->is_valid(boxSet)) {
525 ScdBox* tmp_ptr = NULL;
526 scImpl->mbImpl->tag_set_data(scImpl->box_set_tag(), &boxSet, 1, &tmp_ptr);
527 }
528 else
529 boxSet = 0;
530 }
531
532 scImpl->remove_box(this);
533 }
534
get_vertex_from_seq(int i,int j,int k) const535 EntityHandle ScdBox::get_vertex_from_seq(int i, int j, int k) const
536 {
537 assert(elemSeq);
538 return elemSeq->get_vertex(i, j, k);
539 }
540
box_dimension() const541 int ScdBox::box_dimension() const
542 {
543 return (startElem ? scImpl->mbImpl->dimension_from_handle(startElem) : -1);
544 }
545
add_vbox(ScdBox * vbox,HomCoord from1,HomCoord to1,HomCoord from2,HomCoord to2,HomCoord from3,HomCoord to3,bool bb_input,const HomCoord & bb_min,const HomCoord & bb_max)546 ErrorCode ScdBox::add_vbox(ScdBox *vbox,
547 HomCoord from1, HomCoord to1,
548 HomCoord from2, HomCoord to2,
549 HomCoord from3, HomCoord to3,
550 bool bb_input,
551 const HomCoord &bb_min,
552 const HomCoord &bb_max)
553 {
554 if (!vbox->vertDat) return MB_FAILURE;
555 ScdVertexData *dum_data = dynamic_cast<ScdVertexData*>(vbox->vertDat);
556 ErrorCode rval = elemSeq->sdata()->add_vsequence(dum_data, from1, to1, from2, to2, from3, to3,
557 bb_input, bb_min, bb_max);
558 return rval;
559 }
560
boundary_complete() const561 bool ScdBox::boundary_complete() const
562 {
563 return elemSeq->boundary_complete();
564 }
565
get_coordinate_arrays(double * & xc,double * & yc,double * & zc)566 ErrorCode ScdBox::get_coordinate_arrays(double *&xc, double *&yc, double *&zc)
567 {
568 if (!vertDat) return MB_FAILURE;
569
570 xc = reinterpret_cast<double*>(vertDat->get_sequence_data(0));
571 yc = reinterpret_cast<double*>(vertDat->get_sequence_data(1));
572 zc = reinterpret_cast<double*>(vertDat->get_sequence_data(2));
573 return MB_SUCCESS;
574 }
575
get_coordinate_arrays(const double * & xc,const double * & yc,const double * & zc) const576 ErrorCode ScdBox::get_coordinate_arrays(const double *&xc, const double *&yc, const double *&zc) const
577 {
578 if (!vertDat) return MB_FAILURE;
579 xc = reinterpret_cast<const double*>(vertDat->get_sequence_data(0));
580 yc = reinterpret_cast<const double*>(vertDat->get_sequence_data(1));
581 zc = reinterpret_cast<const double*>(vertDat->get_sequence_data(2));
582 return MB_SUCCESS;
583 }
584
vert_dat(ScdVertexData * vert_dt)585 ErrorCode ScdBox::vert_dat(ScdVertexData *vert_dt)
586 {
587 vertDat = vert_dt;
588 return MB_SUCCESS;
589 }
590
elem_seq(EntitySequence * elem_sq)591 ErrorCode ScdBox::elem_seq(EntitySequence *elem_sq)
592 {
593 elemSeq = dynamic_cast<StructuredElementSeq*>(elem_sq);
594 if (elemSeq) elemSeq->is_periodic(locallyPeriodic);
595
596 if (locallyPeriodic[0])
597 boxSizeIM1 = boxSize[0]-(locallyPeriodic[0] ? 0 : 1);
598 if (locallyPeriodic[0] || locallyPeriodic[1])
599 boxSizeIJM1 = (boxSize[1] ? (boxSize[1]-(locallyPeriodic[1] ? 0 : 1)) : 1) * boxSizeIM1;
600
601 return (elemSeq ? MB_SUCCESS : MB_FAILURE);
602 }
603
get_params(EntityHandle ent,HomCoord & ijkd) const604 ErrorCode ScdBox::get_params(EntityHandle ent, HomCoord &ijkd) const
605 {
606 // check first whether this is an intermediate entity, so we know what to do
607 int dimension = box_dimension();
608 int this_dim = scImpl->impl()->dimension_from_handle(ent);
609
610 if ((0 == this_dim && !vertDat) ||
611 (this_dim && this_dim == dimension)) {
612 assert(elemSeq);
613 return elemSeq->get_params(ent, ijkd[0], ijkd[1], ijkd[2]);
614 }
615
616 else if (!this_dim && vertDat)
617 return vertDat->get_params(ent, ijkd[0], ijkd[1], ijkd[2]);
618
619 else return MB_NOT_IMPLEMENTED;
620 }
621
622 //! Get the entity of specified dimension adjacent to parametric element
623 /**
624 * \param dim Dimension of adjacent entity being requested
625 * \param i Parametric coordinates of cell being evaluated
626 * \param j Parametric coordinates of cell being evaluated
627 * \param k Parametric coordinates of cell being evaluated
628 * \param dir Direction (0, 1, or 2), for getting adjacent edges (2d, 3d) or faces (3d)
629 * \param ent EntityHandle of adjacent entity
630 * \param create_if_missing If true, creates the entity if it doesn't already exist
631 */
get_adj_edge_or_face(int dim,int i,int j,int k,int dir,EntityHandle & ent,bool create_if_missing) const632 ErrorCode ScdBox::get_adj_edge_or_face(int dim, int i, int j, int k, int dir, EntityHandle &ent,
633 bool create_if_missing) const
634 {
635 // describe connectivity of sub-element in static array
636 // subconnect[dim-1][dir][numv][ijk] where dimensions are:
637 // [dim-1]: dim=1 or 2, so this is 0 or 1
638 // [dir]: one of 0..2, for ijk directions in a hex
639 // [numv]: number of vertices describing sub entity = 2*dim <= 4
640 // [ijk]: 3 values for i, j, k
641 int subconnect[2][3][4][3] = {
642 {{{0, 0, 0}, {1, 0, 0}, {-1, -1, -1}, {-1, -1, -1}}, // i edge
643 {{0, 0, 0}, {0, 1, 0}, {-1, -1, -1}, {-1, -1, -1}}, // j edge
644 {{0, 0, 0}, {0, 0, 1}, {-1, -1, -1}, {-1, -1, -1}}}, // k edge
645
646 {{{0, 0, 0}, {0, 1, 0}, {0, 1, 1}, {0, 0, 1}}, // i face
647 {{0, 0, 0}, {1, 0, 0}, {1, 0, 1}, {0, 0, 1}}, // j face
648 {{0, 0, 0}, {1, 0, 0}, {1, 1, 0}, {0, 1, 0}}}}; // k face
649
650 // check proper input dimensions and lower bound
651 if (dim < 1 || dim > 2 || i < boxDims[0] || j < boxDims[1] || k < boxDims[2])
652 return MB_FAILURE;
653
654 // now check upper bound; parameters must be <= upper corner, since edges/faces
655 // follow element parameterization, not vertex parameterization
656 else if ((boxDims[3] != boxDims[0] && i > (locallyPeriodic[0] ? boxDims[3]+1 : boxDims[3])) ||
657 (boxDims[4] != boxDims[1] && j > (locallyPeriodic[1] ? boxDims[4]+1 : boxDims[4])) ||
658 (boxDims[5] != boxDims[2] && k > boxDims[5])) return MB_FAILURE;
659
660 // get the vertices making up this entity
661 EntityHandle verts[4];
662 for (int ind = 0; ind < 2*dim; ind++) {
663 int i1=i+subconnect[dim-1][dir][ind][0];
664 int j1=j+subconnect[dim-1][dir][ind][1];
665 // if periodic in i and i1 is boxDims[3]+1, wrap around
666 if (locallyPeriodic[0] && i1==boxDims[3]+1) i1=boxDims[0];
667 // if periodic in j and j1 is boxDims[4]+1, wrap around
668 if (locallyPeriodic[1] && j1==boxDims[4]+1) j1=boxDims[1];
669 verts[ind] = get_vertex(i1,
670 j1,
671 k+subconnect[dim-1][dir][ind][2]);
672 if (!verts[ind]) return MB_FAILURE;
673 }
674
675 Range ents;
676 ErrorCode rval = scImpl->impl()->get_adjacencies(verts, 2*dim, dim, false, ents);
677 if (MB_SUCCESS != rval) return rval;
678
679 if (ents.size() > 1) return MB_FAILURE;
680
681 else if (ents.size() == 1) {
682 ent = *ents.begin();
683 }
684 else if (create_if_missing)
685 rval = scImpl->impl()->create_element((1 == dim ? MBEDGE : MBQUAD), verts, 2*dim, ent);
686
687 return rval;
688 }
689
690 #ifndef MOAB_HAVE_MPI
tag_shared_vertices(ParallelComm *,ScdBox *)691 ErrorCode ScdInterface::tag_shared_vertices(ParallelComm *, ScdBox *)
692 {
693 return MB_FAILURE;
694 #else
695 ErrorCode ScdInterface::tag_shared_vertices(ParallelComm *pcomm, ScdBox *box)
696 {
697 EntityHandle seth = box->box_set();
698
699 // check the # ents in the box against the num in the set, to make sure it's only 1 box;
700 // reuse tmp_range
701 Range tmp_range;
702 ErrorCode rval = mbImpl->get_entities_by_dimension(seth, box->box_dimension(), tmp_range);
703 if (MB_SUCCESS != rval) return rval;
704 if (box->num_elements() != (int)tmp_range.size()) return MB_FAILURE;
705
706 const int *gdims = box->par_data().gDims;
707 if ((gdims[0] == gdims[3] && gdims[1] == gdims[4] && gdims[2] == gdims[5]) ||
708 -1 == box->par_data().partMethod) return MB_FAILURE;
709
710 // ok, we have a partitioned box; get the vertices shared with other processors
711 std::vector<int> procs, offsets, shared_indices;
712 rval = get_shared_vertices(pcomm, box, procs, offsets, shared_indices);
713 if (MB_SUCCESS != rval) return rval;
714
715 // post receives for start handles once we know how many to look for
716 std::vector<MPI_Request> recv_reqs(procs.size(), MPI_REQUEST_NULL),
717 send_reqs(procs.size(), MPI_REQUEST_NULL);
718 std::vector<EntityHandle> rhandles(4*procs.size()), shandles(4);
719 for (unsigned int i = 0; i < procs.size(); i++) {
720 int success = MPI_Irecv((void*)&rhandles[4*i], 4*sizeof(EntityHandle),
721 MPI_UNSIGNED_CHAR, procs[i],
722 1, pcomm->proc_config().proc_comm(),
723 &recv_reqs[i]);
724 if (success != MPI_SUCCESS) return MB_FAILURE;
725 }
726
727 // send our own start handles
728 shandles[0] = box->start_vertex();
729 shandles[1] = 0;
730 if (box->box_dimension() == 1) {
731 shandles[1] = box->start_element();
732 shandles[2] = 0;
733 shandles[3] = 0;
734 } else if (box->box_dimension() == 2) {
735 shandles[2] = box->start_element();
736 shandles[3] = 0;
737 }
738 else {
739 shandles[2] = 0;
740 shandles[3] = box->start_element();
741 }
742 for (unsigned int i = 0; i < procs.size(); i++) {
743 int success = MPI_Isend((void*)&shandles[0], 4*sizeof(EntityHandle), MPI_UNSIGNED_CHAR, procs[i],
744 1, pcomm->proc_config().proc_comm(), &send_reqs[i]);
745 if (success != MPI_SUCCESS) return MB_FAILURE;
746 }
747
748 // receive start handles and save info to a tuple list
749 int incoming = procs.size();
750 int p, j, k;
751 MPI_Status status;
752 TupleList shared_data;
753 shared_data.initialize(1, 0, 2, 0,
754 shared_indices.size()/2);
755 shared_data.enableWriteAccess();
756
757 j = 0; k = 0;
758 while (incoming) {
759 int success = MPI_Waitany(procs.size(), &recv_reqs[0], &p, &status);
760 if (MPI_SUCCESS != success) return MB_FAILURE;
761 unsigned int num_indices = (offsets[p+1]-offsets[p])/2;
762 int *lh = &shared_indices[offsets[p]], *rh = lh + num_indices;
763 for (unsigned int i = 0; i < num_indices; i++) {
764 shared_data.vi_wr[j++] = procs[p];
765 shared_data.vul_wr[k++] = shandles[0] + lh[i];
766 shared_data.vul_wr[k++] = rhandles[4*p] + rh[i];
767 shared_data.inc_n();
768 }
769 incoming--;
770 }
771
772 // still need to wait for the send requests
773 std::vector<MPI_Status> mult_status(procs.size());
774 int success = MPI_Waitall(procs.size(), &send_reqs[0], &mult_status[0]);
775 if (MPI_SUCCESS != success) {
776 MB_SET_ERR(MB_FAILURE, "Failed in waitall in ScdInterface::tag_shared_vertices");
777 }
778 // sort by local handle
779 TupleList::buffer sort_buffer;
780 sort_buffer.buffer_init(shared_indices.size()/2);
781 shared_data.sort(1, &sort_buffer);
782 sort_buffer.reset();
783
784 // process into sharing data
785 std::map<std::vector<int>, std::vector<EntityHandle> > proc_nvecs;
786 Range dum;
787 rval = pcomm->tag_shared_verts(shared_data, proc_nvecs, dum, 0);
788 if (MB_SUCCESS != rval) return rval;
789
790 // create interface sets
791 rval = pcomm->create_interface_sets(proc_nvecs);
792 if (MB_SUCCESS != rval) return rval;
793
794 // add the box to the PComm's partitionSets
795 pcomm->partition_sets().insert(box->box_set());
796
797 // make sure buffers are allocated for communicating procs
798 for (std::vector<int>::iterator pit = procs.begin(); pit != procs.end(); ++pit)
799 pcomm->get_buffers(*pit);
800
801 if (pcomm->get_debug_verbosity() > 1) pcomm->list_entities(NULL, 1);
802
803 #ifndef NDEBUG
804 rval = pcomm->check_all_shared_handles();
805 if (MB_SUCCESS != rval) return rval;
806 #endif
807
808 return MB_SUCCESS;
809
810 #endif
811 }
812
813 ErrorCode ScdInterface::get_neighbor_alljkbal(int np, int pfrom,
814 const int * const gdims, const int * const gperiodic, const int * const dijk,
815 int &pto, int *rdims, int *facedims, int *across_bdy)
816 {
817 if (dijk[0] != 0) {
818 pto = -1;
819 return MB_SUCCESS;
820 }
821
822 pto = -1;
823 across_bdy[0] = across_bdy[1] = across_bdy[2] = 0;
824
825 int ldims[6], pijk[3], lperiodic[3];
826 ErrorCode rval = compute_partition_alljkbal(np, pfrom, gdims, gperiodic,
827 ldims, lperiodic, pijk);
828 if (MB_SUCCESS != rval) return rval;
829 assert(pijk[1] * pijk[2] == np);
830 pto = -1;
831 bool bot_j = pfrom < pijk[2],
832 top_j = pfrom > np - pijk[2];
833 if ((1 == pijk[2] && dijk[2]) || // 1d in j means no neighbors with dk != 0
834 (!(pfrom%pijk[2]) && -1 == dijk[2]) || // at -k bdy
835 (pfrom%pijk[2] == pijk[2]-1 && 1 == dijk[2]) || // at +k bdy
836 (pfrom < pijk[2] && -1 == dijk[1] && !gperiodic[1]) || // down and not periodic
837 (pfrom >= np-pijk[2] && 1 == dijk[1] && !gperiodic[1])) // up and not periodic
838 return MB_SUCCESS;
839
840 pto = pfrom;
841 std::copy(ldims, ldims+6, rdims);
842 std::copy(ldims, ldims+6, facedims);
843
844 if (0 != dijk[1]) {
845 pto = (pto + dijk[1]*pijk[2] + np) % np;
846 assert (pto >= 0 && pto < np);
847 int dj = (gdims[4] - gdims[1]) / pijk[1], extra = (gdims[4] - gdims[1]) % pijk[1];
848 if (-1 == dijk[1]) {
849 facedims[4] = facedims[1];
850 if (bot_j) {
851 // going across periodic lower bdy in j
852 rdims[4] = gdims[4];
853 across_bdy[1] = -1;
854 }
855 else {
856 rdims[4] = ldims[1];
857 }
858 rdims[1] = rdims[4] - dj;
859 if (pto < extra) rdims[1]--;
860 }
861 else {
862 if (pfrom > np-pijk[2]) facedims[4] = gdims[1];
863 facedims[1] = facedims[4];
864 if (top_j) {
865 // going across periodic upper bdy in j
866 rdims[1] = gdims[1];
867 across_bdy[1] = 1;
868 }
869 else {
870 rdims[1] = ldims[4];
871 }
872 rdims[4] = rdims[1] + dj;
873 if (pto < extra) rdims[4]++;
874 }
875 }
876 if (0 != dijk[2]) {
877 pto = (pto + dijk[2]) % np;
878 assert (pto >= 0 && pto < np);
879 facedims[2] = facedims[5] = (-1 == dijk[2] ? facedims[2] : facedims[5]);
880 int dk = (gdims[5] - gdims[2]) / pijk[2];
881 if (-1 == dijk[2]) {
882 facedims[5] = facedims[2];
883 rdims[5] = ldims[2];
884 rdims[2] = rdims[5] - dk; // never any kextra for alljkbal
885 }
886 else {
887 facedims[2] = facedims[5];
888 rdims[2] = ldims[5];
889 rdims[5] = rdims[2] + dk; // never any kextra for alljkbal
890 }
891 }
892
893 assert(-1 == pto || (rdims[0] >= gdims[0] && rdims[3] <= gdims[3]));
894 assert(-1 == pto || (rdims[1] >= gdims[1] && (rdims[4] <= gdims[4] || (across_bdy[1] && bot_j))));
895 assert(-1 == pto || (rdims[2] >= gdims[2] && rdims[5] <= gdims[5]));
896 assert(-1 == pto || ((facedims[0] >= rdims[0] || (gperiodic[0] && rdims[3] == gdims[3]+1 && facedims[0] == gdims[0]))));
897 assert(-1 == pto || (facedims[3] <= rdims[3]));
898 assert(-1 == pto || ((facedims[1] >= rdims[1] || (gperiodic[1] && rdims[4] == gdims[4]+1 && facedims[1] == gdims[1]))));
899 assert(-1 == pto || (facedims[4] <= rdims[4]));
900 assert(-1 == pto || (facedims[2] >= rdims[2]));
901 assert(-1 == pto || (facedims[5] <= rdims[5]));
902 assert(-1 == pto || (facedims[0] >= ldims[0]));
903 assert(-1 == pto || (facedims[3] <= ldims[3]));
904 assert(-1 == pto || (facedims[1] >= ldims[1]));
905 assert(-1 == pto || (facedims[4] <= ldims[4]));
906 assert(-1 == pto || (facedims[2] >= ldims[2]));
907 assert(-1 == pto || (facedims[5] <= ldims[5]));
908
909 return MB_SUCCESS;
910 }
911
912 ErrorCode ScdInterface::get_neighbor_sqij(int np, int pfrom,
913 const int * const gdims, const int * const gperiodic, const int * const dijk,
914 int &pto, int *rdims, int *facedims, int *across_bdy)
915 {
916 if (dijk[2] != 0) {
917 // for sqij, there is no k neighbor, ever
918 pto = -1;
919 return MB_SUCCESS;
920 }
921
922 pto = -1;
923 across_bdy[0] = across_bdy[1] = across_bdy[2] = 0;
924 int lperiodic[3], pijk[3], ldims[6];
925 ErrorCode rval = compute_partition_sqij(np, pfrom, gdims, gperiodic, ldims, lperiodic, pijk);
926 if (MB_SUCCESS != rval) return rval;
927 assert(pijk[0] * pijk[1] == np);
928 pto = -1;
929 bool top_i = 0, top_j = 0, bot_i = 0, bot_j = 0;
930 int ni = pfrom%pijk[0], nj = pfrom/pijk[0]; // row / column number of me
931 if (ni == pijk[0]-1) top_i = 1;
932 if (nj == pijk[1]-1) top_j = 1;
933 if (!ni) bot_i = 1;
934 if (!nj) bot_j = 1;
935 if ((!gperiodic[0] && bot_i && -1 == dijk[0]) || // left and not periodic
936 (!gperiodic[0] && top_i && 1 == dijk[0]) || // right and not periodic
937 (!gperiodic[1] && bot_j && -1 == dijk[1]) || // bottom and not periodic
938 (!gperiodic[1] && top_j && 1 == dijk[1])) // top and not periodic
939 return MB_SUCCESS;
940
941 std::copy(ldims, ldims+6, facedims);
942 std::copy(ldims, ldims+6, rdims);
943 pto = pfrom;
944 int j = gdims[4] - gdims[1], dj = j / pijk[1], jextra = (gdims[4] - gdims[1]) % dj,
945 i = gdims[3] - gdims[0], di = i / pijk[0], iextra = (gdims[3] - gdims[0]) % di;
946
947 if (0 != dijk[0]) {
948 pto = (ni + dijk[0] + pijk[0]) % pijk[0]; // get pto's ni value
949 pto = nj*pijk[0] + pto; // then convert to pto
950 assert (pto >= 0 && pto < np);
951 if (-1 == dijk[0]) {
952 facedims[3] = facedims[0];
953 if (bot_i) {
954 // going across lower periodic bdy in i
955 across_bdy[0] = -1;
956 rdims[3] = gdims[3]+1; // +1 because ldims[3] on remote proc is gdims[3]+1
957 rdims[0] = rdims[3] - di - 1; // -1 to account for rdims[3] being one larger
958 }
959 else {
960 rdims[3] = ldims[0];
961 rdims[0] = rdims[3] - di;
962 }
963
964 if (pto%pijk[0] < iextra) rdims[0]--;
965 }
966 else {
967 if (top_i) {
968 // going across lower periodic bdy in i
969 facedims[3] = gdims[0];
970 across_bdy[0] = 1;
971 }
972 facedims[0] = facedims[3];
973 rdims[0] = (top_i ? gdims[0] : ldims[3]);
974 rdims[3] = rdims[0] + di;
975 if (pto%pijk[0] < iextra) rdims[3]++;
976 if (gperiodic[0] && ni == pijk[0]-2) rdims[3]++; // remote proc is top_i and periodic
977 }
978 }
979 if (0 != dijk[1]) {
980 pto = (pto + dijk[1]*pijk[0] + np) % np;
981 assert (pto >= 0 && pto < np);
982 if (-1 == dijk[1]) {
983 facedims[4] = facedims[1];
984 if (bot_j) {
985 // going across lower periodic bdy in j
986 rdims[4] = gdims[4]+1; // +1 because ldims[4] on remote proc is gdims[4]+1
987 rdims[1] = rdims[4] - dj - 1; // -1 to account for gdims[4] being one larger
988 across_bdy[1] = -1;
989 }
990 else {
991 rdims[4] = ldims[1];
992 rdims[1] = rdims[4] - dj;
993 }
994 if (pto/pijk[0] < jextra) rdims[1]--;
995 }
996 else {
997 if (top_j) {
998 // going across lower periodic bdy in j
999 facedims[4] = gdims[1];
1000 rdims[1] = gdims[1];
1001 across_bdy[1] = 1;
1002 }
1003 else {
1004 rdims[1] = ldims[4];
1005 }
1006 facedims[1] = facedims[4];
1007 rdims[4] = rdims[1] + dj;
1008 if (nj+1 < jextra) rdims[4]++;
1009 if (gperiodic[1] && nj == pijk[1]-2) rdims[4]++; // remote proc is top_j and periodic
1010 }
1011 }
1012
1013 // rdims within gdims
1014 assert (-1 == pto || (rdims[0] >= gdims[0] && (rdims[3] <= gdims[3] + (gperiodic[0] && pto%pijk[0] == pijk[0]-1 ? 1 : 0))));
1015 assert (-1 == pto || (rdims[1] >= gdims[1] && (rdims[4] <= gdims[4] + (gperiodic[1] && pto/pijk[0] == pijk[1]-1 ? 1 : 0))));
1016 assert (-1 == pto || (rdims[2] >= gdims[2] && rdims[5] <= gdims[5]));
1017 // facedims within rdims
1018 assert (-1 == pto || ((facedims[0] >= rdims[0] || (gperiodic[0] && pto%pijk[0] == pijk[0]-1 && facedims[0] == gdims[0]))));
1019 assert (-1 == pto || (facedims[3] <= rdims[3]));
1020 assert (-1 == pto || ((facedims[1] >= rdims[1] || (gperiodic[1] && pto/pijk[0] == pijk[1]-1 && facedims[1] == gdims[1]))));
1021 assert (-1 == pto || (facedims[4] <= rdims[4]));
1022 assert (-1 == pto || (facedims[2] >= rdims[2] && facedims[5] <= rdims[5]));
1023 // facedims within ldims
1024 assert (-1 == pto || ((facedims[0] >= ldims[0] || (top_i && facedims[0] == gdims[0]))));
1025 assert (-1 == pto || (facedims[3] <= ldims[3]));
1026 assert (-1 == pto || ((facedims[1] >= ldims[1] || (gperiodic[1] && top_j && facedims[1] == gdims[1]))));
1027 assert (-1 == pto || (facedims[4] <= ldims[4]));
1028 assert (-1 == pto || (facedims[2] >= ldims[2] && facedims[5] <= ldims[5]));
1029
1030 return MB_SUCCESS;
1031 }
1032
1033 ErrorCode ScdInterface::get_neighbor_sqjk(int np, int pfrom,
1034 const int * const gdims, const int * const gperiodic, const int * const dijk,
1035 int &pto, int *rdims, int *facedims, int *across_bdy)
1036 {
1037 if (dijk[0] != 0) {
1038 pto = -1;
1039 return MB_SUCCESS;
1040 }
1041
1042 pto = -1;
1043 across_bdy[0] = across_bdy[1] = across_bdy[2] = 0;
1044 int pijk[3], lperiodic[3], ldims[6];
1045 ErrorCode rval = compute_partition_sqjk(np, pfrom, gdims, gperiodic, ldims, lperiodic, pijk);
1046 if (MB_SUCCESS != rval) return rval;
1047 assert(pijk[1] * pijk[2] == np);
1048 pto = -1;
1049 bool top_j = 0, top_k = 0, bot_j = 0, bot_k = 0;
1050 int nj = pfrom%pijk[1], nk = pfrom/pijk[1];
1051 if (nj == pijk[1]-1) top_j = 1;
1052 if (nk == pijk[2]-1) top_k = 1;
1053 if (!nj) bot_j = 1;
1054 if (!nk) bot_k = 1;
1055 if ((!gperiodic[1] && bot_j && -1 == dijk[1]) || // down and not periodic
1056 (!gperiodic[1] && top_j && 1 == dijk[1]) || // up and not periodic
1057 (bot_k && -1 == dijk[2]) || // k- bdy
1058 (top_k && 1 == dijk[2])) // k+ bdy
1059 return MB_SUCCESS;
1060
1061 std::copy(ldims, ldims+6, facedims);
1062 std::copy(ldims, ldims+6, rdims);
1063 pto = pfrom;
1064 int dj = (gdims[4] - gdims[1]) / pijk[1], jextra = (gdims[4] - gdims[1]) % dj,
1065 dk = (gdims[5] == gdims[2] ? 0 : (gdims[5] - gdims[2]) / pijk[2]), kextra = (gdims[5] - gdims[2]) - dk*pijk[2];
1066 assert((dj*pijk[1] + jextra == (gdims[4]-gdims[1])) && (dk*pijk[2] + kextra == (gdims[5]-gdims[2])));
1067 if (0 != dijk[1]) {
1068 pto = (nj + dijk[1] + pijk[1]) % pijk[1]; // get pto's ni value
1069 pto = nk*pijk[1] + pto; // then convert to pto
1070 assert (pto >= 0 && pto < np);
1071 if (-1 == dijk[1]) {
1072 facedims[4] = facedims[1];
1073 if (bot_j) {
1074 // going across lower periodic bdy in j
1075 rdims[4] = gdims[4]+1; // +1 because ldims[4] on remote proc is gdims[4]+1
1076 across_bdy[1] = -1;
1077 }
1078 else {
1079 rdims[4] = ldims[1];
1080 }
1081 rdims[1] = rdims[4] - dj;
1082 if (nj < jextra) rdims[1]--;
1083 }
1084 else {
1085 if (top_j) {
1086 // going across upper periodic bdy in j
1087 rdims[1] = gdims[1];
1088 facedims[4] = gdims[1];
1089 across_bdy[1] = 1;
1090 }
1091 else {
1092 rdims[1] = ldims[4];
1093 }
1094 facedims[1] = facedims[4];
1095 rdims[4] = rdims[1] + dj;
1096 if (nj < jextra) rdims[4]++;
1097 if (gperiodic[1] && nj == dijk[1]-2) rdims[4]++; // +1 because next proc is on periodic bdy
1098 }
1099 }
1100 if (0 != dijk[2]) {
1101 pto = (pto + dijk[2]*pijk[1] + np) % np;
1102 assert (pto >= 0 && pto < np);
1103 if (-1 == dijk[2]) {
1104 facedims[5] = facedims[2];
1105 rdims[5] = ldims[2];
1106 rdims[2] -= dk;
1107 if (pto/pijk[1] < kextra) rdims[2]--;
1108 }
1109 else {
1110 facedims[2] = facedims[5];
1111 rdims[2] = ldims[5];
1112 rdims[5] += dk;
1113 if (pto/pijk[1] < kextra) rdims[5]++;
1114 }
1115 }
1116
1117 assert(-1 == pto || (rdims[0] >= gdims[0] && rdims[3] <= gdims[3]));
1118 assert(-1 == pto || (rdims[1] >= gdims[1] && (rdims[4] <= gdims[4] || (across_bdy[1] && bot_j))));
1119 assert(-1 == pto || (rdims[2] >= gdims[2] && rdims[5] <= gdims[5]));
1120 assert(-1 == pto || (facedims[0] >= rdims[0] && facedims[3] <= rdims[3]));
1121 assert(-1 == pto || ((facedims[1] >= rdims[1] || (gperiodic[1] && rdims[4] == gdims[4] && facedims[1] == gdims[1]))));
1122 assert(-1 == pto || (facedims[4] <= rdims[4]));
1123 assert(-1 == pto || (facedims[2] >= rdims[2] && facedims[5] <= rdims[5]));
1124 assert(-1 == pto || (facedims[0] >= ldims[0] && facedims[3] <= ldims[3]));
1125 assert(-1 == pto || (facedims[1] >= ldims[1] && facedims[4] <= ldims[4]));
1126 assert(-1 == pto || (facedims[2] >= ldims[2] && facedims[5] <= ldims[5]));
1127
1128 return MB_SUCCESS;
1129 }
1130
1131 ErrorCode ScdInterface::get_neighbor_sqijk(int np, int pfrom,
1132 const int * const gdims, const int * const gperiodic, const int * const dijk,
1133 int &pto, int *rdims, int *facedims, int *across_bdy)
1134 {
1135 if (gperiodic[0] || gperiodic[1] || gperiodic[2]) return MB_FAILURE;
1136
1137 pto = -1;
1138 across_bdy[0] = across_bdy[1] = across_bdy[2] = 0;
1139 int pijk[3], lperiodic[3], ldims[6];
1140 ErrorCode rval = compute_partition_sqijk(np, pfrom, gdims, gperiodic, ldims, lperiodic, pijk);
1141 if (MB_SUCCESS != rval) return rval;
1142 assert(pijk[0] * pijk[1] * pijk[2] == np);
1143 pto = -1;
1144 bool top[3] = {false, false, false}, bot[3] = {false, false, false};
1145 // nijk: rank in i/j/k direction
1146 int nijk[3] = {pfrom%pijk[0], (pfrom%(pijk[0]*pijk[1]))/pijk[0], pfrom/(pijk[0]*pijk[1])};
1147
1148 for (int i = 0; i < 3; i++) {
1149 if (nijk[i] == pijk[i]-1) top[i] = true;
1150 if (!nijk[i]) bot[i] = true;
1151 if ((!gperiodic[i] && bot[i] && -1 == dijk[i]) || // downward && not periodic
1152 (!gperiodic[i] && top[i] && 1 == dijk[i])) // upward && not periodic
1153 return MB_SUCCESS;
1154 }
1155
1156 std::copy(ldims, ldims+6, facedims);
1157 std::copy(ldims, ldims+6, rdims);
1158 pto = pfrom;
1159 int delijk[3], extra[3];
1160 // nijk_to: rank of pto in i/j/k direction
1161 int nijk_to[3];
1162 for (int i = 0; i < 3; i++) {
1163 delijk[i] = (gdims[i+3] == gdims[i] ? 0 : (gdims[i+3] - gdims[i])/pijk[i]);
1164 extra[i] = (gdims[i+3]-gdims[i]) % delijk[i];
1165 nijk_to[i] = (nijk[i]+dijk[i]+pijk[i]) % pijk[i];
1166 }
1167 pto = nijk_to[2]*pijk[0]*pijk[1] + nijk_to[1]*pijk[0] + nijk_to[0];
1168 assert (pto >= 0 && pto < np);
1169 for (int i = 0; i < 3; i++) {
1170 if (0 != dijk[i]) {
1171 if (-1 == dijk[i]) {
1172 facedims[i+3] = facedims[i];
1173 if (bot[i]) {
1174 // going across lower periodic bdy in i
1175 rdims[i+3] = gdims[i+3]+1; // +1 because ldims[4] on remote proc is gdims[4]+1
1176 across_bdy[i] = -1;
1177 }
1178 else {
1179 rdims[i+3] = ldims[i];
1180 }
1181 rdims[i] = rdims[i+3] - delijk[i];
1182 if (nijk[i] < extra[i]) rdims[i]--;
1183 }
1184 else {
1185 if (top[i]) {
1186 // going across upper periodic bdy in i
1187 rdims[i] = gdims[i];
1188 facedims[i+3] = gdims[i];
1189 across_bdy[i] = 1;
1190 }
1191 else {
1192 rdims[i] = ldims[i+3];
1193 }
1194 facedims[i] = facedims[i+3];
1195 rdims[i+3] = rdims[i] + delijk[i];
1196 if (nijk[i] < extra[i]) rdims[i+3]++;
1197 if (gperiodic[i] && nijk[i] == dijk[i]-2) rdims[i+3]++; // +1 because next proc is on periodic bdy
1198 }
1199 }
1200 }
1201
1202 assert(-1 != pto);
1203 #ifndef NDEBUG
1204 for (int i = 0; i < 3; i++) {
1205 assert((rdims[i] >= gdims[i] && (rdims[i+3] <= gdims[i+3] || (across_bdy[i] && bot[i]))));
1206 assert(((facedims[i] >= rdims[i] || (gperiodic[i] && rdims[i+3] == gdims[i+3] && facedims[i] == gdims[i]))));
1207 assert((facedims[i] >= ldims[i] && facedims[i+3] <= ldims[i+3]));
1208 }
1209 #endif
1210
1211 return MB_SUCCESS;
1212 }
1213
1214 ErrorCode ScdInterface::get_neighbor_alljorkori(int np, int pfrom,
1215 const int * const gdims, const int * const gperiodic, const int * const dijk,
1216 int &pto, int *rdims, int *facedims, int *across_bdy)
1217 {
1218 ErrorCode rval = MB_SUCCESS;
1219 pto = -1;
1220 if (np == 1) return MB_SUCCESS;
1221
1222 int pijk[3], lperiodic[3], ldims[6];
1223 rval = compute_partition_alljorkori(np, pfrom, gdims, gperiodic, ldims, lperiodic, pijk);
1224 if (MB_SUCCESS != rval) return rval;
1225
1226 int ind = -1;
1227 across_bdy[0] = across_bdy[1] = across_bdy[2] = 0;
1228
1229 for (int i = 0; i < 3; i++) {
1230 if (pijk[i] > 1) {
1231 ind = i;
1232 break;
1233 }
1234 }
1235
1236 assert(-1 < ind);
1237
1238 if (!dijk[ind])
1239 // no neighbor, pto is already -1, return
1240 return MB_SUCCESS;
1241
1242 bool is_periodic = ((gperiodic[0] && ind == 0) || (gperiodic[1] && ind == 1));
1243 if (dijk[(ind+1)%3] || dijk[(ind+2)%3] || // stepping in either other two directions
1244 (!is_periodic && ldims[ind] == gdims[ind] && dijk[ind] == -1) || // lower side and going lower
1245 (!is_periodic && ldims[3+ind] >= gdims[3+ind] && dijk[ind] == 1)) // not >= because ldims is only > gdims when periodic;
1246 // higher side and going higher
1247 return MB_SUCCESS;
1248
1249 std::copy(ldims, ldims+6, facedims);
1250 std::copy(ldims, ldims+6, rdims);
1251
1252 int dind = (gdims[ind+3] - gdims[ind])/np;
1253 int extra = (gdims[ind+3] - gdims[ind])%np;
1254 if (-1 == dijk[ind] && pfrom) {
1255 // actual left neighbor
1256 pto = pfrom-1; // no need for %np, because pfrom > 0
1257 facedims[ind+3] = facedims[ind];
1258 rdims[ind+3] = ldims[ind];
1259 rdims[ind] = rdims[ind+3] - dind - (pto < extra ? 1 : 0);
1260 }
1261 else if (1 == dijk[ind] && pfrom < np-1) {
1262 // actual right neighbor
1263 pto = pfrom+1;
1264 facedims[ind] = facedims[ind+3];
1265 rdims[ind] = ldims[ind+3];
1266 rdims[ind+3] = rdims[ind] + dind + (pto < extra ? 1 : 0);
1267 if (is_periodic && pfrom == np-2) rdims[ind+3]++; // neighbor is on periodic bdy
1268 }
1269 else if (-1 == dijk[ind] && !pfrom && gperiodic[ind]) {
1270 // downward across periodic bdy
1271 pto = np - 1;
1272 facedims[ind+3] = facedims[ind] = gdims[ind]; // by convention, facedims is within gdims, so lower value
1273 rdims[ind+3] = gdims[ind+3] + 1; // by convention, local dims one greater than gdims to indicate global lower value
1274 rdims[ind] = rdims[ind+3] - dind - 1;
1275 across_bdy[ind] = -1;
1276 }
1277 else if (1 == dijk[ind] && pfrom == np-1 && is_periodic) {
1278 // right across periodic bdy
1279 pto = 0;
1280 facedims[ind+3] = facedims[ind] = gdims[ind]; // by convention, facedims is within gdims, so lowest value
1281 rdims[ind] = gdims[ind];
1282 rdims[ind+3] = rdims[ind] + dind + (pto < extra ? 1 : 0);
1283 across_bdy[ind] = 1;
1284 }
1285
1286 assert(-1 == pto || (rdims[0] >= gdims[0] && (rdims[3] <= gdims[3] || (across_bdy[0] && !pfrom))));
1287 assert(-1 == pto || (rdims[1] >= gdims[1] && (rdims[4] <= gdims[4] || (across_bdy[1] && !pfrom))));
1288 assert(-1 == pto || (rdims[2] >= gdims[2] && rdims[5] <= gdims[5]));
1289 assert(-1 == pto || (facedims[0] >= rdims[0] && facedims[3] <= rdims[3]));
1290 assert(-1 == pto || (facedims[1] >= rdims[1] && facedims[4] <= rdims[4]));
1291 assert(-1 == pto || (facedims[2] >= rdims[2] && facedims[5] <= rdims[5]));
1292 assert(-1 == pto || (facedims[0] >= ldims[0] && facedims[3] <= ldims[3]));
1293 assert(-1 == pto || (facedims[1] >= ldims[1] && facedims[4] <= ldims[4]));
1294 assert(-1 == pto || (facedims[2] >= ldims[2] && facedims[5] <= ldims[5]));
1295
1296 return rval;
1297 }
1298
1299 //! get shared vertices for alljorkori partition scheme
1300 #ifndef MOAB_HAVE_MPI
1301 ErrorCode ScdInterface::get_shared_vertices(ParallelComm *, ScdBox *,
1302 std::vector<int> &,
1303 std::vector<int> &, std::vector<int> &)
1304 {
1305 return MB_FAILURE;
1306 #else
1307 ErrorCode ScdInterface::get_shared_vertices(ParallelComm *pcomm, ScdBox *box,
1308 std::vector<int> &procs,
1309 std::vector<int> &offsets, std::vector<int> &shared_indices)
1310 {
1311 // get index of partitioned dimension
1312 const int *ldims = box->box_dims();
1313 ErrorCode rval;
1314 int ijkrem[6], ijkface[6], across_bdy[3];
1315
1316 for (int k = -1; k <= 1; k ++) {
1317 for (int j = -1; j <= 1; j ++) {
1318 for (int i = -1; i <= 1; i ++) {
1319 if (!i && !j && !k) continue;
1320 int pto;
1321 int dijk[] = {i, j, k};
1322 rval = get_neighbor(pcomm->proc_config().proc_size(), pcomm->proc_config().proc_rank(),
1323 box->par_data(), dijk,
1324 pto, ijkrem, ijkface, across_bdy);
1325 if (MB_SUCCESS != rval) return rval;
1326 if (-1 != pto) {
1327 if (procs.empty() || pto != *procs.rbegin()) {
1328 procs.push_back(pto);
1329 offsets.push_back(shared_indices.size());
1330 }
1331 rval = get_indices(ldims, ijkrem, across_bdy, ijkface, shared_indices);
1332 if (MB_SUCCESS != rval) return rval;
1333
1334 // check indices against known #verts on local and remote
1335 // begin of this block is shared_indices[*offsets.rbegin()], end is shared_indices.end(), halfway
1336 // is (shared_indices.size()-*offsets.rbegin())/2
1337 #ifndef NDEBUG
1338 int start_idx = *offsets.rbegin(), end_idx = shared_indices.size(), mid_idx = (start_idx+end_idx)/2;
1339
1340 int num_local_verts = (ldims[3]-ldims[0]+1)*(ldims[4]-ldims[1]+1)*
1341 (-1 == ldims[2] && -1 == ldims[5] ? 1 : (ldims[5]-ldims[2]+1)),
1342 num_remote_verts = (ijkrem[3]-ijkrem[0]+1)*(ijkrem[4]-ijkrem[1]+1)*
1343 (-1 == ijkrem[2] && -1 == ijkrem[5] ? 1 : (ijkrem[5]-ijkrem[2]+1));
1344
1345 assert(*std::min_element(&shared_indices[start_idx], &shared_indices[mid_idx]) >= 0 &&
1346 *std::max_element(&shared_indices[start_idx], &shared_indices[mid_idx]) < num_local_verts &&
1347 *std::min_element(&shared_indices[mid_idx], &shared_indices[end_idx]) >= 0 &&
1348 *std::max_element(&shared_indices[mid_idx], &shared_indices[end_idx]) < num_remote_verts);
1349 #endif
1350 }
1351 }
1352 }
1353 }
1354
1355 offsets.push_back(shared_indices.size());
1356
1357 return MB_SUCCESS;
1358 #endif
1359 }
1360
1361 } // namespace moab
1362