1 // The libMesh Finite Element Library.
2 // Copyright (C) 2002-2020 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3
4 // This library is free software; you can redistribute it and/or
5 // modify it under the terms of the GNU Lesser General Public
6 // License as published by the Free Software Foundation; either
7 // version 2.1 of the License, or (at your option) any later version.
8
9 // This library is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 // Lesser General Public License for more details.
13
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
18
19
20 #ifndef LIBMESH_PARALLEL_GHOST_SYNC_H
21 #define LIBMESH_PARALLEL_GHOST_SYNC_H
22
23 // libMesh includes
24 #include "libmesh/elem.h"
25 #include "libmesh/int_range.h"
26 #include "libmesh/location_maps.h"
27 #include "libmesh/mesh_base.h"
28 #include "libmesh/parallel_algebra.h"
29
30 // TIMPI includes
31 #include "timpi/communicator.h"
32 #include "timpi/parallel_sync.h"
33
34 // C++ includes
35 #include <map> // FIXME - pid > comm.size() breaks with unordered_map
36 #include <vector>
37
38
39 namespace libMesh
40 {
41
42
43
44 //--------------------------------------------------------------------------
45 namespace Parallel {
46
47 //------------------------------------------------------------------------
48 /**
49 * Request data about a range of ghost nodes uniquely identified by
50 * their xyz location or a range of active ghost elements uniquely
51 * identified by their centroids' xyz location. Fulfill requests
52 * with
53 * sync.gather_data(const std::vector<unsigned int> & ids,
54 * std::vector<sync::datum> & data),
55 * by resizing and setting the values of the data vector.
56 * Respond to fulfillment with
57 * sync.act_on_data(const std::vector<unsigned int> & ids,
58 * std::vector<sync::datum> & data)
59 * The user must define Parallel::StandardType<sync::datum> if
60 * sync::datum isn't a built-in type.
61 * The user-provided location_map will be used and left unchanged
62 * if it is provided, or filled and cleared if it is empty.
63 */
64 template <typename Iterator,
65 typename DofObjType,
66 typename SyncFunctor>
67 void sync_dofobject_data_by_xyz(const Communicator & comm,
68 const Iterator & range_begin,
69 const Iterator & range_end,
70 LocationMap<DofObjType> * location_map,
71 SyncFunctor & sync);
72
73 //------------------------------------------------------------------------
74 /**
75 * Request data about a range of ghost dofobjects uniquely
76 * identified by their id. Fulfill requests with
77 * sync.gather_data(const std::vector<dof_id_type> & ids,
78 * std::vector<sync::datum> & data),
79 * by resizing and setting the values of the data vector.
80 * Respond to fulfillment with
81 * sync.act_on_data(const std::vector<dof_id_type> & ids,
82 * std::vector<sync::datum> & data)
83 * The user must define Parallel::StandardType<sync::datum> if
84 * sync::datum isn't a built-in type.
85 */
86 template <typename Iterator,
87 typename SyncFunctor>
88 void sync_dofobject_data_by_id(const Communicator & comm,
89 const Iterator & range_begin,
90 const Iterator & range_end,
91 SyncFunctor & sync);
92
93 /**
94 * Request data about a range of ghost dofobjects uniquely
95 * identified by their id.
96 *
97 * Elements within the range can be excluded from the request by
98 * returning false from dofobj_check(dof_object)
99 */
100 template <typename Iterator,
101 typename DofObjectCheckFunctor,
102 typename SyncFunctor>
103 void sync_dofobject_data_by_id(const Communicator & comm,
104 const Iterator & range_begin,
105 const Iterator & range_end,
106 const DofObjectCheckFunctor & dofobj_check,
107 SyncFunctor & sync);
108
109 //------------------------------------------------------------------------
110 /**
111 * Request data about a range of ghost elements uniquely
112 * identified by their parent id and which child they are.
113 * Fulfill requests with
114 * sync.gather_data(const std::vector<unsigned int> & ids,
115 * std::vector<sync::datum> & data),
116 * by resizing and setting the values of the data vector.
117 * Respond to fulfillment with
118 * sync.act_on_data(const std::vector<unsigned int> & ids,
119 * std::vector<sync::datum> & data)
120 * The user must define Parallel::StandardType<sync::datum> if
121 * sync::datum isn't a built-in type.
122 */
123 template <typename Iterator,
124 typename SyncFunctor>
125 void sync_element_data_by_parent_id(MeshBase & mesh,
126 const Iterator & range_begin,
127 const Iterator & range_end,
128 SyncFunctor & sync);
129
130 //------------------------------------------------------------------------
131 /**
132 * Synchronize data about a range of ghost nodes uniquely identified
133 * by an element id and local node id, assuming a single
134 * synchronization pass is necessary.
135 *
136 * Data for all nodes connected to elements in the given range of
137 * *element* iterators will be requested.
138 *
139 * Elements can be further excluded from the request by returning
140 * false from element_check(elem)
141 *
142 * Nodes can be further excluded from the request by returning false
143 * from node_check(elem, local_node_num)
144 *
145 * Fulfill requests with
146 * sync.gather_data(const std::vector<unsigned int> & ids,
147 * std::vector<sync::datum> & data),
148 * by resizing and setting the values of the data vector.
149 * Respond to fulfillment with
150 * bool sync.act_on_data(const std::vector<unsigned int> & ids,
151 * std::vector<sync::datum> & data)
152 * and return true iff the response changed any data.
153 *
154 * The user must define Parallel::StandardType<sync::datum> if
155 * sync::datum isn't a built-in type.
156 *
157 * This method returns true iff the sync pass changed any data on any
158 * processor.
159 */
160 template <typename ElemCheckFunctor,
161 typename NodeCheckFunctor,
162 typename SyncFunctor>
163 bool sync_node_data_by_element_id_once(MeshBase & mesh,
164 const MeshBase::const_element_iterator & range_begin,
165 const MeshBase::const_element_iterator & range_end,
166 const ElemCheckFunctor & elem_check,
167 const NodeCheckFunctor & node_check,
168 SyncFunctor & sync);
169
170
171
172 //------------------------------------------------------------------------
173 /**
174 * Synchronize data about a range of ghost nodes uniquely identified
175 * by an element id and local node id, iterating until data is
176 * completely in sync and futher synchronization passes cause no
177 * changes.
178 *
179 * Imagine a vertex surrounded by triangles, each on a different
180 * processor, with a ghosting policy that include only face neighbors
181 * and not point neighbors. Then the only way for authoritative
182 * information to trickle out from that vertex is by being passed
183 * along, one neighbor at a time, to processors who mostly don't even
184 * see the node's true owner!
185 *
186 * Data for all nodes connected to elements in the given range of
187 * *element* iterators will be requested.
188 *
189 * Elements can be further excluded from the request by returning
190 * false from element_check(elem)
191 *
192 * Nodes can be further excluded from the request by returning false
193 * from node_check(elem, local_node_num)
194 *
195 * Fulfill requests with
196 * sync.gather_data(const std::vector<unsigned int> & ids,
197 * std::vector<sync::datum> & data),
198 * by resizing and setting the values of the data vector.
199 * Respond to fulfillment with
200 * bool sync.act_on_data(const std::vector<unsigned int> & ids,
201 * std::vector<sync::datum> & data)
202 * and return true iff the response changed any data.
203 *
204 * The user must define Parallel::StandardType<sync::datum> if
205 * sync::datum isn't a built-in type.
206 */
207 template <typename ElemCheckFunctor,
208 typename NodeCheckFunctor,
209 typename SyncFunctor>
210 void sync_node_data_by_element_id(MeshBase & mesh,
211 const MeshBase::const_element_iterator & range_begin,
212 const MeshBase::const_element_iterator & range_end,
213 const ElemCheckFunctor & elem_check,
214 const NodeCheckFunctor & node_check,
215 SyncFunctor & sync);
216
217
218 //------------------------------------------------------------------------
219 // Parallel members
220
221
222 // "Check" Functor to perform sync operations with no exclusions
223 struct SyncEverything
224 {
SyncEverythingSyncEverything225 SyncEverything() {}
226
operatorSyncEverything227 bool operator() (const DofObject *) const { return true; }
228
operatorSyncEverything229 bool operator() (const Elem *, unsigned int) const
230 { return true; }
231 };
232
233
234
235 template <typename Iterator,
236 typename DofObjType,
237 typename SyncFunctor>
sync_dofobject_data_by_xyz(const Communicator & comm,const Iterator & range_begin,const Iterator & range_end,LocationMap<DofObjType> & location_map,SyncFunctor & sync)238 void sync_dofobject_data_by_xyz(const Communicator & comm,
239 const Iterator & range_begin,
240 const Iterator & range_end,
241 LocationMap<DofObjType> & location_map,
242 SyncFunctor & sync)
243 {
244 // This function must be run on all processors at once
245 libmesh_parallel_only(comm);
246
247 // We need a valid location_map
248 #ifdef DEBUG
249 bool need_map_update = (range_begin != range_end && location_map.empty());
250 comm.max(need_map_update);
251 libmesh_assert(!need_map_update);
252 #endif
253
254 // Count the objects to ask each processor about
255 std::map<processor_id_type, dof_id_type>
256 ghost_objects_from_proc;
257
258 for (Iterator it = range_begin; it != range_end; ++it)
259 {
260 DofObjType * obj = *it;
261 libmesh_assert (obj);
262 processor_id_type obj_procid = obj->processor_id();
263 if (obj_procid != DofObject::invalid_processor_id)
264 ghost_objects_from_proc[obj_procid]++;
265 }
266
267 // Request sets to send to each processor
268 std::map<processor_id_type, std::vector<Point>>
269 requested_objs_pt;
270 // Corresponding ids to keep track of
271 std::map<processor_id_type, std::vector<dof_id_type>>
272 requested_objs_id;
273
274 // We know how many objects live on each processor, so reserve()
275 // space for each.
276 for (auto pair : ghost_objects_from_proc)
277 {
278 const processor_id_type p = pair.first;
279 if (p != comm.rank())
280 {
281 requested_objs_pt[p].reserve(pair.second);
282 requested_objs_id[p].reserve(pair.second);
283 }
284 }
285
286 for (Iterator it = range_begin; it != range_end; ++it)
287 {
288 DofObjType * obj = *it;
289 processor_id_type obj_procid = obj->processor_id();
290 if (obj_procid == comm.rank() ||
291 obj_procid == DofObject::invalid_processor_id)
292 continue;
293
294 Point p = location_map.point_of(*obj);
295 requested_objs_pt[obj_procid].push_back(p);
296 requested_objs_id[obj_procid].push_back(obj->id());
297 }
298
299 std::map<const std::vector<Point> *, processor_id_type>
300 requested_objs_pt_inv;
301 for (auto & pair : requested_objs_pt)
302 requested_objs_pt_inv[&pair.second] = pair.first;
303
304 auto gather_functor =
305 [&location_map, &sync]
306 (processor_id_type /*pid*/, const std::vector<Point> & pts,
307 std::vector<typename SyncFunctor::datum> & data)
308 {
309 // Find the local id of each requested object
310 std::size_t query_size = pts.size();
311 std::vector<dof_id_type> query_id(query_size);
312 for (std::size_t i=0; i != query_size; ++i)
313 {
314 Point pt = pts[i];
315
316 // Look for this object in the multimap
317 DofObjType * obj = location_map.find(pt);
318
319 // We'd better find every object we're asked for
320 libmesh_assert (obj);
321
322 // Return the object's correct processor id,
323 // and our (correct if it's local) id for it.
324 query_id[i] = obj->id();
325 }
326
327 // Gather whatever data the user wants
328 sync.gather_data(query_id, data);
329 };
330
331 auto action_functor =
332 [&sync, &requested_objs_id,
333 &requested_objs_pt_inv]
334 (processor_id_type /* pid */, const std::vector<Point> & point_request,
335 const std::vector<typename SyncFunctor::datum> & data)
336 {
337 // With splits working on more pids than ranks, query_pid may not equal pid
338 const processor_id_type query_pid =
339 requested_objs_pt_inv[&point_request];
340
341 // Let the user process the results
342 sync.act_on_data(requested_objs_id[query_pid], data);
343 };
344
345 // Trade requests with other processors
346 typename SyncFunctor::datum * ex = nullptr;
347 pull_parallel_vector_data
348 (comm, requested_objs_pt, gather_functor, action_functor, ex);
349 }
350
351
352
353 template <typename Iterator,
354 typename SyncFunctor>
sync_dofobject_data_by_id(const Communicator & comm,const Iterator & range_begin,const Iterator & range_end,SyncFunctor & sync)355 void sync_dofobject_data_by_id(const Communicator & comm,
356 const Iterator & range_begin,
357 const Iterator & range_end,
358 SyncFunctor & sync)
359 {
360 sync_dofobject_data_by_id(comm, range_begin, range_end, SyncEverything(), sync);
361 }
362
363 template <typename Iterator,
364 typename DofObjectCheckFunctor,
365 typename SyncFunctor>
sync_dofobject_data_by_id(const Communicator & comm,const Iterator & range_begin,const Iterator & range_end,const DofObjectCheckFunctor & dofobj_check,SyncFunctor & sync)366 void sync_dofobject_data_by_id(const Communicator & comm,
367 const Iterator & range_begin,
368 const Iterator & range_end,
369 const DofObjectCheckFunctor & dofobj_check,
370 SyncFunctor & sync)
371 {
372 // This function must be run on all processors at once
373 libmesh_parallel_only(comm);
374
375 // Count the objects to ask each processor about
376 std::map<processor_id_type, dof_id_type>
377 ghost_objects_from_proc;
378
379 for (Iterator it = range_begin; it != range_end; ++it)
380 {
381 DofObject * obj = *it;
382 libmesh_assert (obj);
383
384 // We may want to pass Elem* or Node* to the check function, not
385 // just DofObject*
386 if (!dofobj_check(*it))
387 continue;
388
389 processor_id_type obj_procid = obj->processor_id();
390 if (obj_procid != DofObject::invalid_processor_id)
391 ghost_objects_from_proc[obj_procid]++;
392 }
393
394 // Request sets to send to each processor
395 std::map<processor_id_type, std::vector<dof_id_type>>
396 requested_objs_id;
397
398 // We know how many objects live on each processor, so reserve()
399 // space for each.
400 for (auto pair : ghost_objects_from_proc)
401 {
402 const processor_id_type p = pair.first;
403 if (p != comm.rank())
404 requested_objs_id[p].reserve(pair.second);
405 }
406
407 for (Iterator it = range_begin; it != range_end; ++it)
408 {
409 DofObject * obj = *it;
410
411 if (!dofobj_check(*it))
412 continue;
413
414 processor_id_type obj_procid = obj->processor_id();
415 if (obj_procid == comm.rank() ||
416 obj_procid == DofObject::invalid_processor_id)
417 continue;
418
419 requested_objs_id[obj_procid].push_back(obj->id());
420 }
421
422 auto gather_functor =
423 [&sync]
424 (processor_id_type, const std::vector<dof_id_type> & ids,
425 std::vector<typename SyncFunctor::datum> & data)
426 {
427 sync.gather_data(ids, data);
428 };
429
430 auto action_functor =
431 [&sync]
432 (processor_id_type, const std::vector<dof_id_type> & ids,
433 const std::vector<typename SyncFunctor::datum> & data)
434 {
435 // Let the user process the results
436 sync.act_on_data(ids, data);
437 };
438
439 // Trade requests with other processors
440 typename SyncFunctor::datum * ex = nullptr;
441 pull_parallel_vector_data
442 (comm, requested_objs_id, gather_functor, action_functor, ex);
443 }
444
445
446
447 // If there's no refined elements, there's nothing to sync
448 #ifdef LIBMESH_ENABLE_AMR
449 template <typename Iterator,
450 typename SyncFunctor>
sync_element_data_by_parent_id(MeshBase & mesh,const Iterator & range_begin,const Iterator & range_end,SyncFunctor & sync)451 void sync_element_data_by_parent_id(MeshBase & mesh,
452 const Iterator & range_begin,
453 const Iterator & range_end,
454 SyncFunctor & sync)
455 {
456 const Communicator & comm (mesh.comm());
457
458 // This function must be run on all processors at once
459 libmesh_parallel_only(comm);
460
461 // Count the objects to ask each processor about
462 std::map<processor_id_type, dof_id_type>
463 ghost_objects_from_proc;
464
465 for (Iterator it = range_begin; it != range_end; ++it)
466 {
467 Elem * elem = *it;
468 processor_id_type obj_procid = elem->processor_id();
469 if (obj_procid == comm.rank() ||
470 obj_procid == DofObject::invalid_processor_id)
471 continue;
472 const Elem * parent = elem->parent();
473 if (!parent || !elem->active())
474 continue;
475
476 ghost_objects_from_proc[obj_procid]++;
477 }
478
479 // Request sets to send to each processor
480 std::map<processor_id_type, std::vector<dof_id_type>>
481 requested_objs_id;
482 std::map<processor_id_type, std::vector<std::pair<dof_id_type,unsigned char>>>
483 requested_objs_parent_id_child_num;
484
485 // We know how many objects live on each processor, so reserve()
486 // space for each.
487 for (auto pair : ghost_objects_from_proc)
488 {
489 const processor_id_type p = pair.first;
490 if (p != comm.rank())
491 {
492 requested_objs_id[p].reserve(pair.second);
493 requested_objs_parent_id_child_num[p].reserve(pair.second);
494 }
495 }
496
497 for (Iterator it = range_begin; it != range_end; ++it)
498 {
499 Elem * elem = *it;
500 processor_id_type obj_procid = elem->processor_id();
501 if (obj_procid == comm.rank() ||
502 obj_procid == DofObject::invalid_processor_id)
503 continue;
504 const Elem * parent = elem->parent();
505 if (!parent || !elem->active())
506 continue;
507
508 requested_objs_id[obj_procid].push_back(elem->id());
509 requested_objs_parent_id_child_num[obj_procid].emplace_back
510 (parent->id(), cast_int<unsigned char>(parent->which_child_am_i(elem)));
511 }
512
513 std::map<const std::vector<std::pair<dof_id_type,unsigned char>> *, processor_id_type>
514 requested_objs_parent_id_child_num_inv;
515 for (auto & pair : requested_objs_parent_id_child_num)
516 requested_objs_parent_id_child_num_inv[&pair.second] = pair.first;
517
518 auto gather_functor =
519 [&mesh, &sync]
520 (processor_id_type,
521 const std::vector<std::pair<dof_id_type, unsigned char>> & parent_id_child_num,
522 std::vector<typename SyncFunctor::datum> & data)
523 {
524 // Find the id of each requested element
525 std::size_t query_size = parent_id_child_num.size();
526 std::vector<dof_id_type> query_id(query_size);
527 for (std::size_t i=0; i != query_size; ++i)
528 {
529 Elem & parent = mesh.elem_ref(parent_id_child_num[i].first);
530 libmesh_assert(parent.has_children());
531 Elem * child = parent.child_ptr(parent_id_child_num[i].second);
532 libmesh_assert(child);
533 libmesh_assert(child->active());
534 query_id[i] = child->id();
535 }
536
537 // Gather whatever data the user wants
538 sync.gather_data(query_id, data);
539 };
540
541 auto action_functor =
542 [&sync, &requested_objs_id,
543 &requested_objs_parent_id_child_num_inv]
544 (processor_id_type /* pid */,
545 const std::vector<std::pair<dof_id_type, unsigned char>> & parent_id_child_num_request,
546 const std::vector<typename SyncFunctor::datum> & data)
547 {
548 // With splits working on more pids than ranks, query_pid may not equal pid
549 const processor_id_type query_pid =
550 requested_objs_parent_id_child_num_inv[&parent_id_child_num_request];
551
552 // Let the user process the results
553 sync.act_on_data(requested_objs_id[query_pid], data);
554 };
555
556 // Trade requests with other processors
557 typename SyncFunctor::datum * ex = nullptr;
558 pull_parallel_vector_data
559 (comm, requested_objs_parent_id_child_num, gather_functor,
560 action_functor, ex);
561 }
562 #else
563 template <typename Iterator,
564 typename SyncFunctor>
sync_element_data_by_parent_id(MeshBase &,const Iterator &,const Iterator &,SyncFunctor &)565 void sync_element_data_by_parent_id(MeshBase &,
566 const Iterator &,
567 const Iterator &,
568 SyncFunctor &)
569 {
570 }
571 #endif // LIBMESH_ENABLE_AMR
572
573
574
575 template <typename ElemCheckFunctor,
576 typename NodeCheckFunctor,
577 typename SyncFunctor>
sync_node_data_by_element_id_once(MeshBase & mesh,const MeshBase::const_element_iterator & range_begin,const MeshBase::const_element_iterator & range_end,const ElemCheckFunctor & elem_check,const NodeCheckFunctor & node_check,SyncFunctor & sync)578 bool sync_node_data_by_element_id_once(MeshBase & mesh,
579 const MeshBase::const_element_iterator & range_begin,
580 const MeshBase::const_element_iterator & range_end,
581 const ElemCheckFunctor & elem_check,
582 const NodeCheckFunctor & node_check,
583 SyncFunctor & sync)
584 {
585 const Communicator & comm (mesh.comm());
586
587 // Count the objects to ask each processor about
588 std::map<processor_id_type, dof_id_type>
589 ghost_objects_from_proc;
590
591 for (const auto & elem : as_range(range_begin, range_end))
592 {
593 libmesh_assert (elem);
594
595 if (!elem_check(elem))
596 continue;
597
598 const processor_id_type proc_id = elem->processor_id();
599
600 bool i_have_elem =
601 (proc_id == comm.rank() ||
602 proc_id == DofObject::invalid_processor_id);
603
604 if (elem->active() && i_have_elem)
605 continue;
606
607 for (auto n : elem->node_index_range())
608 {
609 if (!node_check(elem, n))
610 continue;
611
612 const processor_id_type node_pid =
613 elem->node_ref(n).processor_id();
614
615 if (i_have_elem && (node_pid == comm.rank()))
616 continue;
617
618 if (i_have_elem)
619 {
620 libmesh_assert_not_equal_to
621 (node_pid, DofObject::invalid_processor_id);
622 ghost_objects_from_proc[node_pid]++;
623 }
624 else
625 {
626 const processor_id_type request_pid =
627 (node_pid == DofObject::invalid_processor_id) ?
628 proc_id : node_pid;
629 ghost_objects_from_proc[request_pid]++;
630 }
631 }
632 }
633
634 // Now repeat that iteration, filling request sets this time.
635
636 // Request sets to send to each processor
637 std::map<processor_id_type, std::vector<std::pair<dof_id_type, unsigned char>>>
638 requested_objs_elem_id_node_num;
639
640 // We know how many objects live on each processor, so reserve()
641 // space for each.
642 for (auto pair : ghost_objects_from_proc)
643 {
644 const processor_id_type p = pair.first;
645 if (p != comm.rank())
646 requested_objs_elem_id_node_num[p].reserve(ghost_objects_from_proc[p]);
647 }
648
649 for (const auto & elem : as_range(range_begin, range_end))
650 {
651 libmesh_assert (elem);
652
653 if (!elem_check(elem))
654 continue;
655
656 const processor_id_type proc_id = elem->processor_id();
657
658 bool i_have_elem =
659 (proc_id == comm.rank() ||
660 proc_id == DofObject::invalid_processor_id);
661
662 if (elem->active() && i_have_elem)
663 continue;
664
665 const dof_id_type elem_id = elem->id();
666
667 for (auto n : elem->node_index_range())
668 {
669 if (!node_check(elem, n))
670 continue;
671
672 const Node & node = elem->node_ref(n);
673 const processor_id_type node_pid = node.processor_id();
674
675 if (i_have_elem && (node_pid == comm.rank()))
676 continue;
677
678 if (i_have_elem)
679 {
680 libmesh_assert_not_equal_to
681 (node_pid, DofObject::invalid_processor_id);
682 requested_objs_elem_id_node_num[node_pid].emplace_back
683 (elem_id, cast_int<unsigned char>(n));
684 }
685 else
686 {
687 const processor_id_type request_pid =
688 (node_pid == DofObject::invalid_processor_id) ?
689 proc_id : node_pid;
690 requested_objs_elem_id_node_num[request_pid].emplace_back
691 (elem_id,cast_int<unsigned char>(n));
692 }
693 }
694 }
695
696 auto gather_functor =
697 [&mesh, &sync]
698 (processor_id_type,
699 const std::vector<std::pair<dof_id_type, unsigned char>> & elem_id_node_num,
700 std::vector<typename SyncFunctor::datum> & data)
701 {
702 // Find the id of each requested element
703 std::size_t request_size = elem_id_node_num.size();
704 std::vector<dof_id_type> query_id(request_size);
705 for (std::size_t i=0; i != request_size; ++i)
706 {
707 // We might now get queries about remote elements, in which
708 // case we'll have to ignore them and wait for the query
709 // answer to filter to the querier via another source.
710 const Elem * elem = mesh.query_elem_ptr(elem_id_node_num[i].first);
711
712 if (elem)
713 {
714 const unsigned int n = elem_id_node_num[i].second;
715 libmesh_assert_less (n, elem->n_nodes());
716
717 const Node & node = elem->node_ref(n);
718
719 // This isn't a safe assertion in the case where we're
720 // syncing processor ids
721 // libmesh_assert_equal_to (node->processor_id(), comm.rank());
722
723 query_id[i] = node.id();
724 }
725 else
726 query_id[i] = DofObject::invalid_id;
727 }
728
729 // Gather whatever data the user wants
730 sync.gather_data(query_id, data);
731 };
732
733 bool data_changed = false;
734
735 auto action_functor =
736 [&sync, &mesh, &data_changed]
737 (processor_id_type /* pid */,
738 const std::vector<std::pair<dof_id_type, unsigned char>> & elem_id_node_num,
739 const std::vector<typename SyncFunctor::datum> & data)
740 {
741 const std::size_t data_size = data.size();
742
743 libmesh_assert_equal_to(elem_id_node_num.size(), data_size);
744
745 std::vector<dof_id_type> requested_objs_id(data.size());
746
747 for (auto i : IntRange<std::size_t>(0,data_size))
748 {
749 const Elem & elem = mesh.elem_ref(elem_id_node_num[i].first);
750 const Node & node = elem.node_ref(elem_id_node_num[i].second);
751 requested_objs_id[i] = node.id();
752 }
753
754 // Let the user process the results. If any of the results
755 // were different than what the user expected, then we may
756 // need to sync again just in case this processor has to
757 // pass on the changes to yet another processor.
758 if (sync.act_on_data(requested_objs_id, data))
759 data_changed = true;
760 };
761
762 // Trade requests with other processors
763 typename SyncFunctor::datum * ex = nullptr;
764 pull_parallel_vector_data
765 (comm, requested_objs_elem_id_node_num, gather_functor,
766 action_functor, ex);
767
768 comm.max(data_changed);
769
770 return data_changed;
771 }
772
773
774
775 template <typename ElemCheckFunctor,
776 typename NodeCheckFunctor,
777 typename SyncFunctor>
sync_node_data_by_element_id(MeshBase & mesh,const MeshBase::const_element_iterator & range_begin,const MeshBase::const_element_iterator & range_end,const ElemCheckFunctor & elem_check,const NodeCheckFunctor & node_check,SyncFunctor & sync)778 void sync_node_data_by_element_id(MeshBase & mesh,
779 const MeshBase::const_element_iterator & range_begin,
780 const MeshBase::const_element_iterator & range_end,
781 const ElemCheckFunctor & elem_check,
782 const NodeCheckFunctor & node_check,
783 SyncFunctor & sync)
784 {
785 // This function must be run on all processors at once
786 libmesh_parallel_only(mesh.comm());
787
788 bool need_sync = false;
789
790 do
791 {
792 need_sync =
793 sync_node_data_by_element_id_once
794 (mesh, range_begin, range_end, elem_check, node_check,
795 sync);
796 } while (need_sync);
797 }
798
799
800 }
801
802
803
804 // This struct can be created and passed to the
805 // Parallel::sync_dofobject_data_by_id() function.
806 struct SyncNodalPositions
807 {
808 // The constructor. You need a reference to the mesh where you will
809 // be setting/getting nodal positions.
810 explicit
811 SyncNodalPositions(MeshBase & m);
812
813 // The datum typedef is required of this functor, so that the
814 // Parallel::sync_dofobject_data_by_id() function can create e.g.
815 // std::vector<datum>.
816 typedef Point datum;
817
818 // First required interface. This function must fill up the data vector for the
819 // ids specified in the ids vector.
820 void gather_data (const std::vector<dof_id_type> & ids, std::vector<datum> & data) const;
821
822 // Second required interface. This function must do something with the data in
823 // the data vector for the ids in the ids vector.
824 void act_on_data (const std::vector<dof_id_type> & ids, const std::vector<datum> & data) const;
825
826 MeshBase & mesh;
827 };
828
829 // This struct can be created and passed to the
830 // Parallel::sync_dofobject_data_by_id() function.
831 struct SyncSubdomainIds
832 {
833 // The constructor. You need a reference to the mesh where you will
834 // be setting/getting element subdomain IDs.
835 explicit
836 SyncSubdomainIds(MeshBase & m);
837
838 // The datum typedef is required of this functor, so that the
839 // Parallel::sync_dofobject_data_by_id() function can create e.g.
840 // std::vector<datum>.
841 typedef subdomain_id_type datum;
842
843 // First required interface. This function must fill up the data vector for the
844 // ids specified in the ids vector.
845 void gather_data (const std::vector<dof_id_type> & ids, std::vector<datum> & data) const;
846
847 // Second required interface. This function must do something with the data in
848 // the data vector for the ids in the ids vector.
849 void act_on_data (const std::vector<dof_id_type> & ids, const std::vector<datum> & data) const;
850
851 MeshBase & mesh;
852 };
853
854
855 } // namespace libMesh
856
857 #endif // LIBMESH_PARALLEL_GHOST_SYNC_H
858