1 // The libMesh Finite Element Library. 2 // Copyright (C) 2002-2020 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner 3 4 // This library is free software; you can redistribute it and/or 5 // modify it under the terms of the GNU Lesser General Public 6 // License as published by the Free Software Foundation; either 7 // version 2.1 of the License, or (at your option) any later version. 8 9 // This library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 // Lesser General Public License for more details. 13 14 // You should have received a copy of the GNU Lesser General Public 15 // License along with this library; if not, write to the Free Software 16 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 17 18 19 20 #ifndef LIBMESH_MESH_COMMUNICATION_H 21 #define LIBMESH_MESH_COMMUNICATION_H 22 23 // Local Includes 24 #include "libmesh/compare_elems_by_level.h" 25 #include "libmesh/libmesh_common.h" 26 #include "libmesh/mesh_tools.h" 27 28 // C++ Includes 29 #include <unordered_map> 30 31 namespace libMesh 32 { 33 34 // Forward declarations 35 class MeshBase; 36 class DistributedMesh; 37 38 // This is for backwards compatibility, but if your code relies on 39 // forward declarations in our headers then fix it. 40 class ParallelMesh; 41 42 /** 43 * This is the \p MeshCommunication class. It handles all the details 44 * of communicating mesh information from one processor to another. All 45 * parallelization of the \p Mesh data structures is done via this class. 46 * 47 * \author Benjamin S. Kirk 48 * \date 2003 49 */ 50 class MeshCommunication 51 { 52 public: 53 54 /** 55 * Constructor. 56 */ MeshCommunication()57 MeshCommunication () {} 58 59 /** 60 * Destructor. 61 */ ~MeshCommunication()62 ~MeshCommunication () {} 63 64 /** 65 * Clears all data structures and resets to a pristine state. 66 */ 67 void clear (); 68 69 // /** 70 // * Finds all the processors that may contain 71 // * elements that neighbor my elements. This list 72 // * is guaranteed to include all processors that border 73 // * any of my elements, but may include additional ones as 74 // * well. This method computes bounding boxes for the 75 // * elements on each processor and checks for overlaps. 76 // */ 77 // void find_neighboring_processors(const MeshBase &); 78 79 /** 80 * This method takes a mesh (which is assumed to reside on 81 * processor 0) and broadcasts it to all the other processors. 82 * It also broadcasts any boundary information the mesh has 83 * associated with it. 84 */ 85 void broadcast (MeshBase &) const; 86 87 /** 88 * This method takes a parallel distributed mesh and redistributes 89 * the elements. Specifically, any elements stored on a given 90 * processor are sent to the processor which "owns" them. Similarly, 91 * any elements assigned to the current processor but stored on 92 * another are received. Once this step is completed any required ghost 93 * elements are updated. The final result is that each processor stores 94 * only the elements it actually owns and any ghost elements required 95 * to satisfy data dependencies. This method can be invoked after a 96 * partitioning step to affect the new partitioning. 97 * 98 * Redistribution can also be done with newly coarsened elements' 99 * neighbors only. 100 */ 101 void redistribute (DistributedMesh & mesh, 102 bool newly_coarsened_only = false) const; 103 104 /** 105 * 106 */ 107 void gather_neighboring_elements (DistributedMesh &) const; 108 109 /** 110 * Examine a just-coarsened mesh, and for any newly-coarsened elements, 111 * send the associated ghosted elements to the processor which needs them. 112 */ 113 void send_coarse_ghosts (MeshBase &) const; 114 115 /** 116 * This method takes an input \p DistributedMesh which may be 117 * distributed among all the processors. Each processor then 118 * sends its local nodes and elements to processor \p root_id. 119 * The end result is that a previously distributed \p DistributedMesh 120 * will be serialized on processor \p root_id. Since this method is 121 * collective it must be called by all processors. For the special 122 * case of \p root_id equal to \p DofObject::invalid_processor_id 123 * this function performs an allgather. 124 */ 125 void gather (const processor_id_type root_id, DistributedMesh &) const; 126 127 /** 128 * This method takes an input \p DistributedMesh which may be 129 * distributed among all the processors. Each processor then 130 * sends its local nodes and elements to the other processors. 131 * The end result is that a previously distributed \p DistributedMesh 132 * will be serialized on each processor. Since this method is 133 * collective it must be called by all processors. 134 */ allgather(DistributedMesh & mesh)135 void allgather (DistributedMesh & mesh) const 136 { MeshCommunication::gather(DofObject::invalid_processor_id, mesh); } 137 138 /** 139 * This method takes an input \p DistributedMesh which may be 140 * distributed among all the processors. Each processor 141 * deletes all elements which are neither local elements nor "ghost" 142 * elements which touch local elements, and deletes all nodes which 143 * are not contained in local or ghost elements. 144 * The end result is that a previously serial \p DistributedMesh 145 * will be distributed between processors. Since this method is 146 * collective it must be called by all processors. 147 * 148 * The std::set is a list of extra elements that you _don't_ want 149 * to delete. These will be left on the current processor along with 150 * local elements and ghosted neighbors. 151 */ 152 void delete_remote_elements (DistributedMesh &, const std::set<Elem *> &) const; 153 154 /** 155 * This method assigns globally unique, partition-agnostic 156 * indices to the nodes and elements in the mesh. The approach 157 * is to compute the Hilbert space-filling curve key and use its 158 * value to assign an index in [0,N_global). Since the Hilbert key 159 * is unique for each spatial location, two objects occupying the 160 * same location will be assigned the same global id. Thus, this 161 * method can also be useful for identifying duplicate nodes 162 * which may occur during parallel refinement. 163 */ 164 void assign_global_indices (MeshBase &) const; 165 166 /** 167 * Throw an error if we have any index clashes in the numbering used by 168 * assign_global_indices. 169 */ 170 void check_for_duplicate_global_indices (MeshBase & ) const; 171 172 /** 173 * This method determines a locally unique, contiguous 174 * index for each object in the input range. 175 */ 176 template <typename ForwardIterator> 177 void find_local_indices (const libMesh::BoundingBox &, 178 const ForwardIterator &, 179 const ForwardIterator &, 180 std::unordered_map<dof_id_type, dof_id_type> &) const; 181 182 /** 183 * This method determines a globally unique, partition-agnostic 184 * index for each object in the input range. 185 */ 186 template <typename ForwardIterator> 187 void find_global_indices (const Parallel::Communicator & communicator, 188 const libMesh::BoundingBox &, 189 const ForwardIterator &, 190 const ForwardIterator &, 191 std::vector<dof_id_type> &) const; 192 193 /** 194 * Copy ids of ghost elements from their local processors. 195 */ 196 void make_elems_parallel_consistent (MeshBase &); 197 198 #ifdef LIBMESH_ENABLE_AMR 199 /** 200 * Copy p levels of ghost elements from their local processors. 201 */ 202 void make_p_levels_parallel_consistent (MeshBase &); 203 #endif // LIBMESH_ENABLE_AMR 204 205 /** 206 * Assuming all ids on local nodes are globally unique, and 207 * assuming all processor ids are parallel consistent, this function makes 208 * all other ids parallel consistent. 209 */ 210 void make_node_ids_parallel_consistent (MeshBase &); 211 212 /** 213 * Assuming all unique_ids on local nodes are globally unique, and 214 * assuming all processor ids are parallel consistent, this function makes 215 * all ghost unique_ids parallel consistent. 216 */ 217 void make_node_unique_ids_parallel_consistent (MeshBase &); 218 219 /** 220 * Assuming all processor ids on nodes touching local elements 221 * are parallel consistent, this function makes all other processor ids 222 * parallel consistent as well. 223 */ 224 void make_node_proc_ids_parallel_consistent (MeshBase &); 225 226 /** 227 * Assuming all processor ids on nodes touching local elements 228 * are parallel consistent, this function makes processor ids 229 * on new nodes on other processors parallel consistent as well. 230 */ 231 void make_new_node_proc_ids_parallel_consistent (MeshBase &); 232 233 /** 234 * Copy processor_ids and ids on ghost nodes from their 235 * local processors. This is useful for code which wants to add 236 * nodes to a distributed mesh. 237 */ 238 void make_nodes_parallel_consistent (MeshBase &); 239 240 /** 241 * Copy processor_ids and ids on new nodes from their local 242 * processors. 243 */ 244 void make_new_nodes_parallel_consistent (MeshBase &); 245 }; 246 247 248 // Related utilities 249 250 // Ask a mesh's ghosting functors to insert into a set all elements 251 // that are either on or connected to processor id \p pid. Ask only 252 // for elements in the range from \p elem_it before \p elem_end; 253 // typically this may be mesh.active_pid_elements_*(pid) 254 void query_ghosting_functors(const MeshBase & mesh, 255 processor_id_type pid, 256 MeshBase::const_element_iterator elem_it, 257 MeshBase::const_element_iterator elem_end, 258 std::set<const Elem *, CompareElemIdsByLevel> & connected_elements); 259 260 // Take a set of elements and insert all immediate 261 // children of elements in the given range 262 void connect_children(const MeshBase & mesh, 263 MeshBase::const_element_iterator elem_it, 264 MeshBase::const_element_iterator elem_end, 265 std::set<const Elem *, CompareElemIdsByLevel> & connected_elements); 266 267 // Take a set of elements and insert all elements' ancestors and 268 // subactive descendants as well. 269 void connect_families(std::set<const Elem *, CompareElemIdsByLevel> & connected_elements); 270 271 // Take a set of elements and create a set of connected nodes. 272 void reconnect_nodes (const std::set<const Elem *, CompareElemIdsByLevel> & connected_elements, 273 std::set<const Node *> & connected_nodes); 274 275 276 277 //-------------------------------------------------------------- 278 // MeshCommunication inline members 279 280 281 } // namespace libMesh 282 283 #endif // LIBMESH_MESH_COMMUNICATION_H 284