1 /* 2 * Copyright(C) 1999-2020 National Technology & Engineering Solutions 3 * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with 4 * NTESS, the U.S. Government retains certain rights in this software. 5 * 6 * See packages/seacas/LICENSE for details 7 */ 8 #ifndef IOCGNS_DECOMPOSITONDATA_H 9 #define IOCGNS_DECOMPOSITONDATA_H 10 11 #include <string> 12 #include <vector> 13 14 #define CG_USE_ROBIN 15 #if defined CG_USE_STD 16 #include <unordered_map> 17 #elif defined CG_USE_HOPSCOTCH 18 #include <bhopscotch_map.h> 19 #elif defined CG_USE_ROBIN 20 #include <robin_map.h> 21 #endif 22 23 #include <cstddef> 24 #include <cstdint> 25 26 #include <Ioss_CodeTypes.h> 27 #include <Ioss_Decomposition.h> 28 #include <Ioss_FaceGenerator.h> 29 #include <Ioss_Field.h> 30 #include <Ioss_MeshType.h> 31 #include <Ioss_PropertyManager.h> 32 #include <Ioss_StructuredBlock.h> 33 #include <cgns/Iocgns_StructuredZoneData.h> 34 35 #include <cgnslib.h> 36 37 #if 0 38 #if !defined(NO_PARMETIS_SUPPORT) 39 #include <parmetis.h> 40 #endif 41 #endif 42 43 #undef MPICPP 44 #if !defined(NO_ZOLTAN_SUPPORT) 45 #include <zoltan_cpp.h> 46 #endif 47 namespace Ioss { 48 class Field; 49 template <typename INT> class Decomposition; 50 } // namespace Ioss 51 52 namespace Iocgns { 53 54 class ZoneData 55 { 56 public: 57 std::string m_name; 58 size_t m_nodeOffset; 59 size_t m_nodeCount; 60 size_t m_elementOffset; 61 }; 62 63 class DecompositionDataBase 64 { 65 public: 66 DecompositionDataBase() = default; 67 68 virtual ~DecompositionDataBase(); 69 virtual void decompose_model(int filePtr, Ioss::MeshType mesh_type) = 0; 70 virtual size_t ioss_node_count() const = 0; 71 virtual size_t ioss_elem_count() const = 0; 72 virtual int int_size() const = 0; 73 74 virtual int spatial_dimension() const = 0; 75 virtual size_t global_node_count() const = 0; 76 virtual size_t global_elem_count() const = 0; 77 78 virtual size_t decomp_node_offset() const = 0; 79 virtual size_t decomp_node_count() const = 0; 80 virtual size_t decomp_elem_offset() const = 0; 81 virtual size_t decomp_elem_count() const = 0; 82 83 virtual std::vector<double> ¢roids() = 0; 84 85 virtual size_t get_commset_node_size() const = 0; 86 87 virtual void get_node_coordinates(int filePtr, double *ioss_data, 88 const Ioss::Field &field) const = 0; 89 90 void get_block_connectivity(int filePtr, void *data, int blk_seq, bool raw_ids = false) const; 91 92 void get_element_field(int filePtr, int solution_index, int blk_seq, int field_index, 93 double *data) const; 94 95 void get_node_field(int filePtr, int step, int field_index, double *data) const; 96 97 void get_node_entity_proc_data(void *entity_proc, const Ioss::MapContainer &node_map, 98 bool do_map) const; 99 100 template <typename T> 101 void communicate_element_data(T *file_data, T *ioss_data, size_t comp_count) const; 102 103 template <typename T> 104 void communicate_node_data(T *file_data, T *ioss_data, size_t comp_count) const; 105 106 void get_sideset_element_side(int filePtr, const Ioss::SetDecompositionData &sset, 107 void *data) const; 108 109 std::vector<ZoneData> m_zones; 110 std::vector<Ioss::BlockDecompositionData> m_elementBlocks; 111 std::vector<Ioss::SetDecompositionData> m_sideSets; 112 std::vector<Iocgns::StructuredZoneData *> m_structuredZones; 113 114 // Maps nodes shared between zones. 115 // TODO: Currently each processor has same map; need to figure out how to reduce size 116 #if defined CG_USE_STD 117 using ZoneSharedMap = std::unordered_map<cgsize_t, cgsize_t>; 118 #elif defined CG_USE_HOPSCOTCH 119 // using ZoneSharedMap = tsl::hopscotch_map<cgsize_t, cgsize_t>; 120 using ZoneSharedMap = tsl::bhopscotch_map<cgsize_t, cgsize_t>; 121 #elif defined CG_USE_ROBIN 122 using ZoneSharedMap = tsl::robin_map<cgsize_t, cgsize_t>; 123 #endif 124 ZoneSharedMap m_zoneSharedMap; 125 }; 126 127 template <typename INT> class DecompositionData : public DecompositionDataBase 128 { 129 public: 130 DecompositionData(const Ioss::PropertyManager &props, MPI_Comm communicator); 131 ~DecompositionData() override = default; 132 int_size()133 int int_size() const override { return sizeof(INT); } 134 135 void decompose_model(int filePtr, Ioss::MeshType mesh_type) override; 136 spatial_dimension()137 int spatial_dimension() const override { return m_decomposition.m_spatialDimension; } 138 global_node_count()139 size_t global_node_count() const override { return m_decomposition.global_node_count(); } global_elem_count()140 size_t global_elem_count() const override { return m_decomposition.global_elem_count(); } 141 ioss_node_count()142 size_t ioss_node_count() const override { return m_decomposition.ioss_node_count(); } ioss_elem_count()143 size_t ioss_elem_count() const override { return m_decomposition.ioss_elem_count(); } 144 decomp_node_offset()145 size_t decomp_node_offset() const override { return m_decomposition.file_node_offset(); } decomp_node_count()146 size_t decomp_node_count() const override { return m_decomposition.file_node_count(); } decomp_elem_offset()147 size_t decomp_elem_offset() const override { return m_decomposition.file_elem_offset(); } decomp_elem_count()148 size_t decomp_elem_count() const override { return m_decomposition.file_elem_count(); } 149 centroids()150 std::vector<double> ¢roids() override { return m_decomposition.m_centroids; } 151 152 template <typename T> communicate_element_data(T * file_data,T * ioss_data,size_t comp_count)153 void communicate_element_data(T *file_data, T *ioss_data, size_t comp_count) const 154 { 155 m_decomposition.communicate_element_data(file_data, ioss_data, comp_count); 156 } 157 158 template <typename T> communicate_set_data(T * file_data,T * ioss_data,const Ioss::SetDecompositionData & set,size_t comp_count)159 void communicate_set_data(T *file_data, T *ioss_data, const Ioss::SetDecompositionData &set, 160 size_t comp_count) const 161 { 162 m_decomposition.communicate_set_data(file_data, ioss_data, set, comp_count); 163 } 164 165 template <typename T> communicate_node_data(T * file_data,T * ioss_data,size_t comp_count)166 void communicate_node_data(T *file_data, T *ioss_data, size_t comp_count) const 167 { 168 m_decomposition.communicate_node_data(file_data, ioss_data, comp_count); 169 } 170 171 template <typename U, typename T> communicate_block_data(U * file_data,T * ioss_data,const Ioss::BlockDecompositionData & block,size_t comp_count)172 void communicate_block_data(U *file_data, T *ioss_data, 173 const Ioss::BlockDecompositionData &block, size_t comp_count) const 174 { 175 m_decomposition.communicate_block_data(file_data, ioss_data, block, comp_count); 176 } 177 178 void get_block_connectivity(int filePtr, INT *data, int blk_seq, bool raw_ids) const; 179 180 void get_element_field(int filePtr, int solution_index, int blk_seq, int field_index, 181 double *data) const; 182 183 void get_node_field(int filePtr, int step, int field_offset, double *data) const; 184 get_commset_node_size()185 size_t get_commset_node_size() const override 186 { 187 return m_decomposition.m_nodeCommMap.size() / 2; 188 } 189 190 void get_sideset_element_side(int filePtr, const Ioss::SetDecompositionData &sset, 191 INT *data) const; 192 193 private: 194 void decompose_structured(int filePtr); 195 void decompose_unstructured(int filePtr); 196 197 void get_sideset_data(int filePtr); 198 void generate_zone_shared_nodes(int filePtr, INT min_node, INT max_node); 199 i_own_node(size_t node)200 bool i_own_node(size_t node) 201 const // T/F if node with global index node owned by this processors ioss-decomp. 202 { 203 return m_decomposition.i_own_node(node); 204 } 205 i_own_elem(size_t elem)206 bool i_own_elem(size_t elem) 207 const // T/F if node with global index elem owned by this processors ioss-decomp. 208 { 209 return m_decomposition.i_own_elem(elem); 210 } 211 212 // global_index is 1-based index into global list of nodes [1..global_node_count] 213 // return value is 1-based index into local list of nodes on this 214 // processor (ioss-decomposition) node_global_to_local(size_t global_index)215 size_t node_global_to_local(size_t global_index) const 216 { 217 return m_decomposition.node_global_to_local(global_index); 218 } 219 elem_global_to_local(size_t global_index)220 size_t elem_global_to_local(size_t global_index) const 221 { 222 return m_decomposition.elem_global_to_local(global_index); 223 } 224 build_global_to_local_elem_map()225 void build_global_to_local_elem_map() 226 { 227 return m_decomposition.build_global_to_local_elem_map(); 228 } 229 get_element_block_communication()230 void get_element_block_communication() 231 { 232 m_decomposition.get_element_block_communication(m_elementBlocks); 233 } 234 235 void generate_adjacency_list(int filePtr, Ioss::Decomposition<INT> &decomposition); 236 237 void calculate_element_centroids(int filePtr, std::vector<double> ¢roids); 238 get_shared_node_list()239 void get_shared_node_list() { m_decomposition.get_shared_node_list(); } 240 get_local_node_list()241 void get_local_node_list() { m_decomposition.get_local_node_list(); } 242 243 void get_file_node_coordinates(int filePtr, int direction, double *ioss_data) const; 244 void get_node_coordinates(int filePtr, double *ioss_data, 245 const Ioss::Field &field) const override; 246 247 double m_loadBalanceThreshold{1.4}; 248 std::string m_lineDecomposition{}; 249 250 mutable std::map<int, Ioss::FaceUnorderedSet> m_boundaryFaces; 251 252 public: 253 Ioss::Decomposition<INT> m_decomposition; 254 }; 255 256 } // namespace Iocgns 257 #endif 258