1 // Copyright(C) 1999-2021 National Technology & Engineering Solutions 2 // of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with 3 // NTESS, the U.S. Government retains certain rights in this software. 4 // 5 // See packages/seacas/LICENSE for details 6 7 // -*- Mode: c++ -*- 8 #ifndef IOSS_Ioex_ParallelDatabaseIO_h 9 #define IOSS_Ioex_ParallelDatabaseIO_h 10 11 #include "vtk_ioss_mangle.h" 12 #include <vtk_exodusII.h> 13 #if defined PARALLEL_AWARE_EXODUS 14 #include <Ioss_CodeTypes.h> 15 #include <Ioss_DBUsage.h> // for DatabaseUsage 16 #include <Ioss_Map.h> // for Map 17 #include <Ioss_State.h> // for State 18 #include <exodus/Ioex_BaseDatabaseIO.h> // for DatabaseIO 19 #include <functional> // for less 20 #include <map> // for map, map<>::value_compare 21 #include <memory> 22 #include <set> // for set 23 #include <stddef.h> // for size_t 24 #include <stdint.h> // for int64_t 25 #include <string> // for string, operator< 26 #include <time.h> // for nullptr, time_t 27 #include <utility> // for pair 28 #include <vector> // for vector 29 namespace Ioex { 30 class DecompositionDataBase; 31 } 32 namespace Ioex { 33 template <typename INT> class DecompositionData; 34 } 35 36 namespace Ioss { 37 class Assembly; 38 class Blob; 39 class EntityBlock; 40 class ElementTopology; 41 class CommSet; 42 class EdgeBlock; 43 class EdgeSet; 44 class ElementBlock; 45 class ElementSet; 46 class EntitySet; 47 class FaceBlock; 48 class FaceSet; 49 class Field; 50 class GroupingEntity; 51 class NodeBlock; 52 class NodeSet; 53 class PropertyManager; 54 class Region; 55 class SideBlock; 56 class SideSet; 57 class StructuredBlock; 58 } // namespace Ioss 59 60 /** \brief A namespace for the decompose-on-the-fly version of the 61 * parallel exodus database format. 62 */ 63 namespace Ioex { 64 class ParallelDatabaseIO : public Ioex::BaseDatabaseIO 65 { 66 public: 67 ParallelDatabaseIO(Ioss::Region *region, const std::string &filename, 68 Ioss::DatabaseUsage db_usage, MPI_Comm communicator, 69 const Ioss::PropertyManager &properties); 70 ParallelDatabaseIO(const ParallelDatabaseIO &from) = delete; 71 ParallelDatabaseIO &operator=(const ParallelDatabaseIO &from) = delete; 72 ~ParallelDatabaseIO() = default; 73 74 int get_file_pointer() const override; // Open file and set exodusFilePtr. needs_shared_node_information()75 bool needs_shared_node_information() const override { return true; } 76 77 private: 78 void compute_node_status() const; 79 80 void release_memory__() override; 81 82 void get_step_times__() override; 83 84 bool open_input_file(bool write_message, std::string *error_msg, int *bad_count, 85 bool abort_if_error) const override; 86 bool handle_output_file(bool write_message, std::string *error_msg, int *bad_count, 87 bool overwrite, bool abort_if_error) const override; 88 bool check_valid_file_ptr(bool write_message, std::string *error_msg, int *bad_count, 89 bool abort_if_error) const; 90 91 int64_t get_field_internal(const Ioss::Region *reg, const Ioss::Field &field, void *data, 92 size_t data_size) const override; 93 int64_t get_field_internal(const Ioss::NodeBlock *nb, const Ioss::Field &field, void *data, 94 size_t data_size) const override; 95 int64_t get_field_internal(const Ioss::EdgeBlock *eb, const Ioss::Field &field, void *data, 96 size_t data_size) const override; 97 int64_t get_field_internal(const Ioss::FaceBlock *eb, const Ioss::Field &field, void *data, 98 size_t data_size) const override; 99 int64_t get_field_internal(const Ioss::ElementBlock *eb, const Ioss::Field &field, void *data, 100 size_t data_size) const override; get_field_internal(const Ioss::StructuredBlock *,const Ioss::Field &,void *,size_t)101 int64_t get_field_internal(const Ioss::StructuredBlock * /* sb */, 102 const Ioss::Field & /* field */, void * /* data */, 103 size_t /* data_size */) const override 104 { 105 return -1; 106 } 107 int64_t get_field_internal(const Ioss::SideBlock *fb, const Ioss::Field &field, void *data, 108 size_t data_size) const override; 109 int64_t get_field_internal(const Ioss::NodeSet *ns, const Ioss::Field &field, void *data, 110 size_t data_size) const override; 111 int64_t get_field_internal(const Ioss::EdgeSet *ns, const Ioss::Field &field, void *data, 112 size_t data_size) const override; 113 int64_t get_field_internal(const Ioss::FaceSet *ns, const Ioss::Field &field, void *data, 114 size_t data_size) const override; 115 int64_t get_field_internal(const Ioss::ElementSet *ns, const Ioss::Field &field, void *data, 116 size_t data_size) const override; 117 int64_t get_field_internal(const Ioss::SideSet *fs, const Ioss::Field &field, void *data, 118 size_t data_size) const override; 119 int64_t get_field_internal(const Ioss::CommSet *cs, const Ioss::Field &field, void *data, 120 size_t data_size) const override; 121 int64_t get_field_internal(const Ioss::Assembly *sb, const Ioss::Field &field, void *data, 122 size_t data_size) const override; 123 int64_t get_field_internal(const Ioss::Blob *sb, const Ioss::Field &field, void *data, 124 size_t data_size) const override; 125 int64_t put_field_internal(const Ioss::Region *reg, const Ioss::Field &field, void *data, 126 size_t data_size) const override; 127 int64_t put_field_internal(const Ioss::NodeBlock *nb, const Ioss::Field &field, void *data, 128 size_t data_size) const override; 129 int64_t put_field_internal(const Ioss::EdgeBlock *eb, const Ioss::Field &field, void *data, 130 size_t data_size) const override; 131 int64_t put_field_internal(const Ioss::FaceBlock *eb, const Ioss::Field &field, void *data, 132 size_t data_size) const override; 133 int64_t put_field_internal(const Ioss::ElementBlock *eb, const Ioss::Field &field, void *data, 134 size_t data_size) const override; 135 int64_t put_field_internal(const Ioss::SideBlock *fb, const Ioss::Field &field, void *data, 136 size_t data_size) const override; 137 int64_t put_field_internal(const Ioss::NodeSet *ns, const Ioss::Field &field, void *data, 138 size_t data_size) const override; 139 int64_t put_field_internal(const Ioss::EdgeSet *ns, const Ioss::Field &field, void *data, 140 size_t data_size) const override; 141 int64_t put_field_internal(const Ioss::FaceSet *ns, const Ioss::Field &field, void *data, 142 size_t data_size) const override; 143 int64_t put_field_internal(const Ioss::ElementSet *ns, const Ioss::Field &field, void *data, 144 size_t data_size) const override; 145 int64_t put_field_internal(const Ioss::SideSet *fs, const Ioss::Field &field, void *data, 146 size_t data_size) const override; 147 int64_t put_field_internal(const Ioss::CommSet *cs, const Ioss::Field &field, void *data, 148 size_t data_size) const override; 149 int64_t put_field_internal(const Ioss::Assembly *sb, const Ioss::Field &field, void *data, 150 size_t data_size) const override; 151 int64_t put_field_internal(const Ioss::Blob *sb, const Ioss::Field &field, void *data, 152 size_t data_size) const override; put_field_internal(const Ioss::StructuredBlock *,const Ioss::Field &,void *,size_t)153 int64_t put_field_internal(const Ioss::StructuredBlock * /* sb */, 154 const Ioss::Field & /* field */, void * /* data */, 155 size_t /* data_size */) const override 156 { 157 return -1; 158 } 159 160 int64_t put_Xset_field_internal(ex_entity_type type, const Ioss::EntitySet *ns, 161 const Ioss::Field &field, void *data, size_t data_size) const; 162 int64_t get_Xset_field_internal(ex_entity_type type, const Ioss::EntitySet *ns, 163 const Ioss::Field &field, void *data, size_t data_size) const; 164 165 int free_file_pointer() const override; 166 167 int64_t read_nodal_coordinates(); 168 void read_elements(const Ioss::ElementBlock &block); 169 170 void create_implicit_global_map() const; 171 void output_node_map() const; 172 173 // Metadata-related functions. 174 void read_meta_data__() override; 175 176 int64_t read_transient_field(ex_entity_type type, const Ioex::VariableNameMap &variables, 177 const Ioss::Field &field, const Ioss::GroupingEntity *ge, 178 void *data) const; 179 180 int64_t read_attribute_field(ex_entity_type type, const Ioss::Field &field, 181 const Ioss::GroupingEntity *ge, void *data) const; 182 183 int64_t write_attribute_field(ex_entity_type type, const Ioss::Field &field, 184 const Ioss::GroupingEntity *ge, void *data) const; 185 186 // Handles subsetting of side blocks. 187 int64_t read_ss_transient_field(const Ioss::Field &field, int64_t id, void *variables, 188 std::vector<int> &is_valid_side) const; 189 190 // Should be made more generic again so can rejoin with write_element_transient field 191 void write_nodal_transient_field(ex_entity_type type, const Ioss::Field &field, 192 const Ioss::NodeBlock *nb, int64_t count, 193 void *variables) const; 194 // Should be made more generic again so can rejoin with write_nodal_transient field 195 void write_entity_transient_field(ex_entity_type type, const Ioss::Field &field, 196 const Ioss::GroupingEntity *ge, int64_t count, 197 void *variables) const; 198 void write_meta_data(Ioss::IfDatabaseExistsBehavior behavior) override; 199 200 // Read related metadata and store it in the region... 201 void read_region(); 202 void get_edgeblocks(); 203 void get_faceblocks(); 204 void get_elemblocks(); 205 void get_blocks(ex_entity_type entity_type, int rank_offset, const std::string &basename); 206 207 void get_sidesets(); 208 209 template <typename T> 210 void get_sets(ex_entity_type type, int64_t count, const std::string &base, const T *); 211 void get_nodesets(); 212 void get_edgesets(); 213 void get_facesets(); 214 void get_elemsets(); 215 216 void get_commsets(); 217 218 void check_valid_values() const; 219 220 // ID Mapping functions. 221 const Ioss::Map &get_map(ex_entity_type type) const; 222 const Ioss::Map &get_map(Ioss::Map &entity_map, int64_t entityCount, int64_t file_offset, 223 int64_t file_count, ex_entity_type entity_type, 224 ex_inquiry inquiry_type) const; 225 226 // Internal data handling 227 int64_t handle_node_ids(void *ids, int64_t num_to_get, size_t offset, size_t count) const; 228 int64_t handle_element_ids(const Ioss::ElementBlock *eb, void *ids, size_t num_to_get, 229 size_t offset, size_t count) const; 230 int64_t handle_face_ids(const Ioss::FaceBlock *eb, void *ids, size_t num_to_get) const; 231 int64_t handle_edge_ids(const Ioss::EdgeBlock *eb, void *ids, size_t num_to_get) const; 232 233 int64_t get_side_connectivity(const Ioss::SideBlock *fb, int64_t id, int64_t side_count, 234 void *fconnect, bool map_ids) const; 235 int64_t get_side_distributions(const Ioss::SideBlock *fb, int64_t id, int64_t my_side_count, 236 double *dist_fact, size_t data_size) const; 237 238 int64_t get_side_field(const Ioss::SideBlock *ef_blk, const Ioss::Field &field, void *data, 239 size_t data_size) const; 240 int64_t put_side_field(const Ioss::SideBlock *fb, const Ioss::Field &field, void *data, 241 size_t data_size) const; 242 243 // Private member data... 244 mutable std::unique_ptr<DecompositionDataBase> decomp; 245 246 mutable Ioss::IntVector nodeOwningProcessor; // Processor that owns each node on this processor 247 mutable Ioss::Int64Vector 248 nodeGlobalImplicitMap; // Position of this node in the global-implicit ordering 249 mutable Ioss::Int64Vector 250 elemGlobalImplicitMap; // Position of this element in the global-implicit ordering 251 252 // Contains the indices of all owned nodes in each nodeset on this processor to pull data 253 // from the global list down to the file list. 254 // NOTE: Even though map type is GroupingEntity*, it is only valid 255 // for a GroupingEntity* which is a NodeSet* 256 mutable std::map<const Ioss::GroupingEntity *, Ioss::Int64Vector> nodesetOwnedNodes; 257 258 mutable bool metaDataWritten{false}; 259 mutable bool nodeGlobalImplicitMapDefined{false}; 260 mutable bool elemGlobalImplicitMapDefined{false}; 261 }; 262 } // namespace Ioex 263 #endif 264 #endif 265