1 /* ---------------------------------------------------------------------- 2 This is the 3 4 ██╗ ██╗ ██████╗ ██████╗ ██████╗ ██╗ ██╗████████╗███████╗ 5 ██║ ██║██╔════╝ ██╔════╝ ██╔════╝ ██║ ██║╚══██╔══╝██╔════╝ 6 ██║ ██║██║ ███╗██║ ███╗██║ ███╗███████║ ██║ ███████╗ 7 ██║ ██║██║ ██║██║ ██║██║ ██║██╔══██║ ██║ ╚════██║ 8 ███████╗██║╚██████╔╝╚██████╔╝╚██████╔╝██║ ██║ ██║ ███████║ 9 ╚══════╝╚═╝ ╚═════╝ ╚═════╝ ╚═════╝ ╚═╝ ╚═╝ ╚═╝ ╚══════╝® 10 11 DEM simulation engine, released by 12 DCS Computing Gmbh, Linz, Austria 13 http://www.dcs-computing.com, office@dcs-computing.com 14 15 LIGGGHTS® is part of CFDEM®project: 16 http://www.liggghts.com | http://www.cfdem.com 17 18 Core developer and main author: 19 Christoph Kloss, christoph.kloss@dcs-computing.com 20 21 LIGGGHTS® is open-source, distributed under the terms of the GNU Public 22 License, version 2 or later. It is distributed in the hope that it will 23 be useful, but WITHOUT ANY WARRANTY; without even the implied warranty 24 of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. You should have 25 received a copy of the GNU General Public License along with LIGGGHTS®. 26 If not, see http://www.gnu.org/licenses . See also top-level README 27 and LICENSE files. 28 29 LIGGGHTS® and CFDEM® are registered trade marks of DCS Computing GmbH, 30 the producer of the LIGGGHTS® software and the CFDEM®coupling software 31 See http://www.cfdem.com/terms-trademark-policy for details. 32 33 ------------------------------------------------------------------------- 34 Contributing author and copyright for this file: 35 (if not contributing author is listed, this file has been contributed 36 by the core developer) 37 38 Copyright 2012- DCS Computing GmbH, Linz 39 Copyright 2009-2012 JKU Linz 40 ------------------------------------------------------------------------- */ 41 42 #ifndef LMP_MULTI_NODE_MESH_PARALLEL_H 43 #define LMP_MULTI_NODE_MESH_PARALLEL_H 44 45 #include "mpi_liggghts.h" 46 #include "multi_node_mesh.h" 47 #include "comm.h" 48 #include "error.h" 49 #include "vector_liggghts.h" 50 #include "neighbor.h" 51 #include "math_extra_liggghts.h" 52 #include "container_base.h" 53 #include "domain_wedge.h" 54 #include <string> 55 #include <list> 56 #include <cmath> 57 #include <algorithm> 58 59 namespace LAMMPS_NS 60 { 61 62 template<int NUM_NODES> 63 class MultiNodeMeshParallel : public MultiNodeMesh<NUM_NODES> 64 { 65 public: 66 67 void initialSetup(); 68 void pbcExchangeBorders(int setupFlag); 69 void clearReverse(); 70 void forwardComm(std::string property); 71 void forwardComm(std::list<std::string> * properties = NULL); 72 void reverseComm(std::string property); 73 void reverseComm(std::list<std::string> * properties = NULL); 74 75 void writeRestart(FILE *fp); 76 void restart(double *list); 77 78 bool allNodesInsideSimulationBox(); 79 void useAsInsertionMesh(bool parallel); 80 isInsertionMesh()81 inline bool isInsertionMesh() 82 { return isInsertionMesh_; } 83 sizeLocal()84 inline int sizeLocal() 85 { return nLocal_; } 86 sizeGhost()87 inline int sizeGhost() 88 { return nGhost_; } 89 sizeGlobal()90 inline int sizeGlobal() 91 { return nGlobal_; } 92 sizeGlobalOrig()93 inline int sizeGlobalOrig() 94 { return nGlobalOrig_; } 95 isParallel()96 inline bool isParallel() 97 { return isParallel_; } 98 99 virtual int id(int i) = 0; 100 101 protected: 102 103 MultiNodeMeshParallel(LAMMPS *lmp); 104 virtual ~MultiNodeMeshParallel(); 105 106 virtual bool addElement(double **nodeToAdd); 107 108 virtual bool addElement(double **nodeToAdd,int lineNumb) = 0; 109 virtual void deleteElement(int n); 110 111 virtual void buildNeighbours() = 0; 112 113 virtual void refreshOwned(int setupFlag); 114 virtual void refreshGhosts(int setupFlag); 115 116 virtual void qualityCheck() = 0; 117 preSetup()118 virtual void preSetup() {} preInitialSetup()119 virtual void preInitialSetup() {} postInitialSetup()120 virtual void postInitialSetup() {} postBorders()121 virtual void postBorders() {} 122 123 virtual void clearMap() = 0; 124 virtual void generateMap() = 0; 125 virtual void clearGhostForward(bool scale,bool translate,bool rotate); 126 127 // lo-level parallelization also used by derived classes 128 129 virtual int elemListBufSize(int n,int operation,bool scale,bool translate,bool rotate); 130 virtual int pushElemListToBuffer(int n, int *list, int *wraplist, double *buf, int operation, std::list<std::string> * properties, double *dlo, double *dhi, bool scale,bool translate, bool rotate); 131 virtual int popElemListFromBuffer(int first, int n, double *buf, int operation, std::list<std::string> * properties, bool scale,bool translate, bool rotate); 132 virtual int pushElemListToBufferReverse(int first, int n, double *buf, int operation, std::list<std::string> * properties, bool scale,bool translate, bool rotate); 133 virtual int popElemListFromBufferReverse(int n, int *list, double *buf, int operation, std::list<std::string> * properties, bool scale,bool translate, bool rotate); 134 135 virtual int elemBufSize(int operation, std::list<std::string> * properties, bool scale,bool translate,bool rotate); 136 virtual int pushElemToBuffer(int i, double *buf,int operation,bool scale,bool translate,bool rotate); 137 virtual int popElemFromBuffer(double *buf,int operation,bool scale,bool translate,bool rotate); 138 139 virtual int meshPropsBufSize(int operation,bool scale,bool translate,bool rotate) = 0; 140 virtual int pushMeshPropsToBuffer(double *buf, int operation,bool scale,bool translate, bool rotate) = 0; 141 virtual int popMeshPropsFromBuffer(double *buf, int operation,bool scale,bool translate, bool rotate) = 0; 142 143 // flags if mesh should be parallelized 144 bool doParallellization_; 145 146 private: 147 148 // parallelization functions 149 150 void setup(); 151 void deleteUnowned(); 152 void pbc(); 153 void exchange(); 154 void borders(); 155 void clearGhosts(); 156 157 int checkBorderElement (const int, const int, const int, const double, const double) const; 158 int checkBorderElementLeft (const int, const int, const double, const double) const; 159 int checkBorderElementRight(const int, const int, const double, const double) const; 160 161 // lo-level parallelization 162 int pushExchange(int dim); 163 void popExchange(int nrecv,int dim,double *buf); 164 165 int sizeRestartMesh(); 166 int sizeRestartElement(); 167 168 // number of local and ghost elements 169 int nLocal_; 170 int nGhost_; 171 int nGlobal_; 172 173 // initial number of elements 174 int nGlobalOrig_; 175 176 // flags if mesh is parallelized 177 bool isParallel_; 178 179 // flag indicating usage as insertion mesh 180 bool isInsertionMesh_; 181 182 // ************************************* 183 // comm stuff - similar to Comm class 184 // ************************************* 185 186 void grow_swap(int n); 187 void allocate_swap(int n); 188 void free_swap(); 189 void grow_send(int,int); 190 void grow_recv(int); 191 void grow_list(int, int); 192 193 // communication buffers 194 int maxsend_,maxrecv_; // current size of send/recv buffer 195 double *buf_send_, *buf_recv_; 196 197 // comm 198 int sendneed_[3][2]; // # of procs away I send elements to 199 int maxneed_[3]; // max procs away any proc needs, per dim 200 double half_atom_cut_; // half atom neigh cut 201 202 int size_exchange_; // # of per-element datums in exchange 203 int size_forward_; // # of per-element datums in forward comm 204 int size_reverse_; // # of datums in reverse comm 205 int size_border_; // # of datums in forward border comm 206 207 int maxforward_,maxreverse_; // max # of datums in forward/reverse comm 208 209 // comm swaps 210 211 int nswap_; // # of swaps to perform = sum of maxneed 212 int maxswap_; // # of swaps where mem was allocated 213 int *sendnum_,*recvnum_; // # of atoms to send/recv in each swap 214 int *firstrecv_; // where to put 1st recv atom in each swap 215 int *sendproc_,*recvproc_; // proc to send/recv to/from at each swap 216 int *size_forward_recv_; // # of values to recv in each forward comm 217 int *size_reverse_recv_; // # to recv in each reverse comm 218 219 double *slablo_,*slabhi_; 220 int **sendlist_; // list of elements to send in each swap 221 int **sendwraplist_; // whether an element needs to be wrapped or not 222 int *maxsendlist_; // max size of send list for each swap 223 224 int *pbc_flag_; // general flag for sending atoms thru PBC 225 int **pbc_; // dimension flags for PBC adjustments 226 }; 227 228 // ************************************* 229 #include "multi_node_mesh_parallel_I.h" 230 #include "multi_node_mesh_parallel_buffer_I.h" 231 // ************************************* 232 233 } /* LAMMPS_NS */ 234 #endif /* MULTINODEMESH_H_ */ 235