1 /* ----------------------------------------------------------------------
2     This is the
3 
4     ██╗     ██╗ ██████╗  ██████╗  ██████╗ ██╗  ██╗████████╗███████╗
5     ██║     ██║██╔════╝ ██╔════╝ ██╔════╝ ██║  ██║╚══██╔══╝██╔════╝
6     ██║     ██║██║  ███╗██║  ███╗██║  ███╗███████║   ██║   ███████╗
7     ██║     ██║██║   ██║██║   ██║██║   ██║██╔══██║   ██║   ╚════██║
8     ███████╗██║╚██████╔╝╚██████╔╝╚██████╔╝██║  ██║   ██║   ███████║
9     ╚══════╝╚═╝ ╚═════╝  ╚═════╝  ╚═════╝ ╚═╝  ╚═╝   ╚═╝   ╚══════╝®
10 
11     DEM simulation engine, released by
12     DCS Computing Gmbh, Linz, Austria
13     http://www.dcs-computing.com, office@dcs-computing.com
14 
15     LIGGGHTS® is part of CFDEM®project:
16     http://www.liggghts.com | http://www.cfdem.com
17 
18     Core developer and main author:
19     Christoph Kloss, christoph.kloss@dcs-computing.com
20 
21     LIGGGHTS® is open-source, distributed under the terms of the GNU Public
22     License, version 2 or later. It is distributed in the hope that it will
23     be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
24     of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. You should have
25     received a copy of the GNU General Public License along with LIGGGHTS®.
26     If not, see http://www.gnu.org/licenses . See also top-level README
27     and LICENSE files.
28 
29     LIGGGHTS® and CFDEM® are registered trade marks of DCS Computing GmbH,
30     the producer of the LIGGGHTS® software and the CFDEM®coupling software
31     See http://www.cfdem.com/terms-trademark-policy for details.
32 
33 -------------------------------------------------------------------------
34     Contributing author and copyright for this file:
35     (if not contributing author is listed, this file has been contributed
36     by the core developer)
37 
38     Copyright 2012-     DCS Computing GmbH, Linz
39     Copyright 2009-2012 JKU Linz
40 ------------------------------------------------------------------------- */
41 
42 #ifndef LMP_MULTI_NODE_MESH_PARALLEL_H
43 #define LMP_MULTI_NODE_MESH_PARALLEL_H
44 
45 #include "mpi_liggghts.h"
46 #include "multi_node_mesh.h"
47 #include "comm.h"
48 #include "error.h"
49 #include "vector_liggghts.h"
50 #include "neighbor.h"
51 #include "math_extra_liggghts.h"
52 #include "container_base.h"
53 #include "domain_wedge.h"
54 #include <string>
55 #include <list>
56 #include <cmath>
57 #include <algorithm>
58 
59 namespace LAMMPS_NS
60 {
61 
62   template<int NUM_NODES>
63   class MultiNodeMeshParallel : public MultiNodeMesh<NUM_NODES>
64   {
65       public:
66 
67         void initialSetup();
68         void pbcExchangeBorders(int setupFlag);
69         void clearReverse();
70         void forwardComm(std::string property);
71         void forwardComm(std::list<std::string> * properties = NULL);
72         void reverseComm(std::string property);
73         void reverseComm(std::list<std::string> * properties = NULL);
74 
75         void writeRestart(FILE *fp);
76         void restart(double *list);
77 
78         bool allNodesInsideSimulationBox();
79         void useAsInsertionMesh(bool parallel);
80 
isInsertionMesh()81         inline bool isInsertionMesh()
82         { return isInsertionMesh_; }
83 
sizeLocal()84         inline int sizeLocal()
85         { return nLocal_; }
86 
sizeGhost()87         inline int sizeGhost()
88         { return nGhost_; }
89 
sizeGlobal()90         inline int sizeGlobal()
91         { return nGlobal_; }
92 
sizeGlobalOrig()93         inline int sizeGlobalOrig()
94         { return nGlobalOrig_; }
95 
isParallel()96         inline bool isParallel()
97         { return isParallel_; }
98 
99         virtual int id(int i) = 0;
100 
101       protected:
102 
103         MultiNodeMeshParallel(LAMMPS *lmp);
104         virtual ~MultiNodeMeshParallel();
105 
106         virtual bool addElement(double **nodeToAdd);
107 
108         virtual bool addElement(double **nodeToAdd,int lineNumb) = 0;
109         virtual void deleteElement(int n);
110 
111         virtual void buildNeighbours() = 0;
112 
113         virtual void refreshOwned(int setupFlag);
114         virtual void refreshGhosts(int setupFlag);
115 
116         virtual void qualityCheck() = 0;
117 
preSetup()118         virtual void preSetup() {}
preInitialSetup()119         virtual void preInitialSetup() {}
postInitialSetup()120         virtual void postInitialSetup() {}
postBorders()121         virtual void postBorders() {}
122 
123         virtual void clearMap() = 0;
124         virtual void generateMap() = 0;
125         virtual void clearGhostForward(bool scale,bool translate,bool rotate);
126 
127         // lo-level parallelization also used by derived classes
128 
129         virtual int elemListBufSize(int n,int operation,bool scale,bool translate,bool rotate);
130         virtual int pushElemListToBuffer(int n, int *list, int *wraplist, double *buf, int operation, std::list<std::string> * properties, double *dlo, double *dhi, bool scale,bool translate, bool rotate);
131         virtual int popElemListFromBuffer(int first, int n, double *buf, int operation, std::list<std::string> * properties, bool scale,bool translate, bool rotate);
132         virtual int pushElemListToBufferReverse(int first, int n, double *buf, int operation, std::list<std::string> * properties, bool scale,bool translate, bool rotate);
133         virtual int popElemListFromBufferReverse(int n, int *list, double *buf, int operation, std::list<std::string> * properties, bool scale,bool translate, bool rotate);
134 
135         virtual int elemBufSize(int operation, std::list<std::string> * properties, bool scale,bool translate,bool rotate);
136         virtual int pushElemToBuffer(int i, double *buf,int operation,bool scale,bool translate,bool rotate);
137         virtual int popElemFromBuffer(double *buf,int operation,bool scale,bool translate,bool rotate);
138 
139         virtual int meshPropsBufSize(int operation,bool scale,bool translate,bool rotate) = 0;
140         virtual int pushMeshPropsToBuffer(double *buf, int operation,bool scale,bool translate, bool rotate) = 0;
141         virtual int popMeshPropsFromBuffer(double *buf, int operation,bool scale,bool translate, bool rotate) = 0;
142 
143         // flags if mesh should be parallelized
144         bool doParallellization_;
145 
146       private:
147 
148         // parallelization functions
149 
150         void setup();
151         void deleteUnowned();
152         void pbc();
153         void exchange();
154         void borders();
155         void clearGhosts();
156 
157         int checkBorderElement (const int, const int, const int, const double, const double) const;
158         int checkBorderElementLeft (const int, const int, const double, const double) const;
159         int checkBorderElementRight(const int, const int, const double, const double) const;
160 
161         // lo-level parallelization
162         int pushExchange(int dim);
163         void popExchange(int nrecv,int dim,double *buf);
164 
165         int sizeRestartMesh();
166         int sizeRestartElement();
167 
168         // number of local and ghost elements
169         int nLocal_;
170         int nGhost_;
171         int nGlobal_;
172 
173         // initial number of elements
174         int nGlobalOrig_;
175 
176         // flags if mesh is parallelized
177         bool isParallel_;
178 
179         // flag indicating usage as insertion mesh
180         bool isInsertionMesh_;
181 
182         // *************************************
183         // comm stuff - similar to Comm class
184         // *************************************
185 
186         void grow_swap(int n);
187         void allocate_swap(int n);
188         void free_swap();
189         void grow_send(int,int);
190         void grow_recv(int);
191         void grow_list(int, int);
192 
193         // communication buffers
194         int maxsend_,maxrecv_;       // current size of send/recv buffer
195         double *buf_send_, *buf_recv_;
196 
197         // comm
198         int sendneed_[3][2];         // # of procs away I send elements to
199         int maxneed_[3];             // max procs away any proc needs, per dim
200         double half_atom_cut_;       // half atom neigh cut
201 
202         int size_exchange_;          // # of per-element datums in exchange
203         int size_forward_;           // # of per-element datums in forward comm
204         int size_reverse_;           // # of datums in reverse comm
205         int size_border_;            // # of datums in forward border comm
206 
207         int maxforward_,maxreverse_; // max # of datums in forward/reverse comm
208 
209         // comm swaps
210 
211         int nswap_;                  // # of swaps to perform = sum of maxneed
212         int maxswap_;                // # of swaps where mem was allocated
213         int *sendnum_,*recvnum_;     // # of atoms to send/recv in each swap
214         int *firstrecv_;             // where to put 1st recv atom in each swap
215         int *sendproc_,*recvproc_;   // proc to send/recv to/from at each swap
216         int *size_forward_recv_;     // # of values to recv in each forward comm
217         int *size_reverse_recv_;     // # to recv in each reverse comm
218 
219         double *slablo_,*slabhi_;
220         int **sendlist_;             // list of elements to send in each swap
221         int **sendwraplist_;         // whether an element needs to be wrapped or not
222         int *maxsendlist_;           // max size of send list for each swap
223 
224         int *pbc_flag_;              // general flag for sending atoms thru PBC
225         int **pbc_;                  // dimension flags for PBC adjustments
226   };
227 
228   // *************************************
229   #include "multi_node_mesh_parallel_I.h"
230   #include "multi_node_mesh_parallel_buffer_I.h"
231   // *************************************
232 
233 } /* LAMMPS_NS */
234 #endif /* MULTINODEMESH_H_ */
235