1 /* ----------------------------------------------------------------------
2 This is the
3
4 ██╗ ██╗ ██████╗ ██████╗ ██████╗ ██╗ ██╗████████╗███████╗
5 ██║ ██║██╔════╝ ██╔════╝ ██╔════╝ ██║ ██║╚══██╔══╝██╔════╝
6 ██║ ██║██║ ███╗██║ ███╗██║ ███╗███████║ ██║ ███████╗
7 ██║ ██║██║ ██║██║ ██║██║ ██║██╔══██║ ██║ ╚════██║
8 ███████╗██║╚██████╔╝╚██████╔╝╚██████╔╝██║ ██║ ██║ ███████║
9 ╚══════╝╚═╝ ╚═════╝ ╚═════╝ ╚═════╝ ╚═╝ ╚═╝ ╚═╝ ╚══════╝®
10
11 DEM simulation engine, released by
12 DCS Computing Gmbh, Linz, Austria
13 http://www.dcs-computing.com, office@dcs-computing.com
14
15 LIGGGHTS® is part of CFDEM®project:
16 http://www.liggghts.com | http://www.cfdem.com
17
18 Core developer and main author:
19 Christoph Kloss, christoph.kloss@dcs-computing.com
20
21 LIGGGHTS® is open-source, distributed under the terms of the GNU Public
22 License, version 2 or later. It is distributed in the hope that it will
23 be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
24 of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. You should have
25 received a copy of the GNU General Public License along with LIGGGHTS®.
26 If not, see http://www.gnu.org/licenses . See also top-level README
27 and LICENSE files.
28
29 LIGGGHTS® and CFDEM® are registered trade marks of DCS Computing GmbH,
30 the producer of the LIGGGHTS® software and the CFDEM®coupling software
31 See http://www.cfdem.com/terms-trademark-policy for details.
32
33 -------------------------------------------------------------------------
34 Contributing author and copyright for this file:
35 This file is from LAMMPS, but has been modified. Copyright for
36 modification:
37
38 Copyright 2012- DCS Computing GmbH, Linz
39 Copyright 2009-2012 JKU Linz
40
41 Copyright of original file:
42 LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator
43 http://lammps.sandia.gov, Sandia National Laboratories
44 Steve Plimpton, sjplimp@sandia.gov
45
46 Copyright (2003) Sandia Corporation. Under the terms of Contract
47 DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
48 certain rights in this software. This software is distributed under
49 the GNU General Public License.
50 ------------------------------------------------------------------------- */
51
52 #include <mpi.h>
53 #include <stdlib.h>
54 #include <string.h>
55 #include <stdio.h>
56 #include "universe.h"
57 #include "version.h"
58 #include "version_liggghts.h"
59 #include "error.h"
60 #include "force.h"
61 #include "memory.h"
62
63 using namespace LAMMPS_NS;
64
65 #define MAXLINE 256
66
67 /* ----------------------------------------------------------------------
68 create & initialize the universe of processors in communicator
69 ------------------------------------------------------------------------- */
70
Universe(LAMMPS * lmp,MPI_Comm communicator)71 Universe::Universe(LAMMPS *lmp, MPI_Comm communicator) : Pointers(lmp)
72 {
73 version = new char[strlen(LAMMPS_VERSION)+strlen(LIGGGHTS_VERSION)+100];
74
75 //sprintf(version,"Version %s based on LAMMPS %s",LIGGGHTS_VERSION,LAMMPS_VERSION);
76 sprintf(version,"Version %s",LIGGGHTS_VERSION);
77
78 uworld = uorig = communicator;
79 MPI_Comm_rank(uworld,&me);
80 MPI_Comm_size(uworld,&nprocs);
81
82 uscreen = stdout;
83 ulogfile = NULL;
84 uthermofile = NULL;
85
86 existflag = 0;
87 nworlds = 0;
88 procs_per_world = NULL;
89 root_proc = NULL;
90
91 memory->create(uni2orig,nprocs,"universe:uni2orig");
92 for (int i = 0; i < nprocs; i++) uni2orig[i] = i;
93
94 universe_id = 0;
95 }
96
97 /* ---------------------------------------------------------------------- */
98
~Universe()99 Universe::~Universe()
100 {
101 if (uworld != uorig) MPI_Comm_free(&uworld);
102 memory->destroy(procs_per_world);
103 memory->destroy(root_proc);
104 memory->destroy(uni2orig);
105 delete []version;
106 if(universe_id) delete []universe_id;
107 }
108
109 /* ----------------------------------------------------------------------
110 reorder universe processors
111 create uni2orig as inverse mapping
112 re-create uworld communicator with new ordering via Comm_split()
113 style = "nth", arg = N
114 move every Nth proc to end of rankings
115 style = "custom", arg = filename
116 file has nprocs lines with I J
117 I = universe proc ID in original communicator uorig
118 J = universe proc ID in reordered communicator uworld
119 ------------------------------------------------------------------------- */
120
reorder(char * style,char * arg)121 void Universe::reorder(char *style, char *arg)
122 {
123 char line[MAXLINE];
124
125 if (uworld != uorig) MPI_Comm_free(&uworld);
126
127 if (strcmp(style,"nth") == 0) {
128 int n = force->inumeric(FLERR,arg);
129 if (n <= 0)
130 error->universe_all(FLERR,"Invalid -reorder N value");
131 if (nprocs % n)
132 error->universe_all(FLERR,"Nprocs not a multiple of N for -reorder");
133 for (int i = 0; i < nprocs; i++) {
134 if (i < (n-1)*nprocs/n) uni2orig[i] = i/(n-1) * n + (i % (n-1));
135 else uni2orig[i] = (i - (n-1)*nprocs/n) * n + n-1;
136 }
137
138 } else if (strcmp(style,"custom") == 0) {
139
140 if (me == 0) {
141 FILE *fp = fopen(arg,"r");
142 if (fp == NULL) error->universe_one(FLERR,"Cannot open -reorder file");
143
144 // skip header = blank and comment lines
145
146 char *ptr;
147 if (!fgets(line,MAXLINE,fp))
148 error->one(FLERR,"Unexpected end of -reorder file");
149 while (1) {
150 if ((ptr = strchr(line,'#'))) *ptr = '\0';
151 if (strspn(line," \t\n\r") != strlen(line)) break;
152 if (!fgets(line,MAXLINE,fp))
153 error->one(FLERR,"Unexpected end of -reorder file");
154 }
155
156 // read nprocs lines
157 // uni2orig = inverse mapping
158
159 int me_orig,me_new;
160 sscanf(line,"%d %d",&me_orig,&me_new);
161 if (me_orig < 0 || me_orig >= nprocs ||
162 me_new < 0 || me_new >= nprocs)
163 error->one(FLERR,"Invalid entry in -reorder file");
164 uni2orig[me_new] = me_orig;
165
166 for (int i = 1; i < nprocs; i++) {
167 if (!fgets(line,MAXLINE,fp))
168 error->one(FLERR,"Unexpected end of -reorder file");
169 sscanf(line,"%d %d",&me_orig,&me_new);
170 if (me_orig < 0 || me_orig >= nprocs ||
171 me_new < 0 || me_new >= nprocs)
172 error->one(FLERR,"Invalid entry in -reorder file");
173 uni2orig[me_new] = me_orig;
174 }
175 fclose(fp);
176 }
177
178 // bcast uni2org from proc 0 to all other universe procs
179
180 MPI_Bcast(uni2orig,nprocs,MPI_INT,0,uorig);
181
182 } else error->universe_all(FLERR,"Invalid command-line argument");
183
184 // create new uworld communicator
185
186 int ome,key = 0;
187 MPI_Comm_rank(uorig,&ome);
188 for (int i = 0; i < nprocs; i++)
189 if (uni2orig[i] == ome) key = i;
190
191 MPI_Comm_split(uorig,0,key,&uworld);
192 MPI_Comm_rank(uworld,&me);
193 MPI_Comm_size(uworld,&nprocs);
194 }
195
196 /* ----------------------------------------------------------------------
197 add 1 or more worlds to universe
198 str == NULL -> add 1 world with all procs in universe
199 str = NxM -> add N worlds, each with M procs
200 str = P -> add 1 world with P procs
201 ------------------------------------------------------------------------- */
202
add_world(char * str)203 void Universe::add_world(char *str)
204 {
205 int n,nper;
206 char *ptr;
207
208 if (str == NULL) {
209 n = 1;
210 nper = nprocs;
211 } else if ((ptr = strchr(str,'x')) != NULL) {
212 *ptr = '\0';
213 n = atoi(str);
214 nper = atoi(ptr+1);
215 } else {
216 n = 1;
217 nper = atoi(str);
218 }
219
220 memory->grow(procs_per_world,nworlds+n,"universe:procs_per_world");
221 memory->grow(root_proc,(nworlds+n),"universe:root_proc");
222
223 for (int i = 0; i < n; i++) {
224 procs_per_world[nworlds] = nper;
225 if (nworlds == 0) root_proc[nworlds] = 0;
226 else
227 root_proc[nworlds] = root_proc[nworlds-1] + procs_per_world[nworlds-1];
228 if (me >= root_proc[nworlds]) iworld = nworlds;
229 nworlds++;
230 }
231 }
232
233 /* ----------------------------------------------------------------------
234 check if total procs in all worlds = procs in universe
235 ------------------------------------------------------------------------- */
236
consistent()237 int Universe::consistent()
238 {
239 int n = 0;
240 for (int i = 0; i < nworlds; i++) n += procs_per_world[i];
241 if (n == nprocs) return 1;
242 else return 0;
243 }
244
245 /* ----------------------------------------------------------------------
246 give this universe an ID
247 ------------------------------------------------------------------------- */
248
id(char * id)249 void Universe::id(char *id)
250 {
251 universe_id = new char[strlen(id)+1];
252 strcpy(universe_id,id);
253 }
254