1 /* ---------------------------------------------------------------------- 2 This is the 3 4 ██╗ ██╗ ██████╗ ██████╗ ██████╗ ██╗ ██╗████████╗███████╗ 5 ██║ ██║██╔════╝ ██╔════╝ ██╔════╝ ██║ ██║╚══██╔══╝██╔════╝ 6 ██║ ██║██║ ███╗██║ ███╗██║ ███╗███████║ ██║ ███████╗ 7 ██║ ██║██║ ██║██║ ██║██║ ██║██╔══██║ ██║ ╚════██║ 8 ███████╗██║╚██████╔╝╚██████╔╝╚██████╔╝██║ ██║ ██║ ███████║ 9 ╚══════╝╚═╝ ╚═════╝ ╚═════╝ ╚═════╝ ╚═╝ ╚═╝ ╚═╝ ╚══════╝® 10 11 DEM simulation engine, released by 12 DCS Computing Gmbh, Linz, Austria 13 http://www.dcs-computing.com, office@dcs-computing.com 14 15 LIGGGHTS® is part of CFDEM®project: 16 http://www.liggghts.com | http://www.cfdem.com 17 18 Core developer and main author: 19 Christoph Kloss, christoph.kloss@dcs-computing.com 20 21 LIGGGHTS® is open-source, distributed under the terms of the GNU Public 22 License, version 2 or later. It is distributed in the hope that it will 23 be useful, but WITHOUT ANY WARRANTY; without even the implied warranty 24 of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. You should have 25 received a copy of the GNU General Public License along with LIGGGHTS®. 26 If not, see http://www.gnu.org/licenses . See also top-level README 27 and LICENSE files. 28 29 LIGGGHTS® and CFDEM® are registered trade marks of DCS Computing GmbH, 30 the producer of the LIGGGHTS® software and the CFDEM®coupling software 31 See http://www.cfdem.com/terms-trademark-policy for details. 32 33 ------------------------------------------------------------------------- 34 Contributing author and copyright for this file: 35 (if not contributing author is listed, this file has been contributed 36 by the core developer) 37 38 Copyright 2012- DCS Computing GmbH, Linz 39 Copyright 2009-2012 JKU Linz 40 ------------------------------------------------------------------------- */ 41 42 #ifndef LMP_MULTI_NODE_MESH_PARALLEL_BUFFER_I_H 43 #define LMP_MULTI_NODE_MESH_PARALLEL_BUFFER_I_H 44 45 /* ---------------------------------------------------------------------- 46 push / pop for exchange 47 ------------------------------------------------------------------------- */ 48 49 template<int NUM_NODES> pushExchange(int dim)50 int MultiNodeMeshParallel<NUM_NODES>::pushExchange(int dim) 51 { 52 53 // scale translate rotate not needed here 54 bool dummy = false; 55 double checklo,checkhi; 56 57 checklo = this->domain->sublo[dim]; 58 if(this->domain->subhi[dim] == this->domain->boxhi[dim]) 59 checkhi = this->domain->boxhi[dim] + SMALL_DMBRDR; 60 else 61 checkhi = this->domain->subhi[dim]; 62 63 int nsend = 0, nsend_this = 0; 64 int i = 0; 65 while(i < nLocal_) 66 { 67 if(!(this->center_(i)[dim] >= checklo && this->center_(i)[dim] < checkhi)) 68 { 69 nsend_this = pushElemToBuffer(i,&(buf_send_[nsend+1]),OPERATION_COMM_EXCHANGE,dummy,dummy,dummy); 70 buf_send_[nsend] = static_cast<double>(nsend_this+1); 71 nsend += (nsend_this+1); 72 73 if (nsend > maxsend_) 74 grow_send(nsend,1); 75 this->deleteElement(i); 76 } 77 else i++; 78 } 79 return nsend; 80 } 81 82 template<int NUM_NODES> popExchange(int nrecv,int dim,double * buf)83 void MultiNodeMeshParallel<NUM_NODES>::popExchange(int nrecv,int dim,double *buf) 84 { 85 double center_elem[3]; 86 double checklo,checkhi; 87 int m = 0, nrecv_this; 88 89 // scale translate rotate not needed here 90 bool dummy = false; 91 92 checklo = this->domain->sublo[dim]; 93 if(this->domain->subhi[dim] == this->domain->boxhi[dim]) 94 checkhi = this->domain->boxhi[dim] + SMALL_DMBRDR; 95 else 96 checkhi = this->domain->subhi[dim]; 97 98 while (m < nrecv) 99 { 100 // number of values is first in buffer 101 nrecv_this = static_cast<int>(buf[m]); 102 103 // center is next in buffer, test it 104 vectorCopy3D(&(buf[m+1]),center_elem); 105 106 if(center_elem[dim] >= checklo && center_elem[dim] < checkhi) 107 { 108 popElemFromBuffer(&(buf[m+1]),OPERATION_COMM_EXCHANGE,dummy,dummy,dummy); 109 nLocal_++; 110 111 } 112 113 m += nrecv_this; 114 } 115 } 116 117 /* ---------------------------------------------------------------------- 118 restart functionality - write all required data into restart buffer 119 executed on all processes, but only proc 0 writes into writebuf 120 ------------------------------------------------------------------------- */ 121 122 template<int NUM_NODES> writeRestart(FILE * fp)123 void MultiNodeMeshParallel<NUM_NODES>::writeRestart(FILE *fp) 124 { 125 int size_this; 126 127 // # elements 128 int nlocal = this->sizeLocal(); 129 int nglobal = sizeGlobal(); 130 131 // buffer sizes 132 int sizeMesh, sizeElements, sizeElements_all; 133 134 sizeMesh = sizeRestartMesh(); 135 sizeElements = nlocal * (sizeRestartElement() + 1); 136 137 double *bufMesh = NULL, *sendbufElems = NULL, *recvbufElems = NULL; 138 bool dummy = false; 139 140 // pack global data into buffer 141 // do this only on proc 0 142 if(this->comm->me == 0) 143 { 144 this->memory->create(bufMesh,sizeMesh,"MultiNodeMeshParallel::writeRestart:bufMesh"); 145 pushMeshPropsToBuffer(bufMesh, OPERATION_RESTART,dummy,dummy,dummy); 146 } 147 148 // allocate send buffer and pack element data 149 // all local elements are in list 150 this->memory->create(sendbufElems,sizeElements,"MultiNodeMeshParallel::writeRestart:sendbuf"); 151 sizeElements = 0; 152 for(int i = 0; i < nlocal; i++) 153 { 154 size_this = pushElemToBuffer(i,&(sendbufElems[sizeElements+1]),OPERATION_RESTART,dummy,dummy,dummy); 155 sendbufElems[sizeElements] = static_cast<double>(size_this+1); 156 sizeElements += (size_this+1); 157 } 158 159 // gather the per-element data 160 161 sizeElements_all = MPI_Gather0_Vector(sendbufElems,sizeElements,recvbufElems,this->world); 162 163 // actually write data to restart file 164 // do this only on proc 0 165 if(this->comm->me == 0) 166 { 167 double nG = static_cast<double>(nglobal); 168 169 // for error check 170 double sE = static_cast<double>(sizeRestartElement()); 171 double sM = static_cast<double>(sizeRestartMesh()); 172 173 // size with 3 extra values 174 int size = (sizeMesh+sizeElements_all+3) * sizeof(double); 175 176 // write size 177 fwrite(&size,sizeof(int),1,fp); 178 179 // write 3 extra values 180 fwrite(&nG,sizeof(double),1,fp); 181 fwrite(&sE,sizeof(double),1,fp); 182 fwrite(&sM,sizeof(double),1,fp); 183 184 // write per-element and mesh data 185 fwrite(recvbufElems,sizeof(double),sizeElements_all,fp); 186 fwrite(bufMesh,sizeof(double),sizeMesh,fp); 187 } 188 189 // free mem 190 191 if(bufMesh) 192 this->memory->destroy(bufMesh); 193 194 this->memory->destroy(sendbufElems); 195 196 if(recvbufElems) 197 delete []recvbufElems; 198 } 199 200 /* ---------------------------------------------------------------------- 201 restart functionality - read all required data from restart buffer 202 executed on all processes 203 ------------------------------------------------------------------------- */ 204 205 template<int NUM_NODES> restart(double * list)206 void MultiNodeMeshParallel<NUM_NODES>::restart(double *list) 207 { 208 int m, nglobal, nrecv_this, sE, sM; 209 bool dummy = false; 210 211 m = 0; 212 213 nglobal = static_cast<int> (list[m++]); 214 sE = static_cast<int> (list[m++]); 215 sM = static_cast<int> (list[m++]); 216 217 if(sE != sizeRestartElement() || sM != sizeRestartMesh()) 218 this->error->all(FLERR,"Incompatible mesh restart file - mesh has different properties in restarted simulation"); 219 220 for(int i = 0; i < nglobal; i++) 221 { 222 nrecv_this = static_cast<int>(list[m]); 223 224 popElemFromBuffer(&(list[m+1]),OPERATION_RESTART,dummy,dummy,dummy); 225 m += nrecv_this; 226 } 227 228 this->prop().deleteRestartGlobal(dummy,dummy,dummy); 229 popMeshPropsFromBuffer(&list[m],OPERATION_RESTART,dummy,dummy,dummy); 230 } 231 232 /* ---------------------------------------------------------------------- 233 size of restart data 234 ------------------------------------------------------------------------- */ 235 236 template<int NUM_NODES> sizeRestartMesh()237 int MultiNodeMeshParallel<NUM_NODES>::sizeRestartMesh() 238 { 239 240 bool dummy = false; 241 return meshPropsBufSize(OPERATION_RESTART,dummy,dummy,dummy); 242 } 243 244 template<int NUM_NODES> sizeRestartElement()245 int MultiNodeMeshParallel<NUM_NODES>::sizeRestartElement() 246 { 247 248 bool dummy = false; 249 return elemBufSize(OPERATION_RESTART, NULL, dummy,dummy,dummy); 250 } 251 252 /* ---------------------------------------------------------------------- 253 return required buffer size for a list of elements for borders(),forwardComm() 254 must match push / pop implementation 255 depending on operation and if mesh scales, translates or rotates, 256 different properties are communicated 257 ------------------------------------------------------------------------- */ 258 259 template<int NUM_NODES> elemListBufSize(int n,int operation,bool scale,bool translate,bool rotate)260 int MultiNodeMeshParallel<NUM_NODES>::elemListBufSize(int n,int operation,bool scale,bool translate,bool rotate) 261 { 262 return n*elemBufSize(operation, NULL, scale,translate,rotate); 263 } 264 265 /* ---------------------------------------------------------------------- 266 push a list of elements for borders(), forwardComm() 267 depending on operation and if mesh scales, translates or rotates, 268 different properties are communicated 269 ------------------------------------------------------------------------- */ 270 271 template<int NUM_NODES> pushElemListToBuffer(int n,int * list,int * wraplist,double * buf,int operation,std::list<std::string> * properties,double * dlo,double * dhi,bool,bool,bool)272 int MultiNodeMeshParallel<NUM_NODES>::pushElemListToBuffer(int n, int *list, int *wraplist, double *buf, int operation, std::list<std::string> * properties, double *dlo, double *dhi, bool , bool, bool) 273 { 274 275 int nsend = 0; 276 277 if(OPERATION_COMM_EXCHANGE == operation || OPERATION_COMM_BORDERS == operation) 278 { 279 280 if (!properties || MultiNodeMesh<NUM_NODES>::center_.matches_any_id(properties)) 281 nsend += MultiNodeMesh<NUM_NODES>::center_.pushElemListToBuffer(n,list, wraplist, &(buf[nsend]),operation, dlo, dhi); 282 if (!properties || MultiNodeMesh<NUM_NODES>::node_.matches_any_id(properties)) 283 nsend += MultiNodeMesh<NUM_NODES>::node_.pushElemListToBuffer(n,list, wraplist, &(buf[nsend]),operation, dlo, dhi); 284 if (!properties || MultiNodeMesh<NUM_NODES>::rBound_.matches_any_id(properties)) 285 nsend += MultiNodeMesh<NUM_NODES>::rBound_.pushElemListToBuffer(n,list, wraplist, &(buf[nsend]),operation, dlo, dhi); 286 if(this->node_orig_) 287 { 288 if (!properties || MultiNodeMesh<NUM_NODES>::node_orig_->matches_any_id(properties)) 289 nsend += this->node_orig_->pushElemListToBuffer(n,list, wraplist, &(buf[nsend]),operation, dlo, dhi); 290 } 291 return nsend; 292 } 293 294 if(OPERATION_COMM_FORWARD == operation) 295 { 296 297 return nsend; 298 } 299 300 this->error->one(FLERR,"Illegal operation in MultiNodeMeshParallel<NUM_NODES>::pushElemToBuffer"); 301 return 0; 302 } 303 304 /* ---------------------------------------------------------------------- 305 pop a list of elements for borders(), forwardComm() 306 depending on operation and if mesh scales, translates or rotates, 307 different properties are communicated 308 ------------------------------------------------------------------------- */ 309 310 template<int NUM_NODES> popElemListFromBuffer(int first,int n,double * buf,int operation,std::list<std::string> * properties,bool,bool,bool)311 int MultiNodeMeshParallel<NUM_NODES>::popElemListFromBuffer(int first, int n, double *buf, int operation, std::list<std::string> * properties, bool, bool, bool) 312 { 313 int nrecv = 0; 314 315 if(OPERATION_COMM_EXCHANGE == operation || OPERATION_COMM_BORDERS == operation) 316 { 317 if (!properties || MultiNodeMesh<NUM_NODES>::center_.matches_any_id(properties)) 318 nrecv += MultiNodeMesh<NUM_NODES>::center_.popElemListFromBuffer(first,n,&(buf[nrecv]),operation); 319 if (!properties || MultiNodeMesh<NUM_NODES>::node_.matches_any_id(properties)) 320 nrecv += MultiNodeMesh<NUM_NODES>::node_.popElemListFromBuffer(first,n,&(buf[nrecv]),operation); 321 if (!properties || MultiNodeMesh<NUM_NODES>::rBound_.matches_any_id(properties)) 322 nrecv += MultiNodeMesh<NUM_NODES>::rBound_.popElemListFromBuffer(first,n,&(buf[nrecv]),operation); 323 if(MultiNodeMesh<NUM_NODES>::node_orig_) 324 { 325 if (!properties || MultiNodeMesh<NUM_NODES>::node_orig_->matches_any_id(properties)) 326 nrecv += MultiNodeMesh<NUM_NODES>::node_orig_->popElemListFromBuffer(first,n,&(buf[nrecv]),operation); 327 } 328 return nrecv; 329 } 330 331 if(OPERATION_COMM_FORWARD == operation) 332 { 333 334 // nrecv += MultiNodeMesh<NUM_NODES>::node_.popListFromBuffer(first,n,&(buf[nrecv]),operation); 335 return nrecv; 336 } 337 338 this->error->one(FLERR,"Illegal operation in MultiNodeMeshParallel<NUM_NODES>::popElemFromBuffer"); 339 return 0; 340 } 341 342 /* ---------------------------------------------------------------------- 343 push a list of elements for reverseComm() 344 depending on operation and if mesh scales, translates or rotates, 345 different properties are communicated 346 ------------------------------------------------------------------------- */ 347 348 template<int NUM_NODES> pushElemListToBufferReverse(int,int,double *,int operation,std::list<std::string> * properties,bool,bool,bool)349 int MultiNodeMeshParallel<NUM_NODES>::pushElemListToBufferReverse(int, int, double*, int operation, std::list<std::string> * properties, bool, bool, bool) 350 { 351 int nsend = 0; 352 353 if(OPERATION_COMM_REVERSE == operation) 354 { 355 356 return nsend; 357 } 358 359 this->error->one(FLERR,"Illegal operation in MultiNodeMeshParallel<NUM_NODES>::popElemFromBuffer"); 360 return 0; 361 } 362 363 /* ---------------------------------------------------------------------- 364 pop a list of elements for reverseComm() 365 depending on operation and if mesh scales, translates or rotates, 366 different properties are communicated 367 ------------------------------------------------------------------------- */ 368 369 template<int NUM_NODES> popElemListFromBufferReverse(int,int *,double *,int operation,std::list<std::string> * properties,bool,bool,bool)370 int MultiNodeMeshParallel<NUM_NODES>::popElemListFromBufferReverse(int, int*, double*, int operation, std::list<std::string> * properties, bool, bool, bool) 371 { 372 int nrecv = 0; 373 374 if(OPERATION_COMM_REVERSE == operation) 375 { 376 377 return nrecv; 378 } 379 380 this->error->one(FLERR,"Illegal operation in MultiNodeMeshParallel<NUM_NODES>::popElemFromBuffer"); 381 return 0; 382 } 383 384 /* ---------------------------------------------------------------------- 385 return required buffer size for one element for exchange() 386 must match push / pop implementation 387 depending on operation and if mesh scales, translates or rotates, 388 different properties are communicated 389 ------------------------------------------------------------------------- */ 390 391 template<int NUM_NODES> elemBufSize(int operation,std::list<std::string> * properties,bool,bool,bool)392 int MultiNodeMeshParallel<NUM_NODES>::elemBufSize(int operation, std::list<std::string> * properties, bool, bool, bool) 393 { 394 int size_buf = 0; 395 396 if(OPERATION_RESTART == operation) 397 { 398 if (!properties || MultiNodeMesh<NUM_NODES>::node_.matches_any_id(properties)) 399 size_buf += MultiNodeMesh<NUM_NODES>::node_.elemBufSize(); 400 return size_buf; 401 } 402 403 if(OPERATION_COMM_EXCHANGE == operation || OPERATION_COMM_BORDERS == operation) 404 { 405 if (!properties || MultiNodeMesh<NUM_NODES>::center_.matches_any_id(properties)) 406 size_buf += MultiNodeMesh<NUM_NODES>::center_.elemBufSize(); 407 if (!properties || MultiNodeMesh<NUM_NODES>::node_.matches_any_id(properties)) 408 size_buf += MultiNodeMesh<NUM_NODES>::node_.elemBufSize(); 409 if (!properties || MultiNodeMesh<NUM_NODES>::rBound_.matches_any_id(properties)) 410 size_buf += MultiNodeMesh<NUM_NODES>::rBound_.elemBufSize(); 411 if(MultiNodeMesh<NUM_NODES>::node_orig_) 412 { 413 if (!properties || MultiNodeMesh<NUM_NODES>::node_orig_->matches_any_id(properties)) 414 size_buf += MultiNodeMesh<NUM_NODES>::node_orig_->elemBufSize(); 415 } 416 return size_buf; 417 } 418 419 if(OPERATION_COMM_FORWARD == operation) 420 { 421 422 return size_buf; 423 } 424 425 if(OPERATION_COMM_REVERSE == operation) 426 { 427 428 return size_buf; 429 } 430 431 this->error->one(FLERR,"Illegal operation in MultiNodeMeshParallel<NUM_NODES>::elemBufSize"); 432 return 0; 433 } 434 435 /* ---------------------------------------------------------------------- 436 push one element for exchange() 437 depending on operation and if mesh scales, translates or rotates, 438 different properties are communicated 439 ------------------------------------------------------------------------- */ 440 441 template<int NUM_NODES> pushElemToBuffer(int i,double * buf,int operation,bool,bool,bool)442 int MultiNodeMeshParallel<NUM_NODES>::pushElemToBuffer(int i, double *buf, int operation, bool, bool, bool) 443 { 444 int nsend = 0; 445 446 if(OPERATION_RESTART == operation) 447 { 448 nsend += MultiNodeMesh<NUM_NODES>::node_.pushElemToBuffer(i,&(buf[nsend]),operation); 449 450 return nsend; 451 } 452 453 if(OPERATION_COMM_EXCHANGE == operation || OPERATION_COMM_BORDERS == operation) 454 { 455 456 nsend += MultiNodeMesh<NUM_NODES>::center_.pushElemToBuffer(i,&(buf[nsend]),operation); 457 nsend += MultiNodeMesh<NUM_NODES>::node_.pushElemToBuffer(i,&(buf[nsend]),operation); 458 nsend += MultiNodeMesh<NUM_NODES>::rBound_.pushElemToBuffer(i,&(buf[nsend]),operation); 459 if(this->node_orig_) 460 nsend += this->node_orig_->pushElemToBuffer(i,&(buf[nsend]),operation); 461 return nsend; 462 } 463 464 this->error->one(FLERR,"Illegal operation in MultiNodeMeshParallel<NUM_NODES>::pushElemToBuffer"); 465 return 0; 466 } 467 468 /* ---------------------------------------------------------------------- 469 pop one element for exchange, restart or restart 470 depending on operation and if mesh scales, translates or rotates, 471 different properties are communicated 472 ------------------------------------------------------------------------- */ 473 474 template<int NUM_NODES> popElemFromBuffer(double * buf,int operation,bool,bool,bool)475 int MultiNodeMeshParallel<NUM_NODES>::popElemFromBuffer(double *buf, int operation, bool, bool, bool) 476 { 477 int nrecv = 0; 478 479 if(OPERATION_RESTART == operation) 480 { 481 MultiVectorContainer<double,NUM_NODES,3> nodeTmp("nodeTmp"); 482 483 nrecv += nodeTmp.popElemFromBuffer(&(buf[nrecv]),operation); 484 this->addElement(nodeTmp.begin()[0],-1); 485 486 this->prop().deleteRestartElement(nLocal_-1,false,false,false); 487 488 return nrecv; 489 } 490 491 if(OPERATION_COMM_EXCHANGE == operation || OPERATION_COMM_BORDERS == operation) 492 { 493 nrecv += MultiNodeMesh<NUM_NODES>::center_.popElemFromBuffer(&(buf[nrecv]),operation); 494 nrecv += MultiNodeMesh<NUM_NODES>::node_.popElemFromBuffer(&(buf[nrecv]),operation); 495 nrecv += MultiNodeMesh<NUM_NODES>::rBound_.popElemFromBuffer(&(buf[nrecv]),operation); 496 if(MultiNodeMesh<NUM_NODES>::node_orig_) 497 nrecv += MultiNodeMesh<NUM_NODES>::node_orig_->popElemFromBuffer(&(buf[nrecv]),operation); 498 return nrecv; 499 } 500 501 this->error->one(FLERR,"Illegal operation in MultiNodeMeshParallel<NUM_NODES>::popElemFromBuffer"); 502 return 0; 503 } 504 505 #endif 506