1 /*
2
3 This file defines part of the initialization of PETSc
4
5 This file uses regular malloc and free because it cannot be known
6 what malloc is being used until it has already processed the input.
7 */
8
9 #include <petscsys.h> /*I "petscsys.h" I*/
10 #include <petsc/private/petscimpl.h>
11 #include <petscvalgrind.h>
12 #include <petscviewer.h>
13 #if defined(PETSC_USE_LOG)
14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void);
15 #endif
16
17 #if defined(PETSC_HAVE_SYS_SYSINFO_H)
18 #include <sys/sysinfo.h>
19 #endif
20 #if defined(PETSC_HAVE_UNISTD_H)
21 #include <unistd.h>
22 #endif
23
24 #if defined(PETSC_HAVE_CUDA)
25 #include <cuda_runtime.h>
26 #include <petsccublas.h>
27 #endif
28
29 #if defined(PETSC_HAVE_HIP)
30 #include <hip/hip_runtime.h>
31 #endif
32
33 #if defined(PETSC_HAVE_DEVICE)
34 #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION)
35 #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */
36 #endif
37 #endif
38
39 #if defined(PETSC_HAVE_VIENNACL)
40 PETSC_EXTERN PetscErrorCode PetscViennaCLInit();
41 #endif
42
43
44 /* ------------------------Nasty global variables -------------------------------*/
45 /*
46 Indicates if PETSc started up MPI, or it was
47 already started before PETSc was initialized.
48 */
49 PetscBool PetscBeganMPI = PETSC_FALSE;
50 PetscBool PetscErrorHandlingInitialized = PETSC_FALSE;
51 PetscBool PetscInitializeCalled = PETSC_FALSE;
52 PetscBool PetscFinalizeCalled = PETSC_FALSE;
53
54 PetscMPIInt PetscGlobalRank = -1;
55 PetscMPIInt PetscGlobalSize = -1;
56
57 #if defined(PETSC_HAVE_KOKKOS)
58 PetscBool PetscBeganKokkos = PETSC_FALSE;
59 #endif
60
61 PetscBool use_gpu_aware_mpi = PETSC_TRUE;
62 PetscBool PetscCreatedGpuObjects = PETSC_FALSE;
63
64 #if defined(PETSC_HAVE_COMPLEX)
65 #if defined(PETSC_COMPLEX_INSTANTIATE)
66 template <> class std::complex<double>; /* instantiate complex template class */
67 #endif
68 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
69 MPI_Datatype MPIU_C_DOUBLE_COMPLEX;
70 MPI_Datatype MPIU_C_COMPLEX;
71 #endif
72
73 /*MC
74 PETSC_i - the imaginary number i
75
76 Synopsis:
77 #include <petscsys.h>
78 PetscComplex PETSC_i;
79
80 Level: beginner
81
82 Note:
83 Complex numbers are automatically available if PETSc located a working complex implementation
84
85 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex()
86 M*/
87 PetscComplex PETSC_i;
88 #endif
89 #if defined(PETSC_USE_REAL___FLOAT128)
90 MPI_Datatype MPIU___FLOAT128 = 0;
91 #if defined(PETSC_HAVE_COMPLEX)
92 MPI_Datatype MPIU___COMPLEX128 = 0;
93 #endif
94 #elif defined(PETSC_USE_REAL___FP16)
95 MPI_Datatype MPIU___FP16 = 0;
96 #endif
97 MPI_Datatype MPIU_2SCALAR = 0;
98 #if defined(PETSC_USE_64BIT_INDICES)
99 MPI_Datatype MPIU_2INT = 0;
100 #endif
101 MPI_Datatype MPIU_BOOL;
102 MPI_Datatype MPIU_ENUM;
103 MPI_Datatype MPIU_FORTRANADDR;
104 MPI_Datatype MPIU_SIZE_T;
105
106 /*
107 Function that is called to display all error messages
108 */
109 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault;
110 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault;
111 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault;
112 /*
113 This is needed to turn on/off GPU synchronization
114 */
115 PetscBool PetscViennaCLSynchronize = PETSC_FALSE;
116
117 /* ------------------------------------------------------------------------------*/
118 /*
119 Optional file where all PETSc output from various prints is saved
120 */
121 PETSC_INTERN FILE *petsc_history;
122 FILE *petsc_history = NULL;
123
PetscOpenHistoryFile(const char filename[],FILE ** fd)124 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd)
125 {
126 PetscErrorCode ierr;
127 PetscMPIInt rank,size;
128 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
129 char version[256];
130
131 PetscFunctionBegin;
132 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
133 if (!rank) {
134 char arch[10];
135 int err;
136
137 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr);
138 ierr = PetscGetDate(date,64);CHKERRQ(ierr);
139 ierr = PetscGetVersion(version,256);CHKERRQ(ierr);
140 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
141 if (filename) {
142 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr);
143 } else {
144 ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr);
145 ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr);
146 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr);
147 }
148
149 *fd = fopen(fname,"a");
150 if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);
151
152 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr);
153 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr);
154 ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr);
155 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr);
156 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr);
157
158 err = fflush(*fd);
159 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
160 }
161 PetscFunctionReturn(0);
162 }
163
PetscCloseHistoryFile(FILE ** fd)164 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd)
165 {
166 PetscErrorCode ierr;
167 PetscMPIInt rank;
168 char date[64];
169 int err;
170
171 PetscFunctionBegin;
172 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
173 if (!rank) {
174 ierr = PetscGetDate(date,64);CHKERRQ(ierr);
175 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr);
176 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr);
177 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr);
178 err = fflush(*fd);
179 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
180 err = fclose(*fd);
181 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
182 }
183 PetscFunctionReturn(0);
184 }
185
186 /* ------------------------------------------------------------------------------*/
187
188 /*
189 This is ugly and probably belongs somewhere else, but I want to
190 be able to put a true MPI abort error handler with command line args.
191
192 This is so MPI errors in the debugger will leave all the stack
193 frames. The default MP_Abort() cleans up and exits thus providing no useful information
194 in the debugger hence we call abort() instead of MPI_Abort().
195 */
196
Petsc_MPI_AbortOnError(MPI_Comm * comm,PetscMPIInt * flag,...)197 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
198 {
199 PetscFunctionBegin;
200 (*PetscErrorPrintf)("MPI error %d\n",*flag);
201 abort();
202 }
203
Petsc_MPI_DebuggerOnError(MPI_Comm * comm,PetscMPIInt * flag,...)204 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
205 {
206 PetscErrorCode ierr;
207
208 PetscFunctionBegin;
209 (*PetscErrorPrintf)("MPI error %d\n",*flag);
210 ierr = PetscAttachDebugger();
211 if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */
212 }
213
214 /*@C
215 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one
216 wishes a clean exit somewhere deep in the program.
217
218 Collective on PETSC_COMM_WORLD
219
220 Options Database Keys are the same as for PetscFinalize()
221
222 Level: advanced
223
224 Note:
225 See PetscInitialize() for more general runtime options.
226
227 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
228 @*/
PetscEnd(void)229 PetscErrorCode PetscEnd(void)
230 {
231 PetscFunctionBegin;
232 PetscFinalize();
233 exit(0);
234 return 0;
235 }
236
237 PetscBool PetscOptionsPublish = PETSC_FALSE;
238 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void);
239 PETSC_INTERN PetscBool petscsetmallocvisited;
240 static char emacsmachinename[256];
241
242 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL;
243 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = NULL;
244
245 /*@C
246 PetscSetHelpVersionFunctions - Sets functions that print help and version information
247 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
248 This routine enables a "higher-level" package that uses PETSc to print its messages first.
249
250 Input Parameter:
251 + help - the help function (may be NULL)
252 - version - the version function (may be NULL)
253
254 Level: developer
255
256 @*/
PetscSetHelpVersionFunctions(PetscErrorCode (* help)(MPI_Comm),PetscErrorCode (* version)(MPI_Comm))257 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
258 {
259 PetscFunctionBegin;
260 PetscExternalHelpFunction = help;
261 PetscExternalVersionFunction = version;
262 PetscFunctionReturn(0);
263 }
264
265 #if defined(PETSC_USE_LOG)
266 PETSC_INTERN PetscBool PetscObjectsLog;
267 #endif
268
PetscMPI_Comm_eh(MPI_Comm * comm,PetscMPIInt * err,...)269 void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...)
270 {
271 if (PetscUnlikely(*err)) {
272 PetscMPIInt len;
273 char errstring[MPI_MAX_ERROR_STRING];
274
275 MPI_Error_string(*err,errstring,&len);
276 PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring);
277 }
278 return;
279 }
280
281 /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP.
282 Use the following macros to define CUDA/HIP initialization related vars/routines.
283 */
284 #if defined(PETSC_HAVE_CUDA)
285 typedef cudaError_t cupmError_t;
286 typedef struct cudaDeviceProp cupmDeviceProp;
287 #define cupmGetDeviceCount(x) cudaGetDeviceCount(x)
288 #define cupmGetDevice(x) cudaGetDevice(x)
289 #define cupmSetDevice(x) cudaSetDevice(x)
290 #define cupmSetDeviceFlags(x) cudaSetDeviceFlags(x)
291 #define cupmGetDeviceProperties(x,y) cudaGetDeviceProperties(x,y)
292 #define cupmGetLastError() cudaGetLastError()
293 #define cupmDeviceMapHost cudaDeviceMapHost
294 #define cupmSuccess cudaSuccess
295 #define cupmErrorMemoryAllocation cudaErrorMemoryAllocation
296 #define cupmErrorLaunchOutOfResources cudaErrorLaunchOutOfResources
297 #define cupmErrorSetOnActiveProcess cudaErrorSetOnActiveProcess
298 #define CHKERRCUPM(x) CHKERRCUDA(x)
299 #define PetscCUPMBLASInitializeHandle() PetscCUBLASInitializeHandle()
300 #define PetscCUPMSOLVERDnInitializeHandle() PetscCUSOLVERDnInitializeHandle()
301 #define PetscCUPMInitialize PetscCUDAInitialize
302 #define PetscCUPMInitialized PetscCUDAInitialized
303 #define PetscCUPMInitializeCheck PetscCUDAInitializeCheck
304 #define PetscCUPMInitializeAndView PetscCUDAInitializeAndView
305 #define PetscCUPMSynchronize PetscCUDASynchronize
306 #define PetscNotUseCUPM PetscNotUseCUDA
307 #define cupmOptionsStr "CUDA options"
308 #define cupmSetDeviceStr "-cuda_device"
309 #define cupmViewStr "-cuda_view"
310 #define cupmSynchronizeStr "-cuda_synchronize"
311 #define PetscCUPMInitializeStr "PetscCUDAInitialize"
312 #define PetscOptionsCheckCUPM PetscOptionsCheckCUDA
313 #define PetscMPICUPMAwarenessCheck PetscMPICUDAAwarenessCheck
314 #include "cupminit.inc"
315 #endif
316
317 #if defined(PETSC_HAVE_HIP)
318 typedef hipError_t cupmError_t;
319 typedef hipDeviceProp_t cupmDeviceProp;
320 #define cupmGetDeviceCount(x) hipGetDeviceCount(x)
321 #define cupmGetDevice(x) hipGetDevice(x)
322 #define cupmSetDevice(x) hipSetDevice(x)
323 #define cupmSetDeviceFlags(x) hipSetDeviceFlags(x)
324 #define cupmGetDeviceProperties(x,y) hipGetDeviceProperties(x,y)
325 #define cupmGetLastError() hipGetLastError()
326 #define cupmDeviceMapHost hipDeviceMapHost
327 #define cupmSuccess hipSuccess
328 #define cupmErrorMemoryAllocation hipErrorMemoryAllocation
329 #define cupmErrorLaunchOutOfResources hipErrorLaunchOutOfResources
330 #define cupmErrorSetOnActiveProcess hipErrorSetOnActiveProcess
331 #define CHKERRCUPM(x) CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB)
332 #define PetscCUPMBLASInitializeHandle() 0
333 #define PetscCUPMSOLVERDnInitializeHandle() 0
334 #define PetscCUPMInitialize PetscHIPInitialize
335 #define PetscCUPMInitialized PetscHIPInitialized
336 #define PetscCUPMInitializeCheck PetscHIPInitializeCheck
337 #define PetscCUPMInitializeAndView PetscHIPInitializeAndView
338 #define PetscCUPMSynchronize PetscHIPSynchronize
339 #define PetscNotUseCUPM PetscNotUseHIP
340 #define cupmOptionsStr "HIP options"
341 #define cupmSetDeviceStr "-hip_device"
342 #define cupmViewStr "-hip_view"
343 #define cupmSynchronizeStr "-hip_synchronize"
344 #define PetscCUPMInitializeStr "PetscHIPInitialize"
345 #define PetscOptionsCheckCUPM PetscOptionsCheckHIP
346 #define PetscMPICUPMAwarenessCheck PetscMPIHIPAwarenessCheck
347 #include "cupminit.inc"
348 #endif
349
PetscOptionsCheckInitial_Private(const char help[])350 PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char help[])
351 {
352 char string[64];
353 MPI_Comm comm = PETSC_COMM_WORLD;
354 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView;
355 PetscErrorCode ierr;
356 PetscReal si;
357 PetscInt intensity;
358 int i;
359 PetscMPIInt rank;
360 char version[256];
361 #if defined(PETSC_USE_LOG)
362 char mname[PETSC_MAX_PATH_LEN];
363 PetscViewerFormat format;
364 PetscBool flg4 = PETSC_FALSE;
365 #endif
366
367 PetscFunctionBegin;
368 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
369
370 #if !defined(PETSC_HAVE_THREADSAFETY)
371 if (!(PETSC_RUNNING_ON_VALGRIND)) {
372 /*
373 Setup the memory management; support for tracing malloc() usage
374 */
375 PetscBool mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE;
376
377 if (PetscDefined(USE_DEBUG)) {
378 mdebug = PETSC_TRUE;
379 initializenan = PETSC_TRUE;
380 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr);
381 } else {
382 /* don't warn about unused option */
383 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr);
384 flg1 = PETSC_FALSE;
385 }
386 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr);
387 if (flg1 || flg2) {
388 mdebug = PETSC_TRUE;
389 eachcall = PETSC_TRUE;
390 initializenan = PETSC_TRUE;
391 } else if (flg3 && !flg2) {
392 mdebug = PETSC_FALSE;
393 eachcall = PETSC_FALSE;
394 initializenan = PETSC_FALSE;
395 }
396
397 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);CHKERRQ(ierr);
398 if (flg2) {ierr = PetscMallocLogRequestedSizeSet(flg1);CHKERRQ(ierr);}
399
400 ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr);
401 if (mlog) {
402 mdebug = PETSC_TRUE;
403 }
404 /* the next line is deprecated */
405 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr);
406 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr);
407 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr);
408 if (mdebug) {
409 ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr);
410 }
411 if (mlog) {
412 PetscReal logthreshold = 0;
413 ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr);
414 ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr);
415 }
416 #if defined(PETSC_USE_LOG)
417 ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr);
418 #endif
419 }
420
421 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr);
422 if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);}
423 flg1 = PETSC_FALSE;
424 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr);
425 /* ignore this option if malloc is already set */
426 if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);}
427
428 flg1 = PETSC_FALSE;
429 ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr);
430 if (!flg1) {
431 flg1 = PETSC_FALSE;
432 ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr);
433 }
434 if (flg1) {
435 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr);
436 }
437 #endif
438
439 #if defined(PETSC_USE_LOG)
440 ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr);
441 #endif
442
443 /*
444 Set the display variable for graphics
445 */
446 ierr = PetscSetDisplay();CHKERRQ(ierr);
447
448 /*
449 Print main application help message
450 */
451 ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr);
452 if (help && hasHelp) {
453 ierr = PetscPrintf(comm,help);CHKERRQ(ierr);
454 ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr);
455 }
456
457 /*
458 Print the PETSc version information
459 */
460 ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr);
461 if (flg1 || hasHelp) {
462 /*
463 Print "higher-level" package version message
464 */
465 if (PetscExternalVersionFunction) {
466 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr);
467 }
468
469 ierr = PetscGetVersion(version,256);CHKERRQ(ierr);
470 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr);
471 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr);
472 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr);
473 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr);
474 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr);
475 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr);
476 ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr);
477 }
478
479 /*
480 Print "higher-level" package help message
481 */
482 if (hasHelp) {
483 PetscBool hasHelpIntro;
484
485 if (PetscExternalHelpFunction) {
486 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr);
487 }
488 ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr);
489 if (hasHelpIntro) {
490 ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr);
491 ierr = PetscFreeMPIResources();CHKERRQ(ierr);
492 ierr = MPI_Finalize();CHKERRQ(ierr);
493 exit(0);
494 }
495 }
496
497 /*
498 Setup the error handling
499 */
500 flg1 = PETSC_FALSE;
501 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr);
502 if (flg1) {
503 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRQ(ierr);
504 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr);
505 }
506 flg1 = PETSC_FALSE;
507 ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr);
508 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);}
509 flg1 = PETSC_FALSE;
510 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr);
511 if (flg1) {
512 ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRQ(ierr);
513 }
514 /* experimental */
515 flg1 = PETSC_FALSE;
516 ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);CHKERRQ(ierr);
517 if (flg1) {
518 MPI_Errhandler eh;
519
520 ierr = MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);CHKERRQ(ierr);
521 ierr = MPI_Comm_set_errhandler(comm,eh);CHKERRQ(ierr);
522 ierr = MPI_Errhandler_free(&eh);CHKERRQ(ierr);
523 }
524 flg1 = PETSC_FALSE;
525 ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr);
526 if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);}
527
528 /*
529 Setup debugger information
530 */
531 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr);
532 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr);
533 if (flg1) {
534 MPI_Errhandler err_handler;
535
536 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr);
537 ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRQ(ierr);
538 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr);
539 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr);
540 }
541 ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr);
542 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); }
543 ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr);
544 ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr);
545 if (flg1 || flg2) {
546 PetscMPIInt size;
547 PetscInt lsize,*ranks;
548 MPI_Errhandler err_handler;
549 /*
550 we have to make sure that all processors have opened
551 connections to all other processors, otherwise once the
552 debugger has stated it is likely to receive a SIGUSR1
553 and kill the program.
554 */
555 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
556 if (size > 2) {
557 PetscMPIInt dummy = 0;
558 MPI_Status status;
559 for (i=0; i<size; i++) {
560 if (rank != i) {
561 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRQ(ierr);
562 }
563 }
564 for (i=0; i<size; i++) {
565 if (rank != i) {
566 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRQ(ierr);
567 }
568 }
569 }
570 /* check if this processor node should be in debugger */
571 ierr = PetscMalloc1(size,&ranks);CHKERRQ(ierr);
572 lsize = size;
573 /* Deprecated in 3.14 */
574 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);CHKERRQ(ierr);
575 if (flag) {
576 const char * const quietopt="-options_suppress_deprecated_warnings";
577 char msg[4096];
578 PetscBool quiet = PETSC_FALSE;
579
580 ierr = PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);CHKERRQ(ierr);
581 if (!quiet) {
582 ierr = PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");CHKERRQ(ierr);
583 ierr = PetscStrcat(msg,"-debugger_nodes");CHKERRQ(ierr);
584 ierr = PetscStrcat(msg," is deprecated as of version ");CHKERRQ(ierr);
585 ierr = PetscStrcat(msg,"3.14");CHKERRQ(ierr);
586 ierr = PetscStrcat(msg," and will be removed in a future release.");CHKERRQ(ierr);
587 ierr = PetscStrcat(msg," Please use the option ");CHKERRQ(ierr);
588 ierr = PetscStrcat(msg,"-debugger_ranks");CHKERRQ(ierr);
589 ierr = PetscStrcat(msg," instead.");CHKERRQ(ierr);
590 ierr = PetscStrcat(msg," (Silence this warning with ");CHKERRQ(ierr);
591 ierr = PetscStrcat(msg,quietopt);CHKERRQ(ierr);
592 ierr = PetscStrcat(msg,")\n");CHKERRQ(ierr);
593 ierr = PetscPrintf(comm,msg);CHKERRQ(ierr);
594 }
595 } else {
596 lsize = size;
597 ierr = PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);CHKERRQ(ierr);
598 }
599 if (flag) {
600 for (i=0; i<lsize; i++) {
601 if (ranks[i] == rank) { flag = PETSC_FALSE; break; }
602 }
603 }
604 if (!flag) {
605 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr);
606 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr);
607 if (flg1) {
608 ierr = PetscAttachDebugger();CHKERRQ(ierr);
609 } else {
610 ierr = PetscStopForDebugger();CHKERRQ(ierr);
611 }
612 ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRQ(ierr);
613 ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr);
614 } else {
615 ierr = PetscWaitOnError();CHKERRQ(ierr);
616 }
617 ierr = PetscFree(ranks);CHKERRQ(ierr);
618 }
619
620 ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr);
621 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);}
622
623 /*
624 Setup profiling and logging
625 */
626 #if defined(PETSC_USE_INFO)
627 {
628 ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr);
629 }
630 #endif
631 ierr = PetscDetermineInitialFPTrap();
632 flg1 = PETSC_FALSE;
633 ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr);
634 if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);}
635 ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr);
636 if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);}
637 #if defined(PETSC_USE_LOG)
638 mname[0] = 0;
639 ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr);
640 if (flg1) {
641 if (mname[0]) {
642 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr);
643 } else {
644 ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr);
645 }
646 }
647
648 ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr);
649
650 #if defined(PETSC_HAVE_MPE)
651 flg1 = PETSC_FALSE;
652 ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr);
653 if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);}
654 #endif
655 flg1 = PETSC_FALSE;
656 flg3 = PETSC_FALSE;
657 ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr);
658 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr);
659 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); }
660 else if (flg3) { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);}
661
662 ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr);
663 if (flg1) {
664 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
665 FILE *file;
666 if (mname[0]) {
667 PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank);
668 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr);
669 file = fopen(fname,"w");
670 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
671 } else file = PETSC_STDOUT;
672 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr);
673 }
674
675 ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr);
676 if (flg4) {
677 if (format == PETSC_VIEWER_ASCII_XML) {
678 ierr = PetscLogNestedBegin();CHKERRQ(ierr);
679 } else {
680 ierr = PetscLogDefaultBegin();CHKERRQ(ierr);
681 }
682 }
683 if (flg4 && format == PETSC_VIEWER_ASCII_XML) {
684 PetscReal threshold = PetscRealConstant(0.01);
685 ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr);
686 if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);}
687 }
688 #endif
689
690 ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr);
691 ierr = PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);CHKERRQ(ierr);
692 /*
693 If collecting logging information, by default, wait for device to complete its operations
694 before returning to the CPU in order to get accurate timings of each event
695 */
696 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr);
697 if (!logView) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr);}
698
699 #if defined(PETSC_HAVE_CUDA)
700 ierr = PetscOptionsCheckCUDA(logView);CHKERRQ(ierr);
701 #endif
702
703 #if defined(PETSC_HAVE_HIP)
704 ierr = PetscOptionsCheckHIP(logView);CHKERRQ(ierr);
705 #endif
706
707 /*
708 Print basic help message
709 */
710 if (hasHelp) {
711 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr);
712 ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr);
713 ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr);
714 ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr);
715 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr);
716 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr);
717 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr);
718 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr);
719 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr);
720 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr);
721 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr);
722 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr);
723 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr);
724 ierr = (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");CHKERRQ(ierr);
725 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr);
726 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr);
727 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr);
728 ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr);
729 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr);
730 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr);
731 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr);
732 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr);
733 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr);
734 ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr);
735 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr);
736 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr);
737 ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr);
738 ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr);
739 ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr);
740 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr);
741 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr);
742 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr);
743 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr);
744 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr);
745 ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr);
746 #if defined(PETSC_USE_LOG)
747 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr);
748 ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr);
749 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr);
750 ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr);
751 #if defined(PETSC_HAVE_MPE)
752 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr);
753 #endif
754 #endif
755 #if defined(PETSC_USE_INFO)
756 ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr);
757 #endif
758 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr);
759 ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr);
760 ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr);
761 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr);
762 }
763
764 #if defined(PETSC_HAVE_POPEN)
765 {
766 char machine[128];
767 ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr);
768 if (flg1) {
769 ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr);
770 }
771 }
772 #endif
773
774 ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr);
775 if (flg1) {
776 ierr = PetscSleep(si);CHKERRQ(ierr);
777 }
778
779 #if defined(PETSC_HAVE_VIENNACL)
780 ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr);
781 if (!flg3) {
782 ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr);
783 }
784 ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr);
785 PetscViennaCLSynchronize = flg3;
786 ierr = PetscViennaCLInit();CHKERRQ(ierr);
787 #endif
788
789 /*
790 Creates the logging data structures; this is enabled even if logging is not turned on
791 This is the last thing we do before returning to the user code to prevent having the
792 logging numbers contaminated by any startup time associated with MPI and the GPUs
793 */
794 #if defined(PETSC_USE_LOG)
795 ierr = PetscLogInitialize();CHKERRQ(ierr);
796 #endif
797
798 PetscFunctionReturn(0);
799 }
800