1 /*
2 
3    This file defines part of the initialization of PETSc
4 
5   This file uses regular malloc and free because it cannot be known
6   what malloc is being used until it has already processed the input.
7 */
8 
9 #include <petscsys.h>        /*I  "petscsys.h"   I*/
10 #include <petsc/private/petscimpl.h>
11 #include <petscvalgrind.h>
12 #include <petscviewer.h>
13 #if defined(PETSC_USE_LOG)
14 PETSC_INTERN PetscErrorCode PetscLogInitialize(void);
15 #endif
16 
17 #if defined(PETSC_HAVE_SYS_SYSINFO_H)
18 #include <sys/sysinfo.h>
19 #endif
20 #if defined(PETSC_HAVE_UNISTD_H)
21 #include <unistd.h>
22 #endif
23 
24 #if defined(PETSC_HAVE_CUDA)
25   #include <cuda_runtime.h>
26   #include <petsccublas.h>
27 #endif
28 
29 #if defined(PETSC_HAVE_HIP)
30   #include <hip/hip_runtime.h>
31 #endif
32 
33 #if defined(PETSC_HAVE_DEVICE)
34   #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION)
35     #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */
36   #endif
37 #endif
38 
39 #if defined(PETSC_HAVE_VIENNACL)
40 PETSC_EXTERN PetscErrorCode PetscViennaCLInit();
41 #endif
42 
43 
44 /* ------------------------Nasty global variables -------------------------------*/
45 /*
46      Indicates if PETSc started up MPI, or it was
47    already started before PETSc was initialized.
48 */
49 PetscBool   PetscBeganMPI                 = PETSC_FALSE;
50 PetscBool   PetscErrorHandlingInitialized = PETSC_FALSE;
51 PetscBool   PetscInitializeCalled         = PETSC_FALSE;
52 PetscBool   PetscFinalizeCalled           = PETSC_FALSE;
53 
54 PetscMPIInt PetscGlobalRank               = -1;
55 PetscMPIInt PetscGlobalSize               = -1;
56 
57 #if defined(PETSC_HAVE_KOKKOS)
58 PetscBool   PetscBeganKokkos              = PETSC_FALSE;
59 #endif
60 
61 PetscBool   use_gpu_aware_mpi             = PETSC_TRUE;
62 PetscBool   PetscCreatedGpuObjects        = PETSC_FALSE;
63 
64 #if defined(PETSC_HAVE_COMPLEX)
65 #if defined(PETSC_COMPLEX_INSTANTIATE)
66 template <> class std::complex<double>; /* instantiate complex template class */
67 #endif
68 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
69 MPI_Datatype MPIU_C_DOUBLE_COMPLEX;
70 MPI_Datatype MPIU_C_COMPLEX;
71 #endif
72 
73 /*MC
74    PETSC_i - the imaginary number i
75 
76    Synopsis:
77    #include <petscsys.h>
78    PetscComplex PETSC_i;
79 
80    Level: beginner
81 
82    Note:
83    Complex numbers are automatically available if PETSc located a working complex implementation
84 
85 .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex()
86 M*/
87 PetscComplex PETSC_i;
88 #endif
89 #if defined(PETSC_USE_REAL___FLOAT128)
90 MPI_Datatype MPIU___FLOAT128 = 0;
91 #if defined(PETSC_HAVE_COMPLEX)
92 MPI_Datatype MPIU___COMPLEX128 = 0;
93 #endif
94 #elif defined(PETSC_USE_REAL___FP16)
95 MPI_Datatype MPIU___FP16 = 0;
96 #endif
97 MPI_Datatype MPIU_2SCALAR = 0;
98 #if defined(PETSC_USE_64BIT_INDICES)
99 MPI_Datatype MPIU_2INT = 0;
100 #endif
101 MPI_Datatype MPIU_BOOL;
102 MPI_Datatype MPIU_ENUM;
103 MPI_Datatype MPIU_FORTRANADDR;
104 MPI_Datatype MPIU_SIZE_T;
105 
106 /*
107        Function that is called to display all error messages
108 */
109 PetscErrorCode (*PetscErrorPrintf)(const char [],...)          = PetscErrorPrintfDefault;
110 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...)  = PetscHelpPrintfDefault;
111 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintfDefault;
112 /*
113   This is needed to turn on/off GPU synchronization
114 */
115 PetscBool PetscViennaCLSynchronize = PETSC_FALSE;
116 
117 /* ------------------------------------------------------------------------------*/
118 /*
119    Optional file where all PETSc output from various prints is saved
120 */
121 PETSC_INTERN FILE *petsc_history;
122 FILE *petsc_history = NULL;
123 
PetscOpenHistoryFile(const char filename[],FILE ** fd)124 PetscErrorCode  PetscOpenHistoryFile(const char filename[],FILE **fd)
125 {
126   PetscErrorCode ierr;
127   PetscMPIInt    rank,size;
128   char           pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
129   char           version[256];
130 
131   PetscFunctionBegin;
132   ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
133   if (!rank) {
134     char        arch[10];
135     int         err;
136 
137     ierr = PetscGetArchType(arch,10);CHKERRQ(ierr);
138     ierr = PetscGetDate(date,64);CHKERRQ(ierr);
139     ierr = PetscGetVersion(version,256);CHKERRQ(ierr);
140     ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
141     if (filename) {
142       ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr);
143     } else {
144       ierr = PetscGetHomeDirectory(pfile,sizeof(pfile));CHKERRQ(ierr);
145       ierr = PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));CHKERRQ(ierr);
146       ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr);
147     }
148 
149     *fd = fopen(fname,"a");
150     if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);
151 
152     ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr);
153     ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr);
154     ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr);
155     ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr);
156     ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr);
157 
158     err = fflush(*fd);
159     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
160   }
161   PetscFunctionReturn(0);
162 }
163 
PetscCloseHistoryFile(FILE ** fd)164 PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd)
165 {
166   PetscErrorCode ierr;
167   PetscMPIInt    rank;
168   char           date[64];
169   int            err;
170 
171   PetscFunctionBegin;
172   ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
173   if (!rank) {
174     ierr = PetscGetDate(date,64);CHKERRQ(ierr);
175     ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr);
176     ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr);
177     ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");CHKERRQ(ierr);
178     err  = fflush(*fd);
179     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
180     err = fclose(*fd);
181     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
182   }
183   PetscFunctionReturn(0);
184 }
185 
186 /* ------------------------------------------------------------------------------*/
187 
188 /*
189    This is ugly and probably belongs somewhere else, but I want to
190   be able to put a true MPI abort error handler with command line args.
191 
192     This is so MPI errors in the debugger will leave all the stack
193   frames. The default MP_Abort() cleans up and exits thus providing no useful information
194   in the debugger hence we call abort() instead of MPI_Abort().
195 */
196 
Petsc_MPI_AbortOnError(MPI_Comm * comm,PetscMPIInt * flag,...)197 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
198 {
199   PetscFunctionBegin;
200   (*PetscErrorPrintf)("MPI error %d\n",*flag);
201   abort();
202 }
203 
Petsc_MPI_DebuggerOnError(MPI_Comm * comm,PetscMPIInt * flag,...)204 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
205 {
206   PetscErrorCode ierr;
207 
208   PetscFunctionBegin;
209   (*PetscErrorPrintf)("MPI error %d\n",*flag);
210   ierr = PetscAttachDebugger();
211   if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */
212 }
213 
214 /*@C
215    PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one
216      wishes a clean exit somewhere deep in the program.
217 
218    Collective on PETSC_COMM_WORLD
219 
220    Options Database Keys are the same as for PetscFinalize()
221 
222    Level: advanced
223 
224    Note:
225    See PetscInitialize() for more general runtime options.
226 
227 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
228 @*/
PetscEnd(void)229 PetscErrorCode  PetscEnd(void)
230 {
231   PetscFunctionBegin;
232   PetscFinalize();
233   exit(0);
234   return 0;
235 }
236 
237 PetscBool PetscOptionsPublish = PETSC_FALSE;
238 PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void);
239 PETSC_INTERN PetscBool      petscsetmallocvisited;
240 static       char           emacsmachinename[256];
241 
242 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL;
243 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm)    = NULL;
244 
245 /*@C
246    PetscSetHelpVersionFunctions - Sets functions that print help and version information
247    before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
248    This routine enables a "higher-level" package that uses PETSc to print its messages first.
249 
250    Input Parameter:
251 +  help - the help function (may be NULL)
252 -  version - the version function (may be NULL)
253 
254    Level: developer
255 
256 @*/
PetscSetHelpVersionFunctions(PetscErrorCode (* help)(MPI_Comm),PetscErrorCode (* version)(MPI_Comm))257 PetscErrorCode  PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
258 {
259   PetscFunctionBegin;
260   PetscExternalHelpFunction    = help;
261   PetscExternalVersionFunction = version;
262   PetscFunctionReturn(0);
263 }
264 
265 #if defined(PETSC_USE_LOG)
266 PETSC_INTERN PetscBool   PetscObjectsLog;
267 #endif
268 
PetscMPI_Comm_eh(MPI_Comm * comm,PetscMPIInt * err,...)269 void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...)
270 {
271   if (PetscUnlikely(*err)) {
272     PetscMPIInt len;
273     char        errstring[MPI_MAX_ERROR_STRING];
274 
275     MPI_Error_string(*err,errstring,&len);
276     PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring);
277   }
278   return;
279 }
280 
281 /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP.
282    Use the following macros to define CUDA/HIP initialization related vars/routines.
283  */
284 #if defined(PETSC_HAVE_CUDA)
285   typedef cudaError_t                             cupmError_t;
286   typedef struct cudaDeviceProp                   cupmDeviceProp;
287   #define cupmGetDeviceCount(x)                   cudaGetDeviceCount(x)
288   #define cupmGetDevice(x)                        cudaGetDevice(x)
289   #define cupmSetDevice(x)                        cudaSetDevice(x)
290   #define cupmSetDeviceFlags(x)                   cudaSetDeviceFlags(x)
291   #define cupmGetDeviceProperties(x,y)            cudaGetDeviceProperties(x,y)
292   #define cupmGetLastError()                      cudaGetLastError()
293   #define cupmDeviceMapHost                       cudaDeviceMapHost
294   #define cupmSuccess                             cudaSuccess
295   #define cupmErrorMemoryAllocation               cudaErrorMemoryAllocation
296   #define cupmErrorLaunchOutOfResources           cudaErrorLaunchOutOfResources
297   #define cupmErrorSetOnActiveProcess             cudaErrorSetOnActiveProcess
298   #define CHKERRCUPM(x)                           CHKERRCUDA(x)
299   #define PetscCUPMBLASInitializeHandle()         PetscCUBLASInitializeHandle()
300   #define PetscCUPMSOLVERDnInitializeHandle()     PetscCUSOLVERDnInitializeHandle()
301   #define PetscCUPMInitialize                     PetscCUDAInitialize
302   #define PetscCUPMInitialized                    PetscCUDAInitialized
303   #define PetscCUPMInitializeCheck                PetscCUDAInitializeCheck
304   #define PetscCUPMInitializeAndView              PetscCUDAInitializeAndView
305   #define PetscCUPMSynchronize                    PetscCUDASynchronize
306   #define PetscNotUseCUPM                         PetscNotUseCUDA
307   #define cupmOptionsStr                          "CUDA options"
308   #define cupmSetDeviceStr                        "-cuda_device"
309   #define cupmViewStr                             "-cuda_view"
310   #define cupmSynchronizeStr                      "-cuda_synchronize"
311   #define PetscCUPMInitializeStr                  "PetscCUDAInitialize"
312   #define PetscOptionsCheckCUPM                   PetscOptionsCheckCUDA
313   #define PetscMPICUPMAwarenessCheck              PetscMPICUDAAwarenessCheck
314   #include "cupminit.inc"
315 #endif
316 
317 #if defined(PETSC_HAVE_HIP)
318   typedef hipError_t                              cupmError_t;
319   typedef hipDeviceProp_t                         cupmDeviceProp;
320   #define cupmGetDeviceCount(x)                   hipGetDeviceCount(x)
321   #define cupmGetDevice(x)                        hipGetDevice(x)
322   #define cupmSetDevice(x)                        hipSetDevice(x)
323   #define cupmSetDeviceFlags(x)                   hipSetDeviceFlags(x)
324   #define cupmGetDeviceProperties(x,y)            hipGetDeviceProperties(x,y)
325   #define cupmGetLastError()                      hipGetLastError()
326   #define cupmDeviceMapHost                       hipDeviceMapHost
327   #define cupmSuccess                             hipSuccess
328   #define cupmErrorMemoryAllocation               hipErrorMemoryAllocation
329   #define cupmErrorLaunchOutOfResources           hipErrorLaunchOutOfResources
330   #define cupmErrorSetOnActiveProcess             hipErrorSetOnActiveProcess
331   #define CHKERRCUPM(x)                           CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB)
332   #define PetscCUPMBLASInitializeHandle()         0
333   #define PetscCUPMSOLVERDnInitializeHandle()     0
334   #define PetscCUPMInitialize                     PetscHIPInitialize
335   #define PetscCUPMInitialized                    PetscHIPInitialized
336   #define PetscCUPMInitializeCheck                PetscHIPInitializeCheck
337   #define PetscCUPMInitializeAndView              PetscHIPInitializeAndView
338   #define PetscCUPMSynchronize                    PetscHIPSynchronize
339   #define PetscNotUseCUPM                         PetscNotUseHIP
340   #define cupmOptionsStr                          "HIP options"
341   #define cupmSetDeviceStr                        "-hip_device"
342   #define cupmViewStr                             "-hip_view"
343   #define cupmSynchronizeStr                      "-hip_synchronize"
344   #define PetscCUPMInitializeStr                  "PetscHIPInitialize"
345   #define PetscOptionsCheckCUPM                   PetscOptionsCheckHIP
346   #define PetscMPICUPMAwarenessCheck              PetscMPIHIPAwarenessCheck
347   #include "cupminit.inc"
348 #endif
349 
PetscOptionsCheckInitial_Private(const char help[])350 PETSC_INTERN PetscErrorCode  PetscOptionsCheckInitial_Private(const char help[])
351 {
352   char              string[64];
353   MPI_Comm          comm = PETSC_COMM_WORLD;
354   PetscBool         flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView;
355   PetscErrorCode    ierr;
356   PetscReal         si;
357   PetscInt          intensity;
358   int               i;
359   PetscMPIInt       rank;
360   char              version[256];
361 #if defined(PETSC_USE_LOG)
362   char              mname[PETSC_MAX_PATH_LEN];
363   PetscViewerFormat format;
364   PetscBool         flg4 = PETSC_FALSE;
365 #endif
366 
367   PetscFunctionBegin;
368   ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
369 
370 #if !defined(PETSC_HAVE_THREADSAFETY)
371   if (!(PETSC_RUNNING_ON_VALGRIND)) {
372     /*
373       Setup the memory management; support for tracing malloc() usage
374     */
375     PetscBool         mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE;
376 
377     if (PetscDefined(USE_DEBUG)) {
378       mdebug        = PETSC_TRUE;
379       initializenan = PETSC_TRUE;
380       ierr   = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr);
381     } else {
382       /* don't warn about unused option */
383       ierr = PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);CHKERRQ(ierr);
384       flg1 = PETSC_FALSE;
385     }
386     ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);CHKERRQ(ierr);
387     if (flg1 || flg2) {
388       mdebug        = PETSC_TRUE;
389       eachcall      = PETSC_TRUE;
390       initializenan = PETSC_TRUE;
391     } else if (flg3 && !flg2) {
392       mdebug        = PETSC_FALSE;
393       eachcall      = PETSC_FALSE;
394       initializenan = PETSC_FALSE;
395     }
396 
397     ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);CHKERRQ(ierr);
398     if (flg2) {ierr = PetscMallocLogRequestedSizeSet(flg1);CHKERRQ(ierr);}
399 
400     ierr = PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);CHKERRQ(ierr);
401     if (mlog) {
402       mdebug = PETSC_TRUE;
403     }
404     /* the next line is deprecated */
405     ierr = PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);CHKERRQ(ierr);
406     ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);CHKERRQ(ierr);
407     ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);CHKERRQ(ierr);
408     if (mdebug) {
409       ierr = PetscMallocSetDebug(eachcall,initializenan);CHKERRQ(ierr);
410     }
411     if (mlog) {
412       PetscReal logthreshold = 0;
413       ierr = PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);CHKERRQ(ierr);
414       ierr = PetscMallocViewSet(logthreshold);CHKERRQ(ierr);
415     }
416 #if defined(PETSC_USE_LOG)
417     ierr = PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);CHKERRQ(ierr);
418 #endif
419   }
420 
421   ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);CHKERRQ(ierr);
422   if (flg2) {ierr = PetscMallocSetCoalesce(flg1);CHKERRQ(ierr);}
423   flg1 = PETSC_FALSE;
424   ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);CHKERRQ(ierr);
425   /* ignore this option if malloc is already set */
426   if (flg1 && !petscsetmallocvisited) {ierr = PetscSetUseHBWMalloc_Private();CHKERRQ(ierr);}
427 
428   flg1 = PETSC_FALSE;
429   ierr = PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);CHKERRQ(ierr);
430   if (!flg1) {
431     flg1 = PETSC_FALSE;
432     ierr = PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);CHKERRQ(ierr);
433   }
434   if (flg1) {
435     ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr);
436   }
437 #endif
438 
439 #if defined(PETSC_USE_LOG)
440   ierr = PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);CHKERRQ(ierr);
441 #endif
442 
443   /*
444       Set the display variable for graphics
445   */
446   ierr = PetscSetDisplay();CHKERRQ(ierr);
447 
448   /*
449      Print main application help message
450   */
451   ierr = PetscOptionsHasHelp(NULL,&hasHelp);CHKERRQ(ierr);
452   if (help && hasHelp) {
453     ierr = PetscPrintf(comm,help);CHKERRQ(ierr);
454     ierr = PetscPrintf(comm,"----------------------------------------\n");CHKERRQ(ierr);
455   }
456 
457   /*
458       Print the PETSc version information
459   */
460   ierr = PetscOptionsHasName(NULL,NULL,"-version",&flg1);CHKERRQ(ierr);
461   if (flg1 || hasHelp) {
462     /*
463        Print "higher-level" package version message
464     */
465     if (PetscExternalVersionFunction) {
466       ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr);
467     }
468 
469     ierr = PetscGetVersion(version,256);CHKERRQ(ierr);
470     ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr);
471     ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr);
472     ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr);
473     ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr);
474     ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr);
475     ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr);
476     ierr = (*PetscHelpPrintf)(comm,"----------------------------------------\n");CHKERRQ(ierr);
477   }
478 
479   /*
480        Print "higher-level" package help message
481   */
482   if (hasHelp) {
483     PetscBool hasHelpIntro;
484 
485     if (PetscExternalHelpFunction) {
486       ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr);
487     }
488     ierr = PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);CHKERRQ(ierr);
489     if (hasHelpIntro) {
490       ierr = PetscOptionsDestroyDefault();CHKERRQ(ierr);
491       ierr = PetscFreeMPIResources();CHKERRQ(ierr);
492       ierr = MPI_Finalize();CHKERRQ(ierr);
493       exit(0);
494     }
495   }
496 
497   /*
498       Setup the error handling
499   */
500   flg1 = PETSC_FALSE;
501   ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);CHKERRQ(ierr);
502   if (flg1) {
503     ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);CHKERRQ(ierr);
504     ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr);
505   }
506   flg1 = PETSC_FALSE;
507   ierr = PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);CHKERRQ(ierr);
508   if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);CHKERRQ(ierr);}
509   flg1 = PETSC_FALSE;
510   ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);CHKERRQ(ierr);
511   if (flg1) {
512     ierr = MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);CHKERRQ(ierr);
513   }
514   /* experimental */
515   flg1 = PETSC_FALSE;
516   ierr = PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);CHKERRQ(ierr);
517   if (flg1) {
518     MPI_Errhandler eh;
519 
520     ierr = MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);CHKERRQ(ierr);
521     ierr = MPI_Comm_set_errhandler(comm,eh);CHKERRQ(ierr);
522     ierr = MPI_Errhandler_free(&eh);CHKERRQ(ierr);
523   }
524   flg1 = PETSC_FALSE;
525   ierr = PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);CHKERRQ(ierr);
526   if (!flg1) {ierr = PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);CHKERRQ(ierr);}
527 
528   /*
529       Setup debugger information
530   */
531   ierr = PetscSetDefaultDebugger();CHKERRQ(ierr);
532   ierr = PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr);
533   if (flg1) {
534     MPI_Errhandler err_handler;
535 
536     ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr);
537     ierr = MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);CHKERRQ(ierr);
538     ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr);
539     ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);CHKERRQ(ierr);
540   }
541   ierr = PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);CHKERRQ(ierr);
542   if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); }
543   ierr = PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);CHKERRQ(ierr);
544   ierr = PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);CHKERRQ(ierr);
545   if (flg1 || flg2) {
546     PetscMPIInt    size;
547     PetscInt       lsize,*ranks;
548     MPI_Errhandler err_handler;
549     /*
550        we have to make sure that all processors have opened
551        connections to all other processors, otherwise once the
552        debugger has stated it is likely to receive a SIGUSR1
553        and kill the program.
554     */
555     ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
556     if (size > 2) {
557       PetscMPIInt dummy = 0;
558       MPI_Status  status;
559       for (i=0; i<size; i++) {
560         if (rank != i) {
561           ierr = MPI_Send(&dummy,1,MPI_INT,i,109,comm);CHKERRQ(ierr);
562         }
563       }
564       for (i=0; i<size; i++) {
565         if (rank != i) {
566           ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);CHKERRQ(ierr);
567         }
568       }
569     }
570     /* check if this processor node should be in debugger */
571     ierr  = PetscMalloc1(size,&ranks);CHKERRQ(ierr);
572     lsize = size;
573     /* Deprecated in 3.14 */
574     ierr  = PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);CHKERRQ(ierr);
575     if (flag) {
576       const char * const quietopt="-options_suppress_deprecated_warnings";
577       char               msg[4096];
578       PetscBool          quiet = PETSC_FALSE;
579 
580       ierr = PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);CHKERRQ(ierr);
581       if (!quiet) {
582         ierr = PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");CHKERRQ(ierr);
583         ierr = PetscStrcat(msg,"-debugger_nodes");CHKERRQ(ierr);
584         ierr = PetscStrcat(msg," is deprecated as of version ");CHKERRQ(ierr);
585         ierr = PetscStrcat(msg,"3.14");CHKERRQ(ierr);
586         ierr = PetscStrcat(msg," and will be removed in a future release.");CHKERRQ(ierr);
587         ierr = PetscStrcat(msg," Please use the option ");CHKERRQ(ierr);
588         ierr = PetscStrcat(msg,"-debugger_ranks");CHKERRQ(ierr);
589         ierr = PetscStrcat(msg," instead.");CHKERRQ(ierr);
590         ierr = PetscStrcat(msg," (Silence this warning with ");CHKERRQ(ierr);
591         ierr = PetscStrcat(msg,quietopt);CHKERRQ(ierr);
592         ierr = PetscStrcat(msg,")\n");CHKERRQ(ierr);
593         ierr = PetscPrintf(comm,msg);CHKERRQ(ierr);
594       }
595     } else {
596       lsize = size;
597       ierr  = PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);CHKERRQ(ierr);
598     }
599     if (flag) {
600       for (i=0; i<lsize; i++) {
601         if (ranks[i] == rank) { flag = PETSC_FALSE; break; }
602       }
603     }
604     if (!flag) {
605       ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr);
606       ierr = PetscPushErrorHandler(PetscAbortErrorHandler,NULL);CHKERRQ(ierr);
607       if (flg1) {
608         ierr = PetscAttachDebugger();CHKERRQ(ierr);
609       } else {
610         ierr = PetscStopForDebugger();CHKERRQ(ierr);
611       }
612       ierr = MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);CHKERRQ(ierr);
613       ierr = MPI_Comm_set_errhandler(comm,err_handler);CHKERRQ(ierr);
614     } else {
615       ierr = PetscWaitOnError();CHKERRQ(ierr);
616     }
617     ierr = PetscFree(ranks);CHKERRQ(ierr);
618   }
619 
620   ierr = PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);CHKERRQ(ierr);
621   if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);}
622 
623   /*
624         Setup profiling and logging
625   */
626 #if defined(PETSC_USE_INFO)
627   {
628     ierr = PetscInfoSetFromOptions(NULL);CHKERRQ(ierr);
629   }
630 #endif
631   ierr = PetscDetermineInitialFPTrap();
632   flg1 = PETSC_FALSE;
633   ierr = PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);CHKERRQ(ierr);
634   if (flag) {ierr = PetscSetFPTrap((PetscFPTrap)flg1);CHKERRQ(ierr);}
635   ierr = PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);CHKERRQ(ierr);
636   if (flag) {ierr = PetscCheckPointerSetIntensity(intensity);CHKERRQ(ierr);}
637 #if defined(PETSC_USE_LOG)
638   mname[0] = 0;
639   ierr = PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);CHKERRQ(ierr);
640   if (flg1) {
641     if (mname[0]) {
642       ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr);
643     } else {
644       ierr = PetscOpenHistoryFile(NULL,&petsc_history);CHKERRQ(ierr);
645     }
646   }
647 
648   ierr = PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);CHKERRQ(ierr);
649 
650 #if defined(PETSC_HAVE_MPE)
651   flg1 = PETSC_FALSE;
652   ierr = PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);CHKERRQ(ierr);
653   if (flg1) {ierr = PetscLogMPEBegin();CHKERRQ(ierr);}
654 #endif
655   flg1 = PETSC_FALSE;
656   flg3 = PETSC_FALSE;
657   ierr = PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);CHKERRQ(ierr);
658   ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr);
659   if (flg1)                      { ierr = PetscLogAllBegin();CHKERRQ(ierr); }
660   else if (flg3)                 { ierr = PetscLogDefaultBegin();CHKERRQ(ierr);}
661 
662   ierr = PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);CHKERRQ(ierr);
663   if (flg1) {
664     char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
665     FILE *file;
666     if (mname[0]) {
667       PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank);
668       ierr = PetscFixFilename(name,fname);CHKERRQ(ierr);
669       file = fopen(fname,"w");
670       if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
671     } else file = PETSC_STDOUT;
672     ierr = PetscLogTraceBegin(file);CHKERRQ(ierr);
673   }
674 
675   ierr = PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);CHKERRQ(ierr);
676   if (flg4) {
677     if (format == PETSC_VIEWER_ASCII_XML) {
678       ierr = PetscLogNestedBegin();CHKERRQ(ierr);
679     } else {
680       ierr = PetscLogDefaultBegin();CHKERRQ(ierr);
681     }
682   }
683   if (flg4 && format == PETSC_VIEWER_ASCII_XML) {
684     PetscReal threshold = PetscRealConstant(0.01);
685     ierr = PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);CHKERRQ(ierr);
686     if (flg1) {ierr = PetscLogSetThreshold((PetscLogDouble)threshold,NULL);CHKERRQ(ierr);}
687   }
688 #endif
689 
690   ierr = PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);CHKERRQ(ierr);
691   ierr = PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);CHKERRQ(ierr);
692   /*
693     If collecting logging information, by default, wait for device to complete its operations
694     before returning to the CPU in order to get accurate timings of each event
695   */
696   ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);CHKERRQ(ierr);
697   if (!logView) {ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&logView);CHKERRQ(ierr);}
698 
699 #if defined(PETSC_HAVE_CUDA)
700   ierr = PetscOptionsCheckCUDA(logView);CHKERRQ(ierr);
701 #endif
702 
703 #if defined(PETSC_HAVE_HIP)
704   ierr = PetscOptionsCheckHIP(logView);CHKERRQ(ierr);
705 #endif
706 
707   /*
708        Print basic help message
709   */
710   if (hasHelp) {
711     ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr);
712     ierr = (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");CHKERRQ(ierr);
713     ierr = (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");CHKERRQ(ierr);
714     ierr = (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");CHKERRQ(ierr);
715     ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr);
716     ierr = (*PetscHelpPrintf)(comm,"       only when run in the debugger\n");CHKERRQ(ierr);
717     ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr);
718     ierr = (*PetscHelpPrintf)(comm,"       start the debugger in new xterm\n");CHKERRQ(ierr);
719     ierr = (*PetscHelpPrintf)(comm,"       unless noxterm is given\n");CHKERRQ(ierr);
720     ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr);
721     ierr = (*PetscHelpPrintf)(comm,"       start all processes in the debugger\n");CHKERRQ(ierr);
722     ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr);
723     ierr = (*PetscHelpPrintf)(comm,"    emacs jumps to error file\n");CHKERRQ(ierr);
724     ierr = (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");CHKERRQ(ierr);
725     ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr);
726     ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr);
727     ierr = (*PetscHelpPrintf)(comm,"                      waits the delay for you to attach\n");CHKERRQ(ierr);
728     ierr = (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");CHKERRQ(ierr);
729     ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr);
730     ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr);
731     ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr);
732     ierr = (*PetscHelpPrintf)(comm,"           note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr);
733     ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr);
734     ierr = (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");CHKERRQ(ierr);
735     ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");CHKERRQ(ierr);
736     ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr);
737     ierr = (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");CHKERRQ(ierr);
738     ierr = (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");CHKERRQ(ierr);
739     ierr = (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");CHKERRQ(ierr);
740     ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr);
741     ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr);
742     ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr);
743     ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr);
744     ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr);
745     ierr = (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");CHKERRQ(ierr);
746 #if defined(PETSC_USE_LOG)
747     ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr);
748     ierr = (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");CHKERRQ(ierr);
749     ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr);
750     ierr = (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");CHKERRQ(ierr);
751 #if defined(PETSC_HAVE_MPE)
752     ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");CHKERRQ(ierr);
753 #endif
754 #endif
755 #if defined(PETSC_USE_INFO)
756     ierr = (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");CHKERRQ(ierr);
757 #endif
758     ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr);
759     ierr = (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");CHKERRQ(ierr);
760     ierr = (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");CHKERRQ(ierr);
761     ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr);
762   }
763 
764 #if defined(PETSC_HAVE_POPEN)
765   {
766   char machine[128];
767   ierr = PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);CHKERRQ(ierr);
768   if (flg1) {
769     ierr = PetscPOpenSetMachine(machine);CHKERRQ(ierr);
770   }
771   }
772 #endif
773 
774   ierr = PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr);
775   if (flg1) {
776     ierr = PetscSleep(si);CHKERRQ(ierr);
777   }
778 
779 #if defined(PETSC_HAVE_VIENNACL)
780   ierr = PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);CHKERRQ(ierr);
781   if (!flg3) {
782     ierr = PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);CHKERRQ(ierr);
783   }
784   ierr = PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);CHKERRQ(ierr);
785   PetscViennaCLSynchronize = flg3;
786   ierr = PetscViennaCLInit();CHKERRQ(ierr);
787 #endif
788 
789   /*
790      Creates the logging data structures; this is enabled even if logging is not turned on
791      This is the last thing we do before returning to the user code to prevent having the
792      logging numbers contaminated by any startup time associated with MPI and the GPUs
793   */
794 #if defined(PETSC_USE_LOG)
795   ierr = PetscLogInitialize();CHKERRQ(ierr);
796 #endif
797 
798   PetscFunctionReturn(0);
799 }
800