1
2 /******************************************************************************
3 *
4 * This file is part of meryl-utility, a collection of miscellaneous code
5 * used by Meryl, Canu and others.
6 *
7 * This software is based on:
8 * 'Canu' v2.0 (https://github.com/marbl/canu)
9 * which is based on:
10 * 'Celera Assembler' r4587 (http://wgs-assembler.sourceforge.net)
11 * the 'kmer package' r1994 (http://kmer.sourceforge.net)
12 *
13 * Except as indicated otherwise, this is a 'United States Government Work',
14 * and is released in the public domain.
15 *
16 * File 'README.licenses' in the root directory of this distribution
17 * contains full conditions and disclaimers.
18 */
19
20 #include "types.H"
21 #include "system.H"
22
23 #include <sys/types.h>
24 #include <sys/time.h>
25 #include <sys/resource.h>
26
27 #if defined(__FreeBSD__)
28 #include <stdlib.h>
29 #include <malloc_np.h>
30 #endif
31
32 #if defined(JEMALLOC)
33 #include "jemalloc/jemalloc.h"
34 #endif
35
36
37
38 double
getTime(void)39 getTime(void) {
40 struct timeval tp;
41 gettimeofday(&tp, NULL);
42 return(tp.tv_sec + (double)tp.tv_usec / 1000000.0);
43 }
44
45
46
47 static
48 bool
getrusage(struct rusage & ru)49 getrusage(struct rusage &ru) {
50
51 errno = 0;
52
53 if (getrusage(RUSAGE_SELF, &ru) == -1) {
54 fprintf(stderr, "getrusage(RUSAGE_SELF, ...) failed: %s\n",
55 strerror(errno));
56 return(false);
57 }
58
59 return(true);
60 }
61
62
63
64 static
65 bool
getrlimit(struct rlimit & rl)66 getrlimit(struct rlimit &rl) {
67
68 errno = 0;
69
70 if (getrlimit(RLIMIT_DATA, &rl) == -1) {
71 fprintf(stderr, "getrlimit(RLIMIT_DATA, ...) failed: %s\n",
72 strerror(errno));
73 return(false);
74 }
75
76 return(true);
77 }
78
79
80
81 double
getCPUTime(void)82 getCPUTime(void) {
83 struct rusage ru;
84 double tm = 0;
85
86 if (getrusage(ru) == true)
87 tm = ((ru.ru_utime.tv_sec + ru.ru_utime.tv_usec / 1000000.0) +
88 (ru.ru_stime.tv_sec + ru.ru_stime.tv_usec / 1000000.0));
89
90 return(tm);
91 }
92
93
94
95 double
getProcessTime(void)96 getProcessTime(void) {
97 struct timeval tp;
98 static double st = 0.0;
99 double tm = 0;
100
101 if (gettimeofday(&tp, NULL) == 0)
102 tm = tp.tv_sec + tp.tv_usec / 1000000.0;
103
104 if (st == 0.0)
105 st = tm;
106
107 return(tm - st);
108 }
109
110
111
112 uint64
getProcessSize(void)113 getProcessSize(void) {
114 struct rusage ru;
115 uint64 sz = 0;
116
117 if (getrusage(ru) == true) {
118 sz = ru.ru_maxrss;
119 #ifndef __APPLE__ // Everybody but MacOS returns kilobytes.
120 sz *= 1024; // MacOS returns bytes.
121 #endif
122 }
123
124 return(sz);
125 }
126
127
128
129 uint64
getProcessSizeLimit(void)130 getProcessSizeLimit(void) {
131 struct rlimit rl;
132 uint64 sz = uint64max;
133
134 if (getrlimit(rl) == true)
135 sz = rl.rlim_cur;
136
137 return(sz);
138 }
139
140
141
142 uint64
getBytesAllocated(void)143 getBytesAllocated(void) {
144 uint64 epoch = 1;
145 size_t epochLen = sizeof(uint64);
146 size_t active = 0;
147 size_t activeLen = sizeof(size_t);
148
149 #if defined(__FreeBSD__) || defined(JEMALLOC)
150
151 mallctl("epoch", NULL, NULL, &epoch, epochLen);
152 mallctl("stats.active", &active, &activeLen, NULL, 0);
153
154 #else
155
156 active = getProcessSize();
157
158 #endif
159
160 return(active);
161 }
162
163
164
165 uint64
getPhysicalMemorySize(void)166 getPhysicalMemorySize(void) {
167 uint64 physPages = sysconf(_SC_PHYS_PAGES);
168 uint64 pageSize = sysconf(_SC_PAGESIZE);
169 uint64 physMemory = physPages * pageSize;
170
171 return(physMemory);
172 }
173
174
175
176 // Return the size of a page of memory. Every OS we care about (MacOS,
177 // FreeBSD, Linux) claims to have getpagesize().
178 //
179 uint64
getPageSize(void)180 getPageSize(void) {
181 return(getpagesize());
182 }
183
184
185
186 // Query the machine or the environment to find any memory size limit. If
187 // there is no environment limit, the physical memory size is returned.
188 //
189 // Slurm variables (from sbatch man page).
190 // SLURM_MEM_PER_CPU
191 // Set if --mem-per-cpu is supplied to sbatch.
192 // "SLURM_MEM_PER_CPU=2048" for a request of --mem-per-cpu=2g
193 //
194 // SLURM_MEM_PER_NODE
195 // Set if --mem is supplied to sbatch.
196 // "SLURM_MEM_PER_NODE=5120" for a request of --mem=5g
197 //
198 // SLURM_MEM_PER_GPU
199 // Requested memory per allocated GPU.
200 // Only set if the --mem-per-gpu option is specified.
201 // Not checked for below.
202 //
203 // There doesn't appear to be a comparable environment variable for SGE.
204 //
205 // PBS/OpenPBS/PBS Pro variables.
206 // PBS_RESC_MEM
207 // TORQUE_RESC_MEM (probably obsolete)
208 // Potentially memory in bytes.
209 //
210 //
211 uint64
getMaxMemoryAllowed(void)212 getMaxMemoryAllowed(void) {
213 char *env, *cpu;
214 uint64 maxmem = getPhysicalMemorySize();
215
216 cpu = getenv("SLURM_JOB_CPUS_PER_NODE");
217 env = getenv("SLURM_MEM_PER_CPU");
218 if (env && cpu)
219 maxmem = strtouint64(cpu) * strtouint64(env) * 1024 * 1024;
220
221 env = getenv("SLURM_MEM_PER_NODE");
222 if (env)
223 maxmem = strtouint64(env) * 1024 * 1024;
224
225 env = getenv("PBS_RESC_MEM");
226 if (env)
227 maxmem = strtouint64(env);
228
229 return(maxmem);
230 }
231
232
233
234 // There is a bit of a race condition in here. On our grid, at least, a
235 // multi-cpu interactive job sets both SLURM_JOB_CPUS_PER_NODE and
236 // OMP_NUM_THREADS - but sets the former to the correct value and the
237 // latter to one.
238 //
239 // Because of this, we let the grid variables overwrite the OpenMP
240 // variable, and further reset OpenMP to use whatever the grid has
241 // told us to use.
242 //
243 // OpenMP variables.
244 // OMP_NUM_THREADS
245 // - we don't query this, and instead use omp_get_max_threads(),
246 // because if OMP_NUM_THREADS isn't set, the function will
247 // return the number of CPUs on the host.
248 //
249 // Slurm variables (from sbatch man page).
250 // SLURM_CPUS_ON_NODE
251 // - Number of CPUS on the allocated node.
252 //
253 // SLURM_JOB_CPUS_PER_NODE
254 // - --cpus-per-node
255 // - Count of processors available to the job on this node. Note the
256 // select/linear plugin allocates entire nodes to jobs, so the value
257 // indicates the total count of CPUs on the node. The select/cons_res
258 // plugin allocates individual processors to jobs, so this number
259 // indicates the number of processors on this node allocated to the
260 // job.
261 //
262 // SLURM_JOB_NUM_NODES
263 // - total number of nodes in the job's resource allocation
264 //
265 // PBS/OpenPBS/PBS Pro variables (from Torque 9.0.3).
266 // PBS_NUM_NODES - Number of nodes allocated to the job
267 // PBS_NUM_PPN - Number of procs per node allocated to the job
268 // PBS_NCPUS - (older version of PBS_NUM_PPN?)
269 // PBS_NP - Number of execution slots (cores) for the job
270 // TORQUE_RESC_PROC - (can't find any doc on this)
271 //
272 // SGE variables.
273 // NSLOTS
274 //
275 uint32
getMaxThreadsAllowed(uint32 limit)276 getMaxThreadsAllowed(uint32 limit) {
277 char *env;
278 uint32 nAllowed = omp_get_max_threads();
279
280 env = getenv("SLURM_JOB_CPUS_PER_NODE");
281 if (env)
282 nAllowed = strtouint32(env);
283
284 env = getenv("PBS_NCPUS");
285 if (env)
286 nAllowed = strtouint32(env);
287
288 env = getenv("PBS_NUM_PPN");
289 if (env)
290 nAllowed = strtouint32(env);
291
292 env = getenv("NSLOTS");
293 if (env)
294 nAllowed = strtouint32(env);
295
296 nAllowed = std::min(limit, nAllowed);
297
298 return(nAllowed);
299 }
300
301
302
303 uint32
getNumThreads(void)304 getNumThreads(void) {
305 return(omp_get_max_threads());
306 }
307
308
309
310 uint32
getNumThreadsActive(void)311 getNumThreadsActive(void) {
312 return(omp_get_num_threads());
313 }
314
315
316
317 uint32
getThreadNum(void)318 getThreadNum(void) {
319 return(omp_get_thread_num());
320 }
321
322
323
324 uint32
setNumThreads(char const * opt)325 setNumThreads(char const *opt) {
326 return(setNumThreads(strtouint32(opt)));
327 }
328
329
330
331 uint32
setNumThreads(uint32 thr)332 setNumThreads(uint32 thr) {
333 omp_set_num_threads(thr);
334 return(omp_get_max_threads());
335 }
336