1 /*
2 * %CopyrightBegin%
3 *
4 * Copyright Ericsson AB 2005-2020. All Rights Reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * %CopyrightEnd%
19 */
20
21 #ifdef HAVE_CONFIG_H
22 # include "config.h"
23 #endif
24
25 /* These need to be undef:ed to not break activation of
26 * micro level process accounting on /proc/self
27 */
28 #ifdef _LARGEFILE_SOURCE
29 # undef _LARGEFILE_SOURCE
30 #endif
31 #ifdef _FILE_OFFSET_BITS
32 # undef _FILE_OFFSET_BITS
33 #endif
34
35 #include <stdlib.h>
36 #include "sys.h"
37 #include "global.h"
38 #include "erl_os_monotonic_time_extender.h"
39
40 #undef ERTS_HAVE_ERTS_OS_TIMES_IMPL__
41 #undef ERTS_HAVE_ERTS_SYS_HRTIME_IMPL__
42
43 #if defined(OS_MONOTONIC_TIME_USING_MACH_CLOCK_GET_TIME) \
44 || defined(OS_SYSTEM_TIME_USING_MACH_CLOCK_GET_TIME) \
45 || defined(SYS_HRTIME_USING_MACH_CLOCK_GET_TIME)
46 # include <mach/clock.h>
47 # include <mach/mach.h>
48 # define ERTS_MACH_CLOCKS
49 #endif
50
51 #ifdef NO_SYSCONF
52 # define TICKS_PER_SEC() HZ
53 #else
54 #define TICKS_PER_SEC() sysconf(_SC_CLK_TCK)
55 #endif
56
57 #ifdef HAVE_GETHRVTIME_PROCFS_IOCTL
58 # include <unistd.h>
59 # include <sys/types.h>
60 # include <sys/stat.h>
61 # include <sys/signal.h>
62 # include <sys/fault.h>
63 # include <sys/syscall.h>
64 # include <sys/procfs.h>
65 # include <fcntl.h>
66 #endif
67
68 static void init_perf_counter(void);
69
70 /******************* Routines for time measurement *********************/
71
72 #undef ERTS_SYS_TIME_INTERNAL_STATE_WRITE_FREQ__
73 #undef ERTS_SYS_TIME_INTERNAL_STATE_READ_ONLY__
74 #undef ERTS_SYS_TIME_INTERNAL_STATE_READ_MOSTLY__
75
76 #if defined(OS_MONOTONIC_TIME_USING_TIMES)
77
78 static Uint32
get_tick_count(void)79 get_tick_count(void)
80 {
81 struct tms unused;
82 return (Uint32) times(&unused);
83 }
84
85 #define ERTS_SYS_TIME_INTERNAL_STATE_READ_ONLY__
86 #define ERTS_SYS_TIME_INTERNAL_STATE_READ_MOSTLY__
87
88 #endif
89
90 /*
91 * init timers, chose a tick length, and return it.
92 * Unix is priviliged when it comes to time, as erl_time_sup.c
93 * does almost everything. Other platforms have to
94 * emulate Unix in this sense.
95 */
96
97 ErtsSysTimeData__ erts_sys_time_data__ erts_align_attribute(ERTS_CACHE_LINE_SIZE);
98
99 #if defined(__linux__) && defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME)
100
101 #define ERTS_SYS_TIME_INTERNAL_STATE_WRITE_FREQ__
102
103 static ErtsMonotonicTime clock_gettime_monotonic(void);
104 static ErtsMonotonicTime clock_gettime_monotonic_verified(void);
105 #if defined(HAVE_CLOCK_GETTIME_MONOTONIC_RAW)
106 static ErtsMonotonicTime clock_gettime_monotonic_raw(void);
107 #endif
108 #if defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME)
109 static void clock_gettime_times(ErtsMonotonicTime *, ErtsSystemTime *);
110 static void clock_gettime_times_verified(ErtsMonotonicTime *, ErtsSystemTime *);
111 #if defined(HAVE_CLOCK_GETTIME_MONOTONIC_RAW)
112 static void clock_gettime_times_raw(ErtsMonotonicTime *, ErtsSystemTime *);
113 #endif
114 #endif
115
116 #endif /* defined(__linux__) && defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME) */
117
118 #ifdef ERTS_MACH_CLOCKS
119 # define ERTS_SYS_TIME_INTERNAL_STATE_READ_ONLY__
120 typedef struct {
121 clock_id_t id;
122 clock_serv_t srv;
123 char *name;
124 } ErtsMachClock;
125
126 typedef struct {
127 host_name_port_t host;
128 struct {
129 ErtsMachClock monotonic;
130 ErtsMachClock wall;
131 } clock;
132 } ErtsMachClocks;
133 static void mach_clocks_init(void);
134 static void mach_clocks_fini(void);
135 # ifdef HAVE_CLOCK_GET_ATTRIBUTES
136 # define ERTS_HAVE_MACH_CLOCK_GETRES
137 static Sint64
138 mach_clock_getres(ErtsMachClock *clk);
139 # endif
140 #endif /* ERTS_MACH_CLOCKS */
141
142 #ifdef ERTS_SYS_TIME_INTERNAL_STATE_READ_ONLY__
143 struct sys_time_internal_state_read_only__ {
144 #if defined(OS_MONOTONIC_TIME_USING_TIMES)
145 int times_shift;
146 #endif
147 #ifdef ERTS_MACH_CLOCKS
148 ErtsMachClocks mach;
149 #endif
150 };
151 #endif
152
153 #ifdef ERTS_SYS_TIME_INTERNAL_STATE_READ_MOSTLY__
154 struct sys_time_internal_state_read_mostly__ {
155 #if defined(OS_MONOTONIC_TIME_USING_TIMES)
156 ErtsOsMonotonicTimeExtendState os_mtime_xtnd;
157 #endif
158 };
159 #endif
160
161 #ifdef ERTS_SYS_TIME_INTERNAL_STATE_WRITE_FREQ__
162 struct sys_time_internal_state_write_freq__ {
163 erts_mtx_t mtx;
164 #if defined(__linux__) && defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME)
165 ErtsMonotonicTime last_delivered;
166 #endif
167 };
168 #endif
169
170 #if defined(ERTS_SYS_TIME_INTERNAL_STATE_READ_ONLY__) \
171 || defined(ERTS_SYS_TIME_INTERNAL_STATE_WRITE_FREQ__)
172 static struct {
173 #ifdef ERTS_SYS_TIME_INTERNAL_STATE_READ_ONLY__
174 union {
175 struct sys_time_internal_state_read_only__ o;
176 char align__[(((sizeof(struct sys_time_internal_state_read_only__) - 1)
177 / ASSUMED_CACHE_LINE_SIZE) + 1)
178 * ASSUMED_CACHE_LINE_SIZE];
179 } r;
180 #endif
181 #ifdef ERTS_SYS_TIME_INTERNAL_STATE_READ_MOSTLY__
182 union {
183 struct sys_time_internal_state_read_mostly__ m;
184 char align__[(((sizeof(struct sys_time_internal_state_read_mostly__) - 1)
185 / ASSUMED_CACHE_LINE_SIZE) + 1)
186 * ASSUMED_CACHE_LINE_SIZE];
187 } wr;
188 #endif
189 #ifdef ERTS_SYS_TIME_INTERNAL_STATE_WRITE_FREQ__
190 union {
191 struct sys_time_internal_state_write_freq__ f;
192 char align__[(((sizeof(struct sys_time_internal_state_write_freq__) - 1)
193 / ASSUMED_CACHE_LINE_SIZE) + 1)
194 * ASSUMED_CACHE_LINE_SIZE];
195 } w;
196 #endif
197 } internal_state erts_align_attribute(ERTS_CACHE_LINE_SIZE);
198 #endif
199
200 void
sys_init_time(ErtsSysInitTimeResult * init_resp)201 sys_init_time(ErtsSysInitTimeResult *init_resp)
202 {
203 #if defined(ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT)
204 int major, minor, build, vsn;
205 #endif
206 #if defined(ERTS_MACH_CLOCKS)
207 mach_clocks_init();
208 #endif
209 #if !defined(ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT)
210
211 init_resp->have_os_monotonic_time = 0;
212
213 #else /* defined(ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT) */
214
215 #ifdef ERTS_HAVE_CORRECTED_OS_MONOTONIC_TIME
216 init_resp->have_corrected_os_monotonic_time = 1;
217 #else
218 init_resp->have_corrected_os_monotonic_time = 0;
219 #endif
220
221 init_resp->os_monotonic_time_info.resolution = (Uint64) 1000*1000*1000;
222 #if defined(ERTS_HAVE_MACH_CLOCK_GETRES) && defined(MONOTONIC_CLOCK_ID)
223 init_resp->os_monotonic_time_info.resolution
224 = mach_clock_getres(&internal_state.r.o.mach.clock.monotonic);
225 #elif defined(HAVE_CLOCK_GETRES) && defined(MONOTONIC_CLOCK_ID)
226 {
227 struct timespec ts;
228 if (clock_getres(MONOTONIC_CLOCK_ID, &ts) == 0) {
229 if (ts.tv_sec == 0 && ts.tv_nsec != 0)
230 init_resp->os_monotonic_time_info.resolution /= ts.tv_nsec;
231 else if (ts.tv_sec >= 1)
232 init_resp->os_monotonic_time_info.resolution = 1;
233 }
234 }
235 #endif
236
237 #ifdef MONOTONIC_CLOCK_ID_STR
238 init_resp->os_monotonic_time_info.clock_id = MONOTONIC_CLOCK_ID_STR;
239 #else
240 init_resp->os_monotonic_time_info.clock_id = NULL;
241 #endif
242
243 init_resp->os_monotonic_time_info.locked_use = 0;
244
245 #if defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME)
246 init_resp->os_monotonic_time_info.func = "clock_gettime";
247 #elif defined(OS_MONOTONIC_TIME_USING_MACH_CLOCK_GET_TIME)
248 init_resp->os_monotonic_time_info.func = "clock_get_time";
249 #elif defined(OS_MONOTONIC_TIME_USING_GETHRTIME)
250 init_resp->os_monotonic_time_info.func = "gethrtime";
251 #elif defined(OS_MONOTONIC_TIME_USING_TIMES)
252 init_resp->os_monotonic_time_info.func = "times";
253 #else
254 # error Unknown erts_os_monotonic_time() implementation
255 #endif
256
257 init_resp->have_os_monotonic_time = 1;
258
259 os_version(&major, &minor, &build);
260
261 vsn = ERTS_MK_VSN_INT(major, minor, build);
262
263
264 #if defined(__linux__) && defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME)
265 if (vsn >= ERTS_MK_VSN_INT(2, 6, 33)) {
266 erts_sys_time_data__.r.o.os_monotonic_time =
267 clock_gettime_monotonic;
268 #if defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME)
269 erts_sys_time_data__.r.o.os_times =
270 clock_gettime_times;
271 #endif
272 }
273 else {
274 /*
275 * Linux versions prior to 2.6.33 have a
276 * known bug that sometimes cause the NTP
277 * adjusted monotonic clock to take small
278 * steps backwards. Use raw monotonic clock
279 * if it is present; otherwise, fall back
280 * on locked verification of values.
281 */
282 init_resp->have_corrected_os_monotonic_time = 0;
283 #if defined(HAVE_CLOCK_GETTIME_MONOTONIC_RAW)
284 /* We know that CLOCK_MONOTONIC_RAW is defined,
285 but we don't know if we got a kernel that
286 supports it. Support for CLOCK_MONOTONIC_RAW
287 appeared in kernel 2.6.28... */
288 if (vsn >= ERTS_MK_VSN_INT(2, 6, 28)) {
289 erts_sys_time_data__.r.o.os_monotonic_time =
290 clock_gettime_monotonic_raw;
291 #if defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME)
292 erts_sys_time_data__.r.o.os_times =
293 clock_gettime_times_raw;
294 #endif
295 init_resp->os_monotonic_time_info.clock_id =
296 "CLOCK_MONOTONIC_RAW";
297 }
298 else
299 #endif /* defined(HAVE_CLOCK_GETTIME_MONOTONIC_RAW) */
300 {
301 erts_sys_time_data__.r.o.os_monotonic_time =
302 clock_gettime_monotonic_verified;
303 #if defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME)
304 erts_sys_time_data__.r.o.os_times =
305 clock_gettime_times_verified;
306 #endif
307 erts_mtx_init(&internal_state.w.f.mtx, "os_monotonic_time", NIL,
308 ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
309 internal_state.w.f.last_delivered
310 = clock_gettime_monotonic();
311 init_resp->os_monotonic_time_info.locked_use = 1;
312 }
313 }
314 #else /* !(defined(__linux__) && defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME)) */
315 {
316 char flavor[1024];
317
318 os_flavor(flavor, sizeof(flavor));
319
320 if (sys_strcmp(flavor, "sunos") == 0) {
321 /*
322 * Don't trust hrtime on multi processors
323 * on SunOS prior to SunOS 5.8
324 */
325 if (vsn < ERTS_MK_VSN_INT(5, 8, 0)) {
326 #if defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_CONF)
327 if (sysconf(_SC_NPROCESSORS_CONF) > 1)
328 #endif
329 init_resp->have_os_monotonic_time = 0;
330 }
331 }
332 }
333 #endif /* !(defined(__linux__) && defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME)) */
334
335 #endif /* defined(ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT) */
336
337 #ifdef ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
338 init_resp->os_monotonic_time_unit = ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT;
339 #endif
340 init_resp->sys_clock_resolution = SYS_CLOCK_RESOLUTION;
341
342 /*
343 * This (erts_sys_time_data__.r.o.ticks_per_sec) is only for
344 * times() (CLK_TCK), the resolution is always one millisecond..
345 */
346 if ((erts_sys_time_data__.r.o.ticks_per_sec = TICKS_PER_SEC()) < 0)
347 erts_exit(ERTS_ABORT_EXIT, "Can't get clock ticks/sec\n");
348
349 #if defined(OS_MONOTONIC_TIME_USING_TIMES)
350 #if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
351 # error Time unit is supposed to be determined at runtime...
352 #endif
353 {
354 ErtsMonotonicTime resolution = erts_sys_time_data__.r.o.ticks_per_sec;
355 ErtsMonotonicTime time_unit = resolution;
356 int shift = 0;
357
358 while (time_unit < 1000*1000) {
359 time_unit <<= 1;
360 shift++;
361 }
362
363 init_resp->os_monotonic_time_info.resolution = resolution;
364 init_resp->os_monotonic_time_unit = time_unit;
365 init_resp->os_monotonic_time_info.extended = 1;
366 internal_state.r.o.times_shift = shift;
367
368 erts_init_os_monotonic_time_extender(&internal_state.wr.m.os_mtime_xtnd,
369 get_tick_count,
370 (1 << 29) / resolution);
371 }
372 #endif /* defined(OS_MONOTONIC_TIME_USING_TIMES) */
373
374 #ifdef WALL_CLOCK_ID_STR
375 init_resp->os_system_time_info.clock_id = WALL_CLOCK_ID_STR;
376 #else
377 init_resp->os_system_time_info.clock_id = NULL;
378 #endif
379
380 init_resp->os_system_time_info.locked_use = 0;
381 init_resp->os_system_time_info.resolution = (Uint64) 1000*1000*1000;
382 #if defined(ERTS_HAVE_MACH_CLOCK_GETRES) && defined(WALL_CLOCK_ID)
383 init_resp->os_system_time_info.resolution
384 = mach_clock_getres(&internal_state.r.o.mach.clock.wall);
385 #elif defined(HAVE_CLOCK_GETRES) && defined(WALL_CLOCK_ID)
386 {
387 struct timespec ts;
388 if (clock_getres(WALL_CLOCK_ID, &ts) == 0) {
389 if (ts.tv_sec == 0 && ts.tv_nsec != 0)
390 init_resp->os_system_time_info.resolution /= ts.tv_nsec;
391 else if (ts.tv_sec >= 1)
392 init_resp->os_system_time_info.resolution = 1;
393 }
394 }
395 #endif
396
397 #if defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME)
398 init_resp->os_system_time_info.func = "clock_gettime";
399 #elif defined(OS_SYSTEM_TIME_USING_MACH_CLOCK_GET_TIME)
400 init_resp->os_system_time_info.func = "clock_get_time";
401 #elif defined(OS_SYSTEM_TIME_GETTIMEOFDAY)
402 init_resp->os_system_time_info.func = "gettimeofday";
403 init_resp->os_system_time_info.resolution = 1000*1000;
404 init_resp->os_system_time_info.clock_id = NULL;
405 #else
406 # error Missing erts_os_system_time() implementation
407 #endif
408
409 init_perf_counter();
410
411 }
412
413 void
erts_late_sys_init_time(void)414 erts_late_sys_init_time(void)
415 {
416 #if defined(OS_MONOTONIC_TIME_USING_TIMES)
417 erts_late_init_os_monotonic_time_extender(&internal_state.wr.m.os_mtime_xtnd);
418 #endif
419 }
420
421 static ERTS_INLINE ErtsSystemTime
adj_stime_time_unit(ErtsSystemTime stime,Uint32 res)422 adj_stime_time_unit(ErtsSystemTime stime, Uint32 res)
423 {
424 if (res == ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT)
425 return stime;
426 if (res == (Uint32) 1000*1000*1000
427 && ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT == 1000*1000)
428 return stime/1000;
429 if (res == (Uint32) 1000*1000
430 && ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT == 1000*1000*1000)
431 return stime*1000;
432 return ((ErtsSystemTime)
433 erts_time_unit_conversion(stime,
434 (Uint32) res,
435 (Uint32) ERTS_MONOTONIC_TIME_UNIT));
436 }
437
438 #define ERTS_TimeSpec2Sint64(TS) \
439 ((((Sint64) (TS)->tv_sec) * ((Sint64) 1000*1000*1000)) \
440 + ((Sint64) (TS)->tv_nsec))
441
442 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
443 * POSIX clock_gettime() *
444 \* */
445
446 #if defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME) \
447 || defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME)
448
449 static ERTS_INLINE Sint64
posix_clock_gettime(clockid_t id,char * name)450 posix_clock_gettime(clockid_t id, char *name)
451 {
452 struct timespec ts;
453
454 if (clock_gettime(id, &ts) != 0) {
455 int err = errno;
456 const char *errstr = err ? strerror(err) : "unknown";
457 erts_exit(ERTS_ABORT_EXIT,
458 "clock_gettime(%s, _) failed: %s (%d)\n",
459 name, errstr, err);
460 }
461 return ERTS_TimeSpec2Sint64(&ts);
462 }
463
464 #endif /* defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME) \
465 || defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME) */
466
467 #if defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME)
468
469 ErtsSystemTime
erts_os_system_time(void)470 erts_os_system_time(void)
471 {
472 Sint64 stime = posix_clock_gettime(WALL_CLOCK_ID,
473 WALL_CLOCK_ID_STR);
474 return adj_stime_time_unit((ErtsSystemTime) stime,
475 (Uint32) 1000*1000*1000);
476 }
477
478 #endif /* defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME) */
479
480 #if defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME)
481
482 #if defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME)
483
484 #define ERTS_HAVE_ERTS_OS_TIMES_IMPL__
485
486 static ERTS_INLINE void
posix_clock_gettime_times(clockid_t mid,char * mname,ErtsMonotonicTime * mtimep,clockid_t sid,char * sname,ErtsSystemTime * stimep)487 posix_clock_gettime_times(clockid_t mid, char *mname,
488 ErtsMonotonicTime *mtimep,
489 clockid_t sid, char *sname,
490 ErtsSystemTime *stimep)
491 {
492 struct timespec mts, sts;
493 int mres, sres, merr, serr;
494
495 mres = clock_gettime(mid, &mts);
496 merr = errno;
497 sres = clock_gettime(sid, &sts);
498 serr = errno;
499
500 if (mres != 0) {
501 const char *errstr = merr ? strerror(merr) : "unknown";
502 erts_exit(ERTS_ABORT_EXIT,
503 "clock_gettime(%s, _) failed: %s (%d)\n",
504 mname, errstr, merr);
505 }
506 if (sres != 0) {
507 const char *errstr = serr ? strerror(serr) : "unknown";
508 erts_exit(ERTS_ABORT_EXIT,
509 "clock_gettime(%s, _) failed: %s (%d)\n",
510 sname, errstr, serr);
511 }
512
513 *mtimep = (ErtsMonotonicTime) ERTS_TimeSpec2Sint64(&mts);
514 *stimep = (ErtsSystemTime) ERTS_TimeSpec2Sint64(&sts);
515 }
516
517 #endif /* defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME) */
518
519 #if defined(__linux__)
520
clock_gettime_monotonic_verified(void)521 static ErtsMonotonicTime clock_gettime_monotonic_verified(void)
522 {
523 ErtsMonotonicTime mtime;
524
525 mtime = (ErtsMonotonicTime) posix_clock_gettime(MONOTONIC_CLOCK_ID,
526 MONOTONIC_CLOCK_ID_STR);
527
528 erts_mtx_lock(&internal_state.w.f.mtx);
529 if (mtime < internal_state.w.f.last_delivered)
530 mtime = internal_state.w.f.last_delivered;
531 else
532 internal_state.w.f.last_delivered = mtime;
533 erts_mtx_unlock(&internal_state.w.f.mtx);
534
535 return mtime;
536 }
537
538 #if defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME)
539
clock_gettime_times_verified(ErtsMonotonicTime * mtimep,ErtsSystemTime * stimep)540 static void clock_gettime_times_verified(ErtsMonotonicTime *mtimep,
541 ErtsSystemTime *stimep)
542 {
543 posix_clock_gettime_times(MONOTONIC_CLOCK_ID,
544 MONOTONIC_CLOCK_ID_STR,
545 mtimep,
546 WALL_CLOCK_ID,
547 WALL_CLOCK_ID_STR,
548 stimep);
549
550 erts_mtx_lock(&internal_state.w.f.mtx);
551 if (*mtimep < internal_state.w.f.last_delivered)
552 *mtimep = internal_state.w.f.last_delivered;
553 else
554 internal_state.w.f.last_delivered = *mtimep;
555 erts_mtx_unlock(&internal_state.w.f.mtx);
556 }
557
558 #endif /* defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME) */
559
clock_gettime_monotonic(void)560 static ErtsMonotonicTime clock_gettime_monotonic(void)
561 {
562 return (ErtsMonotonicTime) posix_clock_gettime(MONOTONIC_CLOCK_ID,
563 MONOTONIC_CLOCK_ID_STR);
564 }
565
566 #if defined(HAVE_CLOCK_GETTIME_MONOTONIC_RAW)
567
clock_gettime_monotonic_raw(void)568 static ErtsMonotonicTime clock_gettime_monotonic_raw(void)
569 {
570 return (ErtsMonotonicTime) posix_clock_gettime(CLOCK_MONOTONIC_RAW,
571 "CLOCK_MONOTONIC_RAW");
572 }
573
574 #endif /* defined(HAVE_CLOCK_GETTIME_MONOTONIC_RAW) */
575
576 #if defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME)
577
clock_gettime_times(ErtsMonotonicTime * mtimep,ErtsSystemTime * stimep)578 static void clock_gettime_times(ErtsMonotonicTime *mtimep,
579 ErtsSystemTime *stimep)
580 {
581 posix_clock_gettime_times(MONOTONIC_CLOCK_ID,
582 MONOTONIC_CLOCK_ID_STR,
583 mtimep,
584 WALL_CLOCK_ID,
585 WALL_CLOCK_ID_STR,
586 stimep);
587 }
588
589 #if defined(HAVE_CLOCK_GETTIME_MONOTONIC_RAW)
590
clock_gettime_times_raw(ErtsMonotonicTime * mtimep,ErtsSystemTime * stimep)591 static void clock_gettime_times_raw(ErtsMonotonicTime *mtimep,
592 ErtsSystemTime *stimep)
593 {
594 posix_clock_gettime_times(CLOCK_MONOTONIC_RAW,
595 "CLOCK_MONOTONIC_RAW",
596 mtimep,
597 WALL_CLOCK_ID,
598 WALL_CLOCK_ID_STR,
599 stimep);
600 }
601
602 #endif /* defined(HAVE_CLOCK_GETTIME_MONOTONIC_RAW) */
603
604 #endif /* defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME) */
605
606 #else /* !defined(__linux__) */
607
erts_os_monotonic_time(void)608 ErtsMonotonicTime erts_os_monotonic_time(void)
609 {
610 return posix_clock_gettime(MONOTONIC_CLOCK_ID,
611 MONOTONIC_CLOCK_ID_STR);
612 }
613
614 #if defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME)
615
erts_os_times(ErtsMonotonicTime * mtimep,ErtsSystemTime * stimep)616 void erts_os_times(ErtsMonotonicTime *mtimep, ErtsSystemTime *stimep)
617 {
618 posix_clock_gettime_times(MONOTONIC_CLOCK_ID,
619 MONOTONIC_CLOCK_ID_STR,
620 mtimep,
621 WALL_CLOCK_ID,
622 WALL_CLOCK_ID_STR,
623 stimep);
624 }
625
626 #endif /* defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME) */
627
628 #endif /* !defined(__linux__) */
629
630 #endif /* defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME) */
631
632 #if defined(SYS_HRTIME_USING_CLOCK_GETTIME)
633 # define ERTS_HAVE_ERTS_SYS_HRTIME_IMPL__
634
635 ErtsSysHrTime
erts_sys_hrtime(void)636 erts_sys_hrtime(void)
637 {
638 return (ErtsSysHrTime) posix_clock_gettime(HRTIME_CLOCK_ID,
639 HRTIME_CLOCK_ID_STR);
640 }
641
642 #endif /* defined(SYS_HRTIME_USING_CLOCK_GETTIME) */
643
644 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
645 * MACH clock_get_time() *
646 \* */
647
648 #if defined(ERTS_MACH_CLOCKS)
649
650 static void
mach_clocks_fini(void)651 mach_clocks_fini(void)
652 {
653 mach_port_t task = mach_task_self();
654 mach_port_deallocate(task, internal_state.r.o.mach.host);
655 #if defined(OS_MONOTONIC_TIME_USING_MACH_CLOCK_GET_TIME)
656 mach_port_deallocate(task, internal_state.r.o.mach.clock.monotonic.srv);
657 #endif
658 #if defined(OS_SYSTEM_TIME_USING_MACH_CLOCK_GET_TIME)
659 mach_port_deallocate(task, internal_state.r.o.mach.clock.wall.srv);
660 #endif
661 }
662
663 static void
mach_clocks_init(void)664 mach_clocks_init(void)
665 {
666 kern_return_t kret;
667 host_name_port_t host;
668 clock_id_t id;
669 clock_serv_t *clck_srv_p;
670 char *name;
671
672 host = internal_state.r.o.mach.host = mach_host_self();
673
674 #if defined(OS_MONOTONIC_TIME_USING_MACH_CLOCK_GET_TIME) \
675 || defined(SYS_HRTIME_USING_MACH_CLOCK_GET_TIME)
676 id = internal_state.r.o.mach.clock.monotonic.id = MONOTONIC_CLOCK_ID;
677 name = internal_state.r.o.mach.clock.monotonic.name = MONOTONIC_CLOCK_ID_STR;
678 clck_srv_p = &internal_state.r.o.mach.clock.monotonic.srv;
679 kret = host_get_clock_service(host, id, clck_srv_p);
680 if (kret != KERN_SUCCESS) {
681 erts_exit(ERTS_ABORT_EXIT,
682 "host_get_clock_service(_, %s, _) failed\n",
683 name);
684 }
685 #endif
686
687 #if defined(OS_SYSTEM_TIME_USING_MACH_CLOCK_GET_TIME)
688 id = internal_state.r.o.mach.clock.wall.id = WALL_CLOCK_ID;
689 name = internal_state.r.o.mach.clock.wall.name = WALL_CLOCK_ID_STR;
690 clck_srv_p = &internal_state.r.o.mach.clock.wall.srv;
691 kret = host_get_clock_service(host, id, clck_srv_p);
692 if (kret != KERN_SUCCESS) {
693 erts_exit(ERTS_ABORT_EXIT,
694 "host_get_clock_service(_, %s, _) failed\n",
695 name);
696 }
697 #endif
698
699 if (atexit(mach_clocks_fini) != 0) {
700 int err = errno;
701 const char *errstr = err ? strerror(err) : "unknown";
702 erts_exit(ERTS_ABORT_EXIT,
703 "Failed to register mach_clocks_fini() "
704 "for call at exit: %s (%d)\n",
705 errstr, err);
706 }
707 }
708
709 #ifdef ERTS_HAVE_MACH_CLOCK_GETRES
710
711 static Sint64
mach_clock_getres(ErtsMachClock * clk)712 mach_clock_getres(ErtsMachClock *clk)
713 {
714 kern_return_t kret;
715 natural_t attr[1];
716 mach_msg_type_number_t cnt;
717
718 cnt = sizeof(attr);
719 kret = clock_get_attributes(clk->srv,
720 CLOCK_GET_TIME_RES,
721 (clock_attr_t) attr,
722 &cnt);
723 if (kret != KERN_SUCCESS || cnt != 1) {
724 erts_exit(ERTS_ABORT_EXIT,
725 "clock_get_attributes(%s, _) failed\n",
726 clk->name);
727 }
728
729 return (Sint64) attr[0];
730 }
731
732 #endif /* ERTS_HAVE_MACH_CLOCK_GETRES */
733
734 static ERTS_INLINE Sint64
mach_clock_get_time(ErtsMachClock * clk)735 mach_clock_get_time(ErtsMachClock *clk)
736 {
737 kern_return_t kret;
738 mach_timespec_t time_spec;
739
740 kret = clock_get_time(clk->srv, &time_spec);
741 if (kret != KERN_SUCCESS)
742 erts_exit(ERTS_ABORT_EXIT, "clock_get_time(%s, _) failed\n", clk->name);
743
744 return ERTS_TimeSpec2Sint64(&time_spec);
745 }
746
747 #endif /* defined(ERTS_MACH_CLOCKS) */
748
749 #if defined(OS_SYSTEM_TIME_USING_MACH_CLOCK_GET_TIME)
750
751 #define ERTS_HAVE_ERTS_OS_TIMES_IMPL__
752
753 ErtsSystemTime
erts_os_system_time(void)754 erts_os_system_time(void)
755 {
756 Sint64 stime = mach_clock_get_time(&internal_state.r.o.mach.clock.wall);
757 return adj_stime_time_unit((ErtsSystemTime) stime,
758 (Uint32) 1000*1000*1000);
759 }
760
761 #endif /* defined(OS_SYSTEM_TIME_USING_MACH_CLOCK_GET_TIME) */
762
763 #if defined(OS_MONOTONIC_TIME_USING_MACH_CLOCK_GET_TIME)
764
765 ErtsMonotonicTime
erts_os_monotonic_time(void)766 erts_os_monotonic_time(void)
767 {
768 return (ErtsMonotonicTime)
769 mach_clock_get_time(&internal_state.r.o.mach.clock.monotonic);
770 }
771
772 #if defined(OS_SYSTEM_TIME_USING_MACH_CLOCK_GET_TIME)
773
774 #define ERTS_HAVE_ERTS_OS_TIMES_IMPL__
775
776 void
erts_os_times(ErtsMonotonicTime * mtimep,ErtsSystemTime * stimep)777 erts_os_times(ErtsMonotonicTime *mtimep, ErtsSystemTime *stimep)
778 {
779 kern_return_t mkret, skret;
780 mach_timespec_t mon_time_spec, sys_time_spec;
781
782 mkret = clock_get_time(internal_state.r.o.mach.clock.monotonic.srv,
783 &mon_time_spec);
784 skret = clock_get_time(internal_state.r.o.mach.clock.wall.srv,
785 &sys_time_spec);
786
787 if (mkret != KERN_SUCCESS)
788 erts_exit(ERTS_ABORT_EXIT,
789 "clock_get_time(%s, _) failed\n",
790 internal_state.r.o.mach.clock.monotonic.name);
791 if (skret != KERN_SUCCESS)
792 erts_exit(ERTS_ABORT_EXIT,
793 "clock_get_time(%s, _) failed\n",
794 internal_state.r.o.mach.clock.wall.name);
795
796 *mtimep = (ErtsMonotonicTime) ERTS_TimeSpec2Sint64(&mon_time_spec);
797 *stimep = (ErtsSystemTime) ERTS_TimeSpec2Sint64(&sys_time_spec);
798 }
799
800 #endif /* defined(OS_SYSTEM_TIME_USING_MACH_CLOCK_GET_TIME) */
801
802 #endif /* defined(OS_MONOTONIC_TIME_USING_MACH_CLOCK_GET_TIME) */
803
804 #if defined(SYS_HRTIME_USING_MACH_CLOCK_GET_TIME)
805
806 #define ERTS_HAVE_ERTS_SYS_HRTIME_IMPL__
807
808 ErtsSysHrTime
erts_sys_hrtime(void)809 erts_sys_hrtime(void)
810 {
811 return (ErtsSysHrTime)
812 mach_clock_get_time(&internal_state.r.o.mach.clock.monotonic);
813 }
814
815 #endif /* defined(SYS_HRTIME_USING_MACH_CLOCK_GET_TIME) */
816
817 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
818 * Solaris gethrtime() - OS monotonic time *
819 \* */
820
821 #if defined(OS_MONOTONIC_TIME_USING_GETHRTIME)
822
erts_os_monotonic_time(void)823 ErtsMonotonicTime erts_os_monotonic_time(void)
824 {
825 return (ErtsMonotonicTime) gethrtime();
826 }
827
828 #endif /* defined(OS_MONOTONIC_TIME_USING_GETHRTIME) */
829
830 #if defined(SYS_HRTIME_USING_GETHRTIME)
831
832 #define ERTS_HAVE_ERTS_SYS_HRTIME_IMPL__
833
834 ErtsSysHrTime
erts_sys_hrtime(void)835 erts_sys_hrtime(void)
836 {
837 return (ErtsSysHrTime) gethrtime();
838 }
839
840 #endif /* defined(SYS_HRTIME_USING_GETHRTIME) */
841
842 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
843 * gettimeofday() - OS system time *
844 \* */
845
846 #if defined(OS_SYSTEM_TIME_GETTIMEOFDAY)
847
848 ErtsSystemTime
erts_os_system_time(void)849 erts_os_system_time(void)
850 {
851 ErtsSystemTime stime;
852 struct timeval tv;
853
854 if (gettimeofday(&tv, NULL) != 0) {
855 int err = errno;
856 const char *errstr = err ? strerror(err) : "unknown";
857 erts_exit(ERTS_ABORT_EXIT,
858 "gettimeofday(_, NULL) failed: %s (%d)\n",
859 errstr, err);
860 }
861
862 stime = (ErtsSystemTime) tv.tv_sec;
863 stime *= (ErtsSystemTime) 1000*1000;
864 stime += (ErtsSystemTime) tv.tv_usec;
865
866 return adj_stime_time_unit(stime, (Uint32) 1000*1000);
867 }
868
869 #endif /* defined(OS_SYSTEM_TIME_GETTIMEOFDAY) */
870
871 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
872 * times() - OS monotonic time *
873 \* */
874
875 #if defined(OS_MONOTONIC_TIME_USING_TIMES)
876
877 ErtsMonotonicTime
erts_os_monotonic_time(void)878 erts_os_monotonic_time(void)
879 {
880 Uint32 ticks = get_tick_count();
881 return ERTS_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd,
882 ticks) << internal_state.r.o.times_shift;
883 }
884
885 #endif
886
887 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
888 * Fallbacks *
889 \* */
890
891 #ifndef ERTS_HAVE_ERTS_SYS_HRTIME_IMPL__
892
893 ErtsSysHrTime
erts_sys_hrtime(void)894 erts_sys_hrtime(void)
895 {
896 return (ErtsSysHrTime) ERTS_MONOTONIC_TO_NSEC(erts_os_system_time());
897 }
898
899 #endif
900
901 #if !defined(ERTS_HAVE_ERTS_OS_TIMES_IMPL__) \
902 && defined(ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT)
903
904 void
erts_os_times(ErtsMonotonicTime * mtimep,ErtsSystemTime * stimep)905 erts_os_times(ErtsMonotonicTime *mtimep, ErtsSystemTime *stimep)
906 {
907 *mtimep = erts_os_monotonic_time();
908 *stimep = erts_os_system_time();
909 }
910
911 #endif
912
913 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
914 * Performance counter functions *
915 \* */
916
917
918 /* What resolution to spin to in micro seconds */
919 #define RESOLUTION 100
920 /* How many iterations to spin */
921 #define ITERATIONS 1
922 /* How many significant figures to round to */
923 #define SIGFIGS 3
924
calculate_perf_counter_unit(void)925 static ErtsSysPerfCounter calculate_perf_counter_unit(void) {
926 int i;
927 ErtsSysPerfCounter pre, post;
928 double value = 0;
929 double round_factor;
930 #if defined(HAVE_GETHRTIME) && defined(GETHRTIME_WITH_CLOCK_GETTIME)
931 struct timespec basetime,comparetime;
932 #define __GETTIME(arg) clock_gettime(CLOCK_MONOTONIC,arg)
933 #define __GETUSEC(arg) (arg.tv_nsec / 1000)
934 #else
935 SysTimeval basetime,comparetime;
936 #define __GETTIME(arg) sys_gettimeofday(arg)
937 #define __GETUSEC(arg) arg.tv_usec
938 #endif
939
940 for (i = 0; i < ITERATIONS; i++) {
941 /* Make sure usec just flipped over at current resolution */
942 __GETTIME(&basetime);
943 do {
944 __GETTIME(&comparetime);
945 } while ((__GETUSEC(basetime) / RESOLUTION) == (__GETUSEC(comparetime) / RESOLUTION));
946
947 pre = erts_sys_perf_counter();
948
949 __GETTIME(&basetime);
950 do {
951 __GETTIME(&comparetime);
952 } while ((__GETUSEC(basetime) / RESOLUTION) == (__GETUSEC(comparetime) / RESOLUTION));
953
954 post = erts_sys_perf_counter();
955
956 value += post - pre;
957 }
958 /* After this value is ticks per us */
959 value /= (RESOLUTION*ITERATIONS);
960
961 /* We round to 3 significant figures */
962 round_factor = pow(10.0, SIGFIGS - ceil(log10(value)));
963 value = ((ErtsSysPerfCounter)(value * round_factor + 0.5)) / round_factor;
964
965 /* convert to ticks per second */
966 return 1000000 * value;
967 }
968
have_rdtscp(void)969 static int have_rdtscp(void)
970 {
971 #if defined(ETHR_X86_RUNTIME_CONF__)
972 /* On early x86 cpu's the tsc varies with the current speed of the cpu,
973 which means that the time per tick vary depending on the current
974 load of the cpu. We do not want this as it would give very scewed
975 numbers when the cpu is mostly idle.
976 The linux kernel seems to think that checking for constant and
977 reliable is enough to trust the counter so we do the same.
978
979 If this test is not good enough, I don't know what we'll do.
980 Maybe fallback on erts_sys_hrtime always, but that would be a shame as
981 rdtsc is about 3 times faster than hrtime... */
982 return ETHR_X86_RUNTIME_CONF_HAVE_CONSTANT_TSC__ &&
983 ETHR_X86_RUNTIME_CONF_HAVE_TSC_RELIABLE__;
984 #else
985 return 0;
986 #endif
987 }
988
rdtsc(void)989 static ErtsSysPerfCounter rdtsc(void)
990 {
991 /* It may have been a good idea to put the cpuid instruction before
992 the rdtsc, but I decided against it because it is not really
993 needed for msacc, and it slows it down by quite a bit (5-7 times slower).
994 As a result though, this timestamp becomes much less
995 accurate as it might be re-ordered to be executed way before or after this
996 function is called.
997 */
998 ErtsSysPerfCounter ts;
999 #if defined(__x86_64__)
1000 __asm__ __volatile__ ("rdtsc\n\t"
1001 "shl $32, %%rdx\n\t"
1002 "or %%rdx, %0" : "=a" (ts) : : "rdx");
1003 #elif defined(__i386__)
1004 __asm__ __volatile__ ("rdtsc\n\t"
1005 : "=A" (ts) );
1006 #else
1007 ts = 0;
1008 #endif
1009 return ts;
1010 }
1011
init_perf_counter(void)1012 static void init_perf_counter(void)
1013 {
1014 if (have_rdtscp()) {
1015 erts_sys_time_data__.r.o.perf_counter = rdtsc;
1016 erts_sys_time_data__.r.o.perf_counter_unit = calculate_perf_counter_unit();
1017 } else {
1018 erts_sys_time_data__.r.o.perf_counter = erts_sys_hrtime;
1019 erts_sys_time_data__.r.o.perf_counter_unit = ERTS_HRTIME_UNIT;
1020 }
1021 }
1022
1023 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1024
1025 #ifdef HAVE_GETHRVTIME_PROCFS_IOCTL
1026
1027 /* The code below only has effect on solaris < 10,
1028 needed in order for gehhrvtime to work properly */
sys_start_hrvtime(void)1029 int sys_start_hrvtime(void)
1030 {
1031 long msacct = PR_MSACCT;
1032 int fd;
1033
1034 if ( (fd = open("/proc/self", O_WRONLY)) == -1) {
1035 return -1;
1036 }
1037 if (ioctl(fd, PIOCSET, &msacct) < 0) {
1038 close(fd);
1039 return -2;
1040 }
1041 close(fd);
1042 return 0;
1043 }
1044
sys_stop_hrvtime(void)1045 int sys_stop_hrvtime(void)
1046 {
1047 long msacct = PR_MSACCT;
1048 int fd;
1049
1050 if ( (fd = open("/proc/self", O_WRONLY)) == -1) {
1051 return -1;
1052 }
1053 if (ioctl(fd, PIOCRESET, &msacct) < 0) {
1054 close(fd);
1055 return -2;
1056 }
1057 close(fd);
1058 return 0;
1059 }
1060
1061 #endif /* HAVE_GETHRVTIME_PROCFS_IOCTL */
1062
1063
1064