1 /* $NetBSD: kern_clock.c,v 1.150 2023/07/07 12:34:50 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 2000, 2004, 2006, 2007, 2008 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9 * NASA Ames Research Center.
10 * This code is derived from software contributed to The NetBSD Foundation
11 * by Charles M. Hannum.
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
33 */
34
35 /*-
36 * Copyright (c) 1982, 1986, 1991, 1993
37 * The Regents of the University of California. All rights reserved.
38 * (c) UNIX System Laboratories, Inc.
39 * All or some portions of this file are derived from material licensed
40 * to the University of California by American Telephone and Telegraph
41 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
42 * the permission of UNIX System Laboratories, Inc.
43 *
44 * Redistribution and use in source and binary forms, with or without
45 * modification, are permitted provided that the following conditions
46 * are met:
47 * 1. Redistributions of source code must retain the above copyright
48 * notice, this list of conditions and the following disclaimer.
49 * 2. Redistributions in binary form must reproduce the above copyright
50 * notice, this list of conditions and the following disclaimer in the
51 * documentation and/or other materials provided with the distribution.
52 * 3. Neither the name of the University nor the names of its contributors
53 * may be used to endorse or promote products derived from this software
54 * without specific prior written permission.
55 *
56 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
57 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
58 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
59 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
60 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
61 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
62 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
63 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
64 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
65 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
66 * SUCH DAMAGE.
67 *
68 * @(#)kern_clock.c 8.5 (Berkeley) 1/21/94
69 */
70
71 #include <sys/cdefs.h>
72 __KERNEL_RCSID(0, "$NetBSD: kern_clock.c,v 1.150 2023/07/07 12:34:50 riastradh Exp $");
73
74 #ifdef _KERNEL_OPT
75 #include "opt_dtrace.h"
76 #include "opt_gprof.h"
77 #include "opt_heartbeat.h"
78 #include "opt_multiprocessor.h"
79 #endif
80
81 #include <sys/param.h>
82 #include <sys/systm.h>
83 #include <sys/callout.h>
84 #include <sys/kernel.h>
85 #include <sys/proc.h>
86 #include <sys/resourcevar.h>
87 #include <sys/signalvar.h>
88 #include <sys/sysctl.h>
89 #include <sys/timex.h>
90 #include <sys/sched.h>
91 #include <sys/time.h>
92 #include <sys/timetc.h>
93 #include <sys/cpu.h>
94 #include <sys/atomic.h>
95 #include <sys/rndsource.h>
96 #include <sys/heartbeat.h>
97
98 #ifdef GPROF
99 #include <sys/gmon.h>
100 #endif
101
102 #ifdef KDTRACE_HOOKS
103 #include <sys/dtrace_bsd.h>
104 #include <sys/cpu.h>
105
106 cyclic_clock_func_t cyclic_clock_func[MAXCPUS];
107 #endif
108
109 static int sysctl_kern_clockrate(SYSCTLFN_PROTO);
110
111 /*
112 * Clock handling routines.
113 *
114 * This code is written to operate with two timers that run independently of
115 * each other. The main clock, running hz times per second, is used to keep
116 * track of real time. The second timer handles kernel and user profiling,
117 * and does resource use estimation. If the second timer is programmable,
118 * it is randomized to avoid aliasing between the two clocks. For example,
119 * the randomization prevents an adversary from always giving up the CPU
120 * just before its quantum expires. Otherwise, it would never accumulate
121 * CPU ticks. The mean frequency of the second timer is stathz.
122 *
123 * If no second timer exists, stathz will be zero; in this case we drive
124 * profiling and statistics off the main clock. This WILL NOT be accurate;
125 * do not do it unless absolutely necessary.
126 *
127 * The statistics clock may (or may not) be run at a higher rate while
128 * profiling. This profile clock runs at profhz. We require that profhz
129 * be an integral multiple of stathz.
130 *
131 * If the statistics clock is running fast, it must be divided by the ratio
132 * profhz/stathz for statistics. (For profiling, every tick counts.)
133 */
134
135 int stathz;
136 int profhz;
137 int profsrc;
138 int schedhz;
139 int profprocs;
140 static int hardclock_ticks;
141 static int hardscheddiv; /* hard => sched divider (used if schedhz == 0) */
142 static int psdiv; /* prof => stat divider */
143 int psratio; /* ratio: prof / stat */
144
145 struct clockrnd {
146 struct krndsource source;
147 unsigned needed;
148 };
149
150 static struct clockrnd hardclockrnd __aligned(COHERENCY_UNIT);
151 static struct clockrnd statclockrnd __aligned(COHERENCY_UNIT);
152
153 static void
clockrnd_get(size_t needed,void * cookie)154 clockrnd_get(size_t needed, void *cookie)
155 {
156 struct clockrnd *C = cookie;
157
158 /* Start sampling. */
159 atomic_store_relaxed(&C->needed, 2*NBBY*needed);
160 }
161
162 static void
clockrnd_sample(struct clockrnd * C)163 clockrnd_sample(struct clockrnd *C)
164 {
165 struct cpu_info *ci = curcpu();
166
167 /* If there's nothing needed right now, stop here. */
168 if (__predict_true(atomic_load_relaxed(&C->needed) == 0))
169 return;
170
171 /*
172 * If we're not the primary core of a package, we're probably
173 * driven by the same clock as the primary core, so don't
174 * bother.
175 */
176 if (ci != ci->ci_package1st)
177 return;
178
179 /* Take a sample and enter it into the pool. */
180 rnd_add_uint32(&C->source, 0);
181
182 /*
183 * On the primary CPU, count down. Using an atomic decrement
184 * here isn't really necessary -- on every platform we care
185 * about, stores to unsigned int are atomic, and the only other
186 * memory operation that could happen here is for another CPU
187 * to store a higher value for needed. But using an atomic
188 * decrement avoids giving the impression of data races, and is
189 * unlikely to hurt because only one CPU will ever be writing
190 * to the location.
191 */
192 if (CPU_IS_PRIMARY(curcpu())) {
193 unsigned needed __diagused;
194
195 needed = atomic_dec_uint_nv(&C->needed);
196 KASSERT(needed != UINT_MAX);
197 }
198 }
199
200 static u_int get_intr_timecount(struct timecounter *);
201
202 static struct timecounter intr_timecounter = {
203 .tc_get_timecount = get_intr_timecount,
204 .tc_poll_pps = NULL,
205 .tc_counter_mask = ~0u,
206 .tc_frequency = 0,
207 .tc_name = "clockinterrupt",
208 /* quality - minimum implementation level for a clock */
209 .tc_quality = 0,
210 .tc_priv = NULL,
211 };
212
213 static u_int
get_intr_timecount(struct timecounter * tc)214 get_intr_timecount(struct timecounter *tc)
215 {
216
217 return (u_int)getticks();
218 }
219
220 int
getticks(void)221 getticks(void)
222 {
223 return atomic_load_relaxed(&hardclock_ticks);
224 }
225
226 /*
227 * Initialize clock frequencies and start both clocks running.
228 */
229 void
initclocks(void)230 initclocks(void)
231 {
232 static struct sysctllog *clog;
233 int i;
234
235 /*
236 * Set divisors to 1 (normal case) and let the machine-specific
237 * code do its bit.
238 */
239 psdiv = 1;
240
241 /*
242 * Call cpu_initclocks() before registering the default
243 * timecounter, in case it needs to adjust hz.
244 */
245 const int old_hz = hz;
246 cpu_initclocks();
247 if (old_hz != hz) {
248 tick = 1000000 / hz;
249 tickadj = (240000 / (60 * hz)) ? (240000 / (60 * hz)) : 1;
250 }
251
252 /*
253 * provide minimum default time counter
254 * will only run at interrupt resolution
255 */
256 intr_timecounter.tc_frequency = hz;
257 tc_init(&intr_timecounter);
258
259 /*
260 * Compute profhz and stathz, fix profhz if needed.
261 */
262 i = stathz ? stathz : hz;
263 if (profhz == 0)
264 profhz = i;
265 psratio = profhz / i;
266 if (schedhz == 0) {
267 /* 16Hz is best */
268 hardscheddiv = hz / 16;
269 if (hardscheddiv <= 0)
270 panic("hardscheddiv");
271 }
272
273 sysctl_createv(&clog, 0, NULL, NULL,
274 CTLFLAG_PERMANENT,
275 CTLTYPE_STRUCT, "clockrate",
276 SYSCTL_DESCR("Kernel clock rates"),
277 sysctl_kern_clockrate, 0, NULL,
278 sizeof(struct clockinfo),
279 CTL_KERN, KERN_CLOCKRATE, CTL_EOL);
280 sysctl_createv(&clog, 0, NULL, NULL,
281 CTLFLAG_PERMANENT,
282 CTLTYPE_INT, "hardclock_ticks",
283 SYSCTL_DESCR("Number of hardclock ticks"),
284 NULL, 0, &hardclock_ticks, sizeof(hardclock_ticks),
285 CTL_KERN, KERN_HARDCLOCK_TICKS, CTL_EOL);
286
287 rndsource_setcb(&hardclockrnd.source, clockrnd_get, &hardclockrnd);
288 rnd_attach_source(&hardclockrnd.source, "hardclock", RND_TYPE_SKEW,
289 RND_FLAG_COLLECT_TIME|RND_FLAG_ESTIMATE_TIME|RND_FLAG_HASCB);
290 if (stathz) {
291 rndsource_setcb(&statclockrnd.source, clockrnd_get,
292 &statclockrnd);
293 rnd_attach_source(&statclockrnd.source, "statclock",
294 RND_TYPE_SKEW,
295 (RND_FLAG_COLLECT_TIME|RND_FLAG_ESTIMATE_TIME|
296 RND_FLAG_HASCB));
297 }
298 }
299
300 /*
301 * The real-time timer, interrupting hz times per second.
302 */
303 void
hardclock(struct clockframe * frame)304 hardclock(struct clockframe *frame)
305 {
306 struct lwp *l;
307 struct cpu_info *ci;
308
309 clockrnd_sample(&hardclockrnd);
310
311 ci = curcpu();
312 l = ci->ci_onproc;
313
314 ptimer_tick(l, CLKF_USERMODE(frame));
315
316 /*
317 * If no separate statistics clock is available, run it from here.
318 */
319 if (stathz == 0)
320 statclock(frame);
321 /*
322 * If no separate schedclock is provided, call it here
323 * at about 16 Hz.
324 */
325 if (schedhz == 0) {
326 if ((int)(--ci->ci_schedstate.spc_schedticks) <= 0) {
327 schedclock(l);
328 ci->ci_schedstate.spc_schedticks = hardscheddiv;
329 }
330 }
331 if ((--ci->ci_schedstate.spc_ticks) <= 0)
332 sched_tick(ci);
333
334 if (CPU_IS_PRIMARY(ci)) {
335 atomic_store_relaxed(&hardclock_ticks,
336 atomic_load_relaxed(&hardclock_ticks) + 1);
337 tc_ticktock();
338 }
339
340 #ifdef HEARTBEAT
341 /*
342 * Make sure the CPUs and timecounter are making progress.
343 */
344 heartbeat();
345 #endif
346
347 /*
348 * Update real-time timeout queue.
349 */
350 callout_hardclock();
351 }
352
353 /*
354 * Start profiling on a process.
355 *
356 * Kernel profiling passes proc0 which never exits and hence
357 * keeps the profile clock running constantly.
358 */
359 void
startprofclock(struct proc * p)360 startprofclock(struct proc *p)
361 {
362
363 KASSERT(mutex_owned(&p->p_stmutex));
364
365 if ((p->p_stflag & PST_PROFIL) == 0) {
366 p->p_stflag |= PST_PROFIL;
367 /*
368 * This is only necessary if using the clock as the
369 * profiling source.
370 */
371 if (++profprocs == 1 && stathz != 0)
372 psdiv = psratio;
373 }
374 }
375
376 /*
377 * Stop profiling on a process.
378 */
379 void
stopprofclock(struct proc * p)380 stopprofclock(struct proc *p)
381 {
382
383 KASSERT(mutex_owned(&p->p_stmutex));
384
385 if (p->p_stflag & PST_PROFIL) {
386 p->p_stflag &= ~PST_PROFIL;
387 /*
388 * This is only necessary if using the clock as the
389 * profiling source.
390 */
391 if (--profprocs == 0 && stathz != 0)
392 psdiv = 1;
393 }
394 }
395
396 void
schedclock(struct lwp * l)397 schedclock(struct lwp *l)
398 {
399 if ((l->l_flag & LW_IDLE) != 0)
400 return;
401
402 sched_schedclock(l);
403 }
404
405 /*
406 * Statistics clock. Grab profile sample, and if divider reaches 0,
407 * do process and kernel statistics.
408 */
409 void
statclock(struct clockframe * frame)410 statclock(struct clockframe *frame)
411 {
412 #ifdef GPROF
413 struct gmonparam *g;
414 intptr_t i;
415 #endif
416 struct cpu_info *ci = curcpu();
417 struct schedstate_percpu *spc = &ci->ci_schedstate;
418 struct proc *p;
419 struct lwp *l;
420
421 if (stathz)
422 clockrnd_sample(&statclockrnd);
423
424 /*
425 * Notice changes in divisor frequency, and adjust clock
426 * frequency accordingly.
427 */
428 if (spc->spc_psdiv != psdiv) {
429 spc->spc_psdiv = psdiv;
430 spc->spc_pscnt = psdiv;
431 if (psdiv == 1) {
432 setstatclockrate(stathz);
433 } else {
434 setstatclockrate(profhz);
435 }
436 }
437 l = ci->ci_onproc;
438 if ((l->l_flag & LW_IDLE) != 0) {
439 /*
440 * don't account idle lwps as swapper.
441 */
442 p = NULL;
443 } else {
444 p = l->l_proc;
445 mutex_spin_enter(&p->p_stmutex);
446 }
447
448 if (CLKF_USERMODE(frame)) {
449 KASSERT(p != NULL);
450 if ((p->p_stflag & PST_PROFIL) && profsrc == PROFSRC_CLOCK)
451 addupc_intr(l, CLKF_PC(frame));
452 if (--spc->spc_pscnt > 0) {
453 mutex_spin_exit(&p->p_stmutex);
454 return;
455 }
456
457 /*
458 * Came from user mode; CPU was in user state.
459 * If this process is being profiled record the tick.
460 */
461 p->p_uticks++;
462 if (p->p_nice > NZERO)
463 spc->spc_cp_time[CP_NICE]++;
464 else
465 spc->spc_cp_time[CP_USER]++;
466 } else {
467 #ifdef GPROF
468 /*
469 * Kernel statistics are just like addupc_intr, only easier.
470 */
471 #if defined(MULTIPROCESSOR) && !defined(_RUMPKERNEL)
472 g = curcpu()->ci_gmon;
473 if (g != NULL &&
474 profsrc == PROFSRC_CLOCK && g->state == GMON_PROF_ON) {
475 #else
476 g = &_gmonparam;
477 if (profsrc == PROFSRC_CLOCK && g->state == GMON_PROF_ON) {
478 #endif
479 i = CLKF_PC(frame) - g->lowpc;
480 if (i < g->textsize) {
481 i /= HISTFRACTION * sizeof(*g->kcount);
482 g->kcount[i]++;
483 }
484 }
485 #endif
486 #ifdef LWP_PC
487 if (p != NULL && profsrc == PROFSRC_CLOCK &&
488 (p->p_stflag & PST_PROFIL)) {
489 addupc_intr(l, LWP_PC(l));
490 }
491 #endif
492 if (--spc->spc_pscnt > 0) {
493 if (p != NULL)
494 mutex_spin_exit(&p->p_stmutex);
495 return;
496 }
497 /*
498 * Came from kernel mode, so we were:
499 * - handling an interrupt,
500 * - doing syscall or trap work on behalf of the current
501 * user process, or
502 * - spinning in the idle loop.
503 * Whichever it is, charge the time as appropriate.
504 * Note that we charge interrupts to the current process,
505 * regardless of whether they are ``for'' that process,
506 * so that we know how much of its real time was spent
507 * in ``non-process'' (i.e., interrupt) work.
508 */
509 if (CLKF_INTR(frame) || (curlwp->l_pflag & LP_INTR) != 0) {
510 if (p != NULL) {
511 p->p_iticks++;
512 }
513 spc->spc_cp_time[CP_INTR]++;
514 } else if (p != NULL) {
515 p->p_sticks++;
516 spc->spc_cp_time[CP_SYS]++;
517 } else {
518 spc->spc_cp_time[CP_IDLE]++;
519 }
520 }
521 spc->spc_pscnt = psdiv;
522
523 if (p != NULL) {
524 atomic_inc_uint(&l->l_cpticks);
525 mutex_spin_exit(&p->p_stmutex);
526 }
527
528 #ifdef KDTRACE_HOOKS
529 cyclic_clock_func_t func = cyclic_clock_func[cpu_index(ci)];
530 if (func) {
531 (*func)((struct clockframe *)frame);
532 }
533 #endif
534 }
535
536 /*
537 * sysctl helper routine for kern.clockrate. Assembles a struct on
538 * the fly to be returned to the caller.
539 */
540 static int
541 sysctl_kern_clockrate(SYSCTLFN_ARGS)
542 {
543 struct clockinfo clkinfo;
544 struct sysctlnode node;
545
546 clkinfo.tick = tick;
547 clkinfo.tickadj = tickadj;
548 clkinfo.hz = hz;
549 clkinfo.profhz = profhz;
550 clkinfo.stathz = stathz ? stathz : hz;
551
552 node = *rnode;
553 node.sysctl_data = &clkinfo;
554 return (sysctl_lookup(SYSCTLFN_CALL(&node)));
555 }
556