1 /* $NetBSD: kern_cctr.c,v 1.9 2009/01/03 03:31:23 yamt Exp $ */
2
3 /*-
4 * Copyright (c) 2006, 2008 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * re-implementation of TSC for MP systems merging cc_microtime and
8 * TSC for timecounters by Frank Kardel
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /* basic calibration ideas are (kern_microtime.c): */
33 /******************************************************************************
34 * *
35 * Copyright (c) David L. Mills 1993, 1994 *
36 * *
37 * Permission to use, copy, modify, and distribute this software and its *
38 * documentation for any purpose and without fee is hereby granted, provided *
39 * that the above copyright notice appears in all copies and that both the *
40 * copyright notice and this permission notice appear in supporting *
41 * documentation, and that the name University of Delaware not be used in *
42 * advertising or publicity pertaining to distribution of the software *
43 * without specific, written prior permission. The University of Delaware *
44 * makes no representations about the suitability this software for any *
45 * purpose. It is provided "as is" without express or implied warranty. *
46 * *
47 ******************************************************************************/
48
49 /* reminiscents from older version of this file are: */
50 /*-
51 * Copyright (c) 1998-2003 Poul-Henning Kamp
52 * All rights reserved.
53 *
54 * Redistribution and use in source and binary forms, with or without
55 * modification, are permitted provided that the following conditions
56 * are met:
57 * 1. Redistributions of source code must retain the above copyright
58 * notice, this list of conditions and the following disclaimer.
59 * 2. Redistributions in binary form must reproduce the above copyright
60 * notice, this list of conditions and the following disclaimer in the
61 * documentation and/or other materials provided with the distribution.
62 *
63 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
64 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
65 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
66 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
67 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
68 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
69 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
70 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
71 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
72 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
73 * SUCH DAMAGE.
74 */
75
76 #include <sys/cdefs.h>
77 /* __FBSDID("$FreeBSD: src/sys/i386/i386/tsc.c,v 1.204 2003/10/21 18:28:34 silby Exp $"); */
78 __KERNEL_RCSID(0, "$NetBSD: kern_cctr.c,v 1.9 2009/01/03 03:31:23 yamt Exp $");
79
80 #include <sys/param.h>
81 #include <sys/systm.h>
82 #include <sys/sysctl.h>
83 #include <sys/time.h>
84 #include <sys/timetc.h>
85 #include <sys/kernel.h>
86 #include <sys/power.h>
87 #include <sys/cpu.h>
88 #include <machine/cpu_counter.h>
89
90 /* XXX make cc_timecounter.tc_frequency settable by sysctl() */
91
92 static timecounter_pps_t cc_calibrate;
93
94 void cc_calibrate_cpu(struct cpu_info *);
95
96 static int64_t cc_cal_val; /* last calibrate time stamp */
97
98 static struct timecounter cc_timecounter = {
99 .tc_get_timecount = cc_get_timecount,
100 .tc_poll_pps = cc_calibrate,
101 .tc_counter_mask = ~0u,
102 .tc_frequency = 0,
103 .tc_name = "unkown cycle counter",
104 /*
105 * don't pick cycle counter automatically
106 * if frequency changes might affect cycle counter
107 */
108 .tc_quality = -100000,
109
110 .tc_priv = NULL,
111 .tc_next = NULL
112 };
113
114 /*
115 * initialize cycle counter based timecounter
116 */
117 struct timecounter *
cc_init(timecounter_get_t getcc,uint64_t freq,const char * name,int quality)118 cc_init(timecounter_get_t getcc, uint64_t freq, const char *name, int quality)
119 {
120
121 if (getcc != NULL)
122 cc_timecounter.tc_get_timecount = getcc;
123
124 cc_timecounter.tc_frequency = freq;
125 cc_timecounter.tc_name = name;
126 cc_timecounter.tc_quality = quality;
127 tc_init(&cc_timecounter);
128
129 return &cc_timecounter;
130 }
131
132 /*
133 * pick up tick count scaled to reference tick count
134 */
135 u_int
cc_get_timecount(struct timecounter * tc)136 cc_get_timecount(struct timecounter *tc)
137 {
138 struct cpu_info *ci;
139 int64_t rcc, cc, ncsw;
140 u_int gen;
141
142 retry:
143 ncsw = curlwp->l_ncsw;
144 __insn_barrier();
145 ci = curcpu();
146 if (ci->ci_cc.cc_denom == 0) {
147 /*
148 * This is our first time here on this CPU. Just
149 * start with reasonable initial values.
150 */
151 ci->ci_cc.cc_cc = cpu_counter32();
152 ci->ci_cc.cc_val = 0;
153 if (ci->ci_cc.cc_gen == 0)
154 ci->ci_cc.cc_gen++;
155
156 ci->ci_cc.cc_denom = cpu_frequency(ci);
157 if (ci->ci_cc.cc_denom == 0)
158 ci->ci_cc.cc_denom = cc_timecounter.tc_frequency;
159 ci->ci_cc.cc_delta = ci->ci_cc.cc_denom;
160 }
161
162 /*
163 * read counter and re-read when the re-calibration
164 * strikes inbetween
165 */
166 do {
167 /* pick up current generation number */
168 gen = ci->ci_cc.cc_gen;
169
170 /* determine local delta ticks */
171 cc = cpu_counter32() - ci->ci_cc.cc_cc;
172 if (cc < 0)
173 cc += 0x100000000LL;
174
175 /* scale to primary */
176 rcc = (cc * ci->ci_cc.cc_delta) / ci->ci_cc.cc_denom
177 + ci->ci_cc.cc_val;
178 } while (gen == 0 || gen != ci->ci_cc.cc_gen);
179 __insn_barrier();
180 if (ncsw != curlwp->l_ncsw) {
181 /* Was preempted */
182 goto retry;
183 }
184
185 return rcc;
186 }
187
188 /*
189 * called once per clock tick via the pps callback
190 * for the calibration of the TSC counters.
191 * it is called only for the PRIMARY cpu. all
192 * other cpus are called via a broadcast IPI
193 * calibration interval is 1 second - we call
194 * the calibration code only every hz calls
195 */
196 static void
cc_calibrate(struct timecounter * tc)197 cc_calibrate(struct timecounter *tc)
198 {
199 static int calls;
200 struct cpu_info *ci;
201
202 KASSERT(kpreempt_disabled());
203
204 /*
205 * XXX: for high interrupt frequency
206 * support: ++calls < hz / tc_tick
207 */
208 if (++calls < hz)
209 return;
210
211 calls = 0;
212 ci = curcpu();
213 /* pick up reference ticks */
214 cc_cal_val = cpu_counter32();
215
216 #if defined(MULTIPROCESSOR)
217 cc_calibrate_mp(ci);
218 #endif
219 cc_calibrate_cpu(ci);
220 }
221
222 /*
223 * This routine is called about once per second directly by the master
224 * processor and via an interprocessor interrupt for other processors.
225 * It determines the CC frequency of each processor relative to the
226 * master clock and the time this determination is made. These values
227 * are used by cc_get_timecount() to interpolate the ticks between
228 * timer interrupts. Note that we assume the kernel variables have
229 * been zeroed early in life.
230 */
231 void
cc_calibrate_cpu(struct cpu_info * ci)232 cc_calibrate_cpu(struct cpu_info *ci)
233 {
234 u_int gen;
235 int64_t val;
236 int64_t delta, denom;
237 int s;
238 #ifdef TIMECOUNTER_DEBUG
239 int64_t factor, old_factor;
240 #endif
241 val = cc_cal_val;
242
243 s = splhigh();
244 /* create next generation number */
245 gen = ci->ci_cc.cc_gen;
246 gen++;
247 if (gen == 0)
248 gen++;
249
250 /* update in progress */
251 ci->ci_cc.cc_gen = 0;
252
253 denom = ci->ci_cc.cc_cc;
254 ci->ci_cc.cc_cc = cpu_counter32();
255
256 if (ci->ci_cc.cc_denom == 0) {
257 /*
258 * This is our first time here on this CPU. Just
259 * start with reasonable initial values.
260 */
261 ci->ci_cc.cc_val = val;
262 ci->ci_cc.cc_denom = cpu_frequency(ci);
263 if (ci->ci_cc.cc_denom == 0)
264 ci->ci_cc.cc_denom = cc_timecounter.tc_frequency;
265 ci->ci_cc.cc_delta = ci->ci_cc.cc_denom;
266 ci->ci_cc.cc_gen = gen;
267 splx(s);
268 return;
269 }
270
271 #ifdef TIMECOUNTER_DEBUG
272 old_factor = (ci->ci_cc.cc_delta * 1000 ) / ci->ci_cc.cc_denom;
273 #endif
274
275 /* local ticks per period */
276 denom = ci->ci_cc.cc_cc - denom;
277 if (denom < 0)
278 denom += 0x100000000LL;
279
280 ci->ci_cc.cc_denom = denom;
281
282 /* reference ticks per period */
283 delta = val - ci->ci_cc.cc_val;
284 if (delta < 0)
285 delta += 0x100000000LL;
286
287 ci->ci_cc.cc_val = val;
288 ci->ci_cc.cc_delta = delta;
289
290 /* publish new generation number */
291 ci->ci_cc.cc_gen = gen;
292 splx(s);
293
294 #ifdef TIMECOUNTER_DEBUG
295 factor = (delta * 1000) / denom - old_factor;
296 if (factor < 0)
297 factor = -factor;
298
299 if (factor > old_factor / 10)
300 printf("cc_calibrate_cpu[%u]: 10%% exceeded - delta %"
301 PRId64 ", denom %" PRId64 ", factor %" PRId64
302 ", old factor %" PRId64"\n", ci->ci_index,
303 delta, denom, (delta * 1000) / denom, old_factor);
304 #endif /* TIMECOUNTER_DEBUG */
305 }
306