17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
5b0fc0e77Sgovinda * Common Development and Distribution License (the "License").
6b0fc0e77Sgovinda * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
217c478bd9Sstevel@tonic-gate /*
22*113b131bSEric Saxe * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
237c478bd9Sstevel@tonic-gate * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate */
257c478bd9Sstevel@tonic-gate
267c478bd9Sstevel@tonic-gate #include <sys/param.h>
277c478bd9Sstevel@tonic-gate #include <sys/time.h>
287c478bd9Sstevel@tonic-gate #include <sys/systm.h>
297c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
307c478bd9Sstevel@tonic-gate #include <sys/debug.h>
317c478bd9Sstevel@tonic-gate #include <sys/clock.h>
327c478bd9Sstevel@tonic-gate #include <sys/x_call.h>
337c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
347c478bd9Sstevel@tonic-gate #include <sys/promif.h>
357c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
367c478bd9Sstevel@tonic-gate #include <sys/machsystm.h>
377c478bd9Sstevel@tonic-gate #include <sys/ivintr.h>
387c478bd9Sstevel@tonic-gate #include <sys/cyclic.h>
397c478bd9Sstevel@tonic-gate #include <sys/cyclic_impl.h>
407c478bd9Sstevel@tonic-gate
41b0fc0e77Sgovinda uint64_t cbe_level14_inum;
427c478bd9Sstevel@tonic-gate cyclic_id_t cbe_hres_cyclic;
437c478bd9Sstevel@tonic-gate
447c478bd9Sstevel@tonic-gate static hrtime_t cbe_hrtime_max;
457c478bd9Sstevel@tonic-gate static hrtime_t cbe_suspend_delta = 0;
467c478bd9Sstevel@tonic-gate static hrtime_t cbe_suspend_time = 0;
477c478bd9Sstevel@tonic-gate
487c478bd9Sstevel@tonic-gate static uint64_t
hrtime2tick(hrtime_t ts)497c478bd9Sstevel@tonic-gate hrtime2tick(hrtime_t ts)
507c478bd9Sstevel@tonic-gate {
517c478bd9Sstevel@tonic-gate hrtime_t q = ts / NANOSEC;
527c478bd9Sstevel@tonic-gate hrtime_t r = ts - (q * NANOSEC);
537c478bd9Sstevel@tonic-gate
547c478bd9Sstevel@tonic-gate return (q * sys_tick_freq + ((r * sys_tick_freq) / NANOSEC));
557c478bd9Sstevel@tonic-gate }
567c478bd9Sstevel@tonic-gate
57*113b131bSEric Saxe uint64_t
unscalehrtime(hrtime_t ts)587c478bd9Sstevel@tonic-gate unscalehrtime(hrtime_t ts)
597c478bd9Sstevel@tonic-gate {
607c478bd9Sstevel@tonic-gate uint64_t unscale = 0;
617c478bd9Sstevel@tonic-gate hrtime_t rescale;
627c478bd9Sstevel@tonic-gate hrtime_t diff = ts;
637c478bd9Sstevel@tonic-gate
647c478bd9Sstevel@tonic-gate while (diff > nsec_per_sys_tick) {
657c478bd9Sstevel@tonic-gate unscale += hrtime2tick(diff);
667c478bd9Sstevel@tonic-gate rescale = unscale;
677c478bd9Sstevel@tonic-gate scalehrtime(&rescale);
687c478bd9Sstevel@tonic-gate diff = ts - rescale;
697c478bd9Sstevel@tonic-gate }
707c478bd9Sstevel@tonic-gate
717c478bd9Sstevel@tonic-gate return (unscale);
727c478bd9Sstevel@tonic-gate }
737c478bd9Sstevel@tonic-gate
747c478bd9Sstevel@tonic-gate static int
cbe_level1()757c478bd9Sstevel@tonic-gate cbe_level1()
767c478bd9Sstevel@tonic-gate {
777c478bd9Sstevel@tonic-gate cyclic_softint(CPU, CY_LOW_LEVEL);
787c478bd9Sstevel@tonic-gate return (1);
797c478bd9Sstevel@tonic-gate }
807c478bd9Sstevel@tonic-gate
817c478bd9Sstevel@tonic-gate static int
cbe_level10()827c478bd9Sstevel@tonic-gate cbe_level10()
837c478bd9Sstevel@tonic-gate {
847c478bd9Sstevel@tonic-gate cyclic_softint(CPU, CY_LOCK_LEVEL);
857c478bd9Sstevel@tonic-gate return (1);
867c478bd9Sstevel@tonic-gate }
877c478bd9Sstevel@tonic-gate
887c478bd9Sstevel@tonic-gate /*ARGSUSED*/
897c478bd9Sstevel@tonic-gate static void
cbe_enable(cyb_arg_t arg)907c478bd9Sstevel@tonic-gate cbe_enable(cyb_arg_t arg)
917c478bd9Sstevel@tonic-gate {
927c478bd9Sstevel@tonic-gate int pstate_save = disable_vec_intr();
937c478bd9Sstevel@tonic-gate
947c478bd9Sstevel@tonic-gate intr_enqueue_req(PIL_14, cbe_level14_inum);
957c478bd9Sstevel@tonic-gate enable_vec_intr(pstate_save);
967c478bd9Sstevel@tonic-gate }
977c478bd9Sstevel@tonic-gate
987c478bd9Sstevel@tonic-gate /*ARGSUSED*/
997c478bd9Sstevel@tonic-gate static void
cbe_disable(cyb_arg_t arg)1007c478bd9Sstevel@tonic-gate cbe_disable(cyb_arg_t arg)
1017c478bd9Sstevel@tonic-gate {
1027c478bd9Sstevel@tonic-gate int pstate_save = disable_vec_intr();
1037c478bd9Sstevel@tonic-gate
1047c478bd9Sstevel@tonic-gate tickcmpr_disable();
1057c478bd9Sstevel@tonic-gate intr_dequeue_req(PIL_14, cbe_level14_inum);
1067c478bd9Sstevel@tonic-gate enable_vec_intr(pstate_save);
1077c478bd9Sstevel@tonic-gate }
1087c478bd9Sstevel@tonic-gate
1097c478bd9Sstevel@tonic-gate /*ARGSUSED*/
1107c478bd9Sstevel@tonic-gate static void
cbe_reprogram(cyb_arg_t arg,hrtime_t time)1117c478bd9Sstevel@tonic-gate cbe_reprogram(cyb_arg_t arg, hrtime_t time)
1127c478bd9Sstevel@tonic-gate {
1137c478bd9Sstevel@tonic-gate if (time >= cbe_hrtime_max)
1147c478bd9Sstevel@tonic-gate time = cbe_hrtime_max;
1157c478bd9Sstevel@tonic-gate
1167c478bd9Sstevel@tonic-gate tickcmpr_set(unscalehrtime(time));
1177c478bd9Sstevel@tonic-gate }
1187c478bd9Sstevel@tonic-gate
1197c478bd9Sstevel@tonic-gate static void
cbe_softint(cyb_arg_t arg,cyc_level_t level)1207c478bd9Sstevel@tonic-gate cbe_softint(cyb_arg_t arg, cyc_level_t level)
1217c478bd9Sstevel@tonic-gate {
1227c478bd9Sstevel@tonic-gate cbe_data_t *data = (cbe_data_t *)arg;
1237c478bd9Sstevel@tonic-gate
1247c478bd9Sstevel@tonic-gate switch (level) {
1257c478bd9Sstevel@tonic-gate case CY_LOW_LEVEL:
1267c478bd9Sstevel@tonic-gate setsoftint(data->cbe_level1_inum);
1277c478bd9Sstevel@tonic-gate break;
1287c478bd9Sstevel@tonic-gate case CY_LOCK_LEVEL:
1297c478bd9Sstevel@tonic-gate setsoftint(data->cbe_level10_inum);
1307c478bd9Sstevel@tonic-gate break;
1317c478bd9Sstevel@tonic-gate default:
1327c478bd9Sstevel@tonic-gate panic("cbe_softint: unexpected soft level %d", level);
1337c478bd9Sstevel@tonic-gate }
1347c478bd9Sstevel@tonic-gate }
1357c478bd9Sstevel@tonic-gate
1367c478bd9Sstevel@tonic-gate /*ARGSUSED*/
1377c478bd9Sstevel@tonic-gate static cyc_cookie_t
cbe_set_level(cyb_arg_t arg,cyc_level_t level)1387c478bd9Sstevel@tonic-gate cbe_set_level(cyb_arg_t arg, cyc_level_t level)
1397c478bd9Sstevel@tonic-gate {
1407c478bd9Sstevel@tonic-gate int ipl;
1417c478bd9Sstevel@tonic-gate
1427c478bd9Sstevel@tonic-gate switch (level) {
1437c478bd9Sstevel@tonic-gate case CY_LOW_LEVEL:
1447c478bd9Sstevel@tonic-gate ipl = CBE_LOW_PIL;
1457c478bd9Sstevel@tonic-gate break;
1467c478bd9Sstevel@tonic-gate case CY_LOCK_LEVEL:
1477c478bd9Sstevel@tonic-gate ipl = CBE_LOCK_PIL;
1487c478bd9Sstevel@tonic-gate break;
1497c478bd9Sstevel@tonic-gate case CY_HIGH_LEVEL:
1507c478bd9Sstevel@tonic-gate ipl = CBE_HIGH_PIL;
1517c478bd9Sstevel@tonic-gate break;
1527c478bd9Sstevel@tonic-gate default:
1537c478bd9Sstevel@tonic-gate panic("cbe_set_level: unexpected level %d", level);
1547c478bd9Sstevel@tonic-gate }
1557c478bd9Sstevel@tonic-gate
1567c478bd9Sstevel@tonic-gate return (splr(ipl));
1577c478bd9Sstevel@tonic-gate }
1587c478bd9Sstevel@tonic-gate
1597c478bd9Sstevel@tonic-gate /*ARGSUSED*/
1607c478bd9Sstevel@tonic-gate static void
cbe_restore_level(cyb_arg_t arg,cyc_cookie_t cookie)1617c478bd9Sstevel@tonic-gate cbe_restore_level(cyb_arg_t arg, cyc_cookie_t cookie)
1627c478bd9Sstevel@tonic-gate {
1637c478bd9Sstevel@tonic-gate splx(cookie);
1647c478bd9Sstevel@tonic-gate }
1657c478bd9Sstevel@tonic-gate
1667c478bd9Sstevel@tonic-gate static void
cbe_xcall_handler(uint64_t arg1,uint64_t arg2)1677c478bd9Sstevel@tonic-gate cbe_xcall_handler(uint64_t arg1, uint64_t arg2)
1687c478bd9Sstevel@tonic-gate {
1697c478bd9Sstevel@tonic-gate cyc_func_t func = (cyc_func_t)arg1;
1707c478bd9Sstevel@tonic-gate void *arg = (void *)arg2;
1717c478bd9Sstevel@tonic-gate
1727c478bd9Sstevel@tonic-gate (*func)(arg);
1737c478bd9Sstevel@tonic-gate }
1747c478bd9Sstevel@tonic-gate
1757c478bd9Sstevel@tonic-gate /*ARGSUSED*/
1767c478bd9Sstevel@tonic-gate static void
cbe_xcall(cyb_arg_t arg,cpu_t * dest,cyc_func_t func,void * farg)1777c478bd9Sstevel@tonic-gate cbe_xcall(cyb_arg_t arg, cpu_t *dest, cyc_func_t func, void *farg)
1787c478bd9Sstevel@tonic-gate {
1797c478bd9Sstevel@tonic-gate kpreempt_disable();
1807c478bd9Sstevel@tonic-gate xc_one(dest->cpu_id, cbe_xcall_handler, (uint64_t)func, (uint64_t)farg);
1817c478bd9Sstevel@tonic-gate kpreempt_enable();
1827c478bd9Sstevel@tonic-gate }
1837c478bd9Sstevel@tonic-gate
1847c478bd9Sstevel@tonic-gate /*ARGSUSED*/
1857c478bd9Sstevel@tonic-gate static cyb_arg_t
cbe_configure(cpu_t * cpu)1867c478bd9Sstevel@tonic-gate cbe_configure(cpu_t *cpu)
1877c478bd9Sstevel@tonic-gate {
1887c478bd9Sstevel@tonic-gate cbe_data_t *new_data = kmem_alloc(sizeof (cbe_data_t), KM_SLEEP);
1897c478bd9Sstevel@tonic-gate
1907c478bd9Sstevel@tonic-gate /*
1917c478bd9Sstevel@tonic-gate * The setsoftint() code will refuse to post a soft interrupt if
1927c478bd9Sstevel@tonic-gate * one is already pending for the specified inum. Given that we
1937c478bd9Sstevel@tonic-gate * may have disjoint soft interrupts on different CPUs posted
1947c478bd9Sstevel@tonic-gate * simultaneously, we allocate a new set of inums for each CPU.
1957c478bd9Sstevel@tonic-gate */
196b0fc0e77Sgovinda new_data->cbe_level10_inum = add_softintr(PIL_10,
197b0fc0e77Sgovinda (softintrfunc)cbe_level10, 0, SOFTINT_ST);
1987c478bd9Sstevel@tonic-gate
199b0fc0e77Sgovinda new_data->cbe_level1_inum = add_softintr(PIL_1,
200b0fc0e77Sgovinda (softintrfunc)cbe_level1, 0, SOFTINT_ST);
2017c478bd9Sstevel@tonic-gate
2027c478bd9Sstevel@tonic-gate return (new_data);
2037c478bd9Sstevel@tonic-gate }
2047c478bd9Sstevel@tonic-gate
2057c478bd9Sstevel@tonic-gate static void
cbe_unconfigure(cyb_arg_t arg)2067c478bd9Sstevel@tonic-gate cbe_unconfigure(cyb_arg_t arg)
2077c478bd9Sstevel@tonic-gate {
2087c478bd9Sstevel@tonic-gate cbe_data_t *data = (cbe_data_t *)arg;
2097c478bd9Sstevel@tonic-gate
210b0fc0e77Sgovinda (void) rem_softintr(data->cbe_level10_inum);
211b0fc0e77Sgovinda (void) rem_softintr(data->cbe_level1_inum);
2127c478bd9Sstevel@tonic-gate
2137c478bd9Sstevel@tonic-gate kmem_free(data, sizeof (cbe_data_t));
2147c478bd9Sstevel@tonic-gate }
2157c478bd9Sstevel@tonic-gate
2167c478bd9Sstevel@tonic-gate /*ARGSUSED*/
2177c478bd9Sstevel@tonic-gate static void
cbe_suspend(cyb_arg_t arg)2187c478bd9Sstevel@tonic-gate cbe_suspend(cyb_arg_t arg)
2197c478bd9Sstevel@tonic-gate {
2207c478bd9Sstevel@tonic-gate cbe_suspend_time = gethrtime_unscaled();
2217c478bd9Sstevel@tonic-gate cbe_suspend_delta = 0;
2227c478bd9Sstevel@tonic-gate }
2237c478bd9Sstevel@tonic-gate
2247c478bd9Sstevel@tonic-gate /*ARGSUSED*/
2257c478bd9Sstevel@tonic-gate static void
cbe_resume(cyb_arg_t arg)2267c478bd9Sstevel@tonic-gate cbe_resume(cyb_arg_t arg)
2277c478bd9Sstevel@tonic-gate {
2287c478bd9Sstevel@tonic-gate hrtime_t now;
2297c478bd9Sstevel@tonic-gate
2307c478bd9Sstevel@tonic-gate /*
2317c478bd9Sstevel@tonic-gate * If we're actually on a CPU which has apparently had %tick zeroed,
2327c478bd9Sstevel@tonic-gate * we want to add cbe_suspend_delta to %tick.
2337c478bd9Sstevel@tonic-gate */
2347c478bd9Sstevel@tonic-gate if ((now = gethrtime_unscaled()) < cbe_suspend_time) {
2357c478bd9Sstevel@tonic-gate
2367c478bd9Sstevel@tonic-gate if (cbe_suspend_delta == 0) {
2377c478bd9Sstevel@tonic-gate /*
2387c478bd9Sstevel@tonic-gate * We're the first CPU to be resumed. We want %tick
2397c478bd9Sstevel@tonic-gate * to be close to %tick when we suspended the system,
2407c478bd9Sstevel@tonic-gate * so we'll figure out the delta which needs to be
2417c478bd9Sstevel@tonic-gate * written to the register. All subsequent resumed
2427c478bd9Sstevel@tonic-gate * CPUs will write the same delta.
2437c478bd9Sstevel@tonic-gate */
2447c478bd9Sstevel@tonic-gate cbe_suspend_delta = cbe_suspend_time - now;
2457c478bd9Sstevel@tonic-gate }
2467c478bd9Sstevel@tonic-gate
2477c478bd9Sstevel@tonic-gate tick_write_delta(cbe_suspend_delta);
2487c478bd9Sstevel@tonic-gate }
2497c478bd9Sstevel@tonic-gate }
2507c478bd9Sstevel@tonic-gate
2517c478bd9Sstevel@tonic-gate void
cbe_hres_tick(void)2527c478bd9Sstevel@tonic-gate cbe_hres_tick(void)
2537c478bd9Sstevel@tonic-gate {
2547c478bd9Sstevel@tonic-gate dtrace_hres_tick();
2557c478bd9Sstevel@tonic-gate hres_tick();
2567c478bd9Sstevel@tonic-gate }
2577c478bd9Sstevel@tonic-gate
2587c478bd9Sstevel@tonic-gate void
cbe_init_pre(void)259b9499e44Smrj cbe_init_pre(void)
260b9499e44Smrj {
261b9499e44Smrj /* Nothing to do on sparc */
262b9499e44Smrj }
263b9499e44Smrj
264b9499e44Smrj void
cbe_init(void)2657c478bd9Sstevel@tonic-gate cbe_init(void)
2667c478bd9Sstevel@tonic-gate {
2677c478bd9Sstevel@tonic-gate cyc_handler_t hdlr;
2687c478bd9Sstevel@tonic-gate cyc_time_t when;
2697c478bd9Sstevel@tonic-gate hrtime_t resolution = NANOSEC / sys_tick_freq;
2707c478bd9Sstevel@tonic-gate
2717c478bd9Sstevel@tonic-gate cyc_backend_t cbe = {
2727c478bd9Sstevel@tonic-gate cbe_configure, /* cyb_configure */
2737c478bd9Sstevel@tonic-gate cbe_unconfigure, /* cyb_unconfigure */
2747c478bd9Sstevel@tonic-gate cbe_enable, /* cyb_enable */
2757c478bd9Sstevel@tonic-gate cbe_disable, /* cyb_disable */
2767c478bd9Sstevel@tonic-gate cbe_reprogram, /* cyb_reprogram */
2777c478bd9Sstevel@tonic-gate cbe_softint, /* cyb_softint */
2787c478bd9Sstevel@tonic-gate cbe_set_level, /* cyb_set_level */
2797c478bd9Sstevel@tonic-gate cbe_restore_level, /* cyb_restore_level */
2807c478bd9Sstevel@tonic-gate cbe_xcall, /* cyb_xcall */
2817c478bd9Sstevel@tonic-gate cbe_suspend, /* cyb_suspend */
2827c478bd9Sstevel@tonic-gate cbe_resume /* cyb_resume */
2837c478bd9Sstevel@tonic-gate };
2847c478bd9Sstevel@tonic-gate
285b0fc0e77Sgovinda cbe_level14_inum = add_softintr(CBE_HIGH_PIL,
286b0fc0e77Sgovinda (softintrfunc)cbe_level14, 0, SOFTINT_MT);
2877c478bd9Sstevel@tonic-gate cbe_hrtime_max = gethrtime_max();
2887c478bd9Sstevel@tonic-gate
2897c478bd9Sstevel@tonic-gate /*
2907c478bd9Sstevel@tonic-gate * If sys_tick_freq > NANOSEC (i.e. we're on a CPU with a clock rate
2917c478bd9Sstevel@tonic-gate * which exceeds 1 GHz), we'll specify the minimum resolution,
2927c478bd9Sstevel@tonic-gate * 1 nanosecond.
2937c478bd9Sstevel@tonic-gate */
2947c478bd9Sstevel@tonic-gate if (resolution == 0)
2957c478bd9Sstevel@tonic-gate resolution = 1;
2967c478bd9Sstevel@tonic-gate
2977c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock);
2987c478bd9Sstevel@tonic-gate cyclic_init(&cbe, resolution);
2997c478bd9Sstevel@tonic-gate
3007c478bd9Sstevel@tonic-gate /*
3017c478bd9Sstevel@tonic-gate * Initialize hrtime_base and hres_last_tick to reasonable starting
3027c478bd9Sstevel@tonic-gate * values.
3037c478bd9Sstevel@tonic-gate */
3047c478bd9Sstevel@tonic-gate hrtime_base = gethrtime();
3057c478bd9Sstevel@tonic-gate hres_last_tick = gethrtime_unscaled();
3067c478bd9Sstevel@tonic-gate
3077c478bd9Sstevel@tonic-gate hdlr.cyh_level = CY_HIGH_LEVEL;
3087c478bd9Sstevel@tonic-gate hdlr.cyh_func = (cyc_func_t)cbe_hres_tick;
3097c478bd9Sstevel@tonic-gate hdlr.cyh_arg = NULL;
3107c478bd9Sstevel@tonic-gate
3117c478bd9Sstevel@tonic-gate when.cyt_when = 0;
3127c478bd9Sstevel@tonic-gate when.cyt_interval = nsec_per_tick;
3137c478bd9Sstevel@tonic-gate
3147c478bd9Sstevel@tonic-gate cbe_hres_cyclic = cyclic_add(&hdlr, &when);
3157c478bd9Sstevel@tonic-gate
3167c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock);
3177c478bd9Sstevel@tonic-gate
3187c478bd9Sstevel@tonic-gate clkstart();
3197c478bd9Sstevel@tonic-gate }
320