17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*b0fc0e77Sgovinda * Common Development and Distribution License (the "License"). 6*b0fc0e77Sgovinda * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22*b0fc0e77Sgovinda * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 277c478bd9Sstevel@tonic-gate 287c478bd9Sstevel@tonic-gate #include <sys/param.h> 297c478bd9Sstevel@tonic-gate #include <sys/time.h> 307c478bd9Sstevel@tonic-gate #include <sys/systm.h> 317c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 327c478bd9Sstevel@tonic-gate #include <sys/debug.h> 337c478bd9Sstevel@tonic-gate #include <sys/clock.h> 347c478bd9Sstevel@tonic-gate #include <sys/x_call.h> 357c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 367c478bd9Sstevel@tonic-gate #include <sys/promif.h> 377c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 387c478bd9Sstevel@tonic-gate #include <sys/machsystm.h> 397c478bd9Sstevel@tonic-gate #include <sys/ivintr.h> 407c478bd9Sstevel@tonic-gate #include <sys/cyclic.h> 417c478bd9Sstevel@tonic-gate #include <sys/cyclic_impl.h> 427c478bd9Sstevel@tonic-gate 43*b0fc0e77Sgovinda uint64_t cbe_level14_inum; 447c478bd9Sstevel@tonic-gate cyclic_id_t cbe_hres_cyclic; 457c478bd9Sstevel@tonic-gate 467c478bd9Sstevel@tonic-gate static hrtime_t cbe_hrtime_max; 477c478bd9Sstevel@tonic-gate static hrtime_t cbe_suspend_delta = 0; 487c478bd9Sstevel@tonic-gate static hrtime_t cbe_suspend_time = 0; 497c478bd9Sstevel@tonic-gate 507c478bd9Sstevel@tonic-gate static uint64_t 517c478bd9Sstevel@tonic-gate hrtime2tick(hrtime_t ts) 527c478bd9Sstevel@tonic-gate { 537c478bd9Sstevel@tonic-gate hrtime_t q = ts / NANOSEC; 547c478bd9Sstevel@tonic-gate hrtime_t r = ts - (q * NANOSEC); 557c478bd9Sstevel@tonic-gate 567c478bd9Sstevel@tonic-gate return (q * sys_tick_freq + ((r * sys_tick_freq) / NANOSEC)); 577c478bd9Sstevel@tonic-gate } 587c478bd9Sstevel@tonic-gate 597c478bd9Sstevel@tonic-gate uint64_t 607c478bd9Sstevel@tonic-gate unscalehrtime(hrtime_t ts) 617c478bd9Sstevel@tonic-gate { 627c478bd9Sstevel@tonic-gate uint64_t unscale = 0; 637c478bd9Sstevel@tonic-gate hrtime_t rescale; 647c478bd9Sstevel@tonic-gate hrtime_t diff = ts; 657c478bd9Sstevel@tonic-gate 667c478bd9Sstevel@tonic-gate while (diff > nsec_per_sys_tick) { 677c478bd9Sstevel@tonic-gate unscale += hrtime2tick(diff); 687c478bd9Sstevel@tonic-gate rescale = unscale; 697c478bd9Sstevel@tonic-gate scalehrtime(&rescale); 707c478bd9Sstevel@tonic-gate diff = ts - rescale; 717c478bd9Sstevel@tonic-gate } 727c478bd9Sstevel@tonic-gate 737c478bd9Sstevel@tonic-gate return (unscale); 747c478bd9Sstevel@tonic-gate } 757c478bd9Sstevel@tonic-gate 767c478bd9Sstevel@tonic-gate static int 777c478bd9Sstevel@tonic-gate cbe_level1() 787c478bd9Sstevel@tonic-gate { 797c478bd9Sstevel@tonic-gate cyclic_softint(CPU, CY_LOW_LEVEL); 807c478bd9Sstevel@tonic-gate return (1); 817c478bd9Sstevel@tonic-gate } 827c478bd9Sstevel@tonic-gate 837c478bd9Sstevel@tonic-gate static int 847c478bd9Sstevel@tonic-gate cbe_level10() 857c478bd9Sstevel@tonic-gate { 867c478bd9Sstevel@tonic-gate cyclic_softint(CPU, CY_LOCK_LEVEL); 877c478bd9Sstevel@tonic-gate return (1); 887c478bd9Sstevel@tonic-gate } 897c478bd9Sstevel@tonic-gate 907c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 917c478bd9Sstevel@tonic-gate static void 927c478bd9Sstevel@tonic-gate cbe_enable(cyb_arg_t arg) 937c478bd9Sstevel@tonic-gate { 947c478bd9Sstevel@tonic-gate int pstate_save = disable_vec_intr(); 957c478bd9Sstevel@tonic-gate 967c478bd9Sstevel@tonic-gate intr_enqueue_req(PIL_14, cbe_level14_inum); 977c478bd9Sstevel@tonic-gate enable_vec_intr(pstate_save); 987c478bd9Sstevel@tonic-gate } 997c478bd9Sstevel@tonic-gate 1007c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 1017c478bd9Sstevel@tonic-gate static void 1027c478bd9Sstevel@tonic-gate cbe_disable(cyb_arg_t arg) 1037c478bd9Sstevel@tonic-gate { 1047c478bd9Sstevel@tonic-gate int pstate_save = disable_vec_intr(); 1057c478bd9Sstevel@tonic-gate 1067c478bd9Sstevel@tonic-gate tickcmpr_disable(); 1077c478bd9Sstevel@tonic-gate intr_dequeue_req(PIL_14, cbe_level14_inum); 1087c478bd9Sstevel@tonic-gate enable_vec_intr(pstate_save); 1097c478bd9Sstevel@tonic-gate } 1107c478bd9Sstevel@tonic-gate 1117c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 1127c478bd9Sstevel@tonic-gate static void 1137c478bd9Sstevel@tonic-gate cbe_reprogram(cyb_arg_t arg, hrtime_t time) 1147c478bd9Sstevel@tonic-gate { 1157c478bd9Sstevel@tonic-gate if (time >= cbe_hrtime_max) 1167c478bd9Sstevel@tonic-gate time = cbe_hrtime_max; 1177c478bd9Sstevel@tonic-gate 1187c478bd9Sstevel@tonic-gate tickcmpr_set(unscalehrtime(time)); 1197c478bd9Sstevel@tonic-gate } 1207c478bd9Sstevel@tonic-gate 1217c478bd9Sstevel@tonic-gate static void 1227c478bd9Sstevel@tonic-gate cbe_softint(cyb_arg_t arg, cyc_level_t level) 1237c478bd9Sstevel@tonic-gate { 1247c478bd9Sstevel@tonic-gate cbe_data_t *data = (cbe_data_t *)arg; 1257c478bd9Sstevel@tonic-gate 1267c478bd9Sstevel@tonic-gate switch (level) { 1277c478bd9Sstevel@tonic-gate case CY_LOW_LEVEL: 1287c478bd9Sstevel@tonic-gate setsoftint(data->cbe_level1_inum); 1297c478bd9Sstevel@tonic-gate break; 1307c478bd9Sstevel@tonic-gate case CY_LOCK_LEVEL: 1317c478bd9Sstevel@tonic-gate setsoftint(data->cbe_level10_inum); 1327c478bd9Sstevel@tonic-gate break; 1337c478bd9Sstevel@tonic-gate default: 1347c478bd9Sstevel@tonic-gate panic("cbe_softint: unexpected soft level %d", level); 1357c478bd9Sstevel@tonic-gate } 1367c478bd9Sstevel@tonic-gate } 1377c478bd9Sstevel@tonic-gate 1387c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 1397c478bd9Sstevel@tonic-gate static cyc_cookie_t 1407c478bd9Sstevel@tonic-gate cbe_set_level(cyb_arg_t arg, cyc_level_t level) 1417c478bd9Sstevel@tonic-gate { 1427c478bd9Sstevel@tonic-gate int ipl; 1437c478bd9Sstevel@tonic-gate 1447c478bd9Sstevel@tonic-gate switch (level) { 1457c478bd9Sstevel@tonic-gate case CY_LOW_LEVEL: 1467c478bd9Sstevel@tonic-gate ipl = CBE_LOW_PIL; 1477c478bd9Sstevel@tonic-gate break; 1487c478bd9Sstevel@tonic-gate case CY_LOCK_LEVEL: 1497c478bd9Sstevel@tonic-gate ipl = CBE_LOCK_PIL; 1507c478bd9Sstevel@tonic-gate break; 1517c478bd9Sstevel@tonic-gate case CY_HIGH_LEVEL: 1527c478bd9Sstevel@tonic-gate ipl = CBE_HIGH_PIL; 1537c478bd9Sstevel@tonic-gate break; 1547c478bd9Sstevel@tonic-gate default: 1557c478bd9Sstevel@tonic-gate panic("cbe_set_level: unexpected level %d", level); 1567c478bd9Sstevel@tonic-gate } 1577c478bd9Sstevel@tonic-gate 1587c478bd9Sstevel@tonic-gate return (splr(ipl)); 1597c478bd9Sstevel@tonic-gate } 1607c478bd9Sstevel@tonic-gate 1617c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 1627c478bd9Sstevel@tonic-gate static void 1637c478bd9Sstevel@tonic-gate cbe_restore_level(cyb_arg_t arg, cyc_cookie_t cookie) 1647c478bd9Sstevel@tonic-gate { 1657c478bd9Sstevel@tonic-gate splx(cookie); 1667c478bd9Sstevel@tonic-gate } 1677c478bd9Sstevel@tonic-gate 1687c478bd9Sstevel@tonic-gate static void 1697c478bd9Sstevel@tonic-gate cbe_xcall_handler(uint64_t arg1, uint64_t arg2) 1707c478bd9Sstevel@tonic-gate { 1717c478bd9Sstevel@tonic-gate cyc_func_t func = (cyc_func_t)arg1; 1727c478bd9Sstevel@tonic-gate void *arg = (void *)arg2; 1737c478bd9Sstevel@tonic-gate 1747c478bd9Sstevel@tonic-gate (*func)(arg); 1757c478bd9Sstevel@tonic-gate } 1767c478bd9Sstevel@tonic-gate 1777c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 1787c478bd9Sstevel@tonic-gate static void 1797c478bd9Sstevel@tonic-gate cbe_xcall(cyb_arg_t arg, cpu_t *dest, cyc_func_t func, void *farg) 1807c478bd9Sstevel@tonic-gate { 1817c478bd9Sstevel@tonic-gate kpreempt_disable(); 1827c478bd9Sstevel@tonic-gate xc_one(dest->cpu_id, cbe_xcall_handler, (uint64_t)func, (uint64_t)farg); 1837c478bd9Sstevel@tonic-gate kpreempt_enable(); 1847c478bd9Sstevel@tonic-gate } 1857c478bd9Sstevel@tonic-gate 1867c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 1877c478bd9Sstevel@tonic-gate static cyb_arg_t 1887c478bd9Sstevel@tonic-gate cbe_configure(cpu_t *cpu) 1897c478bd9Sstevel@tonic-gate { 1907c478bd9Sstevel@tonic-gate cbe_data_t *new_data = kmem_alloc(sizeof (cbe_data_t), KM_SLEEP); 1917c478bd9Sstevel@tonic-gate 1927c478bd9Sstevel@tonic-gate /* 1937c478bd9Sstevel@tonic-gate * The setsoftint() code will refuse to post a soft interrupt if 1947c478bd9Sstevel@tonic-gate * one is already pending for the specified inum. Given that we 1957c478bd9Sstevel@tonic-gate * may have disjoint soft interrupts on different CPUs posted 1967c478bd9Sstevel@tonic-gate * simultaneously, we allocate a new set of inums for each CPU. 1977c478bd9Sstevel@tonic-gate */ 198*b0fc0e77Sgovinda new_data->cbe_level10_inum = add_softintr(PIL_10, 199*b0fc0e77Sgovinda (softintrfunc)cbe_level10, 0, SOFTINT_ST); 2007c478bd9Sstevel@tonic-gate 201*b0fc0e77Sgovinda new_data->cbe_level1_inum = add_softintr(PIL_1, 202*b0fc0e77Sgovinda (softintrfunc)cbe_level1, 0, SOFTINT_ST); 2037c478bd9Sstevel@tonic-gate 2047c478bd9Sstevel@tonic-gate return (new_data); 2057c478bd9Sstevel@tonic-gate } 2067c478bd9Sstevel@tonic-gate 2077c478bd9Sstevel@tonic-gate static void 2087c478bd9Sstevel@tonic-gate cbe_unconfigure(cyb_arg_t arg) 2097c478bd9Sstevel@tonic-gate { 2107c478bd9Sstevel@tonic-gate cbe_data_t *data = (cbe_data_t *)arg; 2117c478bd9Sstevel@tonic-gate 212*b0fc0e77Sgovinda (void) rem_softintr(data->cbe_level10_inum); 213*b0fc0e77Sgovinda (void) rem_softintr(data->cbe_level1_inum); 2147c478bd9Sstevel@tonic-gate 2157c478bd9Sstevel@tonic-gate kmem_free(data, sizeof (cbe_data_t)); 2167c478bd9Sstevel@tonic-gate } 2177c478bd9Sstevel@tonic-gate 2187c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 2197c478bd9Sstevel@tonic-gate static void 2207c478bd9Sstevel@tonic-gate cbe_suspend(cyb_arg_t arg) 2217c478bd9Sstevel@tonic-gate { 2227c478bd9Sstevel@tonic-gate cbe_suspend_time = gethrtime_unscaled(); 2237c478bd9Sstevel@tonic-gate cbe_suspend_delta = 0; 2247c478bd9Sstevel@tonic-gate } 2257c478bd9Sstevel@tonic-gate 2267c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 2277c478bd9Sstevel@tonic-gate static void 2287c478bd9Sstevel@tonic-gate cbe_resume(cyb_arg_t arg) 2297c478bd9Sstevel@tonic-gate { 2307c478bd9Sstevel@tonic-gate hrtime_t now; 2317c478bd9Sstevel@tonic-gate 2327c478bd9Sstevel@tonic-gate /* 2337c478bd9Sstevel@tonic-gate * If we're actually on a CPU which has apparently had %tick zeroed, 2347c478bd9Sstevel@tonic-gate * we want to add cbe_suspend_delta to %tick. 2357c478bd9Sstevel@tonic-gate */ 2367c478bd9Sstevel@tonic-gate if ((now = gethrtime_unscaled()) < cbe_suspend_time) { 2377c478bd9Sstevel@tonic-gate 2387c478bd9Sstevel@tonic-gate if (cbe_suspend_delta == 0) { 2397c478bd9Sstevel@tonic-gate /* 2407c478bd9Sstevel@tonic-gate * We're the first CPU to be resumed. We want %tick 2417c478bd9Sstevel@tonic-gate * to be close to %tick when we suspended the system, 2427c478bd9Sstevel@tonic-gate * so we'll figure out the delta which needs to be 2437c478bd9Sstevel@tonic-gate * written to the register. All subsequent resumed 2447c478bd9Sstevel@tonic-gate * CPUs will write the same delta. 2457c478bd9Sstevel@tonic-gate */ 2467c478bd9Sstevel@tonic-gate cbe_suspend_delta = cbe_suspend_time - now; 2477c478bd9Sstevel@tonic-gate } 2487c478bd9Sstevel@tonic-gate 2497c478bd9Sstevel@tonic-gate tick_write_delta(cbe_suspend_delta); 2507c478bd9Sstevel@tonic-gate } 2517c478bd9Sstevel@tonic-gate } 2527c478bd9Sstevel@tonic-gate 2537c478bd9Sstevel@tonic-gate void 2547c478bd9Sstevel@tonic-gate cbe_hres_tick(void) 2557c478bd9Sstevel@tonic-gate { 2567c478bd9Sstevel@tonic-gate dtrace_hres_tick(); 2577c478bd9Sstevel@tonic-gate hres_tick(); 2587c478bd9Sstevel@tonic-gate } 2597c478bd9Sstevel@tonic-gate 2607c478bd9Sstevel@tonic-gate void 2617c478bd9Sstevel@tonic-gate cbe_init(void) 2627c478bd9Sstevel@tonic-gate { 2637c478bd9Sstevel@tonic-gate cyc_handler_t hdlr; 2647c478bd9Sstevel@tonic-gate cyc_time_t when; 2657c478bd9Sstevel@tonic-gate hrtime_t resolution = NANOSEC / sys_tick_freq; 2667c478bd9Sstevel@tonic-gate 2677c478bd9Sstevel@tonic-gate cyc_backend_t cbe = { 2687c478bd9Sstevel@tonic-gate cbe_configure, /* cyb_configure */ 2697c478bd9Sstevel@tonic-gate cbe_unconfigure, /* cyb_unconfigure */ 2707c478bd9Sstevel@tonic-gate cbe_enable, /* cyb_enable */ 2717c478bd9Sstevel@tonic-gate cbe_disable, /* cyb_disable */ 2727c478bd9Sstevel@tonic-gate cbe_reprogram, /* cyb_reprogram */ 2737c478bd9Sstevel@tonic-gate cbe_softint, /* cyb_softint */ 2747c478bd9Sstevel@tonic-gate cbe_set_level, /* cyb_set_level */ 2757c478bd9Sstevel@tonic-gate cbe_restore_level, /* cyb_restore_level */ 2767c478bd9Sstevel@tonic-gate cbe_xcall, /* cyb_xcall */ 2777c478bd9Sstevel@tonic-gate cbe_suspend, /* cyb_suspend */ 2787c478bd9Sstevel@tonic-gate cbe_resume /* cyb_resume */ 2797c478bd9Sstevel@tonic-gate }; 2807c478bd9Sstevel@tonic-gate 281*b0fc0e77Sgovinda cbe_level14_inum = add_softintr(CBE_HIGH_PIL, 282*b0fc0e77Sgovinda (softintrfunc)cbe_level14, 0, SOFTINT_MT); 2837c478bd9Sstevel@tonic-gate cbe_hrtime_max = gethrtime_max(); 2847c478bd9Sstevel@tonic-gate 2857c478bd9Sstevel@tonic-gate /* 2867c478bd9Sstevel@tonic-gate * If sys_tick_freq > NANOSEC (i.e. we're on a CPU with a clock rate 2877c478bd9Sstevel@tonic-gate * which exceeds 1 GHz), we'll specify the minimum resolution, 2887c478bd9Sstevel@tonic-gate * 1 nanosecond. 2897c478bd9Sstevel@tonic-gate */ 2907c478bd9Sstevel@tonic-gate if (resolution == 0) 2917c478bd9Sstevel@tonic-gate resolution = 1; 2927c478bd9Sstevel@tonic-gate 2937c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 2947c478bd9Sstevel@tonic-gate cyclic_init(&cbe, resolution); 2957c478bd9Sstevel@tonic-gate 2967c478bd9Sstevel@tonic-gate /* 2977c478bd9Sstevel@tonic-gate * Initialize hrtime_base and hres_last_tick to reasonable starting 2987c478bd9Sstevel@tonic-gate * values. 2997c478bd9Sstevel@tonic-gate */ 3007c478bd9Sstevel@tonic-gate hrtime_base = gethrtime(); 3017c478bd9Sstevel@tonic-gate hres_last_tick = gethrtime_unscaled(); 3027c478bd9Sstevel@tonic-gate 3037c478bd9Sstevel@tonic-gate hdlr.cyh_level = CY_HIGH_LEVEL; 3047c478bd9Sstevel@tonic-gate hdlr.cyh_func = (cyc_func_t)cbe_hres_tick; 3057c478bd9Sstevel@tonic-gate hdlr.cyh_arg = NULL; 3067c478bd9Sstevel@tonic-gate 3077c478bd9Sstevel@tonic-gate when.cyt_when = 0; 3087c478bd9Sstevel@tonic-gate when.cyt_interval = nsec_per_tick; 3097c478bd9Sstevel@tonic-gate 3107c478bd9Sstevel@tonic-gate cbe_hres_cyclic = cyclic_add(&hdlr, &when); 3117c478bd9Sstevel@tonic-gate 3127c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 3137c478bd9Sstevel@tonic-gate 3147c478bd9Sstevel@tonic-gate clkstart(); 3157c478bd9Sstevel@tonic-gate } 316