17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*f635d46aSqiao * Common Development and Distribution License (the "License"). 6*f635d46aSqiao * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22*f635d46aSqiao * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 277c478bd9Sstevel@tonic-gate 287c478bd9Sstevel@tonic-gate #include <sys/callo.h> 297c478bd9Sstevel@tonic-gate #include <sys/param.h> 307c478bd9Sstevel@tonic-gate #include <sys/types.h> 317c478bd9Sstevel@tonic-gate #include <sys/systm.h> 327c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 337c478bd9Sstevel@tonic-gate #include <sys/thread.h> 347c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 357c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 367c478bd9Sstevel@tonic-gate #include <sys/callb.h> 377c478bd9Sstevel@tonic-gate #include <sys/debug.h> 387c478bd9Sstevel@tonic-gate #include <sys/vtrace.h> 397c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 407c478bd9Sstevel@tonic-gate #include <sys/sdt.h> 417c478bd9Sstevel@tonic-gate 427c478bd9Sstevel@tonic-gate /* 437c478bd9Sstevel@tonic-gate * Callout tables. See timeout(9F) for details. 447c478bd9Sstevel@tonic-gate */ 457c478bd9Sstevel@tonic-gate static int cpr_stop_callout; 467c478bd9Sstevel@tonic-gate static int callout_fanout; 477c478bd9Sstevel@tonic-gate static int ncallout; 487c478bd9Sstevel@tonic-gate static callout_table_t *callout_table[CALLOUT_TABLES]; 497c478bd9Sstevel@tonic-gate 507c478bd9Sstevel@tonic-gate #define CALLOUT_HASH_INSERT(cthead, cp, cnext, cprev) \ 517c478bd9Sstevel@tonic-gate { \ 527c478bd9Sstevel@tonic-gate callout_t **headpp = &cthead; \ 537c478bd9Sstevel@tonic-gate callout_t *headp = *headpp; \ 547c478bd9Sstevel@tonic-gate cp->cnext = headp; \ 557c478bd9Sstevel@tonic-gate cp->cprev = NULL; \ 567c478bd9Sstevel@tonic-gate if (headp != NULL) \ 577c478bd9Sstevel@tonic-gate headp->cprev = cp; \ 587c478bd9Sstevel@tonic-gate *headpp = cp; \ 597c478bd9Sstevel@tonic-gate } 607c478bd9Sstevel@tonic-gate 617c478bd9Sstevel@tonic-gate #define CALLOUT_HASH_DELETE(cthead, cp, cnext, cprev) \ 627c478bd9Sstevel@tonic-gate { \ 637c478bd9Sstevel@tonic-gate callout_t *nextp = cp->cnext; \ 647c478bd9Sstevel@tonic-gate callout_t *prevp = cp->cprev; \ 657c478bd9Sstevel@tonic-gate if (nextp != NULL) \ 667c478bd9Sstevel@tonic-gate nextp->cprev = prevp; \ 677c478bd9Sstevel@tonic-gate if (prevp != NULL) \ 687c478bd9Sstevel@tonic-gate prevp->cnext = nextp; \ 697c478bd9Sstevel@tonic-gate else \ 707c478bd9Sstevel@tonic-gate cthead = nextp; \ 717c478bd9Sstevel@tonic-gate } 727c478bd9Sstevel@tonic-gate 73*f635d46aSqiao #define CALLOUT_HASH_UPDATE(INSDEL, ct, cp, id, runtime, runhrtime) \ 747c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ct->ct_lock)); \ 75*f635d46aSqiao ASSERT(cp->c_xid == id && ((cp->c_runtime == runtime) || \ 76*f635d46aSqiao (cp->c_runhrtime <= runhrtime))); \ 777c478bd9Sstevel@tonic-gate CALLOUT_HASH_##INSDEL(ct->ct_idhash[CALLOUT_IDHASH(id)], \ 787c478bd9Sstevel@tonic-gate cp, c_idnext, c_idprev) \ 797c478bd9Sstevel@tonic-gate CALLOUT_HASH_##INSDEL(ct->ct_lbhash[CALLOUT_LBHASH(runtime)], \ 807c478bd9Sstevel@tonic-gate cp, c_lbnext, c_lbprev) 817c478bd9Sstevel@tonic-gate 827c478bd9Sstevel@tonic-gate /* 837c478bd9Sstevel@tonic-gate * Allocate a callout structure. We try quite hard because we 847c478bd9Sstevel@tonic-gate * can't sleep, and if we can't do the allocation, we're toast. 857c478bd9Sstevel@tonic-gate * Failing all, we try a KM_PANIC allocation. 867c478bd9Sstevel@tonic-gate */ 877c478bd9Sstevel@tonic-gate static callout_t * 887c478bd9Sstevel@tonic-gate callout_alloc(callout_table_t *ct) 897c478bd9Sstevel@tonic-gate { 907c478bd9Sstevel@tonic-gate size_t size = 0; 917c478bd9Sstevel@tonic-gate callout_t *cp = NULL; 927c478bd9Sstevel@tonic-gate 937c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 947c478bd9Sstevel@tonic-gate cp = kmem_alloc_tryhard(sizeof (callout_t), &size, 957c478bd9Sstevel@tonic-gate KM_NOSLEEP | KM_PANIC); 967c478bd9Sstevel@tonic-gate bzero(cp, sizeof (callout_t)); 977c478bd9Sstevel@tonic-gate ncallout++; 987c478bd9Sstevel@tonic-gate mutex_enter(&ct->ct_lock); 997c478bd9Sstevel@tonic-gate return (cp); 1007c478bd9Sstevel@tonic-gate } 1017c478bd9Sstevel@tonic-gate 1027c478bd9Sstevel@tonic-gate /* 1037c478bd9Sstevel@tonic-gate * Arrange that func(arg) be called after delta clock ticks. 1047c478bd9Sstevel@tonic-gate */ 1057c478bd9Sstevel@tonic-gate static timeout_id_t 1067c478bd9Sstevel@tonic-gate timeout_common(void (*func)(void *), void *arg, clock_t delta, 1077c478bd9Sstevel@tonic-gate callout_table_t *ct) 1087c478bd9Sstevel@tonic-gate { 1097c478bd9Sstevel@tonic-gate callout_t *cp; 1107c478bd9Sstevel@tonic-gate callout_id_t id; 1117c478bd9Sstevel@tonic-gate clock_t runtime; 112*f635d46aSqiao timestruc_t start; 113*f635d46aSqiao int64_t runhrtime; 114*f635d46aSqiao 115*f635d46aSqiao gethrestime_lasttick(&start); 1167c478bd9Sstevel@tonic-gate 1177c478bd9Sstevel@tonic-gate mutex_enter(&ct->ct_lock); 1187c478bd9Sstevel@tonic-gate 1197c478bd9Sstevel@tonic-gate if ((cp = ct->ct_freelist) == NULL) 1207c478bd9Sstevel@tonic-gate cp = callout_alloc(ct); 1217c478bd9Sstevel@tonic-gate else 1227c478bd9Sstevel@tonic-gate ct->ct_freelist = cp->c_idnext; 1237c478bd9Sstevel@tonic-gate 1247c478bd9Sstevel@tonic-gate cp->c_func = func; 1257c478bd9Sstevel@tonic-gate cp->c_arg = arg; 1267c478bd9Sstevel@tonic-gate 1277c478bd9Sstevel@tonic-gate /* 1287c478bd9Sstevel@tonic-gate * Make sure the callout runs at least 1 tick in the future. 1297c478bd9Sstevel@tonic-gate */ 1307c478bd9Sstevel@tonic-gate if (delta <= 0) 1317c478bd9Sstevel@tonic-gate delta = 1; 1327c478bd9Sstevel@tonic-gate cp->c_runtime = runtime = lbolt + delta; 133*f635d46aSqiao cp->c_runhrtime = runhrtime = delta + timespectohz64(&start); 1347c478bd9Sstevel@tonic-gate 13530392143Sqiao /* 1367c478bd9Sstevel@tonic-gate * Assign an ID to this callout 1377c478bd9Sstevel@tonic-gate */ 1387c478bd9Sstevel@tonic-gate if (delta > CALLOUT_LONGTERM_TICKS) 1397c478bd9Sstevel@tonic-gate ct->ct_long_id = id = (ct->ct_long_id - CALLOUT_COUNTER_LOW) | 1407c478bd9Sstevel@tonic-gate CALLOUT_COUNTER_HIGH; 1417c478bd9Sstevel@tonic-gate else 1427c478bd9Sstevel@tonic-gate ct->ct_short_id = id = (ct->ct_short_id - CALLOUT_COUNTER_LOW) | 1437c478bd9Sstevel@tonic-gate CALLOUT_COUNTER_HIGH; 1447c478bd9Sstevel@tonic-gate 1457c478bd9Sstevel@tonic-gate cp->c_xid = id; 1467c478bd9Sstevel@tonic-gate 147*f635d46aSqiao CALLOUT_HASH_UPDATE(INSERT, ct, cp, id, runtime, runhrtime); 1487c478bd9Sstevel@tonic-gate 1497c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 1507c478bd9Sstevel@tonic-gate 1517c478bd9Sstevel@tonic-gate TRACE_4(TR_FAC_CALLOUT, TR_TIMEOUT, 1527c478bd9Sstevel@tonic-gate "timeout:%K(%p) in %ld ticks, cp %p", 1537c478bd9Sstevel@tonic-gate func, arg, delta, cp); 1547c478bd9Sstevel@tonic-gate 1557c478bd9Sstevel@tonic-gate return ((timeout_id_t)id); 1567c478bd9Sstevel@tonic-gate } 1577c478bd9Sstevel@tonic-gate 1587c478bd9Sstevel@tonic-gate timeout_id_t 1597c478bd9Sstevel@tonic-gate timeout(void (*func)(void *), void *arg, clock_t delta) 1607c478bd9Sstevel@tonic-gate { 1617c478bd9Sstevel@tonic-gate return (timeout_common(func, arg, delta, 1627c478bd9Sstevel@tonic-gate callout_table[CALLOUT_TABLE(CALLOUT_NORMAL, CPU->cpu_seqid)])); 1637c478bd9Sstevel@tonic-gate 1647c478bd9Sstevel@tonic-gate } 1657c478bd9Sstevel@tonic-gate 1667c478bd9Sstevel@tonic-gate timeout_id_t 1677c478bd9Sstevel@tonic-gate realtime_timeout(void (*func)(void *), void *arg, clock_t delta) 1687c478bd9Sstevel@tonic-gate { 1697c478bd9Sstevel@tonic-gate return (timeout_common(func, arg, delta, 1707c478bd9Sstevel@tonic-gate callout_table[CALLOUT_TABLE(CALLOUT_REALTIME, CPU->cpu_seqid)])); 1717c478bd9Sstevel@tonic-gate } 1727c478bd9Sstevel@tonic-gate 1737c478bd9Sstevel@tonic-gate clock_t 1747c478bd9Sstevel@tonic-gate untimeout(timeout_id_t id_arg) 1757c478bd9Sstevel@tonic-gate { 1767c478bd9Sstevel@tonic-gate callout_id_t id = (callout_id_t)id_arg; 1777c478bd9Sstevel@tonic-gate callout_table_t *ct; 1787c478bd9Sstevel@tonic-gate callout_t *cp; 1797c478bd9Sstevel@tonic-gate callout_id_t xid; 1807c478bd9Sstevel@tonic-gate 1817c478bd9Sstevel@tonic-gate ct = callout_table[id & CALLOUT_TABLE_MASK]; 1827c478bd9Sstevel@tonic-gate 1837c478bd9Sstevel@tonic-gate mutex_enter(&ct->ct_lock); 1847c478bd9Sstevel@tonic-gate 1857c478bd9Sstevel@tonic-gate for (cp = ct->ct_idhash[CALLOUT_IDHASH(id)]; cp; cp = cp->c_idnext) { 1867c478bd9Sstevel@tonic-gate 1877c478bd9Sstevel@tonic-gate if ((xid = cp->c_xid) == id) { 1887c478bd9Sstevel@tonic-gate clock_t runtime = cp->c_runtime; 189*f635d46aSqiao int64_t runhrtime = cp->c_runhrtime; 1907c478bd9Sstevel@tonic-gate clock_t time_left = runtime - lbolt; 1917c478bd9Sstevel@tonic-gate 192*f635d46aSqiao CALLOUT_HASH_UPDATE(DELETE, ct, cp, id, 193*f635d46aSqiao runtime, runhrtime); 194*f635d46aSqiao 1957c478bd9Sstevel@tonic-gate cp->c_idnext = ct->ct_freelist; 1967c478bd9Sstevel@tonic-gate ct->ct_freelist = cp; 1977c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 1987c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_CALLOUT, TR_UNTIMEOUT, 1997c478bd9Sstevel@tonic-gate "untimeout:ID %lx ticks_left %ld", id, time_left); 2007c478bd9Sstevel@tonic-gate return (time_left < 0 ? 0 : time_left); 2017c478bd9Sstevel@tonic-gate } 2027c478bd9Sstevel@tonic-gate 2037c478bd9Sstevel@tonic-gate if (xid != (id | CALLOUT_EXECUTING)) 2047c478bd9Sstevel@tonic-gate continue; 2057c478bd9Sstevel@tonic-gate 2067c478bd9Sstevel@tonic-gate /* 2077c478bd9Sstevel@tonic-gate * The callout we want to delete is currently executing. 2087c478bd9Sstevel@tonic-gate * The DDI states that we must wait until the callout 2097c478bd9Sstevel@tonic-gate * completes before returning, so we block on c_done until 2107c478bd9Sstevel@tonic-gate * the callout ID changes (to zero if it's on the freelist, 2117c478bd9Sstevel@tonic-gate * or to a new callout ID if it's in use). This implicitly 2127c478bd9Sstevel@tonic-gate * assumes that callout structures are persistent (they are). 2137c478bd9Sstevel@tonic-gate */ 2147c478bd9Sstevel@tonic-gate if (cp->c_executor == curthread) { 2157c478bd9Sstevel@tonic-gate /* 2167c478bd9Sstevel@tonic-gate * The timeout handler called untimeout() on itself. 2177c478bd9Sstevel@tonic-gate * Stupid, but legal. We can't wait for the timeout 2187c478bd9Sstevel@tonic-gate * to complete without deadlocking, so we just return. 2197c478bd9Sstevel@tonic-gate */ 2207c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 2217c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_CALLOUT, TR_UNTIMEOUT_SELF, 2227c478bd9Sstevel@tonic-gate "untimeout_self:ID %x", id); 2237c478bd9Sstevel@tonic-gate return (-1); 2247c478bd9Sstevel@tonic-gate } 2257c478bd9Sstevel@tonic-gate while (cp->c_xid == xid) 2267c478bd9Sstevel@tonic-gate cv_wait(&cp->c_done, &ct->ct_lock); 2277c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 2287c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_CALLOUT, TR_UNTIMEOUT_EXECUTING, 2297c478bd9Sstevel@tonic-gate "untimeout_executing:ID %lx", id); 2307c478bd9Sstevel@tonic-gate return (-1); 2317c478bd9Sstevel@tonic-gate } 2327c478bd9Sstevel@tonic-gate 2337c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 2347c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_CALLOUT, TR_UNTIMEOUT_BOGUS_ID, 2357c478bd9Sstevel@tonic-gate "untimeout_bogus_id:ID %lx", id); 2367c478bd9Sstevel@tonic-gate 2377c478bd9Sstevel@tonic-gate /* 2387c478bd9Sstevel@tonic-gate * We didn't find the specified callout ID. This means either 2397c478bd9Sstevel@tonic-gate * (1) the callout already fired, or (2) the caller passed us 2407c478bd9Sstevel@tonic-gate * a bogus value. Perform a sanity check to detect case (2). 2417c478bd9Sstevel@tonic-gate */ 2427c478bd9Sstevel@tonic-gate if (id != 0 && (id & (CALLOUT_COUNTER_HIGH | CALLOUT_EXECUTING)) != 2437c478bd9Sstevel@tonic-gate CALLOUT_COUNTER_HIGH) 2447c478bd9Sstevel@tonic-gate panic("untimeout: impossible timeout id %lx", id); 2457c478bd9Sstevel@tonic-gate 2467c478bd9Sstevel@tonic-gate return (-1); 2477c478bd9Sstevel@tonic-gate } 2487c478bd9Sstevel@tonic-gate 2497c478bd9Sstevel@tonic-gate /* 2507c478bd9Sstevel@tonic-gate * Do the actual work of executing callouts. This routine is called either 2517c478bd9Sstevel@tonic-gate * by a taskq_thread (normal case), or by softcall (realtime case). 2527c478bd9Sstevel@tonic-gate */ 2537c478bd9Sstevel@tonic-gate static void 2547c478bd9Sstevel@tonic-gate callout_execute(callout_table_t *ct) 2557c478bd9Sstevel@tonic-gate { 2567c478bd9Sstevel@tonic-gate callout_t *cp; 2577c478bd9Sstevel@tonic-gate callout_id_t xid; 2587c478bd9Sstevel@tonic-gate clock_t runtime; 259*f635d46aSqiao int64_t curhrtime; 2607c478bd9Sstevel@tonic-gate 2617c478bd9Sstevel@tonic-gate mutex_enter(&ct->ct_lock); 2627c478bd9Sstevel@tonic-gate 263*f635d46aSqiao /* 264*f635d46aSqiao * Assuming the system time can be set forward and backward 265*f635d46aSqiao * at any time. If it is set backward, we will measure the 266*f635d46aSqiao * c_runtime; otherwise, we will compare c_runhrtime with 267*f635d46aSqiao * ct_curhrtime. 268*f635d46aSqiao */ 269*f635d46aSqiao curhrtime = ct->ct_curhrtime; 2707c478bd9Sstevel@tonic-gate while (((runtime = ct->ct_runtime) - ct->ct_curtime) <= 0) { 2717c478bd9Sstevel@tonic-gate for (cp = ct->ct_lbhash[CALLOUT_LBHASH(runtime)]; 2727c478bd9Sstevel@tonic-gate cp != NULL; cp = cp->c_lbnext) { 2737c478bd9Sstevel@tonic-gate xid = cp->c_xid; 274*f635d46aSqiao if ((cp->c_runtime != runtime && 275*f635d46aSqiao cp->c_runhrtime > curhrtime) || 2767c478bd9Sstevel@tonic-gate (xid & CALLOUT_EXECUTING)) 2777c478bd9Sstevel@tonic-gate continue; 2787c478bd9Sstevel@tonic-gate cp->c_executor = curthread; 2797c478bd9Sstevel@tonic-gate cp->c_xid = xid |= CALLOUT_EXECUTING; 2807c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 2817c478bd9Sstevel@tonic-gate DTRACE_PROBE1(callout__start, callout_t *, cp); 2827c478bd9Sstevel@tonic-gate (*cp->c_func)(cp->c_arg); 2837c478bd9Sstevel@tonic-gate DTRACE_PROBE1(callout__end, callout_t *, cp); 2847c478bd9Sstevel@tonic-gate mutex_enter(&ct->ct_lock); 2857c478bd9Sstevel@tonic-gate 2867c478bd9Sstevel@tonic-gate /* 2873348528fSdm120769 * Delete callout from hash tables, return to freelist, 2883348528fSdm120769 * and tell anyone who cares that we're done. 2897c478bd9Sstevel@tonic-gate * Even though we dropped and reacquired ct->ct_lock, 2907c478bd9Sstevel@tonic-gate * it's OK to pick up where we left off because only 2917c478bd9Sstevel@tonic-gate * newly-created timeouts can precede cp on ct_lbhash, 2927c478bd9Sstevel@tonic-gate * and those timeouts cannot be due on this tick. 2937c478bd9Sstevel@tonic-gate */ 294*f635d46aSqiao CALLOUT_HASH_UPDATE(DELETE, ct, cp, xid, 295*f635d46aSqiao runtime, curhrtime); 296*f635d46aSqiao 2977c478bd9Sstevel@tonic-gate cp->c_idnext = ct->ct_freelist; 2987c478bd9Sstevel@tonic-gate ct->ct_freelist = cp; 2997c478bd9Sstevel@tonic-gate cp->c_xid = 0; /* Indicate completion for c_done */ 3007c478bd9Sstevel@tonic-gate cv_broadcast(&cp->c_done); 3017c478bd9Sstevel@tonic-gate } 3027c478bd9Sstevel@tonic-gate /* 3037c478bd9Sstevel@tonic-gate * We have completed all callouts that were scheduled to 3047c478bd9Sstevel@tonic-gate * run at "runtime". If the global run time still matches 3057c478bd9Sstevel@tonic-gate * our local copy, then we advance the global run time; 3067c478bd9Sstevel@tonic-gate * otherwise, another callout thread must have already done so. 3077c478bd9Sstevel@tonic-gate */ 3087c478bd9Sstevel@tonic-gate if (ct->ct_runtime == runtime) 3097c478bd9Sstevel@tonic-gate ct->ct_runtime = runtime + 1; 3107c478bd9Sstevel@tonic-gate } 3117c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 3127c478bd9Sstevel@tonic-gate } 3137c478bd9Sstevel@tonic-gate 3147c478bd9Sstevel@tonic-gate /* 3157c478bd9Sstevel@tonic-gate * Schedule any callouts that are due on or before this tick. 3167c478bd9Sstevel@tonic-gate */ 3177c478bd9Sstevel@tonic-gate static void 3187c478bd9Sstevel@tonic-gate callout_schedule_1(callout_table_t *ct) 3197c478bd9Sstevel@tonic-gate { 3207c478bd9Sstevel@tonic-gate callout_t *cp; 3217c478bd9Sstevel@tonic-gate clock_t curtime, runtime; 322*f635d46aSqiao timestruc_t now; 323*f635d46aSqiao int64_t curhrtime; 324*f635d46aSqiao 325*f635d46aSqiao gethrestime(&now); 326*f635d46aSqiao curhrtime = timespectohz64(&now); 3277c478bd9Sstevel@tonic-gate 3287c478bd9Sstevel@tonic-gate mutex_enter(&ct->ct_lock); 3297c478bd9Sstevel@tonic-gate ct->ct_curtime = curtime = lbolt; 330*f635d46aSqiao 331*f635d46aSqiao /* 332*f635d46aSqiao * We use both the conditions cp->c_runtime == runtime and 333*f635d46aSqiao * cp->c_runhrtime <= curhrtime to determine a timeout is 334*f635d46aSqiao * premature or not. If the system time has been set backwards, 335*f635d46aSqiao * then cp->c_runtime == runtime will become true first. 336*f635d46aSqiao * Otherwise, we test cp->c_runhrtime <= curhrtime 337*f635d46aSqiao */ 338*f635d46aSqiao ct->ct_curhrtime = curhrtime; 3397c478bd9Sstevel@tonic-gate while (((runtime = ct->ct_runtime) - curtime) <= 0) { 3407c478bd9Sstevel@tonic-gate for (cp = ct->ct_lbhash[CALLOUT_LBHASH(runtime)]; 3417c478bd9Sstevel@tonic-gate cp != NULL; cp = cp->c_lbnext) { 342*f635d46aSqiao if ((cp->c_runtime != runtime && 343*f635d46aSqiao cp->c_runhrtime > curhrtime) || 3447c478bd9Sstevel@tonic-gate (cp->c_xid & CALLOUT_EXECUTING)) 3457c478bd9Sstevel@tonic-gate continue; 3467c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 3477c478bd9Sstevel@tonic-gate if (ct->ct_taskq == NULL) 3487c478bd9Sstevel@tonic-gate softcall((void (*)(void *))callout_execute, ct); 3497c478bd9Sstevel@tonic-gate else 3507c478bd9Sstevel@tonic-gate (void) taskq_dispatch(ct->ct_taskq, 3517c478bd9Sstevel@tonic-gate (task_func_t *)callout_execute, ct, 3527c478bd9Sstevel@tonic-gate KM_NOSLEEP); 3537c478bd9Sstevel@tonic-gate return; 3547c478bd9Sstevel@tonic-gate } 3557c478bd9Sstevel@tonic-gate ct->ct_runtime++; 3567c478bd9Sstevel@tonic-gate } 3577c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 3587c478bd9Sstevel@tonic-gate } 3597c478bd9Sstevel@tonic-gate 3607c478bd9Sstevel@tonic-gate /* 3617c478bd9Sstevel@tonic-gate * Schedule callouts for all callout tables. Called by clock() on each tick. 3627c478bd9Sstevel@tonic-gate */ 3637c478bd9Sstevel@tonic-gate void 3647c478bd9Sstevel@tonic-gate callout_schedule(void) 3657c478bd9Sstevel@tonic-gate { 3667c478bd9Sstevel@tonic-gate int f, t; 3677c478bd9Sstevel@tonic-gate 3687c478bd9Sstevel@tonic-gate if (cpr_stop_callout) 3697c478bd9Sstevel@tonic-gate return; 3707c478bd9Sstevel@tonic-gate 3717c478bd9Sstevel@tonic-gate for (t = 0; t < CALLOUT_NTYPES; t++) 3727c478bd9Sstevel@tonic-gate for (f = 0; f < callout_fanout; f++) 3737c478bd9Sstevel@tonic-gate callout_schedule_1(callout_table[CALLOUT_TABLE(t, f)]); 3747c478bd9Sstevel@tonic-gate } 3757c478bd9Sstevel@tonic-gate 3767c478bd9Sstevel@tonic-gate /* 3777c478bd9Sstevel@tonic-gate * Callback handler used by CPR to stop and resume callouts. 3787c478bd9Sstevel@tonic-gate */ 3797c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 3807c478bd9Sstevel@tonic-gate static boolean_t 3817c478bd9Sstevel@tonic-gate callout_cpr_callb(void *arg, int code) 3827c478bd9Sstevel@tonic-gate { 3837c478bd9Sstevel@tonic-gate cpr_stop_callout = (code == CB_CODE_CPR_CHKPT); 3847c478bd9Sstevel@tonic-gate return (B_TRUE); 3857c478bd9Sstevel@tonic-gate } 3867c478bd9Sstevel@tonic-gate 3877c478bd9Sstevel@tonic-gate /* 3887c478bd9Sstevel@tonic-gate * Initialize all callout tables. Called at boot time just before clkstart(). 3897c478bd9Sstevel@tonic-gate */ 3907c478bd9Sstevel@tonic-gate void 3917c478bd9Sstevel@tonic-gate callout_init(void) 3927c478bd9Sstevel@tonic-gate { 3937c478bd9Sstevel@tonic-gate int f, t; 3947c478bd9Sstevel@tonic-gate int table_id; 3957c478bd9Sstevel@tonic-gate callout_table_t *ct; 3967c478bd9Sstevel@tonic-gate 3977c478bd9Sstevel@tonic-gate callout_fanout = MIN(CALLOUT_FANOUT, max_ncpus); 3987c478bd9Sstevel@tonic-gate 3997c478bd9Sstevel@tonic-gate for (t = 0; t < CALLOUT_NTYPES; t++) { 4007c478bd9Sstevel@tonic-gate for (f = 0; f < CALLOUT_FANOUT; f++) { 4017c478bd9Sstevel@tonic-gate table_id = CALLOUT_TABLE(t, f); 4027c478bd9Sstevel@tonic-gate if (f >= callout_fanout) { 4037c478bd9Sstevel@tonic-gate callout_table[table_id] = 4047c478bd9Sstevel@tonic-gate callout_table[table_id - callout_fanout]; 4057c478bd9Sstevel@tonic-gate continue; 4067c478bd9Sstevel@tonic-gate } 4077c478bd9Sstevel@tonic-gate ct = kmem_zalloc(sizeof (callout_table_t), KM_SLEEP); 4087c478bd9Sstevel@tonic-gate callout_table[table_id] = ct; 4097c478bd9Sstevel@tonic-gate ct->ct_short_id = (callout_id_t)table_id | 4107c478bd9Sstevel@tonic-gate CALLOUT_COUNTER_HIGH; 4117c478bd9Sstevel@tonic-gate ct->ct_long_id = ct->ct_short_id | CALLOUT_LONGTERM; 4127c478bd9Sstevel@tonic-gate ct->ct_curtime = ct->ct_runtime = lbolt; 413*f635d46aSqiao 414*f635d46aSqiao /* 415*f635d46aSqiao * We can not call gethrestime() at this moment 416*f635d46aSqiao * since the system time has not been validated. 417*f635d46aSqiao * So Set ct_curhrtime to zero. 418*f635d46aSqiao */ 419*f635d46aSqiao ct->ct_curhrtime = 0; 420*f635d46aSqiao 4217c478bd9Sstevel@tonic-gate if (t == CALLOUT_NORMAL) { 4227c478bd9Sstevel@tonic-gate /* 4237c478bd9Sstevel@tonic-gate * Each callout thread consumes exactly one 4247c478bd9Sstevel@tonic-gate * task structure while active. Therefore, 4257c478bd9Sstevel@tonic-gate * prepopulating with 2 * CALLOUT_THREADS tasks 4267c478bd9Sstevel@tonic-gate * ensures that there's at least one task per 4277c478bd9Sstevel@tonic-gate * thread that's either scheduled or on the 4287c478bd9Sstevel@tonic-gate * freelist. In turn, this guarantees that 4297c478bd9Sstevel@tonic-gate * taskq_dispatch() will always either succeed 4307c478bd9Sstevel@tonic-gate * (because there's a free task structure) or 4317c478bd9Sstevel@tonic-gate * be unnecessary (because "callout_excute(ct)" 4327c478bd9Sstevel@tonic-gate * has already scheduled). 4337c478bd9Sstevel@tonic-gate */ 4347c478bd9Sstevel@tonic-gate ct->ct_taskq = 4357c478bd9Sstevel@tonic-gate taskq_create_instance("callout_taskq", f, 4367c478bd9Sstevel@tonic-gate CALLOUT_THREADS, maxclsyspri, 4377c478bd9Sstevel@tonic-gate 2 * CALLOUT_THREADS, 2 * CALLOUT_THREADS, 4387c478bd9Sstevel@tonic-gate TASKQ_PREPOPULATE | TASKQ_CPR_SAFE); 4397c478bd9Sstevel@tonic-gate } 4407c478bd9Sstevel@tonic-gate } 4417c478bd9Sstevel@tonic-gate } 4427c478bd9Sstevel@tonic-gate (void) callb_add(callout_cpr_callb, 0, CB_CL_CPR_CALLOUT, "callout"); 4437c478bd9Sstevel@tonic-gate } 444