1*7c478bd9Sstevel@tonic-gate /* 2*7c478bd9Sstevel@tonic-gate * CDDL HEADER START 3*7c478bd9Sstevel@tonic-gate * 4*7c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*7c478bd9Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*7c478bd9Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*7c478bd9Sstevel@tonic-gate * with the License. 8*7c478bd9Sstevel@tonic-gate * 9*7c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*7c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*7c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 12*7c478bd9Sstevel@tonic-gate * and limitations under the License. 13*7c478bd9Sstevel@tonic-gate * 14*7c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*7c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*7c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*7c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*7c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*7c478bd9Sstevel@tonic-gate * 20*7c478bd9Sstevel@tonic-gate * CDDL HEADER END 21*7c478bd9Sstevel@tonic-gate */ 22*7c478bd9Sstevel@tonic-gate /* 23*7c478bd9Sstevel@tonic-gate * Copyright 2004 Sun Microsystems, Inc. All rights reserved. 24*7c478bd9Sstevel@tonic-gate * Use is subject to license terms. 25*7c478bd9Sstevel@tonic-gate */ 26*7c478bd9Sstevel@tonic-gate 27*7c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 28*7c478bd9Sstevel@tonic-gate 29*7c478bd9Sstevel@tonic-gate #include <sys/callo.h> 30*7c478bd9Sstevel@tonic-gate #include <sys/param.h> 31*7c478bd9Sstevel@tonic-gate #include <sys/types.h> 32*7c478bd9Sstevel@tonic-gate #include <sys/systm.h> 33*7c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 34*7c478bd9Sstevel@tonic-gate #include <sys/thread.h> 35*7c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 36*7c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 37*7c478bd9Sstevel@tonic-gate #include <sys/callb.h> 38*7c478bd9Sstevel@tonic-gate #include <sys/debug.h> 39*7c478bd9Sstevel@tonic-gate #include <sys/vtrace.h> 40*7c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 41*7c478bd9Sstevel@tonic-gate #include <sys/sdt.h> 42*7c478bd9Sstevel@tonic-gate 43*7c478bd9Sstevel@tonic-gate /* 44*7c478bd9Sstevel@tonic-gate * Callout tables. See timeout(9F) for details. 45*7c478bd9Sstevel@tonic-gate */ 46*7c478bd9Sstevel@tonic-gate static int cpr_stop_callout; 47*7c478bd9Sstevel@tonic-gate static int callout_fanout; 48*7c478bd9Sstevel@tonic-gate static int ncallout; 49*7c478bd9Sstevel@tonic-gate static callout_table_t *callout_table[CALLOUT_TABLES]; 50*7c478bd9Sstevel@tonic-gate 51*7c478bd9Sstevel@tonic-gate #define CALLOUT_HASH_INSERT(cthead, cp, cnext, cprev) \ 52*7c478bd9Sstevel@tonic-gate { \ 53*7c478bd9Sstevel@tonic-gate callout_t **headpp = &cthead; \ 54*7c478bd9Sstevel@tonic-gate callout_t *headp = *headpp; \ 55*7c478bd9Sstevel@tonic-gate cp->cnext = headp; \ 56*7c478bd9Sstevel@tonic-gate cp->cprev = NULL; \ 57*7c478bd9Sstevel@tonic-gate if (headp != NULL) \ 58*7c478bd9Sstevel@tonic-gate headp->cprev = cp; \ 59*7c478bd9Sstevel@tonic-gate *headpp = cp; \ 60*7c478bd9Sstevel@tonic-gate } 61*7c478bd9Sstevel@tonic-gate 62*7c478bd9Sstevel@tonic-gate #define CALLOUT_HASH_DELETE(cthead, cp, cnext, cprev) \ 63*7c478bd9Sstevel@tonic-gate { \ 64*7c478bd9Sstevel@tonic-gate callout_t *nextp = cp->cnext; \ 65*7c478bd9Sstevel@tonic-gate callout_t *prevp = cp->cprev; \ 66*7c478bd9Sstevel@tonic-gate if (nextp != NULL) \ 67*7c478bd9Sstevel@tonic-gate nextp->cprev = prevp; \ 68*7c478bd9Sstevel@tonic-gate if (prevp != NULL) \ 69*7c478bd9Sstevel@tonic-gate prevp->cnext = nextp; \ 70*7c478bd9Sstevel@tonic-gate else \ 71*7c478bd9Sstevel@tonic-gate cthead = nextp; \ 72*7c478bd9Sstevel@tonic-gate } 73*7c478bd9Sstevel@tonic-gate 74*7c478bd9Sstevel@tonic-gate #define CALLOUT_HASH_UPDATE(INSDEL, ct, cp, id, runtime) \ 75*7c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ct->ct_lock)); \ 76*7c478bd9Sstevel@tonic-gate ASSERT(cp->c_xid == id && cp->c_runtime == runtime); \ 77*7c478bd9Sstevel@tonic-gate CALLOUT_HASH_##INSDEL(ct->ct_idhash[CALLOUT_IDHASH(id)], \ 78*7c478bd9Sstevel@tonic-gate cp, c_idnext, c_idprev) \ 79*7c478bd9Sstevel@tonic-gate CALLOUT_HASH_##INSDEL(ct->ct_lbhash[CALLOUT_LBHASH(runtime)], \ 80*7c478bd9Sstevel@tonic-gate cp, c_lbnext, c_lbprev) 81*7c478bd9Sstevel@tonic-gate 82*7c478bd9Sstevel@tonic-gate /* 83*7c478bd9Sstevel@tonic-gate * Allocate a callout structure. We try quite hard because we 84*7c478bd9Sstevel@tonic-gate * can't sleep, and if we can't do the allocation, we're toast. 85*7c478bd9Sstevel@tonic-gate * Failing all, we try a KM_PANIC allocation. 86*7c478bd9Sstevel@tonic-gate */ 87*7c478bd9Sstevel@tonic-gate static callout_t * 88*7c478bd9Sstevel@tonic-gate callout_alloc(callout_table_t *ct) 89*7c478bd9Sstevel@tonic-gate { 90*7c478bd9Sstevel@tonic-gate size_t size = 0; 91*7c478bd9Sstevel@tonic-gate callout_t *cp = NULL; 92*7c478bd9Sstevel@tonic-gate 93*7c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 94*7c478bd9Sstevel@tonic-gate cp = kmem_alloc_tryhard(sizeof (callout_t), &size, 95*7c478bd9Sstevel@tonic-gate KM_NOSLEEP | KM_PANIC); 96*7c478bd9Sstevel@tonic-gate bzero(cp, sizeof (callout_t)); 97*7c478bd9Sstevel@tonic-gate ncallout++; 98*7c478bd9Sstevel@tonic-gate mutex_enter(&ct->ct_lock); 99*7c478bd9Sstevel@tonic-gate return (cp); 100*7c478bd9Sstevel@tonic-gate } 101*7c478bd9Sstevel@tonic-gate 102*7c478bd9Sstevel@tonic-gate /* 103*7c478bd9Sstevel@tonic-gate * Arrange that func(arg) be called after delta clock ticks. 104*7c478bd9Sstevel@tonic-gate */ 105*7c478bd9Sstevel@tonic-gate static timeout_id_t 106*7c478bd9Sstevel@tonic-gate timeout_common(void (*func)(void *), void *arg, clock_t delta, 107*7c478bd9Sstevel@tonic-gate callout_table_t *ct) 108*7c478bd9Sstevel@tonic-gate { 109*7c478bd9Sstevel@tonic-gate callout_t *cp; 110*7c478bd9Sstevel@tonic-gate callout_id_t id; 111*7c478bd9Sstevel@tonic-gate clock_t runtime; 112*7c478bd9Sstevel@tonic-gate 113*7c478bd9Sstevel@tonic-gate mutex_enter(&ct->ct_lock); 114*7c478bd9Sstevel@tonic-gate 115*7c478bd9Sstevel@tonic-gate if ((cp = ct->ct_freelist) == NULL) 116*7c478bd9Sstevel@tonic-gate cp = callout_alloc(ct); 117*7c478bd9Sstevel@tonic-gate else 118*7c478bd9Sstevel@tonic-gate ct->ct_freelist = cp->c_idnext; 119*7c478bd9Sstevel@tonic-gate 120*7c478bd9Sstevel@tonic-gate cp->c_func = func; 121*7c478bd9Sstevel@tonic-gate cp->c_arg = arg; 122*7c478bd9Sstevel@tonic-gate 123*7c478bd9Sstevel@tonic-gate /* 124*7c478bd9Sstevel@tonic-gate * Make sure the callout runs at least 1 tick in the future. 125*7c478bd9Sstevel@tonic-gate */ 126*7c478bd9Sstevel@tonic-gate if (delta <= 0) 127*7c478bd9Sstevel@tonic-gate delta = 1; 128*7c478bd9Sstevel@tonic-gate cp->c_runtime = runtime = lbolt + delta; 129*7c478bd9Sstevel@tonic-gate 130*7c478bd9Sstevel@tonic-gate /* 131*7c478bd9Sstevel@tonic-gate * Assign an ID to this callout 132*7c478bd9Sstevel@tonic-gate */ 133*7c478bd9Sstevel@tonic-gate if (delta > CALLOUT_LONGTERM_TICKS) 134*7c478bd9Sstevel@tonic-gate ct->ct_long_id = id = (ct->ct_long_id - CALLOUT_COUNTER_LOW) | 135*7c478bd9Sstevel@tonic-gate CALLOUT_COUNTER_HIGH; 136*7c478bd9Sstevel@tonic-gate else 137*7c478bd9Sstevel@tonic-gate ct->ct_short_id = id = (ct->ct_short_id - CALLOUT_COUNTER_LOW) | 138*7c478bd9Sstevel@tonic-gate CALLOUT_COUNTER_HIGH; 139*7c478bd9Sstevel@tonic-gate 140*7c478bd9Sstevel@tonic-gate cp->c_xid = id; 141*7c478bd9Sstevel@tonic-gate 142*7c478bd9Sstevel@tonic-gate CALLOUT_HASH_UPDATE(INSERT, ct, cp, id, runtime); 143*7c478bd9Sstevel@tonic-gate 144*7c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 145*7c478bd9Sstevel@tonic-gate 146*7c478bd9Sstevel@tonic-gate TRACE_4(TR_FAC_CALLOUT, TR_TIMEOUT, 147*7c478bd9Sstevel@tonic-gate "timeout:%K(%p) in %ld ticks, cp %p", 148*7c478bd9Sstevel@tonic-gate func, arg, delta, cp); 149*7c478bd9Sstevel@tonic-gate 150*7c478bd9Sstevel@tonic-gate return ((timeout_id_t)id); 151*7c478bd9Sstevel@tonic-gate } 152*7c478bd9Sstevel@tonic-gate 153*7c478bd9Sstevel@tonic-gate timeout_id_t 154*7c478bd9Sstevel@tonic-gate timeout(void (*func)(void *), void *arg, clock_t delta) 155*7c478bd9Sstevel@tonic-gate { 156*7c478bd9Sstevel@tonic-gate return (timeout_common(func, arg, delta, 157*7c478bd9Sstevel@tonic-gate callout_table[CALLOUT_TABLE(CALLOUT_NORMAL, CPU->cpu_seqid)])); 158*7c478bd9Sstevel@tonic-gate 159*7c478bd9Sstevel@tonic-gate } 160*7c478bd9Sstevel@tonic-gate 161*7c478bd9Sstevel@tonic-gate timeout_id_t 162*7c478bd9Sstevel@tonic-gate realtime_timeout(void (*func)(void *), void *arg, clock_t delta) 163*7c478bd9Sstevel@tonic-gate { 164*7c478bd9Sstevel@tonic-gate return (timeout_common(func, arg, delta, 165*7c478bd9Sstevel@tonic-gate callout_table[CALLOUT_TABLE(CALLOUT_REALTIME, CPU->cpu_seqid)])); 166*7c478bd9Sstevel@tonic-gate } 167*7c478bd9Sstevel@tonic-gate 168*7c478bd9Sstevel@tonic-gate clock_t 169*7c478bd9Sstevel@tonic-gate untimeout(timeout_id_t id_arg) 170*7c478bd9Sstevel@tonic-gate { 171*7c478bd9Sstevel@tonic-gate callout_id_t id = (callout_id_t)id_arg; 172*7c478bd9Sstevel@tonic-gate callout_table_t *ct; 173*7c478bd9Sstevel@tonic-gate callout_t *cp; 174*7c478bd9Sstevel@tonic-gate callout_id_t xid; 175*7c478bd9Sstevel@tonic-gate 176*7c478bd9Sstevel@tonic-gate ct = callout_table[id & CALLOUT_TABLE_MASK]; 177*7c478bd9Sstevel@tonic-gate 178*7c478bd9Sstevel@tonic-gate mutex_enter(&ct->ct_lock); 179*7c478bd9Sstevel@tonic-gate 180*7c478bd9Sstevel@tonic-gate for (cp = ct->ct_idhash[CALLOUT_IDHASH(id)]; cp; cp = cp->c_idnext) { 181*7c478bd9Sstevel@tonic-gate 182*7c478bd9Sstevel@tonic-gate if ((xid = cp->c_xid) == id) { 183*7c478bd9Sstevel@tonic-gate clock_t runtime = cp->c_runtime; 184*7c478bd9Sstevel@tonic-gate clock_t time_left = runtime - lbolt; 185*7c478bd9Sstevel@tonic-gate 186*7c478bd9Sstevel@tonic-gate CALLOUT_HASH_UPDATE(DELETE, ct, cp, id, runtime); 187*7c478bd9Sstevel@tonic-gate cp->c_idnext = ct->ct_freelist; 188*7c478bd9Sstevel@tonic-gate ct->ct_freelist = cp; 189*7c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 190*7c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_CALLOUT, TR_UNTIMEOUT, 191*7c478bd9Sstevel@tonic-gate "untimeout:ID %lx ticks_left %ld", id, time_left); 192*7c478bd9Sstevel@tonic-gate return (time_left < 0 ? 0 : time_left); 193*7c478bd9Sstevel@tonic-gate } 194*7c478bd9Sstevel@tonic-gate 195*7c478bd9Sstevel@tonic-gate if (xid != (id | CALLOUT_EXECUTING)) 196*7c478bd9Sstevel@tonic-gate continue; 197*7c478bd9Sstevel@tonic-gate 198*7c478bd9Sstevel@tonic-gate /* 199*7c478bd9Sstevel@tonic-gate * The callout we want to delete is currently executing. 200*7c478bd9Sstevel@tonic-gate * The DDI states that we must wait until the callout 201*7c478bd9Sstevel@tonic-gate * completes before returning, so we block on c_done until 202*7c478bd9Sstevel@tonic-gate * the callout ID changes (to zero if it's on the freelist, 203*7c478bd9Sstevel@tonic-gate * or to a new callout ID if it's in use). This implicitly 204*7c478bd9Sstevel@tonic-gate * assumes that callout structures are persistent (they are). 205*7c478bd9Sstevel@tonic-gate */ 206*7c478bd9Sstevel@tonic-gate if (cp->c_executor == curthread) { 207*7c478bd9Sstevel@tonic-gate /* 208*7c478bd9Sstevel@tonic-gate * The timeout handler called untimeout() on itself. 209*7c478bd9Sstevel@tonic-gate * Stupid, but legal. We can't wait for the timeout 210*7c478bd9Sstevel@tonic-gate * to complete without deadlocking, so we just return. 211*7c478bd9Sstevel@tonic-gate */ 212*7c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 213*7c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_CALLOUT, TR_UNTIMEOUT_SELF, 214*7c478bd9Sstevel@tonic-gate "untimeout_self:ID %x", id); 215*7c478bd9Sstevel@tonic-gate return (-1); 216*7c478bd9Sstevel@tonic-gate } 217*7c478bd9Sstevel@tonic-gate while (cp->c_xid == xid) 218*7c478bd9Sstevel@tonic-gate cv_wait(&cp->c_done, &ct->ct_lock); 219*7c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 220*7c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_CALLOUT, TR_UNTIMEOUT_EXECUTING, 221*7c478bd9Sstevel@tonic-gate "untimeout_executing:ID %lx", id); 222*7c478bd9Sstevel@tonic-gate return (-1); 223*7c478bd9Sstevel@tonic-gate } 224*7c478bd9Sstevel@tonic-gate 225*7c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 226*7c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_CALLOUT, TR_UNTIMEOUT_BOGUS_ID, 227*7c478bd9Sstevel@tonic-gate "untimeout_bogus_id:ID %lx", id); 228*7c478bd9Sstevel@tonic-gate 229*7c478bd9Sstevel@tonic-gate /* 230*7c478bd9Sstevel@tonic-gate * We didn't find the specified callout ID. This means either 231*7c478bd9Sstevel@tonic-gate * (1) the callout already fired, or (2) the caller passed us 232*7c478bd9Sstevel@tonic-gate * a bogus value. Perform a sanity check to detect case (2). 233*7c478bd9Sstevel@tonic-gate */ 234*7c478bd9Sstevel@tonic-gate if (id != 0 && (id & (CALLOUT_COUNTER_HIGH | CALLOUT_EXECUTING)) != 235*7c478bd9Sstevel@tonic-gate CALLOUT_COUNTER_HIGH) 236*7c478bd9Sstevel@tonic-gate panic("untimeout: impossible timeout id %lx", id); 237*7c478bd9Sstevel@tonic-gate 238*7c478bd9Sstevel@tonic-gate return (-1); 239*7c478bd9Sstevel@tonic-gate } 240*7c478bd9Sstevel@tonic-gate 241*7c478bd9Sstevel@tonic-gate /* 242*7c478bd9Sstevel@tonic-gate * Do the actual work of executing callouts. This routine is called either 243*7c478bd9Sstevel@tonic-gate * by a taskq_thread (normal case), or by softcall (realtime case). 244*7c478bd9Sstevel@tonic-gate */ 245*7c478bd9Sstevel@tonic-gate static void 246*7c478bd9Sstevel@tonic-gate callout_execute(callout_table_t *ct) 247*7c478bd9Sstevel@tonic-gate { 248*7c478bd9Sstevel@tonic-gate callout_t *cp; 249*7c478bd9Sstevel@tonic-gate callout_id_t xid; 250*7c478bd9Sstevel@tonic-gate clock_t runtime; 251*7c478bd9Sstevel@tonic-gate 252*7c478bd9Sstevel@tonic-gate mutex_enter(&ct->ct_lock); 253*7c478bd9Sstevel@tonic-gate 254*7c478bd9Sstevel@tonic-gate while (((runtime = ct->ct_runtime) - ct->ct_curtime) <= 0) { 255*7c478bd9Sstevel@tonic-gate for (cp = ct->ct_lbhash[CALLOUT_LBHASH(runtime)]; 256*7c478bd9Sstevel@tonic-gate cp != NULL; cp = cp->c_lbnext) { 257*7c478bd9Sstevel@tonic-gate xid = cp->c_xid; 258*7c478bd9Sstevel@tonic-gate if (cp->c_runtime != runtime || 259*7c478bd9Sstevel@tonic-gate (xid & CALLOUT_EXECUTING)) 260*7c478bd9Sstevel@tonic-gate continue; 261*7c478bd9Sstevel@tonic-gate cp->c_executor = curthread; 262*7c478bd9Sstevel@tonic-gate cp->c_xid = xid |= CALLOUT_EXECUTING; 263*7c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 264*7c478bd9Sstevel@tonic-gate DTRACE_PROBE1(callout__start, callout_t *, cp); 265*7c478bd9Sstevel@tonic-gate (*cp->c_func)(cp->c_arg); 266*7c478bd9Sstevel@tonic-gate DTRACE_PROBE1(callout__end, callout_t *, cp); 267*7c478bd9Sstevel@tonic-gate mutex_enter(&ct->ct_lock); 268*7c478bd9Sstevel@tonic-gate 269*7c478bd9Sstevel@tonic-gate /* 270*7c478bd9Sstevel@tonic-gate * Delete callout from hash tables, return to freelist, 271*7c478bd9Sstevel@tonic-gate * and tell anyone who cares that we're done. 272*7c478bd9Sstevel@tonic-gate * Even though we dropped and reacquired ct->ct_lock, 273*7c478bd9Sstevel@tonic-gate * it's OK to pick up where we left off because only 274*7c478bd9Sstevel@tonic-gate * newly-created timeouts can precede cp on ct_lbhash, 275*7c478bd9Sstevel@tonic-gate * and those timeouts cannot be due on this tick. 276*7c478bd9Sstevel@tonic-gate */ 277*7c478bd9Sstevel@tonic-gate CALLOUT_HASH_UPDATE(DELETE, ct, cp, xid, runtime); 278*7c478bd9Sstevel@tonic-gate cp->c_idnext = ct->ct_freelist; 279*7c478bd9Sstevel@tonic-gate ct->ct_freelist = cp; 280*7c478bd9Sstevel@tonic-gate cp->c_xid = 0; /* Indicate completion for c_done */ 281*7c478bd9Sstevel@tonic-gate cv_broadcast(&cp->c_done); 282*7c478bd9Sstevel@tonic-gate } 283*7c478bd9Sstevel@tonic-gate /* 284*7c478bd9Sstevel@tonic-gate * We have completed all callouts that were scheduled to 285*7c478bd9Sstevel@tonic-gate * run at "runtime". If the global run time still matches 286*7c478bd9Sstevel@tonic-gate * our local copy, then we advance the global run time; 287*7c478bd9Sstevel@tonic-gate * otherwise, another callout thread must have already done so. 288*7c478bd9Sstevel@tonic-gate */ 289*7c478bd9Sstevel@tonic-gate if (ct->ct_runtime == runtime) 290*7c478bd9Sstevel@tonic-gate ct->ct_runtime = runtime + 1; 291*7c478bd9Sstevel@tonic-gate } 292*7c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 293*7c478bd9Sstevel@tonic-gate } 294*7c478bd9Sstevel@tonic-gate 295*7c478bd9Sstevel@tonic-gate /* 296*7c478bd9Sstevel@tonic-gate * Schedule any callouts that are due on or before this tick. 297*7c478bd9Sstevel@tonic-gate */ 298*7c478bd9Sstevel@tonic-gate static void 299*7c478bd9Sstevel@tonic-gate callout_schedule_1(callout_table_t *ct) 300*7c478bd9Sstevel@tonic-gate { 301*7c478bd9Sstevel@tonic-gate callout_t *cp; 302*7c478bd9Sstevel@tonic-gate clock_t curtime, runtime; 303*7c478bd9Sstevel@tonic-gate 304*7c478bd9Sstevel@tonic-gate mutex_enter(&ct->ct_lock); 305*7c478bd9Sstevel@tonic-gate ct->ct_curtime = curtime = lbolt; 306*7c478bd9Sstevel@tonic-gate while (((runtime = ct->ct_runtime) - curtime) <= 0) { 307*7c478bd9Sstevel@tonic-gate for (cp = ct->ct_lbhash[CALLOUT_LBHASH(runtime)]; 308*7c478bd9Sstevel@tonic-gate cp != NULL; cp = cp->c_lbnext) { 309*7c478bd9Sstevel@tonic-gate if (cp->c_runtime != runtime || 310*7c478bd9Sstevel@tonic-gate (cp->c_xid & CALLOUT_EXECUTING)) 311*7c478bd9Sstevel@tonic-gate continue; 312*7c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 313*7c478bd9Sstevel@tonic-gate if (ct->ct_taskq == NULL) 314*7c478bd9Sstevel@tonic-gate softcall((void (*)(void *))callout_execute, ct); 315*7c478bd9Sstevel@tonic-gate else 316*7c478bd9Sstevel@tonic-gate (void) taskq_dispatch(ct->ct_taskq, 317*7c478bd9Sstevel@tonic-gate (task_func_t *)callout_execute, ct, 318*7c478bd9Sstevel@tonic-gate KM_NOSLEEP); 319*7c478bd9Sstevel@tonic-gate return; 320*7c478bd9Sstevel@tonic-gate } 321*7c478bd9Sstevel@tonic-gate ct->ct_runtime++; 322*7c478bd9Sstevel@tonic-gate } 323*7c478bd9Sstevel@tonic-gate mutex_exit(&ct->ct_lock); 324*7c478bd9Sstevel@tonic-gate } 325*7c478bd9Sstevel@tonic-gate 326*7c478bd9Sstevel@tonic-gate /* 327*7c478bd9Sstevel@tonic-gate * Schedule callouts for all callout tables. Called by clock() on each tick. 328*7c478bd9Sstevel@tonic-gate */ 329*7c478bd9Sstevel@tonic-gate void 330*7c478bd9Sstevel@tonic-gate callout_schedule(void) 331*7c478bd9Sstevel@tonic-gate { 332*7c478bd9Sstevel@tonic-gate int f, t; 333*7c478bd9Sstevel@tonic-gate 334*7c478bd9Sstevel@tonic-gate if (cpr_stop_callout) 335*7c478bd9Sstevel@tonic-gate return; 336*7c478bd9Sstevel@tonic-gate 337*7c478bd9Sstevel@tonic-gate for (t = 0; t < CALLOUT_NTYPES; t++) 338*7c478bd9Sstevel@tonic-gate for (f = 0; f < callout_fanout; f++) 339*7c478bd9Sstevel@tonic-gate callout_schedule_1(callout_table[CALLOUT_TABLE(t, f)]); 340*7c478bd9Sstevel@tonic-gate } 341*7c478bd9Sstevel@tonic-gate 342*7c478bd9Sstevel@tonic-gate /* 343*7c478bd9Sstevel@tonic-gate * Callback handler used by CPR to stop and resume callouts. 344*7c478bd9Sstevel@tonic-gate */ 345*7c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 346*7c478bd9Sstevel@tonic-gate static boolean_t 347*7c478bd9Sstevel@tonic-gate callout_cpr_callb(void *arg, int code) 348*7c478bd9Sstevel@tonic-gate { 349*7c478bd9Sstevel@tonic-gate cpr_stop_callout = (code == CB_CODE_CPR_CHKPT); 350*7c478bd9Sstevel@tonic-gate return (B_TRUE); 351*7c478bd9Sstevel@tonic-gate } 352*7c478bd9Sstevel@tonic-gate 353*7c478bd9Sstevel@tonic-gate /* 354*7c478bd9Sstevel@tonic-gate * Initialize all callout tables. Called at boot time just before clkstart(). 355*7c478bd9Sstevel@tonic-gate */ 356*7c478bd9Sstevel@tonic-gate void 357*7c478bd9Sstevel@tonic-gate callout_init(void) 358*7c478bd9Sstevel@tonic-gate { 359*7c478bd9Sstevel@tonic-gate int f, t; 360*7c478bd9Sstevel@tonic-gate int table_id; 361*7c478bd9Sstevel@tonic-gate callout_table_t *ct; 362*7c478bd9Sstevel@tonic-gate 363*7c478bd9Sstevel@tonic-gate callout_fanout = MIN(CALLOUT_FANOUT, max_ncpus); 364*7c478bd9Sstevel@tonic-gate 365*7c478bd9Sstevel@tonic-gate for (t = 0; t < CALLOUT_NTYPES; t++) { 366*7c478bd9Sstevel@tonic-gate for (f = 0; f < CALLOUT_FANOUT; f++) { 367*7c478bd9Sstevel@tonic-gate table_id = CALLOUT_TABLE(t, f); 368*7c478bd9Sstevel@tonic-gate if (f >= callout_fanout) { 369*7c478bd9Sstevel@tonic-gate callout_table[table_id] = 370*7c478bd9Sstevel@tonic-gate callout_table[table_id - callout_fanout]; 371*7c478bd9Sstevel@tonic-gate continue; 372*7c478bd9Sstevel@tonic-gate } 373*7c478bd9Sstevel@tonic-gate ct = kmem_zalloc(sizeof (callout_table_t), KM_SLEEP); 374*7c478bd9Sstevel@tonic-gate callout_table[table_id] = ct; 375*7c478bd9Sstevel@tonic-gate ct->ct_short_id = (callout_id_t)table_id | 376*7c478bd9Sstevel@tonic-gate CALLOUT_COUNTER_HIGH; 377*7c478bd9Sstevel@tonic-gate ct->ct_long_id = ct->ct_short_id | CALLOUT_LONGTERM; 378*7c478bd9Sstevel@tonic-gate ct->ct_curtime = ct->ct_runtime = lbolt; 379*7c478bd9Sstevel@tonic-gate if (t == CALLOUT_NORMAL) { 380*7c478bd9Sstevel@tonic-gate /* 381*7c478bd9Sstevel@tonic-gate * Each callout thread consumes exactly one 382*7c478bd9Sstevel@tonic-gate * task structure while active. Therefore, 383*7c478bd9Sstevel@tonic-gate * prepopulating with 2 * CALLOUT_THREADS tasks 384*7c478bd9Sstevel@tonic-gate * ensures that there's at least one task per 385*7c478bd9Sstevel@tonic-gate * thread that's either scheduled or on the 386*7c478bd9Sstevel@tonic-gate * freelist. In turn, this guarantees that 387*7c478bd9Sstevel@tonic-gate * taskq_dispatch() will always either succeed 388*7c478bd9Sstevel@tonic-gate * (because there's a free task structure) or 389*7c478bd9Sstevel@tonic-gate * be unnecessary (because "callout_excute(ct)" 390*7c478bd9Sstevel@tonic-gate * has already scheduled). 391*7c478bd9Sstevel@tonic-gate */ 392*7c478bd9Sstevel@tonic-gate ct->ct_taskq = 393*7c478bd9Sstevel@tonic-gate taskq_create_instance("callout_taskq", f, 394*7c478bd9Sstevel@tonic-gate CALLOUT_THREADS, maxclsyspri, 395*7c478bd9Sstevel@tonic-gate 2 * CALLOUT_THREADS, 2 * CALLOUT_THREADS, 396*7c478bd9Sstevel@tonic-gate TASKQ_PREPOPULATE | TASKQ_CPR_SAFE); 397*7c478bd9Sstevel@tonic-gate } 398*7c478bd9Sstevel@tonic-gate } 399*7c478bd9Sstevel@tonic-gate } 400*7c478bd9Sstevel@tonic-gate (void) callb_add(callout_cpr_callb, 0, CB_CL_CPR_CALLOUT, "callout"); 401*7c478bd9Sstevel@tonic-gate } 402