17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
5f635d46aSqiao * Common Development and Distribution License (the "License").
6f635d46aSqiao * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
217c478bd9Sstevel@tonic-gate /*
226e00b116SPeter Telford * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23*48bbca81SDaniel Hoffman * Copyright (c) 2016 by Delphix. All rights reserved.
247c478bd9Sstevel@tonic-gate */
257c478bd9Sstevel@tonic-gate
267c478bd9Sstevel@tonic-gate #include <sys/callo.h>
277c478bd9Sstevel@tonic-gate #include <sys/param.h>
287c478bd9Sstevel@tonic-gate #include <sys/types.h>
297c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
307c478bd9Sstevel@tonic-gate #include <sys/thread.h>
317c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
3287a18d3fSMadhavan Venkataraman #include <sys/kmem_impl.h>
337c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
347c478bd9Sstevel@tonic-gate #include <sys/callb.h>
357c478bd9Sstevel@tonic-gate #include <sys/debug.h>
367c478bd9Sstevel@tonic-gate #include <sys/vtrace.h>
377c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
387c478bd9Sstevel@tonic-gate #include <sys/sdt.h>
397c478bd9Sstevel@tonic-gate
406e00b116SPeter Telford int callout_init_done; /* useful during boot */
416e00b116SPeter Telford
427c478bd9Sstevel@tonic-gate /*
437c478bd9Sstevel@tonic-gate * Callout tables. See timeout(9F) for details.
447c478bd9Sstevel@tonic-gate */
4551b32bddSMadhavan Venkataraman static int callout_threads; /* callout normal threads */
4687a18d3fSMadhavan Venkataraman static hrtime_t callout_debug_hrtime; /* debugger entry time */
47060cedfbSMadhavan Venkataraman static int callout_chunk; /* callout heap chunk size */
4851b32bddSMadhavan Venkataraman static int callout_min_reap; /* callout minimum reap count */
4951b32bddSMadhavan Venkataraman static int callout_tolerance; /* callout hires tolerance */
5087a18d3fSMadhavan Venkataraman static callout_table_t *callout_boot_ct; /* Boot CPU's callout tables */
51454ab202SMadhavan Venkataraman static clock_t callout_max_ticks; /* max interval */
5287a18d3fSMadhavan Venkataraman static hrtime_t callout_longterm; /* longterm nanoseconds */
5387a18d3fSMadhavan Venkataraman static ulong_t callout_counter_low; /* callout ID increment */
5487a18d3fSMadhavan Venkataraman static ulong_t callout_table_bits; /* number of table bits in ID */
5587a18d3fSMadhavan Venkataraman static ulong_t callout_table_mask; /* mask for the table bits */
5687a18d3fSMadhavan Venkataraman static callout_cache_t *callout_caches; /* linked list of caches */
5787a18d3fSMadhavan Venkataraman #pragma align 64(callout_table)
5887a18d3fSMadhavan Venkataraman static callout_table_t *callout_table; /* global callout table array */
597c478bd9Sstevel@tonic-gate
6007247649SMadhavan Venkataraman /*
614c06356bSdh142964 * We run 'realtime' callouts at PIL 1 (CY_LOW_LEVEL). For 'normal'
624c06356bSdh142964 * callouts, from PIL 10 (CY_LOCK_LEVEL) we dispatch the callout,
634c06356bSdh142964 * via taskq, to a thread that executes at PIL 0 - so we end up running
644c06356bSdh142964 * 'normal' callouts at PIL 0.
6507247649SMadhavan Venkataraman */
6651b32bddSMadhavan Venkataraman static volatile int callout_realtime_level = CY_LOW_LEVEL;
6751b32bddSMadhavan Venkataraman static volatile int callout_normal_level = CY_LOCK_LEVEL;
6807247649SMadhavan Venkataraman
6987a18d3fSMadhavan Venkataraman static char *callout_kstat_names[] = {
7087a18d3fSMadhavan Venkataraman "callout_timeouts",
7187a18d3fSMadhavan Venkataraman "callout_timeouts_pending",
7287a18d3fSMadhavan Venkataraman "callout_untimeouts_unexpired",
7387a18d3fSMadhavan Venkataraman "callout_untimeouts_executing",
7487a18d3fSMadhavan Venkataraman "callout_untimeouts_expired",
7587a18d3fSMadhavan Venkataraman "callout_expirations",
7687a18d3fSMadhavan Venkataraman "callout_allocations",
7751b32bddSMadhavan Venkataraman "callout_cleanups",
7887a18d3fSMadhavan Venkataraman };
7987a18d3fSMadhavan Venkataraman
8051b32bddSMadhavan Venkataraman static hrtime_t callout_heap_process(callout_table_t *, hrtime_t, int);
8151b32bddSMadhavan Venkataraman
8287a18d3fSMadhavan Venkataraman #define CALLOUT_HASH_INSERT(hash, cp, cnext, cprev) \
837c478bd9Sstevel@tonic-gate { \
8487a18d3fSMadhavan Venkataraman callout_hash_t *hashp = &(hash); \
8587a18d3fSMadhavan Venkataraman \
867c478bd9Sstevel@tonic-gate cp->cprev = NULL; \
8787a18d3fSMadhavan Venkataraman cp->cnext = hashp->ch_head; \
8887a18d3fSMadhavan Venkataraman if (hashp->ch_head == NULL) \
8987a18d3fSMadhavan Venkataraman hashp->ch_tail = cp; \
907c478bd9Sstevel@tonic-gate else \
9187a18d3fSMadhavan Venkataraman cp->cnext->cprev = cp; \
9287a18d3fSMadhavan Venkataraman hashp->ch_head = cp; \
937c478bd9Sstevel@tonic-gate }
947c478bd9Sstevel@tonic-gate
9587a18d3fSMadhavan Venkataraman #define CALLOUT_HASH_APPEND(hash, cp, cnext, cprev) \
9687a18d3fSMadhavan Venkataraman { \
9787a18d3fSMadhavan Venkataraman callout_hash_t *hashp = &(hash); \
9887a18d3fSMadhavan Venkataraman \
9987a18d3fSMadhavan Venkataraman cp->cnext = NULL; \
10087a18d3fSMadhavan Venkataraman cp->cprev = hashp->ch_tail; \
10187a18d3fSMadhavan Venkataraman if (hashp->ch_tail == NULL) \
10287a18d3fSMadhavan Venkataraman hashp->ch_head = cp; \
10387a18d3fSMadhavan Venkataraman else \
10487a18d3fSMadhavan Venkataraman cp->cprev->cnext = cp; \
10587a18d3fSMadhavan Venkataraman hashp->ch_tail = cp; \
10687a18d3fSMadhavan Venkataraman }
10787a18d3fSMadhavan Venkataraman
10887a18d3fSMadhavan Venkataraman #define CALLOUT_HASH_DELETE(hash, cp, cnext, cprev) \
10987a18d3fSMadhavan Venkataraman { \
11087a18d3fSMadhavan Venkataraman callout_hash_t *hashp = &(hash); \
11187a18d3fSMadhavan Venkataraman \
11287a18d3fSMadhavan Venkataraman if (cp->cnext == NULL) \
11387a18d3fSMadhavan Venkataraman hashp->ch_tail = cp->cprev; \
11487a18d3fSMadhavan Venkataraman else \
11587a18d3fSMadhavan Venkataraman cp->cnext->cprev = cp->cprev; \
11687a18d3fSMadhavan Venkataraman if (cp->cprev == NULL) \
11787a18d3fSMadhavan Venkataraman hashp->ch_head = cp->cnext; \
11887a18d3fSMadhavan Venkataraman else \
11987a18d3fSMadhavan Venkataraman cp->cprev->cnext = cp->cnext; \
12087a18d3fSMadhavan Venkataraman }
12187a18d3fSMadhavan Venkataraman
12287a18d3fSMadhavan Venkataraman /*
12387a18d3fSMadhavan Venkataraman * These definitions help us queue callouts and callout lists. Here is
12487a18d3fSMadhavan Venkataraman * the queueing rationale:
12587a18d3fSMadhavan Venkataraman *
12687a18d3fSMadhavan Venkataraman * - callouts are queued in a FIFO manner in the ID hash table.
12787a18d3fSMadhavan Venkataraman * TCP timers are typically cancelled in the same order that they
12887a18d3fSMadhavan Venkataraman * were issued. The FIFO queueing shortens the search for a callout
12987a18d3fSMadhavan Venkataraman * during untimeout().
13087a18d3fSMadhavan Venkataraman *
13187a18d3fSMadhavan Venkataraman * - callouts are queued in a FIFO manner in their callout lists.
13287a18d3fSMadhavan Venkataraman * This ensures that the callouts are executed in the same order that
13387a18d3fSMadhavan Venkataraman * they were queued. This is fair. Plus, it helps to make each
13487a18d3fSMadhavan Venkataraman * callout expiration timely. It also favors cancellations.
13587a18d3fSMadhavan Venkataraman *
13651b32bddSMadhavan Venkataraman * - callout lists are queued in the following manner in the callout
13751b32bddSMadhavan Venkataraman * hash table buckets:
13851b32bddSMadhavan Venkataraman *
13951b32bddSMadhavan Venkataraman * - appended, if the callout list is a 1-nanosecond resolution
14051b32bddSMadhavan Venkataraman * callout list. When a callout is created, we first look for
14151b32bddSMadhavan Venkataraman * a callout list that has the same expiration so we can avoid
14251b32bddSMadhavan Venkataraman * allocating a callout list and inserting the expiration into
14351b32bddSMadhavan Venkataraman * the heap. However, we do not want to look at 1-nanosecond
14451b32bddSMadhavan Venkataraman * resolution callout lists as we will seldom find a match in
14551b32bddSMadhavan Venkataraman * them. Keeping these callout lists in the rear of the hash
14651b32bddSMadhavan Venkataraman * buckets allows us to skip these during the lookup.
14751b32bddSMadhavan Venkataraman *
14851b32bddSMadhavan Venkataraman * - inserted at the beginning, if the callout list is not a
14951b32bddSMadhavan Venkataraman * 1-nanosecond resolution callout list. This also has the
15051b32bddSMadhavan Venkataraman * side-effect of keeping the long term timers away from the
15151b32bddSMadhavan Venkataraman * front of the buckets.
15287a18d3fSMadhavan Venkataraman *
15387a18d3fSMadhavan Venkataraman * - callout lists are queued in a FIFO manner in the expired callouts
15487a18d3fSMadhavan Venkataraman * list. This ensures that callout lists are executed in the order
15587a18d3fSMadhavan Venkataraman * of expiration.
15687a18d3fSMadhavan Venkataraman */
15787a18d3fSMadhavan Venkataraman #define CALLOUT_APPEND(ct, cp) \
15887a18d3fSMadhavan Venkataraman CALLOUT_HASH_APPEND(ct->ct_idhash[CALLOUT_IDHASH(cp->c_xid)], \
15987a18d3fSMadhavan Venkataraman cp, c_idnext, c_idprev); \
16087a18d3fSMadhavan Venkataraman CALLOUT_HASH_APPEND(cp->c_list->cl_callouts, cp, c_clnext, c_clprev)
16187a18d3fSMadhavan Venkataraman
16287a18d3fSMadhavan Venkataraman #define CALLOUT_DELETE(ct, cp) \
16387a18d3fSMadhavan Venkataraman CALLOUT_HASH_DELETE(ct->ct_idhash[CALLOUT_IDHASH(cp->c_xid)], \
16487a18d3fSMadhavan Venkataraman cp, c_idnext, c_idprev); \
16587a18d3fSMadhavan Venkataraman CALLOUT_HASH_DELETE(cp->c_list->cl_callouts, cp, c_clnext, c_clprev)
16687a18d3fSMadhavan Venkataraman
16787a18d3fSMadhavan Venkataraman #define CALLOUT_LIST_INSERT(hash, cl) \
16887a18d3fSMadhavan Venkataraman CALLOUT_HASH_INSERT(hash, cl, cl_next, cl_prev)
16987a18d3fSMadhavan Venkataraman
17087a18d3fSMadhavan Venkataraman #define CALLOUT_LIST_APPEND(hash, cl) \
17187a18d3fSMadhavan Venkataraman CALLOUT_HASH_APPEND(hash, cl, cl_next, cl_prev)
17287a18d3fSMadhavan Venkataraman
17387a18d3fSMadhavan Venkataraman #define CALLOUT_LIST_DELETE(hash, cl) \
17487a18d3fSMadhavan Venkataraman CALLOUT_HASH_DELETE(hash, cl, cl_next, cl_prev)
1757c478bd9Sstevel@tonic-gate
176060cedfbSMadhavan Venkataraman #define CALLOUT_LIST_BEFORE(cl, nextcl) \
177060cedfbSMadhavan Venkataraman { \
178060cedfbSMadhavan Venkataraman (cl)->cl_prev = (nextcl)->cl_prev; \
179060cedfbSMadhavan Venkataraman (cl)->cl_next = (nextcl); \
180060cedfbSMadhavan Venkataraman (nextcl)->cl_prev = (cl); \
181060cedfbSMadhavan Venkataraman if (cl->cl_prev != NULL) \
182060cedfbSMadhavan Venkataraman cl->cl_prev->cl_next = cl; \
183060cedfbSMadhavan Venkataraman }
184060cedfbSMadhavan Venkataraman
1857c478bd9Sstevel@tonic-gate /*
18607247649SMadhavan Venkataraman * For normal callouts, there is a deadlock scenario if two callouts that
18707247649SMadhavan Venkataraman * have an inter-dependency end up on the same callout list. To break the
18807247649SMadhavan Venkataraman * deadlock, you need two taskq threads running in parallel. We compute
18907247649SMadhavan Venkataraman * the number of taskq threads here using a bunch of conditions to make
19007247649SMadhavan Venkataraman * it optimal for the common case. This is an ugly hack, but one that is
19107247649SMadhavan Venkataraman * necessary (sigh).
19207247649SMadhavan Venkataraman */
19307247649SMadhavan Venkataraman #define CALLOUT_THRESHOLD 100000000
194060cedfbSMadhavan Venkataraman #define CALLOUT_EXEC_COMPUTE(ct, nextexp, exec) \
19507247649SMadhavan Venkataraman { \
19607247649SMadhavan Venkataraman callout_list_t *cl; \
19707247649SMadhavan Venkataraman \
19807247649SMadhavan Venkataraman cl = ct->ct_expired.ch_head; \
19907247649SMadhavan Venkataraman if (cl == NULL) { \
20007247649SMadhavan Venkataraman /* \
20107247649SMadhavan Venkataraman * If the expired list is NULL, there is nothing to \
20207247649SMadhavan Venkataraman * process. \
20307247649SMadhavan Venkataraman */ \
20407247649SMadhavan Venkataraman exec = 0; \
20507247649SMadhavan Venkataraman } else if ((cl->cl_next == NULL) && \
20607247649SMadhavan Venkataraman (cl->cl_callouts.ch_head == cl->cl_callouts.ch_tail)) { \
20707247649SMadhavan Venkataraman /* \
20807247649SMadhavan Venkataraman * If there is only one callout list and it contains \
20907247649SMadhavan Venkataraman * only one callout, there is no need for two threads. \
21007247649SMadhavan Venkataraman */ \
21107247649SMadhavan Venkataraman exec = 1; \
212060cedfbSMadhavan Venkataraman } else if ((nextexp) > (gethrtime() + CALLOUT_THRESHOLD)) { \
21307247649SMadhavan Venkataraman /* \
214060cedfbSMadhavan Venkataraman * If the next expiration of the cyclic is way out into \
215060cedfbSMadhavan Venkataraman * the future, we need two threads. \
21607247649SMadhavan Venkataraman */ \
21707247649SMadhavan Venkataraman exec = 2; \
21807247649SMadhavan Venkataraman } else { \
21907247649SMadhavan Venkataraman /* \
22007247649SMadhavan Venkataraman * We have multiple callouts to process. But the cyclic \
22107247649SMadhavan Venkataraman * will fire in the near future. So, we only need one \
22207247649SMadhavan Venkataraman * thread for now. \
22307247649SMadhavan Venkataraman */ \
22407247649SMadhavan Venkataraman exec = 1; \
22507247649SMadhavan Venkataraman } \
22607247649SMadhavan Venkataraman }
22707247649SMadhavan Venkataraman
22807247649SMadhavan Venkataraman /*
22951b32bddSMadhavan Venkataraman * Macro to swap two heap items.
23051b32bddSMadhavan Venkataraman */
23151b32bddSMadhavan Venkataraman #define CALLOUT_SWAP(h1, h2) \
23251b32bddSMadhavan Venkataraman { \
23351b32bddSMadhavan Venkataraman callout_heap_t tmp; \
23451b32bddSMadhavan Venkataraman \
23551b32bddSMadhavan Venkataraman tmp = *h1; \
23651b32bddSMadhavan Venkataraman *h1 = *h2; \
23751b32bddSMadhavan Venkataraman *h2 = tmp; \
23851b32bddSMadhavan Venkataraman }
23951b32bddSMadhavan Venkataraman
24051b32bddSMadhavan Venkataraman /*
24151b32bddSMadhavan Venkataraman * Macro to free a callout list.
24251b32bddSMadhavan Venkataraman */
24351b32bddSMadhavan Venkataraman #define CALLOUT_LIST_FREE(ct, cl) \
24451b32bddSMadhavan Venkataraman { \
24551b32bddSMadhavan Venkataraman cl->cl_next = ct->ct_lfree; \
24651b32bddSMadhavan Venkataraman ct->ct_lfree = cl; \
24751b32bddSMadhavan Venkataraman cl->cl_flags |= CALLOUT_LIST_FLAG_FREE; \
24851b32bddSMadhavan Venkataraman }
24951b32bddSMadhavan Venkataraman
25051b32bddSMadhavan Venkataraman /*
251060cedfbSMadhavan Venkataraman * Macro to free a callout.
252060cedfbSMadhavan Venkataraman */
253060cedfbSMadhavan Venkataraman #define CALLOUT_FREE(ct, cl) \
254060cedfbSMadhavan Venkataraman { \
255060cedfbSMadhavan Venkataraman cp->c_idnext = ct->ct_free; \
256060cedfbSMadhavan Venkataraman ct->ct_free = cp; \
257060cedfbSMadhavan Venkataraman cp->c_xid |= CALLOUT_ID_FREE; \
258060cedfbSMadhavan Venkataraman }
259060cedfbSMadhavan Venkataraman
260060cedfbSMadhavan Venkataraman /*
2617c478bd9Sstevel@tonic-gate * Allocate a callout structure. We try quite hard because we
2627c478bd9Sstevel@tonic-gate * can't sleep, and if we can't do the allocation, we're toast.
26387a18d3fSMadhavan Venkataraman * Failing all, we try a KM_PANIC allocation. Note that we never
26487a18d3fSMadhavan Venkataraman * deallocate a callout. See untimeout() for the reasoning.
2657c478bd9Sstevel@tonic-gate */
2667c478bd9Sstevel@tonic-gate static callout_t *
callout_alloc(callout_table_t * ct)2677c478bd9Sstevel@tonic-gate callout_alloc(callout_table_t *ct)
2687c478bd9Sstevel@tonic-gate {
26987a18d3fSMadhavan Venkataraman size_t size;
27087a18d3fSMadhavan Venkataraman callout_t *cp;
2717c478bd9Sstevel@tonic-gate
27287a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex));
27387a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
27487a18d3fSMadhavan Venkataraman
27587a18d3fSMadhavan Venkataraman cp = kmem_cache_alloc(ct->ct_cache, KM_NOSLEEP);
27687a18d3fSMadhavan Venkataraman if (cp == NULL) {
27787a18d3fSMadhavan Venkataraman size = sizeof (callout_t);
27887a18d3fSMadhavan Venkataraman cp = kmem_alloc_tryhard(size, &size, KM_NOSLEEP | KM_PANIC);
27987a18d3fSMadhavan Venkataraman }
28087a18d3fSMadhavan Venkataraman cp->c_xid = 0;
28107247649SMadhavan Venkataraman cp->c_executor = NULL;
28207247649SMadhavan Venkataraman cv_init(&cp->c_done, NULL, CV_DEFAULT, NULL);
28307247649SMadhavan Venkataraman cp->c_waiting = 0;
28487a18d3fSMadhavan Venkataraman
28587a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex);
28687a18d3fSMadhavan Venkataraman ct->ct_allocations++;
2877c478bd9Sstevel@tonic-gate return (cp);
2887c478bd9Sstevel@tonic-gate }
2897c478bd9Sstevel@tonic-gate
2907c478bd9Sstevel@tonic-gate /*
29187a18d3fSMadhavan Venkataraman * Allocate a callout list structure. We try quite hard because we
29287a18d3fSMadhavan Venkataraman * can't sleep, and if we can't do the allocation, we're toast.
29387a18d3fSMadhavan Venkataraman * Failing all, we try a KM_PANIC allocation. Note that we never
29487a18d3fSMadhavan Venkataraman * deallocate a callout list.
2957c478bd9Sstevel@tonic-gate */
29687a18d3fSMadhavan Venkataraman static void
callout_list_alloc(callout_table_t * ct)29787a18d3fSMadhavan Venkataraman callout_list_alloc(callout_table_t *ct)
2987c478bd9Sstevel@tonic-gate {
29987a18d3fSMadhavan Venkataraman size_t size;
30087a18d3fSMadhavan Venkataraman callout_list_t *cl;
30187a18d3fSMadhavan Venkataraman
30287a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex));
30387a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
30487a18d3fSMadhavan Venkataraman
30587a18d3fSMadhavan Venkataraman cl = kmem_cache_alloc(ct->ct_lcache, KM_NOSLEEP);
30687a18d3fSMadhavan Venkataraman if (cl == NULL) {
30787a18d3fSMadhavan Venkataraman size = sizeof (callout_list_t);
30887a18d3fSMadhavan Venkataraman cl = kmem_alloc_tryhard(size, &size, KM_NOSLEEP | KM_PANIC);
30987a18d3fSMadhavan Venkataraman }
31087a18d3fSMadhavan Venkataraman bzero(cl, sizeof (callout_list_t));
31187a18d3fSMadhavan Venkataraman
31287a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex);
31351b32bddSMadhavan Venkataraman CALLOUT_LIST_FREE(ct, cl);
31487a18d3fSMadhavan Venkataraman }
31587a18d3fSMadhavan Venkataraman
31687a18d3fSMadhavan Venkataraman /*
31751b32bddSMadhavan Venkataraman * Find a callout list that corresponds to an expiration and matching flags.
31887a18d3fSMadhavan Venkataraman */
31987a18d3fSMadhavan Venkataraman static callout_list_t *
callout_list_get(callout_table_t * ct,hrtime_t expiration,int flags,int hash)32007247649SMadhavan Venkataraman callout_list_get(callout_table_t *ct, hrtime_t expiration, int flags, int hash)
32187a18d3fSMadhavan Venkataraman {
32287a18d3fSMadhavan Venkataraman callout_list_t *cl;
32351b32bddSMadhavan Venkataraman int clflags;
32487a18d3fSMadhavan Venkataraman
32587a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex));
32687a18d3fSMadhavan Venkataraman
32751b32bddSMadhavan Venkataraman if (flags & CALLOUT_LIST_FLAG_NANO) {
32851b32bddSMadhavan Venkataraman /*
32951b32bddSMadhavan Venkataraman * This is a 1-nanosecond resolution callout. We will rarely
33051b32bddSMadhavan Venkataraman * find a match for this. So, bail out.
33151b32bddSMadhavan Venkataraman */
33251b32bddSMadhavan Venkataraman return (NULL);
33351b32bddSMadhavan Venkataraman }
33451b32bddSMadhavan Venkataraman
33551b32bddSMadhavan Venkataraman clflags = (CALLOUT_LIST_FLAG_ABSOLUTE | CALLOUT_LIST_FLAG_HRESTIME);
33687a18d3fSMadhavan Venkataraman for (cl = ct->ct_clhash[hash].ch_head; (cl != NULL); cl = cl->cl_next) {
33751b32bddSMadhavan Venkataraman /*
33851b32bddSMadhavan Venkataraman * If we have reached a 1-nanosecond resolution callout list,
33951b32bddSMadhavan Venkataraman * we don't have much hope of finding a match in this hash
34051b32bddSMadhavan Venkataraman * bucket. So, just bail out.
34151b32bddSMadhavan Venkataraman */
34251b32bddSMadhavan Venkataraman if (cl->cl_flags & CALLOUT_LIST_FLAG_NANO)
34351b32bddSMadhavan Venkataraman return (NULL);
34451b32bddSMadhavan Venkataraman
34507247649SMadhavan Venkataraman if ((cl->cl_expiration == expiration) &&
34651b32bddSMadhavan Venkataraman ((cl->cl_flags & clflags) == (flags & clflags)))
34787a18d3fSMadhavan Venkataraman return (cl);
34887a18d3fSMadhavan Venkataraman }
34987a18d3fSMadhavan Venkataraman
35087a18d3fSMadhavan Venkataraman return (NULL);
35187a18d3fSMadhavan Venkataraman }
35287a18d3fSMadhavan Venkataraman
35387a18d3fSMadhavan Venkataraman /*
354060cedfbSMadhavan Venkataraman * Add a new callout list into a callout table's queue in sorted order by
355060cedfbSMadhavan Venkataraman * expiration.
356060cedfbSMadhavan Venkataraman */
357060cedfbSMadhavan Venkataraman static int
callout_queue_add(callout_table_t * ct,callout_list_t * cl)358060cedfbSMadhavan Venkataraman callout_queue_add(callout_table_t *ct, callout_list_t *cl)
359060cedfbSMadhavan Venkataraman {
360060cedfbSMadhavan Venkataraman callout_list_t *nextcl;
361060cedfbSMadhavan Venkataraman hrtime_t expiration;
362060cedfbSMadhavan Venkataraman
363060cedfbSMadhavan Venkataraman expiration = cl->cl_expiration;
364060cedfbSMadhavan Venkataraman nextcl = ct->ct_queue.ch_head;
365060cedfbSMadhavan Venkataraman if ((nextcl == NULL) || (expiration < nextcl->cl_expiration)) {
366060cedfbSMadhavan Venkataraman CALLOUT_LIST_INSERT(ct->ct_queue, cl);
367060cedfbSMadhavan Venkataraman return (1);
368060cedfbSMadhavan Venkataraman }
369060cedfbSMadhavan Venkataraman
370060cedfbSMadhavan Venkataraman while (nextcl != NULL) {
371060cedfbSMadhavan Venkataraman if (expiration < nextcl->cl_expiration) {
372060cedfbSMadhavan Venkataraman CALLOUT_LIST_BEFORE(cl, nextcl);
373060cedfbSMadhavan Venkataraman return (0);
374060cedfbSMadhavan Venkataraman }
375060cedfbSMadhavan Venkataraman nextcl = nextcl->cl_next;
376060cedfbSMadhavan Venkataraman }
377060cedfbSMadhavan Venkataraman CALLOUT_LIST_APPEND(ct->ct_queue, cl);
378060cedfbSMadhavan Venkataraman
379060cedfbSMadhavan Venkataraman return (0);
380060cedfbSMadhavan Venkataraman }
381060cedfbSMadhavan Venkataraman
382060cedfbSMadhavan Venkataraman /*
383060cedfbSMadhavan Venkataraman * Insert a callout list into a callout table's queue and reprogram the queue
384060cedfbSMadhavan Venkataraman * cyclic if needed.
385060cedfbSMadhavan Venkataraman */
386060cedfbSMadhavan Venkataraman static void
callout_queue_insert(callout_table_t * ct,callout_list_t * cl)387060cedfbSMadhavan Venkataraman callout_queue_insert(callout_table_t *ct, callout_list_t *cl)
388060cedfbSMadhavan Venkataraman {
389060cedfbSMadhavan Venkataraman cl->cl_flags |= CALLOUT_LIST_FLAG_QUEUED;
390060cedfbSMadhavan Venkataraman
391060cedfbSMadhavan Venkataraman /*
392060cedfbSMadhavan Venkataraman * Add the callout to the callout queue. If it ends up at the head,
393060cedfbSMadhavan Venkataraman * the cyclic needs to be reprogrammed as we have an earlier
394060cedfbSMadhavan Venkataraman * expiration.
395060cedfbSMadhavan Venkataraman *
396060cedfbSMadhavan Venkataraman * Also, during the CPR suspend phase, do not reprogram the cyclic.
397060cedfbSMadhavan Venkataraman * We don't want any callout activity. When the CPR resume phase is
398060cedfbSMadhavan Venkataraman * entered, the cyclic will be programmed for the earliest expiration
399060cedfbSMadhavan Venkataraman * in the queue.
400060cedfbSMadhavan Venkataraman */
401060cedfbSMadhavan Venkataraman if (callout_queue_add(ct, cl) && (ct->ct_suspend == 0))
402060cedfbSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_qcyclic, cl->cl_expiration);
403060cedfbSMadhavan Venkataraman }
404060cedfbSMadhavan Venkataraman
405060cedfbSMadhavan Venkataraman /*
406060cedfbSMadhavan Venkataraman * Delete and handle all past expirations in a callout table's queue.
407060cedfbSMadhavan Venkataraman */
408060cedfbSMadhavan Venkataraman static hrtime_t
callout_queue_delete(callout_table_t * ct)409060cedfbSMadhavan Venkataraman callout_queue_delete(callout_table_t *ct)
410060cedfbSMadhavan Venkataraman {
411060cedfbSMadhavan Venkataraman callout_list_t *cl;
412060cedfbSMadhavan Venkataraman hrtime_t now;
413060cedfbSMadhavan Venkataraman
414060cedfbSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex));
415060cedfbSMadhavan Venkataraman
416060cedfbSMadhavan Venkataraman now = gethrtime();
417060cedfbSMadhavan Venkataraman while ((cl = ct->ct_queue.ch_head) != NULL) {
418060cedfbSMadhavan Venkataraman if (cl->cl_expiration > now)
419060cedfbSMadhavan Venkataraman break;
420060cedfbSMadhavan Venkataraman cl->cl_flags &= ~CALLOUT_LIST_FLAG_QUEUED;
421060cedfbSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_queue, cl);
422060cedfbSMadhavan Venkataraman CALLOUT_LIST_APPEND(ct->ct_expired, cl);
423060cedfbSMadhavan Venkataraman }
424060cedfbSMadhavan Venkataraman
425060cedfbSMadhavan Venkataraman /*
426060cedfbSMadhavan Venkataraman * If this callout queue is empty or callouts have been suspended,
427060cedfbSMadhavan Venkataraman * just return.
428060cedfbSMadhavan Venkataraman */
429060cedfbSMadhavan Venkataraman if ((cl == NULL) || (ct->ct_suspend > 0))
430060cedfbSMadhavan Venkataraman return (CY_INFINITY);
431060cedfbSMadhavan Venkataraman
432060cedfbSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_qcyclic, cl->cl_expiration);
433060cedfbSMadhavan Venkataraman
434060cedfbSMadhavan Venkataraman return (cl->cl_expiration);
435060cedfbSMadhavan Venkataraman }
436060cedfbSMadhavan Venkataraman
437060cedfbSMadhavan Venkataraman static hrtime_t
callout_queue_process(callout_table_t * ct,hrtime_t delta,int timechange)438060cedfbSMadhavan Venkataraman callout_queue_process(callout_table_t *ct, hrtime_t delta, int timechange)
439060cedfbSMadhavan Venkataraman {
440060cedfbSMadhavan Venkataraman callout_list_t *firstcl, *cl;
441060cedfbSMadhavan Venkataraman hrtime_t expiration, now;
442060cedfbSMadhavan Venkataraman int clflags;
443060cedfbSMadhavan Venkataraman callout_hash_t temp;
444060cedfbSMadhavan Venkataraman
445060cedfbSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex));
446060cedfbSMadhavan Venkataraman
447060cedfbSMadhavan Venkataraman firstcl = ct->ct_queue.ch_head;
448060cedfbSMadhavan Venkataraman if (firstcl == NULL)
449060cedfbSMadhavan Venkataraman return (CY_INFINITY);
450060cedfbSMadhavan Venkataraman
451060cedfbSMadhavan Venkataraman /*
452060cedfbSMadhavan Venkataraman * We walk the callout queue. If we encounter a hrestime entry that
453060cedfbSMadhavan Venkataraman * must be removed, we clean it out. Otherwise, we apply any
454060cedfbSMadhavan Venkataraman * adjustments needed to it. Because of the latter, we need to
455060cedfbSMadhavan Venkataraman * recreate the list as we go along.
456060cedfbSMadhavan Venkataraman */
457060cedfbSMadhavan Venkataraman temp = ct->ct_queue;
458060cedfbSMadhavan Venkataraman ct->ct_queue.ch_head = NULL;
459060cedfbSMadhavan Venkataraman ct->ct_queue.ch_tail = NULL;
460060cedfbSMadhavan Venkataraman
461060cedfbSMadhavan Venkataraman clflags = (CALLOUT_LIST_FLAG_HRESTIME | CALLOUT_LIST_FLAG_ABSOLUTE);
462060cedfbSMadhavan Venkataraman now = gethrtime();
463060cedfbSMadhavan Venkataraman while ((cl = temp.ch_head) != NULL) {
464060cedfbSMadhavan Venkataraman CALLOUT_LIST_DELETE(temp, cl);
465060cedfbSMadhavan Venkataraman
466060cedfbSMadhavan Venkataraman /*
467060cedfbSMadhavan Venkataraman * Delete the callout and expire it, if one of the following
468060cedfbSMadhavan Venkataraman * is true:
469060cedfbSMadhavan Venkataraman * - the callout has expired
470060cedfbSMadhavan Venkataraman * - the callout is an absolute hrestime one and
471060cedfbSMadhavan Venkataraman * there has been a system time change
472060cedfbSMadhavan Venkataraman */
473060cedfbSMadhavan Venkataraman if ((cl->cl_expiration <= now) ||
474060cedfbSMadhavan Venkataraman (timechange && ((cl->cl_flags & clflags) == clflags))) {
475060cedfbSMadhavan Venkataraman cl->cl_flags &= ~CALLOUT_LIST_FLAG_QUEUED;
476060cedfbSMadhavan Venkataraman CALLOUT_LIST_APPEND(ct->ct_expired, cl);
477060cedfbSMadhavan Venkataraman continue;
478060cedfbSMadhavan Venkataraman }
479060cedfbSMadhavan Venkataraman
480060cedfbSMadhavan Venkataraman /*
481060cedfbSMadhavan Venkataraman * Apply adjustments, if any. Adjustments are applied after
482060cedfbSMadhavan Venkataraman * the system returns from KMDB or OBP. They are only applied
483060cedfbSMadhavan Venkataraman * to relative callout lists.
484060cedfbSMadhavan Venkataraman */
485060cedfbSMadhavan Venkataraman if (delta && !(cl->cl_flags & CALLOUT_LIST_FLAG_ABSOLUTE)) {
486060cedfbSMadhavan Venkataraman expiration = cl->cl_expiration + delta;
487060cedfbSMadhavan Venkataraman if (expiration <= 0)
488060cedfbSMadhavan Venkataraman expiration = CY_INFINITY;
489060cedfbSMadhavan Venkataraman cl->cl_expiration = expiration;
490060cedfbSMadhavan Venkataraman }
491060cedfbSMadhavan Venkataraman
492060cedfbSMadhavan Venkataraman (void) callout_queue_add(ct, cl);
493060cedfbSMadhavan Venkataraman }
494060cedfbSMadhavan Venkataraman
495060cedfbSMadhavan Venkataraman /*
496060cedfbSMadhavan Venkataraman * We need to return the expiration to help program the cyclic.
497060cedfbSMadhavan Venkataraman * If there are expired callouts, the cyclic needs to go off
498060cedfbSMadhavan Venkataraman * immediately. If the queue has become empty, then we return infinity.
499060cedfbSMadhavan Venkataraman * Else, we return the expiration of the earliest callout in the queue.
500060cedfbSMadhavan Venkataraman */
501060cedfbSMadhavan Venkataraman if (ct->ct_expired.ch_head != NULL)
502060cedfbSMadhavan Venkataraman return (gethrtime());
503060cedfbSMadhavan Venkataraman
504060cedfbSMadhavan Venkataraman cl = ct->ct_queue.ch_head;
505060cedfbSMadhavan Venkataraman if (cl == NULL)
506060cedfbSMadhavan Venkataraman return (CY_INFINITY);
507060cedfbSMadhavan Venkataraman
508060cedfbSMadhavan Venkataraman return (cl->cl_expiration);
509060cedfbSMadhavan Venkataraman }
510060cedfbSMadhavan Venkataraman
511060cedfbSMadhavan Venkataraman /*
51287a18d3fSMadhavan Venkataraman * Initialize a callout table's heap, if necessary. Preallocate some free
51387a18d3fSMadhavan Venkataraman * entries so we don't have to check for NULL elsewhere.
51487a18d3fSMadhavan Venkataraman */
51587a18d3fSMadhavan Venkataraman static void
callout_heap_init(callout_table_t * ct)51687a18d3fSMadhavan Venkataraman callout_heap_init(callout_table_t *ct)
51787a18d3fSMadhavan Venkataraman {
51887a18d3fSMadhavan Venkataraman size_t size;
51987a18d3fSMadhavan Venkataraman
52087a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex));
52187a18d3fSMadhavan Venkataraman ASSERT(ct->ct_heap == NULL);
52287a18d3fSMadhavan Venkataraman
52387a18d3fSMadhavan Venkataraman ct->ct_heap_num = 0;
524060cedfbSMadhavan Venkataraman ct->ct_heap_max = callout_chunk;
525060cedfbSMadhavan Venkataraman size = sizeof (callout_heap_t) * callout_chunk;
52687a18d3fSMadhavan Venkataraman ct->ct_heap = kmem_alloc(size, KM_SLEEP);
52787a18d3fSMadhavan Venkataraman }
52887a18d3fSMadhavan Venkataraman
52987a18d3fSMadhavan Venkataraman /*
530060cedfbSMadhavan Venkataraman * Reallocate the heap. Return 0 if the heap is still full at the end of it.
531060cedfbSMadhavan Venkataraman * Return 1 otherwise. Note that the heap only expands, it never contracts.
53287a18d3fSMadhavan Venkataraman */
533060cedfbSMadhavan Venkataraman static int
callout_heap_expand(callout_table_t * ct)53487a18d3fSMadhavan Venkataraman callout_heap_expand(callout_table_t *ct)
53587a18d3fSMadhavan Venkataraman {
53687a18d3fSMadhavan Venkataraman size_t max, size, osize;
53751b32bddSMadhavan Venkataraman callout_heap_t *heap;
53887a18d3fSMadhavan Venkataraman
53987a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex));
54087a18d3fSMadhavan Venkataraman ASSERT(ct->ct_heap_num <= ct->ct_heap_max);
54187a18d3fSMadhavan Venkataraman
54287a18d3fSMadhavan Venkataraman while (ct->ct_heap_num == ct->ct_heap_max) {
54387a18d3fSMadhavan Venkataraman max = ct->ct_heap_max;
54487a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
54587a18d3fSMadhavan Venkataraman
54651b32bddSMadhavan Venkataraman osize = sizeof (callout_heap_t) * max;
547060cedfbSMadhavan Venkataraman size = sizeof (callout_heap_t) * (max + callout_chunk);
548060cedfbSMadhavan Venkataraman heap = kmem_alloc(size, KM_NOSLEEP);
54987a18d3fSMadhavan Venkataraman
55087a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex);
551060cedfbSMadhavan Venkataraman if (heap == NULL) {
552060cedfbSMadhavan Venkataraman /*
553060cedfbSMadhavan Venkataraman * We could not allocate memory. If we can free up
554060cedfbSMadhavan Venkataraman * some entries, that would be great.
555060cedfbSMadhavan Venkataraman */
556060cedfbSMadhavan Venkataraman if (ct->ct_nreap > 0)
557060cedfbSMadhavan Venkataraman (void) callout_heap_process(ct, 0, 0);
558060cedfbSMadhavan Venkataraman /*
559060cedfbSMadhavan Venkataraman * If we still have no space in the heap, inform the
560060cedfbSMadhavan Venkataraman * caller.
561060cedfbSMadhavan Venkataraman */
562060cedfbSMadhavan Venkataraman if (ct->ct_heap_num == ct->ct_heap_max)
563060cedfbSMadhavan Venkataraman return (0);
564060cedfbSMadhavan Venkataraman return (1);
565060cedfbSMadhavan Venkataraman }
56687a18d3fSMadhavan Venkataraman if (max < ct->ct_heap_max) {
56787a18d3fSMadhavan Venkataraman /*
56887a18d3fSMadhavan Venkataraman * Someone beat us to the allocation. Free what we
56987a18d3fSMadhavan Venkataraman * just allocated and proceed.
57087a18d3fSMadhavan Venkataraman */
57187a18d3fSMadhavan Venkataraman kmem_free(heap, size);
57287a18d3fSMadhavan Venkataraman continue;
57387a18d3fSMadhavan Venkataraman }
57487a18d3fSMadhavan Venkataraman
57587a18d3fSMadhavan Venkataraman bcopy(ct->ct_heap, heap, osize);
57687a18d3fSMadhavan Venkataraman kmem_free(ct->ct_heap, osize);
57787a18d3fSMadhavan Venkataraman ct->ct_heap = heap;
57851b32bddSMadhavan Venkataraman ct->ct_heap_max = size / sizeof (callout_heap_t);
57987a18d3fSMadhavan Venkataraman }
580060cedfbSMadhavan Venkataraman
581060cedfbSMadhavan Venkataraman return (1);
58287a18d3fSMadhavan Venkataraman }
58387a18d3fSMadhavan Venkataraman
58487a18d3fSMadhavan Venkataraman /*
58587a18d3fSMadhavan Venkataraman * Move an expiration from the bottom of the heap to its correct place
58687a18d3fSMadhavan Venkataraman * in the heap. If we reached the root doing this, return 1. Else,
58787a18d3fSMadhavan Venkataraman * return 0.
58887a18d3fSMadhavan Venkataraman */
58987a18d3fSMadhavan Venkataraman static int
callout_upheap(callout_table_t * ct)59087a18d3fSMadhavan Venkataraman callout_upheap(callout_table_t *ct)
59187a18d3fSMadhavan Venkataraman {
59287a18d3fSMadhavan Venkataraman int current, parent;
59351b32bddSMadhavan Venkataraman callout_heap_t *heap, *hcurrent, *hparent;
59487a18d3fSMadhavan Venkataraman
59587a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex));
59687a18d3fSMadhavan Venkataraman ASSERT(ct->ct_heap_num >= 1);
59787a18d3fSMadhavan Venkataraman
59887a18d3fSMadhavan Venkataraman if (ct->ct_heap_num == 1) {
59987a18d3fSMadhavan Venkataraman return (1);
60087a18d3fSMadhavan Venkataraman }
60187a18d3fSMadhavan Venkataraman
60287a18d3fSMadhavan Venkataraman heap = ct->ct_heap;
60387a18d3fSMadhavan Venkataraman current = ct->ct_heap_num - 1;
60487a18d3fSMadhavan Venkataraman
60587a18d3fSMadhavan Venkataraman for (;;) {
60687a18d3fSMadhavan Venkataraman parent = CALLOUT_HEAP_PARENT(current);
60751b32bddSMadhavan Venkataraman hparent = &heap[parent];
60851b32bddSMadhavan Venkataraman hcurrent = &heap[current];
60987a18d3fSMadhavan Venkataraman
61087a18d3fSMadhavan Venkataraman /*
61187a18d3fSMadhavan Venkataraman * We have an expiration later than our parent; we're done.
61287a18d3fSMadhavan Venkataraman */
61351b32bddSMadhavan Venkataraman if (hcurrent->ch_expiration >= hparent->ch_expiration) {
61487a18d3fSMadhavan Venkataraman return (0);
61587a18d3fSMadhavan Venkataraman }
61687a18d3fSMadhavan Venkataraman
61787a18d3fSMadhavan Venkataraman /*
61887a18d3fSMadhavan Venkataraman * We need to swap with our parent, and continue up the heap.
61987a18d3fSMadhavan Venkataraman */
62051b32bddSMadhavan Venkataraman CALLOUT_SWAP(hparent, hcurrent);
62187a18d3fSMadhavan Venkataraman
62287a18d3fSMadhavan Venkataraman /*
62387a18d3fSMadhavan Venkataraman * If we just reached the root, we're done.
62487a18d3fSMadhavan Venkataraman */
62587a18d3fSMadhavan Venkataraman if (parent == 0) {
62687a18d3fSMadhavan Venkataraman return (1);
62787a18d3fSMadhavan Venkataraman }
62887a18d3fSMadhavan Venkataraman
62987a18d3fSMadhavan Venkataraman current = parent;
63087a18d3fSMadhavan Venkataraman }
63187a18d3fSMadhavan Venkataraman /*NOTREACHED*/
63287a18d3fSMadhavan Venkataraman }
63387a18d3fSMadhavan Venkataraman
63487a18d3fSMadhavan Venkataraman /*
63551b32bddSMadhavan Venkataraman * Insert a new heap item into a callout table's heap.
63687a18d3fSMadhavan Venkataraman */
63787a18d3fSMadhavan Venkataraman static void
callout_heap_insert(callout_table_t * ct,callout_list_t * cl)63851b32bddSMadhavan Venkataraman callout_heap_insert(callout_table_t *ct, callout_list_t *cl)
63987a18d3fSMadhavan Venkataraman {
64087a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex));
64187a18d3fSMadhavan Venkataraman ASSERT(ct->ct_heap_num < ct->ct_heap_max);
64287a18d3fSMadhavan Venkataraman
643060cedfbSMadhavan Venkataraman cl->cl_flags |= CALLOUT_LIST_FLAG_HEAPED;
64487a18d3fSMadhavan Venkataraman /*
64551b32bddSMadhavan Venkataraman * First, copy the expiration and callout list pointer to the bottom
64651b32bddSMadhavan Venkataraman * of the heap.
64787a18d3fSMadhavan Venkataraman */
64851b32bddSMadhavan Venkataraman ct->ct_heap[ct->ct_heap_num].ch_expiration = cl->cl_expiration;
64951b32bddSMadhavan Venkataraman ct->ct_heap[ct->ct_heap_num].ch_list = cl;
65087a18d3fSMadhavan Venkataraman ct->ct_heap_num++;
65187a18d3fSMadhavan Venkataraman
65287a18d3fSMadhavan Venkataraman /*
65387a18d3fSMadhavan Venkataraman * Now, perform an upheap operation. If we reached the root, then
65487a18d3fSMadhavan Venkataraman * the cyclic needs to be reprogrammed as we have an earlier
65587a18d3fSMadhavan Venkataraman * expiration.
65687a18d3fSMadhavan Venkataraman *
65787a18d3fSMadhavan Venkataraman * Also, during the CPR suspend phase, do not reprogram the cyclic.
65887a18d3fSMadhavan Venkataraman * We don't want any callout activity. When the CPR resume phase is
65987a18d3fSMadhavan Venkataraman * entered, the cyclic will be programmed for the earliest expiration
66087a18d3fSMadhavan Venkataraman * in the heap.
66187a18d3fSMadhavan Venkataraman */
662454ab202SMadhavan Venkataraman if (callout_upheap(ct) && (ct->ct_suspend == 0))
66351b32bddSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_cyclic, cl->cl_expiration);
66487a18d3fSMadhavan Venkataraman }
66587a18d3fSMadhavan Venkataraman
66687a18d3fSMadhavan Venkataraman /*
66787a18d3fSMadhavan Venkataraman * Move an expiration from the top of the heap to its correct place
66887a18d3fSMadhavan Venkataraman * in the heap.
66987a18d3fSMadhavan Venkataraman */
67087a18d3fSMadhavan Venkataraman static void
callout_downheap(callout_table_t * ct)67187a18d3fSMadhavan Venkataraman callout_downheap(callout_table_t *ct)
67287a18d3fSMadhavan Venkataraman {
67351b32bddSMadhavan Venkataraman int current, left, right, nelems;
67451b32bddSMadhavan Venkataraman callout_heap_t *heap, *hleft, *hright, *hcurrent;
67587a18d3fSMadhavan Venkataraman
67687a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex));
67787a18d3fSMadhavan Venkataraman ASSERT(ct->ct_heap_num >= 1);
67887a18d3fSMadhavan Venkataraman
67987a18d3fSMadhavan Venkataraman heap = ct->ct_heap;
68087a18d3fSMadhavan Venkataraman current = 0;
68187a18d3fSMadhavan Venkataraman nelems = ct->ct_heap_num;
68287a18d3fSMadhavan Venkataraman
68387a18d3fSMadhavan Venkataraman for (;;) {
68487a18d3fSMadhavan Venkataraman /*
68587a18d3fSMadhavan Venkataraman * If we don't have a left child (i.e., we're a leaf), we're
68687a18d3fSMadhavan Venkataraman * done.
68787a18d3fSMadhavan Venkataraman */
68887a18d3fSMadhavan Venkataraman if ((left = CALLOUT_HEAP_LEFT(current)) >= nelems)
68987a18d3fSMadhavan Venkataraman return;
69087a18d3fSMadhavan Venkataraman
69151b32bddSMadhavan Venkataraman hleft = &heap[left];
69251b32bddSMadhavan Venkataraman hcurrent = &heap[current];
69387a18d3fSMadhavan Venkataraman
69487a18d3fSMadhavan Venkataraman right = CALLOUT_HEAP_RIGHT(current);
69587a18d3fSMadhavan Venkataraman
69687a18d3fSMadhavan Venkataraman /*
69787a18d3fSMadhavan Venkataraman * Even if we don't have a right child, we still need to compare
69887a18d3fSMadhavan Venkataraman * our expiration against that of our left child.
69987a18d3fSMadhavan Venkataraman */
70087a18d3fSMadhavan Venkataraman if (right >= nelems)
70187a18d3fSMadhavan Venkataraman goto comp_left;
70287a18d3fSMadhavan Venkataraman
70351b32bddSMadhavan Venkataraman hright = &heap[right];
70487a18d3fSMadhavan Venkataraman
70587a18d3fSMadhavan Venkataraman /*
70687a18d3fSMadhavan Venkataraman * We have both a left and a right child. We need to compare
70787a18d3fSMadhavan Venkataraman * the expiration of the children to determine which
70887a18d3fSMadhavan Venkataraman * expires earlier.
70987a18d3fSMadhavan Venkataraman */
71051b32bddSMadhavan Venkataraman if (hright->ch_expiration < hleft->ch_expiration) {
71187a18d3fSMadhavan Venkataraman /*
71287a18d3fSMadhavan Venkataraman * Our right child is the earlier of our children.
71387a18d3fSMadhavan Venkataraman * We'll now compare our expiration to its expiration.
71487a18d3fSMadhavan Venkataraman * If ours is the earlier one, we're done.
71587a18d3fSMadhavan Venkataraman */
71651b32bddSMadhavan Venkataraman if (hcurrent->ch_expiration <= hright->ch_expiration)
71787a18d3fSMadhavan Venkataraman return;
71887a18d3fSMadhavan Venkataraman
71987a18d3fSMadhavan Venkataraman /*
72087a18d3fSMadhavan Venkataraman * Our right child expires earlier than we do; swap
72187a18d3fSMadhavan Venkataraman * with our right child, and descend right.
72287a18d3fSMadhavan Venkataraman */
72351b32bddSMadhavan Venkataraman CALLOUT_SWAP(hright, hcurrent);
72487a18d3fSMadhavan Venkataraman current = right;
72587a18d3fSMadhavan Venkataraman continue;
72687a18d3fSMadhavan Venkataraman }
72787a18d3fSMadhavan Venkataraman
72887a18d3fSMadhavan Venkataraman comp_left:
72987a18d3fSMadhavan Venkataraman /*
73087a18d3fSMadhavan Venkataraman * Our left child is the earlier of our children (or we have
73187a18d3fSMadhavan Venkataraman * no right child). We'll now compare our expiration
73287a18d3fSMadhavan Venkataraman * to its expiration. If ours is the earlier one, we're done.
73387a18d3fSMadhavan Venkataraman */
73451b32bddSMadhavan Venkataraman if (hcurrent->ch_expiration <= hleft->ch_expiration)
73587a18d3fSMadhavan Venkataraman return;
73687a18d3fSMadhavan Venkataraman
73787a18d3fSMadhavan Venkataraman /*
73887a18d3fSMadhavan Venkataraman * Our left child expires earlier than we do; swap with our
73987a18d3fSMadhavan Venkataraman * left child, and descend left.
74087a18d3fSMadhavan Venkataraman */
74151b32bddSMadhavan Venkataraman CALLOUT_SWAP(hleft, hcurrent);
74287a18d3fSMadhavan Venkataraman current = left;
74387a18d3fSMadhavan Venkataraman }
74487a18d3fSMadhavan Venkataraman }
74587a18d3fSMadhavan Venkataraman
74687a18d3fSMadhavan Venkataraman /*
74787a18d3fSMadhavan Venkataraman * Delete and handle all past expirations in a callout table's heap.
74887a18d3fSMadhavan Venkataraman */
749060cedfbSMadhavan Venkataraman static hrtime_t
callout_heap_delete(callout_table_t * ct)75087a18d3fSMadhavan Venkataraman callout_heap_delete(callout_table_t *ct)
75187a18d3fSMadhavan Venkataraman {
75251b32bddSMadhavan Venkataraman hrtime_t now, expiration, next;
75387a18d3fSMadhavan Venkataraman callout_list_t *cl;
75451b32bddSMadhavan Venkataraman callout_heap_t *heap;
75587a18d3fSMadhavan Venkataraman int hash;
75687a18d3fSMadhavan Venkataraman
75787a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex));
75887a18d3fSMadhavan Venkataraman
75951b32bddSMadhavan Venkataraman if (CALLOUT_CLEANUP(ct)) {
76051b32bddSMadhavan Venkataraman /*
76151b32bddSMadhavan Venkataraman * There are too many heap elements pointing to empty callout
76251b32bddSMadhavan Venkataraman * lists. Clean them out.
76351b32bddSMadhavan Venkataraman */
76451b32bddSMadhavan Venkataraman (void) callout_heap_process(ct, 0, 0);
76551b32bddSMadhavan Venkataraman }
76651b32bddSMadhavan Venkataraman
76787a18d3fSMadhavan Venkataraman now = gethrtime();
76851b32bddSMadhavan Venkataraman heap = ct->ct_heap;
76987a18d3fSMadhavan Venkataraman
77087a18d3fSMadhavan Venkataraman while (ct->ct_heap_num > 0) {
77151b32bddSMadhavan Venkataraman expiration = heap->ch_expiration;
77287a18d3fSMadhavan Venkataraman hash = CALLOUT_CLHASH(expiration);
77351b32bddSMadhavan Venkataraman cl = heap->ch_list;
77451b32bddSMadhavan Venkataraman ASSERT(expiration == cl->cl_expiration);
77551b32bddSMadhavan Venkataraman
77651b32bddSMadhavan Venkataraman if (cl->cl_callouts.ch_head == NULL) {
77787a18d3fSMadhavan Venkataraman /*
77851b32bddSMadhavan Venkataraman * If the callout list is empty, reap it.
77951b32bddSMadhavan Venkataraman * Decrement the reap count.
78051b32bddSMadhavan Venkataraman */
78151b32bddSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_clhash[hash], cl);
78251b32bddSMadhavan Venkataraman CALLOUT_LIST_FREE(ct, cl);
78351b32bddSMadhavan Venkataraman ct->ct_nreap--;
78451b32bddSMadhavan Venkataraman } else {
78551b32bddSMadhavan Venkataraman /*
78651b32bddSMadhavan Venkataraman * If the root of the heap expires in the future,
78751b32bddSMadhavan Venkataraman * bail out.
78887a18d3fSMadhavan Venkataraman */
78987a18d3fSMadhavan Venkataraman if (expiration > now)
79087a18d3fSMadhavan Venkataraman break;
79187a18d3fSMadhavan Venkataraman
79287a18d3fSMadhavan Venkataraman /*
79387a18d3fSMadhavan Venkataraman * Move the callout list for this expiration to the
79487a18d3fSMadhavan Venkataraman * list of expired callout lists. It will be processed
79587a18d3fSMadhavan Venkataraman * by the callout executor.
79687a18d3fSMadhavan Venkataraman */
797060cedfbSMadhavan Venkataraman cl->cl_flags &= ~CALLOUT_LIST_FLAG_HEAPED;
79887a18d3fSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_clhash[hash], cl);
79987a18d3fSMadhavan Venkataraman CALLOUT_LIST_APPEND(ct->ct_expired, cl);
80087a18d3fSMadhavan Venkataraman }
80187a18d3fSMadhavan Venkataraman
80287a18d3fSMadhavan Venkataraman /*
80387a18d3fSMadhavan Venkataraman * Now delete the root. This is done by swapping the root with
80487a18d3fSMadhavan Venkataraman * the last item in the heap and downheaping the item.
80587a18d3fSMadhavan Venkataraman */
80687a18d3fSMadhavan Venkataraman ct->ct_heap_num--;
80787a18d3fSMadhavan Venkataraman if (ct->ct_heap_num > 0) {
80851b32bddSMadhavan Venkataraman heap[0] = heap[ct->ct_heap_num];
80987a18d3fSMadhavan Venkataraman callout_downheap(ct);
81087a18d3fSMadhavan Venkataraman }
81187a18d3fSMadhavan Venkataraman }
81287a18d3fSMadhavan Venkataraman
81387a18d3fSMadhavan Venkataraman /*
81451b32bddSMadhavan Venkataraman * If this callout table is empty or callouts have been suspended,
81551b32bddSMadhavan Venkataraman * just return. The cyclic has already been programmed to
81687a18d3fSMadhavan Venkataraman * infinity by the cyclic subsystem.
81787a18d3fSMadhavan Venkataraman */
818454ab202SMadhavan Venkataraman if ((ct->ct_heap_num == 0) || (ct->ct_suspend > 0))
819060cedfbSMadhavan Venkataraman return (CY_INFINITY);
82087a18d3fSMadhavan Venkataraman
82151b32bddSMadhavan Venkataraman /*
82251b32bddSMadhavan Venkataraman * If the top expirations are within callout_tolerance of each other,
82351b32bddSMadhavan Venkataraman * delay the cyclic expire so that they can be processed together.
82451b32bddSMadhavan Venkataraman * This is to prevent high resolution timers from swamping the system
82551b32bddSMadhavan Venkataraman * with cyclic activity.
82651b32bddSMadhavan Venkataraman */
82751b32bddSMadhavan Venkataraman if (ct->ct_heap_num > 2) {
82851b32bddSMadhavan Venkataraman next = expiration + callout_tolerance;
82951b32bddSMadhavan Venkataraman if ((heap[1].ch_expiration < next) ||
83051b32bddSMadhavan Venkataraman (heap[2].ch_expiration < next))
83151b32bddSMadhavan Venkataraman expiration = next;
83251b32bddSMadhavan Venkataraman }
83351b32bddSMadhavan Venkataraman
83487a18d3fSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_cyclic, expiration);
835060cedfbSMadhavan Venkataraman
836060cedfbSMadhavan Venkataraman return (expiration);
83787a18d3fSMadhavan Venkataraman }
83887a18d3fSMadhavan Venkataraman
839454ab202SMadhavan Venkataraman /*
84051b32bddSMadhavan Venkataraman * There are some situations when the entire heap is walked and processed.
84151b32bddSMadhavan Venkataraman * This function is called to do the processing. These are the situations:
84251b32bddSMadhavan Venkataraman *
84351b32bddSMadhavan Venkataraman * 1. When the reap count reaches its threshold, the heap has to be cleared
84451b32bddSMadhavan Venkataraman * of all empty callout lists.
84551b32bddSMadhavan Venkataraman *
84651b32bddSMadhavan Venkataraman * 2. When the system enters and exits KMDB/OBP, all entries in the heap
84751b32bddSMadhavan Venkataraman * need to be adjusted by the interval spent in KMDB/OBP.
84851b32bddSMadhavan Venkataraman *
84951b32bddSMadhavan Venkataraman * 3. When system time is changed, the heap has to be scanned for
85051b32bddSMadhavan Venkataraman * absolute hrestime timers. These need to be removed from the heap
85151b32bddSMadhavan Venkataraman * and expired immediately.
85251b32bddSMadhavan Venkataraman *
85351b32bddSMadhavan Venkataraman * In cases 2 and 3, it is a good idea to do 1 as well since we are
85451b32bddSMadhavan Venkataraman * scanning the heap anyway.
85551b32bddSMadhavan Venkataraman *
85651b32bddSMadhavan Venkataraman * If the root gets changed and/or callout lists are expired, return the
857*48bbca81SDaniel Hoffman * new expiration to the caller so it can reprogram the cyclic accordingly.
85851b32bddSMadhavan Venkataraman */
85951b32bddSMadhavan Venkataraman static hrtime_t
callout_heap_process(callout_table_t * ct,hrtime_t delta,int timechange)86051b32bddSMadhavan Venkataraman callout_heap_process(callout_table_t *ct, hrtime_t delta, int timechange)
86151b32bddSMadhavan Venkataraman {
86251b32bddSMadhavan Venkataraman callout_heap_t *heap;
863060cedfbSMadhavan Venkataraman callout_list_t *cl;
86451b32bddSMadhavan Venkataraman hrtime_t expiration, now;
865060cedfbSMadhavan Venkataraman int i, hash, clflags;
86651b32bddSMadhavan Venkataraman ulong_t num;
86751b32bddSMadhavan Venkataraman
86851b32bddSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex));
86951b32bddSMadhavan Venkataraman
87051b32bddSMadhavan Venkataraman if (ct->ct_heap_num == 0)
871060cedfbSMadhavan Venkataraman return (CY_INFINITY);
87251b32bddSMadhavan Venkataraman
87351b32bddSMadhavan Venkataraman if (ct->ct_nreap > 0)
87451b32bddSMadhavan Venkataraman ct->ct_cleanups++;
87551b32bddSMadhavan Venkataraman
87651b32bddSMadhavan Venkataraman heap = ct->ct_heap;
87751b32bddSMadhavan Venkataraman
87851b32bddSMadhavan Venkataraman /*
87951b32bddSMadhavan Venkataraman * We walk the heap from the top to the bottom. If we encounter
88051b32bddSMadhavan Venkataraman * a heap item that points to an empty callout list, we clean
88151b32bddSMadhavan Venkataraman * it out. If we encounter a hrestime entry that must be removed,
88251b32bddSMadhavan Venkataraman * again we clean it out. Otherwise, we apply any adjustments needed
88351b32bddSMadhavan Venkataraman * to an element.
88451b32bddSMadhavan Venkataraman *
88551b32bddSMadhavan Venkataraman * During the walk, we also compact the heap from the bottom and
88651b32bddSMadhavan Venkataraman * reconstruct the heap using upheap operations. This is very
88751b32bddSMadhavan Venkataraman * efficient if the number of elements to be cleaned is greater than
88851b32bddSMadhavan Venkataraman * or equal to half the heap. This is the common case.
88951b32bddSMadhavan Venkataraman *
89051b32bddSMadhavan Venkataraman * Even in the non-common case, the upheap operations should be short
89151b32bddSMadhavan Venkataraman * as the entries below generally tend to be bigger than the entries
89251b32bddSMadhavan Venkataraman * above.
89351b32bddSMadhavan Venkataraman */
89451b32bddSMadhavan Venkataraman num = ct->ct_heap_num;
89551b32bddSMadhavan Venkataraman ct->ct_heap_num = 0;
89651b32bddSMadhavan Venkataraman clflags = (CALLOUT_LIST_FLAG_HRESTIME | CALLOUT_LIST_FLAG_ABSOLUTE);
89751b32bddSMadhavan Venkataraman now = gethrtime();
89851b32bddSMadhavan Venkataraman for (i = 0; i < num; i++) {
89951b32bddSMadhavan Venkataraman cl = heap[i].ch_list;
90051b32bddSMadhavan Venkataraman /*
90151b32bddSMadhavan Venkataraman * If the callout list is empty, delete the heap element and
90251b32bddSMadhavan Venkataraman * free the callout list.
90351b32bddSMadhavan Venkataraman */
90451b32bddSMadhavan Venkataraman if (cl->cl_callouts.ch_head == NULL) {
90551b32bddSMadhavan Venkataraman hash = CALLOUT_CLHASH(cl->cl_expiration);
90651b32bddSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_clhash[hash], cl);
90751b32bddSMadhavan Venkataraman CALLOUT_LIST_FREE(ct, cl);
90851b32bddSMadhavan Venkataraman continue;
90951b32bddSMadhavan Venkataraman }
91051b32bddSMadhavan Venkataraman
91151b32bddSMadhavan Venkataraman /*
91251b32bddSMadhavan Venkataraman * Delete the heap element and expire the callout list, if
91351b32bddSMadhavan Venkataraman * one of the following is true:
91451b32bddSMadhavan Venkataraman * - the callout list has expired
91551b32bddSMadhavan Venkataraman * - the callout list is an absolute hrestime one and
91651b32bddSMadhavan Venkataraman * there has been a system time change
91751b32bddSMadhavan Venkataraman */
91851b32bddSMadhavan Venkataraman if ((cl->cl_expiration <= now) ||
91951b32bddSMadhavan Venkataraman (timechange && ((cl->cl_flags & clflags) == clflags))) {
92051b32bddSMadhavan Venkataraman hash = CALLOUT_CLHASH(cl->cl_expiration);
921060cedfbSMadhavan Venkataraman cl->cl_flags &= ~CALLOUT_LIST_FLAG_HEAPED;
92251b32bddSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_clhash[hash], cl);
92351b32bddSMadhavan Venkataraman CALLOUT_LIST_APPEND(ct->ct_expired, cl);
92451b32bddSMadhavan Venkataraman continue;
92551b32bddSMadhavan Venkataraman }
92651b32bddSMadhavan Venkataraman
92751b32bddSMadhavan Venkataraman /*
92851b32bddSMadhavan Venkataraman * Apply adjustments, if any. Adjustments are applied after
92951b32bddSMadhavan Venkataraman * the system returns from KMDB or OBP. They are only applied
93051b32bddSMadhavan Venkataraman * to relative callout lists.
93151b32bddSMadhavan Venkataraman */
93251b32bddSMadhavan Venkataraman if (delta && !(cl->cl_flags & CALLOUT_LIST_FLAG_ABSOLUTE)) {
93351b32bddSMadhavan Venkataraman hash = CALLOUT_CLHASH(cl->cl_expiration);
93451b32bddSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_clhash[hash], cl);
93551b32bddSMadhavan Venkataraman expiration = cl->cl_expiration + delta;
93651b32bddSMadhavan Venkataraman if (expiration <= 0)
93751b32bddSMadhavan Venkataraman expiration = CY_INFINITY;
93851b32bddSMadhavan Venkataraman heap[i].ch_expiration = expiration;
93951b32bddSMadhavan Venkataraman cl->cl_expiration = expiration;
94051b32bddSMadhavan Venkataraman hash = CALLOUT_CLHASH(cl->cl_expiration);
94151b32bddSMadhavan Venkataraman if (cl->cl_flags & CALLOUT_LIST_FLAG_NANO) {
94251b32bddSMadhavan Venkataraman CALLOUT_LIST_APPEND(ct->ct_clhash[hash], cl);
94351b32bddSMadhavan Venkataraman } else {
94451b32bddSMadhavan Venkataraman CALLOUT_LIST_INSERT(ct->ct_clhash[hash], cl);
94551b32bddSMadhavan Venkataraman }
94651b32bddSMadhavan Venkataraman }
94751b32bddSMadhavan Venkataraman
94851b32bddSMadhavan Venkataraman heap[ct->ct_heap_num] = heap[i];
94951b32bddSMadhavan Venkataraman ct->ct_heap_num++;
95051b32bddSMadhavan Venkataraman (void) callout_upheap(ct);
95151b32bddSMadhavan Venkataraman }
95251b32bddSMadhavan Venkataraman
95351b32bddSMadhavan Venkataraman ct->ct_nreap = 0;
95451b32bddSMadhavan Venkataraman
955060cedfbSMadhavan Venkataraman /*
956060cedfbSMadhavan Venkataraman * We need to return the expiration to help program the cyclic.
957060cedfbSMadhavan Venkataraman * If there are expired callouts, the cyclic needs to go off
958060cedfbSMadhavan Venkataraman * immediately. If the heap has become empty, then we return infinity.
959060cedfbSMadhavan Venkataraman * Else, return the expiration of the earliest callout in the heap.
960060cedfbSMadhavan Venkataraman */
961060cedfbSMadhavan Venkataraman if (ct->ct_expired.ch_head != NULL)
962060cedfbSMadhavan Venkataraman return (gethrtime());
96351b32bddSMadhavan Venkataraman
964060cedfbSMadhavan Venkataraman if (ct->ct_heap_num == 0)
965060cedfbSMadhavan Venkataraman return (CY_INFINITY);
966060cedfbSMadhavan Venkataraman
967060cedfbSMadhavan Venkataraman return (heap->ch_expiration);
96851b32bddSMadhavan Venkataraman }
96951b32bddSMadhavan Venkataraman
97051b32bddSMadhavan Venkataraman /*
971454ab202SMadhavan Venkataraman * Common function used to create normal and realtime callouts.
972454ab202SMadhavan Venkataraman *
973454ab202SMadhavan Venkataraman * Realtime callouts are handled at CY_LOW_PIL by a cyclic handler. So,
974454ab202SMadhavan Venkataraman * there is one restriction on a realtime callout handler - it should not
975454ab202SMadhavan Venkataraman * directly or indirectly acquire cpu_lock. CPU offline waits for pending
976454ab202SMadhavan Venkataraman * cyclic handlers to complete while holding cpu_lock. So, if a realtime
977454ab202SMadhavan Venkataraman * callout handler were to try to get cpu_lock, there would be a deadlock
978454ab202SMadhavan Venkataraman * during CPU offline.
979454ab202SMadhavan Venkataraman */
98087a18d3fSMadhavan Venkataraman callout_id_t
timeout_generic(int type,void (* func)(void *),void * arg,hrtime_t expiration,hrtime_t resolution,int flags)98187a18d3fSMadhavan Venkataraman timeout_generic(int type, void (*func)(void *), void *arg,
98287a18d3fSMadhavan Venkataraman hrtime_t expiration, hrtime_t resolution, int flags)
98387a18d3fSMadhavan Venkataraman {
98487a18d3fSMadhavan Venkataraman callout_table_t *ct;
9857c478bd9Sstevel@tonic-gate callout_t *cp;
9867c478bd9Sstevel@tonic-gate callout_id_t id;
98787a18d3fSMadhavan Venkataraman callout_list_t *cl;
988060cedfbSMadhavan Venkataraman hrtime_t now, interval;
98951b32bddSMadhavan Venkataraman int hash, clflags;
990f635d46aSqiao
99187a18d3fSMadhavan Venkataraman ASSERT(resolution > 0);
99287a18d3fSMadhavan Venkataraman ASSERT(func != NULL);
9937c478bd9Sstevel@tonic-gate
99487a18d3fSMadhavan Venkataraman /*
99551b32bddSMadhavan Venkataraman * We get the current hrtime right upfront so that latencies in
99651b32bddSMadhavan Venkataraman * this function do not affect the accuracy of the callout.
99787a18d3fSMadhavan Venkataraman */
99851b32bddSMadhavan Venkataraman now = gethrtime();
9997c478bd9Sstevel@tonic-gate
100087a18d3fSMadhavan Venkataraman /*
100187a18d3fSMadhavan Venkataraman * We disable kernel preemption so that we remain on the same CPU
100287a18d3fSMadhavan Venkataraman * throughout. If we needed to reprogram the callout table's cyclic,
100387a18d3fSMadhavan Venkataraman * we can avoid X-calls if we are on the same CPU.
100487a18d3fSMadhavan Venkataraman *
100587a18d3fSMadhavan Venkataraman * Note that callout_alloc() releases and reacquires the callout
100687a18d3fSMadhavan Venkataraman * table mutex. While reacquiring the mutex, it is possible for us
100787a18d3fSMadhavan Venkataraman * to go to sleep and later migrate to another CPU. This should be
100887a18d3fSMadhavan Venkataraman * pretty rare, though.
100987a18d3fSMadhavan Venkataraman */
101087a18d3fSMadhavan Venkataraman kpreempt_disable();
101187a18d3fSMadhavan Venkataraman
101287a18d3fSMadhavan Venkataraman ct = &callout_table[CALLOUT_TABLE(type, CPU->cpu_seqid)];
101387a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex);
101487a18d3fSMadhavan Venkataraman
101587a18d3fSMadhavan Venkataraman if (ct->ct_cyclic == CYCLIC_NONE) {
101687a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
101787a18d3fSMadhavan Venkataraman /*
101887a18d3fSMadhavan Venkataraman * The callout table has not yet been initialized fully.
101987a18d3fSMadhavan Venkataraman * So, put this one on the boot callout table which is
102087a18d3fSMadhavan Venkataraman * always initialized.
102187a18d3fSMadhavan Venkataraman */
102287a18d3fSMadhavan Venkataraman ct = &callout_boot_ct[type];
102387a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex);
102487a18d3fSMadhavan Venkataraman }
102587a18d3fSMadhavan Venkataraman
102651b32bddSMadhavan Venkataraman if (CALLOUT_CLEANUP(ct)) {
102751b32bddSMadhavan Venkataraman /*
102851b32bddSMadhavan Venkataraman * There are too many heap elements pointing to empty callout
1029060cedfbSMadhavan Venkataraman * lists. Clean them out. Since cleanup is only done once
1030060cedfbSMadhavan Venkataraman * in a while, no need to reprogram the cyclic if the root
1031060cedfbSMadhavan Venkataraman * of the heap gets cleaned out.
103251b32bddSMadhavan Venkataraman */
1033060cedfbSMadhavan Venkataraman (void) callout_heap_process(ct, 0, 0);
103451b32bddSMadhavan Venkataraman }
103551b32bddSMadhavan Venkataraman
103687a18d3fSMadhavan Venkataraman if ((cp = ct->ct_free) == NULL)
10377c478bd9Sstevel@tonic-gate cp = callout_alloc(ct);
10387c478bd9Sstevel@tonic-gate else
103987a18d3fSMadhavan Venkataraman ct->ct_free = cp->c_idnext;
10407c478bd9Sstevel@tonic-gate
10417c478bd9Sstevel@tonic-gate cp->c_func = func;
10427c478bd9Sstevel@tonic-gate cp->c_arg = arg;
10437c478bd9Sstevel@tonic-gate
10447c478bd9Sstevel@tonic-gate /*
104587a18d3fSMadhavan Venkataraman * Compute the expiration hrtime.
104687a18d3fSMadhavan Venkataraman */
104787a18d3fSMadhavan Venkataraman if (flags & CALLOUT_FLAG_ABSOLUTE) {
104887a18d3fSMadhavan Venkataraman interval = expiration - now;
104987a18d3fSMadhavan Venkataraman } else {
105087a18d3fSMadhavan Venkataraman interval = expiration;
105187a18d3fSMadhavan Venkataraman expiration += now;
105287a18d3fSMadhavan Venkataraman }
105351b32bddSMadhavan Venkataraman
105451b32bddSMadhavan Venkataraman if (resolution > 1) {
105551b32bddSMadhavan Venkataraman /*
105651b32bddSMadhavan Venkataraman * Align expiration to the specified resolution.
105751b32bddSMadhavan Venkataraman */
105887a18d3fSMadhavan Venkataraman if (flags & CALLOUT_FLAG_ROUNDUP)
105987a18d3fSMadhavan Venkataraman expiration += resolution - 1;
106087a18d3fSMadhavan Venkataraman expiration = (expiration / resolution) * resolution;
106151b32bddSMadhavan Venkataraman }
106251b32bddSMadhavan Venkataraman
1063454ab202SMadhavan Venkataraman if (expiration <= 0) {
1064454ab202SMadhavan Venkataraman /*
1065454ab202SMadhavan Venkataraman * expiration hrtime overflow has occurred. Just set the
1066454ab202SMadhavan Venkataraman * expiration to infinity.
1067454ab202SMadhavan Venkataraman */
1068454ab202SMadhavan Venkataraman expiration = CY_INFINITY;
1069454ab202SMadhavan Venkataraman }
107087a18d3fSMadhavan Venkataraman
107187a18d3fSMadhavan Venkataraman /*
107287a18d3fSMadhavan Venkataraman * Assign an ID to this callout
107387a18d3fSMadhavan Venkataraman */
107487a18d3fSMadhavan Venkataraman if (flags & CALLOUT_FLAG_32BIT) {
107587a18d3fSMadhavan Venkataraman if (interval > callout_longterm) {
107687a18d3fSMadhavan Venkataraman id = (ct->ct_long_id - callout_counter_low);
107787a18d3fSMadhavan Venkataraman id |= CALLOUT_COUNTER_HIGH;
107887a18d3fSMadhavan Venkataraman ct->ct_long_id = id;
107987a18d3fSMadhavan Venkataraman } else {
108087a18d3fSMadhavan Venkataraman id = (ct->ct_short_id - callout_counter_low);
108187a18d3fSMadhavan Venkataraman id |= CALLOUT_COUNTER_HIGH;
108287a18d3fSMadhavan Venkataraman ct->ct_short_id = id;
108387a18d3fSMadhavan Venkataraman }
108487a18d3fSMadhavan Venkataraman } else {
108587a18d3fSMadhavan Venkataraman id = (ct->ct_gen_id - callout_counter_low);
108687a18d3fSMadhavan Venkataraman if ((id & CALLOUT_COUNTER_HIGH) == 0) {
108787a18d3fSMadhavan Venkataraman id |= CALLOUT_COUNTER_HIGH;
108887a18d3fSMadhavan Venkataraman id += CALLOUT_GENERATION_LOW;
108987a18d3fSMadhavan Venkataraman }
109087a18d3fSMadhavan Venkataraman ct->ct_gen_id = id;
109187a18d3fSMadhavan Venkataraman }
109287a18d3fSMadhavan Venkataraman
109387a18d3fSMadhavan Venkataraman cp->c_xid = id;
109487a18d3fSMadhavan Venkataraman
109551b32bddSMadhavan Venkataraman clflags = 0;
109651b32bddSMadhavan Venkataraman if (flags & CALLOUT_FLAG_ABSOLUTE)
109751b32bddSMadhavan Venkataraman clflags |= CALLOUT_LIST_FLAG_ABSOLUTE;
109851b32bddSMadhavan Venkataraman if (flags & CALLOUT_FLAG_HRESTIME)
109951b32bddSMadhavan Venkataraman clflags |= CALLOUT_LIST_FLAG_HRESTIME;
110051b32bddSMadhavan Venkataraman if (resolution == 1)
110151b32bddSMadhavan Venkataraman clflags |= CALLOUT_LIST_FLAG_NANO;
110287a18d3fSMadhavan Venkataraman hash = CALLOUT_CLHASH(expiration);
110387a18d3fSMadhavan Venkataraman
110487a18d3fSMadhavan Venkataraman again:
110587a18d3fSMadhavan Venkataraman /*
110687a18d3fSMadhavan Venkataraman * Try to see if a callout list already exists for this expiration.
110787a18d3fSMadhavan Venkataraman */
110851b32bddSMadhavan Venkataraman cl = callout_list_get(ct, expiration, clflags, hash);
110987a18d3fSMadhavan Venkataraman if (cl == NULL) {
111087a18d3fSMadhavan Venkataraman /*
111187a18d3fSMadhavan Venkataraman * Check the free list. If we don't find one, we have to
111287a18d3fSMadhavan Venkataraman * take the slow path and allocate from kmem.
111387a18d3fSMadhavan Venkataraman */
111487a18d3fSMadhavan Venkataraman if ((cl = ct->ct_lfree) == NULL) {
111587a18d3fSMadhavan Venkataraman callout_list_alloc(ct);
111687a18d3fSMadhavan Venkataraman /*
111787a18d3fSMadhavan Venkataraman * In the above call, we drop the lock, allocate and
111887a18d3fSMadhavan Venkataraman * reacquire the lock. So, we could have been away
111987a18d3fSMadhavan Venkataraman * for a while. In the meantime, someone could have
112087a18d3fSMadhavan Venkataraman * inserted a callout list with the same expiration.
112187a18d3fSMadhavan Venkataraman * Plus, the heap could have become full. So, the best
112287a18d3fSMadhavan Venkataraman * course is to repeat the steps. This should be an
112387a18d3fSMadhavan Venkataraman * infrequent event.
112487a18d3fSMadhavan Venkataraman */
112587a18d3fSMadhavan Venkataraman goto again;
112687a18d3fSMadhavan Venkataraman }
112787a18d3fSMadhavan Venkataraman ct->ct_lfree = cl->cl_next;
112887a18d3fSMadhavan Venkataraman cl->cl_expiration = expiration;
112951b32bddSMadhavan Venkataraman cl->cl_flags = clflags;
113087a18d3fSMadhavan Venkataraman
1131060cedfbSMadhavan Venkataraman /*
1132060cedfbSMadhavan Venkataraman * Check if we have enough space in the heap to insert one
1133060cedfbSMadhavan Venkataraman * expiration. If not, expand the heap.
1134060cedfbSMadhavan Venkataraman */
1135060cedfbSMadhavan Venkataraman if (ct->ct_heap_num == ct->ct_heap_max) {
1136060cedfbSMadhavan Venkataraman if (callout_heap_expand(ct) == 0) {
1137060cedfbSMadhavan Venkataraman /*
1138060cedfbSMadhavan Venkataraman * Could not expand the heap. Just queue it.
1139060cedfbSMadhavan Venkataraman */
1140060cedfbSMadhavan Venkataraman callout_queue_insert(ct, cl);
1141060cedfbSMadhavan Venkataraman goto out;
1142060cedfbSMadhavan Venkataraman }
1143060cedfbSMadhavan Venkataraman
1144060cedfbSMadhavan Venkataraman /*
1145060cedfbSMadhavan Venkataraman * In the above call, we drop the lock, allocate and
1146060cedfbSMadhavan Venkataraman * reacquire the lock. So, we could have been away
1147060cedfbSMadhavan Venkataraman * for a while. In the meantime, someone could have
1148060cedfbSMadhavan Venkataraman * inserted a callout list with the same expiration.
1149060cedfbSMadhavan Venkataraman * But we will not go back and check for it as this
1150060cedfbSMadhavan Venkataraman * should be a really infrequent event. There is no
1151060cedfbSMadhavan Venkataraman * point.
1152060cedfbSMadhavan Venkataraman */
1153060cedfbSMadhavan Venkataraman }
1154060cedfbSMadhavan Venkataraman
115551b32bddSMadhavan Venkataraman if (clflags & CALLOUT_LIST_FLAG_NANO) {
115651b32bddSMadhavan Venkataraman CALLOUT_LIST_APPEND(ct->ct_clhash[hash], cl);
115751b32bddSMadhavan Venkataraman } else {
115887a18d3fSMadhavan Venkataraman CALLOUT_LIST_INSERT(ct->ct_clhash[hash], cl);
115951b32bddSMadhavan Venkataraman }
116087a18d3fSMadhavan Venkataraman
116187a18d3fSMadhavan Venkataraman /*
116287a18d3fSMadhavan Venkataraman * This is a new expiration. So, insert it into the heap.
116387a18d3fSMadhavan Venkataraman * This will also reprogram the cyclic, if the expiration
116487a18d3fSMadhavan Venkataraman * propagated to the root of the heap.
116587a18d3fSMadhavan Venkataraman */
116651b32bddSMadhavan Venkataraman callout_heap_insert(ct, cl);
116751b32bddSMadhavan Venkataraman } else {
116851b32bddSMadhavan Venkataraman /*
116951b32bddSMadhavan Venkataraman * If the callout list was empty, untimeout_generic() would
117051b32bddSMadhavan Venkataraman * have incremented a reap count. Decrement the reap count
117151b32bddSMadhavan Venkataraman * as we are going to insert a callout into this list.
117251b32bddSMadhavan Venkataraman */
117351b32bddSMadhavan Venkataraman if (cl->cl_callouts.ch_head == NULL)
117451b32bddSMadhavan Venkataraman ct->ct_nreap--;
117587a18d3fSMadhavan Venkataraman }
1176060cedfbSMadhavan Venkataraman out:
117787a18d3fSMadhavan Venkataraman cp->c_list = cl;
117887a18d3fSMadhavan Venkataraman CALLOUT_APPEND(ct, cp);
117987a18d3fSMadhavan Venkataraman
118087a18d3fSMadhavan Venkataraman ct->ct_timeouts++;
118187a18d3fSMadhavan Venkataraman ct->ct_timeouts_pending++;
118287a18d3fSMadhavan Venkataraman
118387a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
118487a18d3fSMadhavan Venkataraman
118587a18d3fSMadhavan Venkataraman kpreempt_enable();
118687a18d3fSMadhavan Venkataraman
118787a18d3fSMadhavan Venkataraman TRACE_4(TR_FAC_CALLOUT, TR_TIMEOUT,
118887a18d3fSMadhavan Venkataraman "timeout:%K(%p) in %llx expiration, cp %p", func, arg, expiration,
118987a18d3fSMadhavan Venkataraman cp);
119087a18d3fSMadhavan Venkataraman
119187a18d3fSMadhavan Venkataraman return (id);
119287a18d3fSMadhavan Venkataraman }
119387a18d3fSMadhavan Venkataraman
119487a18d3fSMadhavan Venkataraman timeout_id_t
timeout(void (* func)(void *),void * arg,clock_t delta)119587a18d3fSMadhavan Venkataraman timeout(void (*func)(void *), void *arg, clock_t delta)
119687a18d3fSMadhavan Venkataraman {
119787a18d3fSMadhavan Venkataraman ulong_t id;
119887a18d3fSMadhavan Venkataraman
119987a18d3fSMadhavan Venkataraman /*
12007c478bd9Sstevel@tonic-gate * Make sure the callout runs at least 1 tick in the future.
12017c478bd9Sstevel@tonic-gate */
12027c478bd9Sstevel@tonic-gate if (delta <= 0)
12037c478bd9Sstevel@tonic-gate delta = 1;
1204454ab202SMadhavan Venkataraman else if (delta > callout_max_ticks)
1205454ab202SMadhavan Venkataraman delta = callout_max_ticks;
12067c478bd9Sstevel@tonic-gate
120787a18d3fSMadhavan Venkataraman id = (ulong_t)timeout_generic(CALLOUT_NORMAL, func, arg,
120887a18d3fSMadhavan Venkataraman TICK_TO_NSEC(delta), nsec_per_tick, CALLOUT_LEGACY);
12097c478bd9Sstevel@tonic-gate
12107c478bd9Sstevel@tonic-gate return ((timeout_id_t)id);
12117c478bd9Sstevel@tonic-gate }
12127c478bd9Sstevel@tonic-gate
121387a18d3fSMadhavan Venkataraman /*
121487a18d3fSMadhavan Venkataraman * Convenience function that creates a normal callout with default parameters
121587a18d3fSMadhavan Venkataraman * and returns a full ID.
121687a18d3fSMadhavan Venkataraman */
121787a18d3fSMadhavan Venkataraman callout_id_t
timeout_default(void (* func)(void *),void * arg,clock_t delta)121887a18d3fSMadhavan Venkataraman timeout_default(void (*func)(void *), void *arg, clock_t delta)
12197c478bd9Sstevel@tonic-gate {
122087a18d3fSMadhavan Venkataraman callout_id_t id;
12217c478bd9Sstevel@tonic-gate
122287a18d3fSMadhavan Venkataraman /*
122387a18d3fSMadhavan Venkataraman * Make sure the callout runs at least 1 tick in the future.
122487a18d3fSMadhavan Venkataraman */
122587a18d3fSMadhavan Venkataraman if (delta <= 0)
122687a18d3fSMadhavan Venkataraman delta = 1;
1227454ab202SMadhavan Venkataraman else if (delta > callout_max_ticks)
1228454ab202SMadhavan Venkataraman delta = callout_max_ticks;
122987a18d3fSMadhavan Venkataraman
123087a18d3fSMadhavan Venkataraman id = timeout_generic(CALLOUT_NORMAL, func, arg, TICK_TO_NSEC(delta),
123187a18d3fSMadhavan Venkataraman nsec_per_tick, 0);
123287a18d3fSMadhavan Venkataraman
123387a18d3fSMadhavan Venkataraman return (id);
12347c478bd9Sstevel@tonic-gate }
12357c478bd9Sstevel@tonic-gate
12367c478bd9Sstevel@tonic-gate timeout_id_t
realtime_timeout(void (* func)(void *),void * arg,clock_t delta)12377c478bd9Sstevel@tonic-gate realtime_timeout(void (*func)(void *), void *arg, clock_t delta)
12387c478bd9Sstevel@tonic-gate {
123987a18d3fSMadhavan Venkataraman ulong_t id;
124087a18d3fSMadhavan Venkataraman
124187a18d3fSMadhavan Venkataraman /*
124287a18d3fSMadhavan Venkataraman * Make sure the callout runs at least 1 tick in the future.
124387a18d3fSMadhavan Venkataraman */
124487a18d3fSMadhavan Venkataraman if (delta <= 0)
124587a18d3fSMadhavan Venkataraman delta = 1;
1246454ab202SMadhavan Venkataraman else if (delta > callout_max_ticks)
1247454ab202SMadhavan Venkataraman delta = callout_max_ticks;
124887a18d3fSMadhavan Venkataraman
124987a18d3fSMadhavan Venkataraman id = (ulong_t)timeout_generic(CALLOUT_REALTIME, func, arg,
125087a18d3fSMadhavan Venkataraman TICK_TO_NSEC(delta), nsec_per_tick, CALLOUT_LEGACY);
125187a18d3fSMadhavan Venkataraman
125287a18d3fSMadhavan Venkataraman return ((timeout_id_t)id);
12537c478bd9Sstevel@tonic-gate }
12547c478bd9Sstevel@tonic-gate
125587a18d3fSMadhavan Venkataraman /*
125687a18d3fSMadhavan Venkataraman * Convenience function that creates a realtime callout with default parameters
125787a18d3fSMadhavan Venkataraman * and returns a full ID.
125887a18d3fSMadhavan Venkataraman */
125987a18d3fSMadhavan Venkataraman callout_id_t
realtime_timeout_default(void (* func)(void *),void * arg,clock_t delta)126087a18d3fSMadhavan Venkataraman realtime_timeout_default(void (*func)(void *), void *arg, clock_t delta)
12617c478bd9Sstevel@tonic-gate {
126287a18d3fSMadhavan Venkataraman callout_id_t id;
126387a18d3fSMadhavan Venkataraman
126487a18d3fSMadhavan Venkataraman /*
126587a18d3fSMadhavan Venkataraman * Make sure the callout runs at least 1 tick in the future.
126687a18d3fSMadhavan Venkataraman */
126787a18d3fSMadhavan Venkataraman if (delta <= 0)
126887a18d3fSMadhavan Venkataraman delta = 1;
1269454ab202SMadhavan Venkataraman else if (delta > callout_max_ticks)
1270454ab202SMadhavan Venkataraman delta = callout_max_ticks;
127187a18d3fSMadhavan Venkataraman
127287a18d3fSMadhavan Venkataraman id = timeout_generic(CALLOUT_REALTIME, func, arg, TICK_TO_NSEC(delta),
127387a18d3fSMadhavan Venkataraman nsec_per_tick, 0);
127487a18d3fSMadhavan Venkataraman
127587a18d3fSMadhavan Venkataraman return (id);
127687a18d3fSMadhavan Venkataraman }
127787a18d3fSMadhavan Venkataraman
127887a18d3fSMadhavan Venkataraman hrtime_t
untimeout_generic(callout_id_t id,int nowait)127987a18d3fSMadhavan Venkataraman untimeout_generic(callout_id_t id, int nowait)
128087a18d3fSMadhavan Venkataraman {
12817c478bd9Sstevel@tonic-gate callout_table_t *ct;
12827c478bd9Sstevel@tonic-gate callout_t *cp;
12837c478bd9Sstevel@tonic-gate callout_id_t xid;
128451b32bddSMadhavan Venkataraman callout_list_t *cl;
1285060cedfbSMadhavan Venkataraman int hash, flags;
128687a18d3fSMadhavan Venkataraman callout_id_t bogus;
12877c478bd9Sstevel@tonic-gate
128887a18d3fSMadhavan Venkataraman ct = &callout_table[CALLOUT_ID_TO_TABLE(id)];
128987a18d3fSMadhavan Venkataraman hash = CALLOUT_IDHASH(id);
12907c478bd9Sstevel@tonic-gate
129187a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex);
12927c478bd9Sstevel@tonic-gate
129387a18d3fSMadhavan Venkataraman /*
129487a18d3fSMadhavan Venkataraman * Search the ID hash table for the callout.
129587a18d3fSMadhavan Venkataraman */
129687a18d3fSMadhavan Venkataraman for (cp = ct->ct_idhash[hash].ch_head; cp; cp = cp->c_idnext) {
12977c478bd9Sstevel@tonic-gate
129887a18d3fSMadhavan Venkataraman xid = cp->c_xid;
12997c478bd9Sstevel@tonic-gate
130087a18d3fSMadhavan Venkataraman /*
130187a18d3fSMadhavan Venkataraman * Match the ID and generation number.
130287a18d3fSMadhavan Venkataraman */
130387a18d3fSMadhavan Venkataraman if ((xid & CALLOUT_ID_MASK) != id)
13047c478bd9Sstevel@tonic-gate continue;
13057c478bd9Sstevel@tonic-gate
130687a18d3fSMadhavan Venkataraman if ((xid & CALLOUT_EXECUTING) == 0) {
130787a18d3fSMadhavan Venkataraman hrtime_t expiration;
130887a18d3fSMadhavan Venkataraman
130987a18d3fSMadhavan Venkataraman /*
131087a18d3fSMadhavan Venkataraman * Delete the callout. If the callout list becomes
131187a18d3fSMadhavan Venkataraman * NULL, we don't remove it from the table. This is
131287a18d3fSMadhavan Venkataraman * so it can be reused. If the empty callout list
131387a18d3fSMadhavan Venkataraman * corresponds to the top of the the callout heap, we
131487a18d3fSMadhavan Venkataraman * don't reprogram the table cyclic here. This is in
131587a18d3fSMadhavan Venkataraman * order to avoid lots of X-calls to the CPU associated
131687a18d3fSMadhavan Venkataraman * with the callout table.
131787a18d3fSMadhavan Venkataraman */
131851b32bddSMadhavan Venkataraman cl = cp->c_list;
131951b32bddSMadhavan Venkataraman expiration = cl->cl_expiration;
132087a18d3fSMadhavan Venkataraman CALLOUT_DELETE(ct, cp);
1321060cedfbSMadhavan Venkataraman CALLOUT_FREE(ct, cp);
132287a18d3fSMadhavan Venkataraman ct->ct_untimeouts_unexpired++;
132387a18d3fSMadhavan Venkataraman ct->ct_timeouts_pending--;
132451b32bddSMadhavan Venkataraman
132551b32bddSMadhavan Venkataraman /*
1326060cedfbSMadhavan Venkataraman * If the callout list has become empty, there are 3
1327060cedfbSMadhavan Venkataraman * possibilities. If it is present:
1328060cedfbSMadhavan Venkataraman * - in the heap, it needs to be cleaned along
1329060cedfbSMadhavan Venkataraman * with its heap entry. Increment a reap count.
1330060cedfbSMadhavan Venkataraman * - in the callout queue, free it.
1331060cedfbSMadhavan Venkataraman * - in the expired list, free it.
133251b32bddSMadhavan Venkataraman */
1333060cedfbSMadhavan Venkataraman if (cl->cl_callouts.ch_head == NULL) {
1334060cedfbSMadhavan Venkataraman flags = cl->cl_flags;
1335060cedfbSMadhavan Venkataraman if (flags & CALLOUT_LIST_FLAG_HEAPED) {
133651b32bddSMadhavan Venkataraman ct->ct_nreap++;
1337060cedfbSMadhavan Venkataraman } else if (flags & CALLOUT_LIST_FLAG_QUEUED) {
1338060cedfbSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_queue, cl);
1339060cedfbSMadhavan Venkataraman CALLOUT_LIST_FREE(ct, cl);
1340060cedfbSMadhavan Venkataraman } else {
1341060cedfbSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_expired, cl);
1342060cedfbSMadhavan Venkataraman CALLOUT_LIST_FREE(ct, cl);
1343060cedfbSMadhavan Venkataraman }
1344060cedfbSMadhavan Venkataraman }
134587a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
134687a18d3fSMadhavan Venkataraman
134787a18d3fSMadhavan Venkataraman expiration -= gethrtime();
134887a18d3fSMadhavan Venkataraman TRACE_2(TR_FAC_CALLOUT, TR_UNTIMEOUT,
134987a18d3fSMadhavan Venkataraman "untimeout:ID %lx hrtime left %llx", id,
135087a18d3fSMadhavan Venkataraman expiration);
135187a18d3fSMadhavan Venkataraman return (expiration < 0 ? 0 : expiration);
135287a18d3fSMadhavan Venkataraman }
135387a18d3fSMadhavan Venkataraman
135487a18d3fSMadhavan Venkataraman ct->ct_untimeouts_executing++;
13557c478bd9Sstevel@tonic-gate /*
13567c478bd9Sstevel@tonic-gate * The callout we want to delete is currently executing.
13577c478bd9Sstevel@tonic-gate * The DDI states that we must wait until the callout
135807247649SMadhavan Venkataraman * completes before returning, so we block on c_done until the
135987a18d3fSMadhavan Venkataraman * callout ID changes (to the old ID if it's on the freelist,
13607c478bd9Sstevel@tonic-gate * or to a new callout ID if it's in use). This implicitly
13617c478bd9Sstevel@tonic-gate * assumes that callout structures are persistent (they are).
13627c478bd9Sstevel@tonic-gate */
136307247649SMadhavan Venkataraman if (cp->c_executor == curthread) {
13647c478bd9Sstevel@tonic-gate /*
13657c478bd9Sstevel@tonic-gate * The timeout handler called untimeout() on itself.
13667c478bd9Sstevel@tonic-gate * Stupid, but legal. We can't wait for the timeout
13677c478bd9Sstevel@tonic-gate * to complete without deadlocking, so we just return.
13687c478bd9Sstevel@tonic-gate */
136987a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
13707c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_CALLOUT, TR_UNTIMEOUT_SELF,
13717c478bd9Sstevel@tonic-gate "untimeout_self:ID %x", id);
13727c478bd9Sstevel@tonic-gate return (-1);
13737c478bd9Sstevel@tonic-gate }
137487a18d3fSMadhavan Venkataraman if (nowait == 0) {
137587a18d3fSMadhavan Venkataraman /*
137687a18d3fSMadhavan Venkataraman * We need to wait. Indicate that we are waiting by
137707247649SMadhavan Venkataraman * incrementing c_waiting. This prevents the executor
137807247649SMadhavan Venkataraman * from doing a wakeup on c_done if there are no
137987a18d3fSMadhavan Venkataraman * waiters.
138087a18d3fSMadhavan Venkataraman */
138187a18d3fSMadhavan Venkataraman while (cp->c_xid == xid) {
138207247649SMadhavan Venkataraman cp->c_waiting = 1;
138307247649SMadhavan Venkataraman cv_wait(&cp->c_done, &ct->ct_mutex);
138487a18d3fSMadhavan Venkataraman }
138587a18d3fSMadhavan Venkataraman }
138687a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
13877c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_CALLOUT, TR_UNTIMEOUT_EXECUTING,
13887c478bd9Sstevel@tonic-gate "untimeout_executing:ID %lx", id);
13897c478bd9Sstevel@tonic-gate return (-1);
13907c478bd9Sstevel@tonic-gate }
139187a18d3fSMadhavan Venkataraman ct->ct_untimeouts_expired++;
13927c478bd9Sstevel@tonic-gate
139387a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
13947c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_CALLOUT, TR_UNTIMEOUT_BOGUS_ID,
13957c478bd9Sstevel@tonic-gate "untimeout_bogus_id:ID %lx", id);
13967c478bd9Sstevel@tonic-gate
13977c478bd9Sstevel@tonic-gate /*
13987c478bd9Sstevel@tonic-gate * We didn't find the specified callout ID. This means either
13997c478bd9Sstevel@tonic-gate * (1) the callout already fired, or (2) the caller passed us
14007c478bd9Sstevel@tonic-gate * a bogus value. Perform a sanity check to detect case (2).
14017c478bd9Sstevel@tonic-gate */
140251b32bddSMadhavan Venkataraman bogus = (CALLOUT_ID_FLAGS | CALLOUT_COUNTER_HIGH);
140387a18d3fSMadhavan Venkataraman if (((id & bogus) != CALLOUT_COUNTER_HIGH) && (id != 0))
140487a18d3fSMadhavan Venkataraman panic("untimeout: impossible timeout id %llx",
140587a18d3fSMadhavan Venkataraman (unsigned long long)id);
14067c478bd9Sstevel@tonic-gate
14077c478bd9Sstevel@tonic-gate return (-1);
14087c478bd9Sstevel@tonic-gate }
14097c478bd9Sstevel@tonic-gate
141087a18d3fSMadhavan Venkataraman clock_t
untimeout(timeout_id_t id_arg)141187a18d3fSMadhavan Venkataraman untimeout(timeout_id_t id_arg)
141287a18d3fSMadhavan Venkataraman {
141387a18d3fSMadhavan Venkataraman hrtime_t hleft;
141487a18d3fSMadhavan Venkataraman clock_t tleft;
141587a18d3fSMadhavan Venkataraman callout_id_t id;
141687a18d3fSMadhavan Venkataraman
141787a18d3fSMadhavan Venkataraman id = (ulong_t)id_arg;
141887a18d3fSMadhavan Venkataraman hleft = untimeout_generic(id, 0);
141987a18d3fSMadhavan Venkataraman if (hleft < 0)
142087a18d3fSMadhavan Venkataraman tleft = -1;
142187a18d3fSMadhavan Venkataraman else if (hleft == 0)
142287a18d3fSMadhavan Venkataraman tleft = 0;
142387a18d3fSMadhavan Venkataraman else
142487a18d3fSMadhavan Venkataraman tleft = NSEC_TO_TICK(hleft);
142587a18d3fSMadhavan Venkataraman
142687a18d3fSMadhavan Venkataraman return (tleft);
142787a18d3fSMadhavan Venkataraman }
142887a18d3fSMadhavan Venkataraman
14297c478bd9Sstevel@tonic-gate /*
143087a18d3fSMadhavan Venkataraman * Convenience function to untimeout a timeout with a full ID with default
143187a18d3fSMadhavan Venkataraman * parameters.
143287a18d3fSMadhavan Venkataraman */
143387a18d3fSMadhavan Venkataraman clock_t
untimeout_default(callout_id_t id,int nowait)143487a18d3fSMadhavan Venkataraman untimeout_default(callout_id_t id, int nowait)
143587a18d3fSMadhavan Venkataraman {
143687a18d3fSMadhavan Venkataraman hrtime_t hleft;
143787a18d3fSMadhavan Venkataraman clock_t tleft;
143887a18d3fSMadhavan Venkataraman
143987a18d3fSMadhavan Venkataraman hleft = untimeout_generic(id, nowait);
144087a18d3fSMadhavan Venkataraman if (hleft < 0)
144187a18d3fSMadhavan Venkataraman tleft = -1;
144287a18d3fSMadhavan Venkataraman else if (hleft == 0)
144387a18d3fSMadhavan Venkataraman tleft = 0;
144487a18d3fSMadhavan Venkataraman else
144587a18d3fSMadhavan Venkataraman tleft = NSEC_TO_TICK(hleft);
144687a18d3fSMadhavan Venkataraman
144787a18d3fSMadhavan Venkataraman return (tleft);
144887a18d3fSMadhavan Venkataraman }
144987a18d3fSMadhavan Venkataraman
145087a18d3fSMadhavan Venkataraman /*
145187a18d3fSMadhavan Venkataraman * Expire all the callouts queued in the specified callout list.
14527c478bd9Sstevel@tonic-gate */
14537c478bd9Sstevel@tonic-gate static void
callout_list_expire(callout_table_t * ct,callout_list_t * cl)145487a18d3fSMadhavan Venkataraman callout_list_expire(callout_table_t *ct, callout_list_t *cl)
14557c478bd9Sstevel@tonic-gate {
145607247649SMadhavan Venkataraman callout_t *cp, *cnext;
14577c478bd9Sstevel@tonic-gate
145887a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex));
145987a18d3fSMadhavan Venkataraman ASSERT(cl != NULL);
14607c478bd9Sstevel@tonic-gate
146107247649SMadhavan Venkataraman for (cp = cl->cl_callouts.ch_head; cp != NULL; cp = cnext) {
146207247649SMadhavan Venkataraman /*
146307247649SMadhavan Venkataraman * Multiple executor threads could be running at the same
146407247649SMadhavan Venkataraman * time. If this callout is already being executed,
146507247649SMadhavan Venkataraman * go on to the next one.
146607247649SMadhavan Venkataraman */
146707247649SMadhavan Venkataraman if (cp->c_xid & CALLOUT_EXECUTING) {
146807247649SMadhavan Venkataraman cnext = cp->c_clnext;
146907247649SMadhavan Venkataraman continue;
147007247649SMadhavan Venkataraman }
147187a18d3fSMadhavan Venkataraman
1472f635d46aSqiao /*
147387a18d3fSMadhavan Venkataraman * Indicate to untimeout() that a callout is
147487a18d3fSMadhavan Venkataraman * being expired by the executor.
1475f635d46aSqiao */
147687a18d3fSMadhavan Venkataraman cp->c_xid |= CALLOUT_EXECUTING;
147707247649SMadhavan Venkataraman cp->c_executor = curthread;
147887a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
147987a18d3fSMadhavan Venkataraman
14807c478bd9Sstevel@tonic-gate DTRACE_PROBE1(callout__start, callout_t *, cp);
14817c478bd9Sstevel@tonic-gate (*cp->c_func)(cp->c_arg);
14827c478bd9Sstevel@tonic-gate DTRACE_PROBE1(callout__end, callout_t *, cp);
14837c478bd9Sstevel@tonic-gate
148487a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex);
148587a18d3fSMadhavan Venkataraman
148687a18d3fSMadhavan Venkataraman ct->ct_expirations++;
148787a18d3fSMadhavan Venkataraman ct->ct_timeouts_pending--;
14887c478bd9Sstevel@tonic-gate /*
148907247649SMadhavan Venkataraman * Indicate completion for c_done.
14907c478bd9Sstevel@tonic-gate */
149187a18d3fSMadhavan Venkataraman cp->c_xid &= ~CALLOUT_EXECUTING;
149207247649SMadhavan Venkataraman cp->c_executor = NULL;
149307247649SMadhavan Venkataraman cnext = cp->c_clnext;
1494f635d46aSqiao
14957c478bd9Sstevel@tonic-gate /*
149687a18d3fSMadhavan Venkataraman * Delete callout from ID hash table and the callout
149787a18d3fSMadhavan Venkataraman * list, return to freelist, and tell any untimeout() that
149887a18d3fSMadhavan Venkataraman * cares that we're done.
14997c478bd9Sstevel@tonic-gate */
150087a18d3fSMadhavan Venkataraman CALLOUT_DELETE(ct, cp);
1501060cedfbSMadhavan Venkataraman CALLOUT_FREE(ct, cp);
150287a18d3fSMadhavan Venkataraman
150307247649SMadhavan Venkataraman if (cp->c_waiting) {
150407247649SMadhavan Venkataraman cp->c_waiting = 0;
150507247649SMadhavan Venkataraman cv_broadcast(&cp->c_done);
15067c478bd9Sstevel@tonic-gate }
150787a18d3fSMadhavan Venkataraman }
15087c478bd9Sstevel@tonic-gate }
15097c478bd9Sstevel@tonic-gate
15107c478bd9Sstevel@tonic-gate /*
151187a18d3fSMadhavan Venkataraman * Execute all expired callout lists for a callout table.
15127c478bd9Sstevel@tonic-gate */
15137c478bd9Sstevel@tonic-gate static void
callout_expire(callout_table_t * ct)151487a18d3fSMadhavan Venkataraman callout_expire(callout_table_t *ct)
15157c478bd9Sstevel@tonic-gate {
151687a18d3fSMadhavan Venkataraman callout_list_t *cl, *clnext;
1517f635d46aSqiao
151887a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex));
15197c478bd9Sstevel@tonic-gate
152087a18d3fSMadhavan Venkataraman for (cl = ct->ct_expired.ch_head; (cl != NULL); cl = clnext) {
1521f635d46aSqiao /*
152287a18d3fSMadhavan Venkataraman * Expire all the callouts in this callout list.
152387a18d3fSMadhavan Venkataraman */
152487a18d3fSMadhavan Venkataraman callout_list_expire(ct, cl);
152587a18d3fSMadhavan Venkataraman
152607247649SMadhavan Venkataraman clnext = cl->cl_next;
152707247649SMadhavan Venkataraman if (cl->cl_callouts.ch_head == NULL) {
152887a18d3fSMadhavan Venkataraman /*
152987a18d3fSMadhavan Venkataraman * Free the callout list.
153087a18d3fSMadhavan Venkataraman */
153187a18d3fSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_expired, cl);
153251b32bddSMadhavan Venkataraman CALLOUT_LIST_FREE(ct, cl);
153387a18d3fSMadhavan Venkataraman }
153487a18d3fSMadhavan Venkataraman }
153507247649SMadhavan Venkataraman }
153687a18d3fSMadhavan Venkataraman
153787a18d3fSMadhavan Venkataraman /*
153887a18d3fSMadhavan Venkataraman * The cyclic handlers below process callouts in two steps:
153987a18d3fSMadhavan Venkataraman *
154087a18d3fSMadhavan Venkataraman * 1. Find all expired callout lists and queue them in a separate
154187a18d3fSMadhavan Venkataraman * list of expired callouts.
154287a18d3fSMadhavan Venkataraman * 2. Execute the expired callout lists.
154387a18d3fSMadhavan Venkataraman *
154487a18d3fSMadhavan Venkataraman * This is done for two reasons:
154587a18d3fSMadhavan Venkataraman *
154687a18d3fSMadhavan Venkataraman * 1. We want to quickly find the next earliest expiration to program
154787a18d3fSMadhavan Venkataraman * the cyclic to and reprogram it. We can do this right at the end
154887a18d3fSMadhavan Venkataraman * of step 1.
154987a18d3fSMadhavan Venkataraman * 2. The realtime cyclic handler expires callouts in place. However,
155087a18d3fSMadhavan Venkataraman * for normal callouts, callouts are expired by a taskq thread.
155187a18d3fSMadhavan Venkataraman * So, it is simpler and more robust to have the taskq thread just
155287a18d3fSMadhavan Venkataraman * do step 2.
155387a18d3fSMadhavan Venkataraman */
155487a18d3fSMadhavan Venkataraman
155587a18d3fSMadhavan Venkataraman /*
1556060cedfbSMadhavan Venkataraman * Realtime callout cyclic handlers.
15577c478bd9Sstevel@tonic-gate */
15587c478bd9Sstevel@tonic-gate void
callout_realtime(callout_table_t * ct)155987a18d3fSMadhavan Venkataraman callout_realtime(callout_table_t *ct)
15607c478bd9Sstevel@tonic-gate {
156187a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex);
1562060cedfbSMadhavan Venkataraman (void) callout_heap_delete(ct);
1563060cedfbSMadhavan Venkataraman callout_expire(ct);
1564060cedfbSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
1565060cedfbSMadhavan Venkataraman }
1566060cedfbSMadhavan Venkataraman
1567060cedfbSMadhavan Venkataraman void
callout_queue_realtime(callout_table_t * ct)1568060cedfbSMadhavan Venkataraman callout_queue_realtime(callout_table_t *ct)
1569060cedfbSMadhavan Venkataraman {
1570060cedfbSMadhavan Venkataraman mutex_enter(&ct->ct_mutex);
1571060cedfbSMadhavan Venkataraman (void) callout_queue_delete(ct);
157287a18d3fSMadhavan Venkataraman callout_expire(ct);
157387a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
157487a18d3fSMadhavan Venkataraman }
15757c478bd9Sstevel@tonic-gate
157687a18d3fSMadhavan Venkataraman void
callout_execute(callout_table_t * ct)157787a18d3fSMadhavan Venkataraman callout_execute(callout_table_t *ct)
157887a18d3fSMadhavan Venkataraman {
157987a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex);
158087a18d3fSMadhavan Venkataraman callout_expire(ct);
158187a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
158287a18d3fSMadhavan Venkataraman }
15837c478bd9Sstevel@tonic-gate
158487a18d3fSMadhavan Venkataraman /*
1585060cedfbSMadhavan Venkataraman * Normal callout cyclic handlers.
158687a18d3fSMadhavan Venkataraman */
158787a18d3fSMadhavan Venkataraman void
callout_normal(callout_table_t * ct)158887a18d3fSMadhavan Venkataraman callout_normal(callout_table_t *ct)
158987a18d3fSMadhavan Venkataraman {
159007247649SMadhavan Venkataraman int i, exec;
1591060cedfbSMadhavan Venkataraman hrtime_t exp;
159287a18d3fSMadhavan Venkataraman
159387a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex);
1594060cedfbSMadhavan Venkataraman exp = callout_heap_delete(ct);
1595060cedfbSMadhavan Venkataraman CALLOUT_EXEC_COMPUTE(ct, exp, exec);
1596060cedfbSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
1597060cedfbSMadhavan Venkataraman
1598060cedfbSMadhavan Venkataraman for (i = 0; i < exec; i++) {
1599060cedfbSMadhavan Venkataraman ASSERT(ct->ct_taskq != NULL);
1600060cedfbSMadhavan Venkataraman (void) taskq_dispatch(ct->ct_taskq,
1601060cedfbSMadhavan Venkataraman (task_func_t *)callout_execute, ct, TQ_NOSLEEP);
1602060cedfbSMadhavan Venkataraman }
1603060cedfbSMadhavan Venkataraman }
1604060cedfbSMadhavan Venkataraman
1605060cedfbSMadhavan Venkataraman void
callout_queue_normal(callout_table_t * ct)1606060cedfbSMadhavan Venkataraman callout_queue_normal(callout_table_t *ct)
1607060cedfbSMadhavan Venkataraman {
1608060cedfbSMadhavan Venkataraman int i, exec;
1609060cedfbSMadhavan Venkataraman hrtime_t exp;
1610060cedfbSMadhavan Venkataraman
1611060cedfbSMadhavan Venkataraman mutex_enter(&ct->ct_mutex);
1612060cedfbSMadhavan Venkataraman exp = callout_queue_delete(ct);
1613060cedfbSMadhavan Venkataraman CALLOUT_EXEC_COMPUTE(ct, exp, exec);
161487a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
161587a18d3fSMadhavan Venkataraman
161607247649SMadhavan Venkataraman for (i = 0; i < exec; i++) {
161787a18d3fSMadhavan Venkataraman ASSERT(ct->ct_taskq != NULL);
161887a18d3fSMadhavan Venkataraman (void) taskq_dispatch(ct->ct_taskq,
161987a18d3fSMadhavan Venkataraman (task_func_t *)callout_execute, ct, TQ_NOSLEEP);
162087a18d3fSMadhavan Venkataraman }
162187a18d3fSMadhavan Venkataraman }
162287a18d3fSMadhavan Venkataraman
162387a18d3fSMadhavan Venkataraman /*
162487a18d3fSMadhavan Venkataraman * Suspend callout processing.
162587a18d3fSMadhavan Venkataraman */
162687a18d3fSMadhavan Venkataraman static void
callout_suspend(void)162787a18d3fSMadhavan Venkataraman callout_suspend(void)
162887a18d3fSMadhavan Venkataraman {
162987a18d3fSMadhavan Venkataraman int t, f;
163087a18d3fSMadhavan Venkataraman callout_table_t *ct;
163187a18d3fSMadhavan Venkataraman
163287a18d3fSMadhavan Venkataraman /*
163387a18d3fSMadhavan Venkataraman * Traverse every callout table in the system and suspend callout
163487a18d3fSMadhavan Venkataraman * processing.
163587a18d3fSMadhavan Venkataraman *
163687a18d3fSMadhavan Venkataraman * We need to suspend all the tables (including the inactive ones)
163787a18d3fSMadhavan Venkataraman * so that if a table is made active while the suspend is still on,
163887a18d3fSMadhavan Venkataraman * the table remains suspended.
163987a18d3fSMadhavan Venkataraman */
164087a18d3fSMadhavan Venkataraman for (f = 0; f < max_ncpus; f++) {
164187a18d3fSMadhavan Venkataraman for (t = 0; t < CALLOUT_NTYPES; t++) {
164287a18d3fSMadhavan Venkataraman ct = &callout_table[CALLOUT_TABLE(t, f)];
164387a18d3fSMadhavan Venkataraman
164487a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex);
1645454ab202SMadhavan Venkataraman ct->ct_suspend++;
164687a18d3fSMadhavan Venkataraman if (ct->ct_cyclic == CYCLIC_NONE) {
164787a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
164887a18d3fSMadhavan Venkataraman continue;
164987a18d3fSMadhavan Venkataraman }
1650060cedfbSMadhavan Venkataraman if (ct->ct_suspend == 1) {
1651454ab202SMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_cyclic,
1652454ab202SMadhavan Venkataraman CY_INFINITY);
1653060cedfbSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_qcyclic,
1654060cedfbSMadhavan Venkataraman CY_INFINITY);
1655060cedfbSMadhavan Venkataraman }
165687a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
165787a18d3fSMadhavan Venkataraman }
165887a18d3fSMadhavan Venkataraman }
165987a18d3fSMadhavan Venkataraman }
166087a18d3fSMadhavan Venkataraman
166187a18d3fSMadhavan Venkataraman /*
166287a18d3fSMadhavan Venkataraman * Resume callout processing.
166387a18d3fSMadhavan Venkataraman */
166487a18d3fSMadhavan Venkataraman static void
callout_resume(hrtime_t delta,int timechange)166551b32bddSMadhavan Venkataraman callout_resume(hrtime_t delta, int timechange)
166687a18d3fSMadhavan Venkataraman {
1667060cedfbSMadhavan Venkataraman hrtime_t hexp, qexp;
166887a18d3fSMadhavan Venkataraman int t, f;
166987a18d3fSMadhavan Venkataraman callout_table_t *ct;
167087a18d3fSMadhavan Venkataraman
167187a18d3fSMadhavan Venkataraman /*
167287a18d3fSMadhavan Venkataraman * Traverse every callout table in the system and resume callout
167387a18d3fSMadhavan Venkataraman * processing. For active tables, perform any hrtime adjustments
167487a18d3fSMadhavan Venkataraman * necessary.
167587a18d3fSMadhavan Venkataraman */
167687a18d3fSMadhavan Venkataraman for (f = 0; f < max_ncpus; f++) {
167787a18d3fSMadhavan Venkataraman for (t = 0; t < CALLOUT_NTYPES; t++) {
167887a18d3fSMadhavan Venkataraman ct = &callout_table[CALLOUT_TABLE(t, f)];
167987a18d3fSMadhavan Venkataraman
168087a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex);
168187a18d3fSMadhavan Venkataraman if (ct->ct_cyclic == CYCLIC_NONE) {
1682454ab202SMadhavan Venkataraman ct->ct_suspend--;
168387a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
168487a18d3fSMadhavan Venkataraman continue;
168587a18d3fSMadhavan Venkataraman }
168687a18d3fSMadhavan Venkataraman
168751b32bddSMadhavan Venkataraman /*
168851b32bddSMadhavan Venkataraman * If a delta is specified, adjust the expirations in
168951b32bddSMadhavan Venkataraman * the heap by delta. Also, if the caller indicates
169051b32bddSMadhavan Venkataraman * a timechange, process that. This step also cleans
169151b32bddSMadhavan Venkataraman * out any empty callout lists that might happen to
169251b32bddSMadhavan Venkataraman * be there.
169351b32bddSMadhavan Venkataraman */
1694060cedfbSMadhavan Venkataraman hexp = callout_heap_process(ct, delta, timechange);
1695060cedfbSMadhavan Venkataraman qexp = callout_queue_process(ct, delta, timechange);
169687a18d3fSMadhavan Venkataraman
1697454ab202SMadhavan Venkataraman ct->ct_suspend--;
1698454ab202SMadhavan Venkataraman if (ct->ct_suspend == 0) {
1699060cedfbSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_cyclic, hexp);
1700060cedfbSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_qcyclic, qexp);
1701454ab202SMadhavan Venkataraman }
170251b32bddSMadhavan Venkataraman
170387a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
170487a18d3fSMadhavan Venkataraman }
170587a18d3fSMadhavan Venkataraman }
17067c478bd9Sstevel@tonic-gate }
17077c478bd9Sstevel@tonic-gate
17087c478bd9Sstevel@tonic-gate /*
17097c478bd9Sstevel@tonic-gate * Callback handler used by CPR to stop and resume callouts.
171051b32bddSMadhavan Venkataraman * The cyclic subsystem saves and restores hrtime during CPR.
171151b32bddSMadhavan Venkataraman * That is why callout_resume() is called with a 0 delta.
171251b32bddSMadhavan Venkataraman * Although hrtime is the same, hrestime (system time) has
171351b32bddSMadhavan Venkataraman * progressed during CPR. So, we have to indicate a time change
171451b32bddSMadhavan Venkataraman * to expire the absolute hrestime timers.
17157c478bd9Sstevel@tonic-gate */
17167c478bd9Sstevel@tonic-gate /*ARGSUSED*/
17177c478bd9Sstevel@tonic-gate static boolean_t
callout_cpr_callb(void * arg,int code)17187c478bd9Sstevel@tonic-gate callout_cpr_callb(void *arg, int code)
17197c478bd9Sstevel@tonic-gate {
172087a18d3fSMadhavan Venkataraman if (code == CB_CODE_CPR_CHKPT)
172187a18d3fSMadhavan Venkataraman callout_suspend();
172287a18d3fSMadhavan Venkataraman else
172351b32bddSMadhavan Venkataraman callout_resume(0, 1);
172487a18d3fSMadhavan Venkataraman
17257c478bd9Sstevel@tonic-gate return (B_TRUE);
17267c478bd9Sstevel@tonic-gate }
17277c478bd9Sstevel@tonic-gate
17287c478bd9Sstevel@tonic-gate /*
172987a18d3fSMadhavan Venkataraman * Callback handler invoked when the debugger is entered or exited.
17307c478bd9Sstevel@tonic-gate */
173187a18d3fSMadhavan Venkataraman /*ARGSUSED*/
173287a18d3fSMadhavan Venkataraman static boolean_t
callout_debug_callb(void * arg,int code)173387a18d3fSMadhavan Venkataraman callout_debug_callb(void *arg, int code)
17347c478bd9Sstevel@tonic-gate {
173587a18d3fSMadhavan Venkataraman hrtime_t delta;
1736f635d46aSqiao
1737f635d46aSqiao /*
173887a18d3fSMadhavan Venkataraman * When the system enters the debugger. make a note of the hrtime.
173987a18d3fSMadhavan Venkataraman * When it is resumed, compute how long the system was in the
174087a18d3fSMadhavan Venkataraman * debugger. This interval should not be counted for callouts.
1741f635d46aSqiao */
174287a18d3fSMadhavan Venkataraman if (code == 0) {
174387a18d3fSMadhavan Venkataraman callout_suspend();
174487a18d3fSMadhavan Venkataraman callout_debug_hrtime = gethrtime();
174587a18d3fSMadhavan Venkataraman } else {
174687a18d3fSMadhavan Venkataraman delta = gethrtime() - callout_debug_hrtime;
174751b32bddSMadhavan Venkataraman callout_resume(delta, 0);
174887a18d3fSMadhavan Venkataraman }
1749f635d46aSqiao
175087a18d3fSMadhavan Venkataraman return (B_TRUE);
175187a18d3fSMadhavan Venkataraman }
175287a18d3fSMadhavan Venkataraman
175387a18d3fSMadhavan Venkataraman /*
175407247649SMadhavan Venkataraman * Move the absolute hrestime callouts to the expired list. Then program the
175507247649SMadhavan Venkataraman * table's cyclic to expire immediately so that the callouts can be executed
175687a18d3fSMadhavan Venkataraman * immediately.
175787a18d3fSMadhavan Venkataraman */
175887a18d3fSMadhavan Venkataraman static void
callout_hrestime_one(callout_table_t * ct)175987a18d3fSMadhavan Venkataraman callout_hrestime_one(callout_table_t *ct)
176087a18d3fSMadhavan Venkataraman {
1761060cedfbSMadhavan Venkataraman hrtime_t hexp, qexp;
176287a18d3fSMadhavan Venkataraman
176387a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex);
1764060cedfbSMadhavan Venkataraman if (ct->ct_cyclic == CYCLIC_NONE) {
176587a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
176687a18d3fSMadhavan Venkataraman return;
176787a18d3fSMadhavan Venkataraman }
176887a18d3fSMadhavan Venkataraman
176951b32bddSMadhavan Venkataraman /*
177051b32bddSMadhavan Venkataraman * Walk the heap and process all the absolute hrestime entries.
177151b32bddSMadhavan Venkataraman */
1772060cedfbSMadhavan Venkataraman hexp = callout_heap_process(ct, 0, 1);
1773060cedfbSMadhavan Venkataraman qexp = callout_queue_process(ct, 0, 1);
177487a18d3fSMadhavan Venkataraman
1775060cedfbSMadhavan Venkataraman if (ct->ct_suspend == 0) {
1776060cedfbSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_cyclic, hexp);
1777060cedfbSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_qcyclic, qexp);
1778060cedfbSMadhavan Venkataraman }
177907247649SMadhavan Venkataraman
178087a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
178187a18d3fSMadhavan Venkataraman }
178287a18d3fSMadhavan Venkataraman
178387a18d3fSMadhavan Venkataraman /*
178487a18d3fSMadhavan Venkataraman * This function is called whenever system time (hrestime) is changed
178587a18d3fSMadhavan Venkataraman * explicitly. All the HRESTIME callouts must be expired at once.
178687a18d3fSMadhavan Venkataraman */
178787a18d3fSMadhavan Venkataraman /*ARGSUSED*/
178887a18d3fSMadhavan Venkataraman void
callout_hrestime(void)178987a18d3fSMadhavan Venkataraman callout_hrestime(void)
179087a18d3fSMadhavan Venkataraman {
179187a18d3fSMadhavan Venkataraman int t, f;
179287a18d3fSMadhavan Venkataraman callout_table_t *ct;
179387a18d3fSMadhavan Venkataraman
179487a18d3fSMadhavan Venkataraman /*
179587a18d3fSMadhavan Venkataraman * Traverse every callout table in the system and process the hrestime
179687a18d3fSMadhavan Venkataraman * callouts therein.
179787a18d3fSMadhavan Venkataraman *
179887a18d3fSMadhavan Venkataraman * We look at all the tables because we don't know which ones were
179987a18d3fSMadhavan Venkataraman * onlined and offlined in the past. The offlined tables may still
180087a18d3fSMadhavan Venkataraman * have active cyclics processing timers somewhere.
180187a18d3fSMadhavan Venkataraman */
180287a18d3fSMadhavan Venkataraman for (f = 0; f < max_ncpus; f++) {
180387a18d3fSMadhavan Venkataraman for (t = 0; t < CALLOUT_NTYPES; t++) {
180487a18d3fSMadhavan Venkataraman ct = &callout_table[CALLOUT_TABLE(t, f)];
180587a18d3fSMadhavan Venkataraman callout_hrestime_one(ct);
180687a18d3fSMadhavan Venkataraman }
180787a18d3fSMadhavan Venkataraman }
180887a18d3fSMadhavan Venkataraman }
180987a18d3fSMadhavan Venkataraman
181087a18d3fSMadhavan Venkataraman /*
181187a18d3fSMadhavan Venkataraman * Create the hash tables for this callout table.
181287a18d3fSMadhavan Venkataraman */
181387a18d3fSMadhavan Venkataraman static void
callout_hash_init(callout_table_t * ct)181487a18d3fSMadhavan Venkataraman callout_hash_init(callout_table_t *ct)
181587a18d3fSMadhavan Venkataraman {
181687a18d3fSMadhavan Venkataraman size_t size;
181787a18d3fSMadhavan Venkataraman
181887a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex));
181987a18d3fSMadhavan Venkataraman ASSERT((ct->ct_idhash == NULL) && (ct->ct_clhash == NULL));
182087a18d3fSMadhavan Venkataraman
182187a18d3fSMadhavan Venkataraman size = sizeof (callout_hash_t) * CALLOUT_BUCKETS;
182287a18d3fSMadhavan Venkataraman ct->ct_idhash = kmem_zalloc(size, KM_SLEEP);
182387a18d3fSMadhavan Venkataraman ct->ct_clhash = kmem_zalloc(size, KM_SLEEP);
182487a18d3fSMadhavan Venkataraman }
182587a18d3fSMadhavan Venkataraman
182687a18d3fSMadhavan Venkataraman /*
182787a18d3fSMadhavan Venkataraman * Create per-callout table kstats.
182887a18d3fSMadhavan Venkataraman */
182987a18d3fSMadhavan Venkataraman static void
callout_kstat_init(callout_table_t * ct)183087a18d3fSMadhavan Venkataraman callout_kstat_init(callout_table_t *ct)
183187a18d3fSMadhavan Venkataraman {
183287a18d3fSMadhavan Venkataraman callout_stat_type_t stat;
183387a18d3fSMadhavan Venkataraman kstat_t *ct_kstats;
183487a18d3fSMadhavan Venkataraman int ndx;
183587a18d3fSMadhavan Venkataraman
183687a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex));
183787a18d3fSMadhavan Venkataraman ASSERT(ct->ct_kstats == NULL);
183887a18d3fSMadhavan Venkataraman
183987a18d3fSMadhavan Venkataraman ndx = ct - callout_table;
184087a18d3fSMadhavan Venkataraman ct_kstats = kstat_create("unix", ndx, "callout",
184187a18d3fSMadhavan Venkataraman "misc", KSTAT_TYPE_NAMED, CALLOUT_NUM_STATS, KSTAT_FLAG_VIRTUAL);
184287a18d3fSMadhavan Venkataraman
184387a18d3fSMadhavan Venkataraman if (ct_kstats == NULL) {
184487a18d3fSMadhavan Venkataraman cmn_err(CE_WARN, "kstat_create for callout table %p failed",
184587a18d3fSMadhavan Venkataraman (void *)ct);
184687a18d3fSMadhavan Venkataraman } else {
184787a18d3fSMadhavan Venkataraman ct_kstats->ks_data = ct->ct_kstat_data;
184887a18d3fSMadhavan Venkataraman for (stat = 0; stat < CALLOUT_NUM_STATS; stat++)
184987a18d3fSMadhavan Venkataraman kstat_named_init(&ct->ct_kstat_data[stat],
185087a18d3fSMadhavan Venkataraman callout_kstat_names[stat], KSTAT_DATA_INT64);
185187a18d3fSMadhavan Venkataraman ct->ct_kstats = ct_kstats;
185287a18d3fSMadhavan Venkataraman kstat_install(ct_kstats);
185387a18d3fSMadhavan Venkataraman }
185487a18d3fSMadhavan Venkataraman }
185587a18d3fSMadhavan Venkataraman
185687a18d3fSMadhavan Venkataraman static void
callout_cyclic_init(callout_table_t * ct)185787a18d3fSMadhavan Venkataraman callout_cyclic_init(callout_table_t *ct)
185887a18d3fSMadhavan Venkataraman {
185987a18d3fSMadhavan Venkataraman cyc_handler_t hdlr;
186087a18d3fSMadhavan Venkataraman cyc_time_t when;
186187a18d3fSMadhavan Venkataraman processorid_t seqid;
186287a18d3fSMadhavan Venkataraman int t;
1863060cedfbSMadhavan Venkataraman cyclic_id_t cyclic, qcyclic;
186487a18d3fSMadhavan Venkataraman
186587a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex));
186687a18d3fSMadhavan Venkataraman
1867060cedfbSMadhavan Venkataraman t = ct->ct_type;
186887a18d3fSMadhavan Venkataraman seqid = CALLOUT_TABLE_SEQID(ct);
186987a18d3fSMadhavan Venkataraman
187087a18d3fSMadhavan Venkataraman /*
187187a18d3fSMadhavan Venkataraman * Create the taskq thread if the table type is normal.
187287a18d3fSMadhavan Venkataraman * Realtime tables are handled at PIL1 by a softint
187387a18d3fSMadhavan Venkataraman * handler.
187487a18d3fSMadhavan Venkataraman */
18757c478bd9Sstevel@tonic-gate if (t == CALLOUT_NORMAL) {
187687a18d3fSMadhavan Venkataraman ASSERT(ct->ct_taskq == NULL);
18777c478bd9Sstevel@tonic-gate /*
18787c478bd9Sstevel@tonic-gate * Each callout thread consumes exactly one
18797c478bd9Sstevel@tonic-gate * task structure while active. Therefore,
188051b32bddSMadhavan Venkataraman * prepopulating with 2 * callout_threads tasks
18817c478bd9Sstevel@tonic-gate * ensures that there's at least one task per
18827c478bd9Sstevel@tonic-gate * thread that's either scheduled or on the
18837c478bd9Sstevel@tonic-gate * freelist. In turn, this guarantees that
18847c478bd9Sstevel@tonic-gate * taskq_dispatch() will always either succeed
18857c478bd9Sstevel@tonic-gate * (because there's a free task structure) or
18867c478bd9Sstevel@tonic-gate * be unnecessary (because "callout_excute(ct)"
18877c478bd9Sstevel@tonic-gate * has already scheduled).
18887c478bd9Sstevel@tonic-gate */
18897c478bd9Sstevel@tonic-gate ct->ct_taskq =
189087a18d3fSMadhavan Venkataraman taskq_create_instance("callout_taskq", seqid,
189151b32bddSMadhavan Venkataraman callout_threads, maxclsyspri,
189251b32bddSMadhavan Venkataraman 2 * callout_threads, 2 * callout_threads,
18937c478bd9Sstevel@tonic-gate TASKQ_PREPOPULATE | TASKQ_CPR_SAFE);
18947c478bd9Sstevel@tonic-gate }
189587a18d3fSMadhavan Venkataraman
189687a18d3fSMadhavan Venkataraman /*
189787a18d3fSMadhavan Venkataraman * callouts can only be created in a table whose
189887a18d3fSMadhavan Venkataraman * cyclic has been initialized.
189987a18d3fSMadhavan Venkataraman */
190087a18d3fSMadhavan Venkataraman ASSERT(ct->ct_heap_num == 0);
190187a18d3fSMadhavan Venkataraman
190287a18d3fSMadhavan Venkataraman /*
1903113d3ed7SMadhavan Venkataraman * Drop the mutex before creating the callout cyclics. cyclic_add()
1904113d3ed7SMadhavan Venkataraman * could potentially expand the cyclic heap. We don't want to be
1905113d3ed7SMadhavan Venkataraman * holding the callout table mutex in that case. Note that this
1906113d3ed7SMadhavan Venkataraman * function is called during CPU online. cpu_lock is held at this
1907113d3ed7SMadhavan Venkataraman * point. So, only one thread can be executing the cyclic add logic
1908113d3ed7SMadhavan Venkataraman * below at any time.
1909113d3ed7SMadhavan Venkataraman */
1910113d3ed7SMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
1911113d3ed7SMadhavan Venkataraman
1912113d3ed7SMadhavan Venkataraman /*
191387a18d3fSMadhavan Venkataraman * Create the callout table cyclics.
191407247649SMadhavan Venkataraman *
191507247649SMadhavan Venkataraman * The realtime cyclic handler executes at low PIL. The normal cyclic
191607247649SMadhavan Venkataraman * handler executes at lock PIL. This is because there are cases
191707247649SMadhavan Venkataraman * where code can block at PIL > 1 waiting for a normal callout handler
191807247649SMadhavan Venkataraman * to unblock it directly or indirectly. If the normal cyclic were to
191907247649SMadhavan Venkataraman * be executed at low PIL, it could get blocked out by the waiter
192007247649SMadhavan Venkataraman * and cause a deadlock.
192187a18d3fSMadhavan Venkataraman */
192287a18d3fSMadhavan Venkataraman ASSERT(ct->ct_cyclic == CYCLIC_NONE);
192387a18d3fSMadhavan Venkataraman
1924060cedfbSMadhavan Venkataraman if (t == CALLOUT_REALTIME) {
192507247649SMadhavan Venkataraman hdlr.cyh_level = callout_realtime_level;
1926060cedfbSMadhavan Venkataraman hdlr.cyh_func = (cyc_func_t)callout_realtime;
1927060cedfbSMadhavan Venkataraman } else {
192807247649SMadhavan Venkataraman hdlr.cyh_level = callout_normal_level;
1929060cedfbSMadhavan Venkataraman hdlr.cyh_func = (cyc_func_t)callout_normal;
1930060cedfbSMadhavan Venkataraman }
193187a18d3fSMadhavan Venkataraman hdlr.cyh_arg = ct;
193287a18d3fSMadhavan Venkataraman when.cyt_when = CY_INFINITY;
193387a18d3fSMadhavan Venkataraman when.cyt_interval = CY_INFINITY;
193487a18d3fSMadhavan Venkataraman
1935113d3ed7SMadhavan Venkataraman cyclic = cyclic_add(&hdlr, &when);
1936113d3ed7SMadhavan Venkataraman
1937060cedfbSMadhavan Venkataraman if (t == CALLOUT_REALTIME)
1938060cedfbSMadhavan Venkataraman hdlr.cyh_func = (cyc_func_t)callout_queue_realtime;
1939060cedfbSMadhavan Venkataraman else
1940060cedfbSMadhavan Venkataraman hdlr.cyh_func = (cyc_func_t)callout_queue_normal;
1941060cedfbSMadhavan Venkataraman
1942060cedfbSMadhavan Venkataraman qcyclic = cyclic_add(&hdlr, &when);
1943060cedfbSMadhavan Venkataraman
1944113d3ed7SMadhavan Venkataraman mutex_enter(&ct->ct_mutex);
1945113d3ed7SMadhavan Venkataraman ct->ct_cyclic = cyclic;
1946060cedfbSMadhavan Venkataraman ct->ct_qcyclic = qcyclic;
194787a18d3fSMadhavan Venkataraman }
194887a18d3fSMadhavan Venkataraman
194987a18d3fSMadhavan Venkataraman void
callout_cpu_online(cpu_t * cp)195087a18d3fSMadhavan Venkataraman callout_cpu_online(cpu_t *cp)
195187a18d3fSMadhavan Venkataraman {
195287a18d3fSMadhavan Venkataraman lgrp_handle_t hand;
195387a18d3fSMadhavan Venkataraman callout_cache_t *cache;
195487a18d3fSMadhavan Venkataraman char s[KMEM_CACHE_NAMELEN];
195587a18d3fSMadhavan Venkataraman callout_table_t *ct;
195687a18d3fSMadhavan Venkataraman processorid_t seqid;
195787a18d3fSMadhavan Venkataraman int t;
195887a18d3fSMadhavan Venkataraman
195987a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&cpu_lock));
196087a18d3fSMadhavan Venkataraman
196187a18d3fSMadhavan Venkataraman /*
196287a18d3fSMadhavan Venkataraman * Locate the cache corresponding to the onlined CPU's lgroup.
196387a18d3fSMadhavan Venkataraman * Note that access to callout_caches is protected by cpu_lock.
196487a18d3fSMadhavan Venkataraman */
196587a18d3fSMadhavan Venkataraman hand = lgrp_plat_cpu_to_hand(cp->cpu_id);
196687a18d3fSMadhavan Venkataraman for (cache = callout_caches; cache != NULL; cache = cache->cc_next) {
196787a18d3fSMadhavan Venkataraman if (cache->cc_hand == hand)
196887a18d3fSMadhavan Venkataraman break;
196987a18d3fSMadhavan Venkataraman }
197087a18d3fSMadhavan Venkataraman
197187a18d3fSMadhavan Venkataraman /*
197287a18d3fSMadhavan Venkataraman * If not found, create one. The caches are never destroyed.
197387a18d3fSMadhavan Venkataraman */
197487a18d3fSMadhavan Venkataraman if (cache == NULL) {
197587a18d3fSMadhavan Venkataraman cache = kmem_alloc(sizeof (callout_cache_t), KM_SLEEP);
197687a18d3fSMadhavan Venkataraman cache->cc_hand = hand;
197787a18d3fSMadhavan Venkataraman (void) snprintf(s, KMEM_CACHE_NAMELEN, "callout_cache%lx",
197887a18d3fSMadhavan Venkataraman (long)hand);
197987a18d3fSMadhavan Venkataraman cache->cc_cache = kmem_cache_create(s, sizeof (callout_t),
198087a18d3fSMadhavan Venkataraman CALLOUT_ALIGN, NULL, NULL, NULL, NULL, NULL, 0);
198187a18d3fSMadhavan Venkataraman (void) snprintf(s, KMEM_CACHE_NAMELEN, "callout_lcache%lx",
198287a18d3fSMadhavan Venkataraman (long)hand);
198387a18d3fSMadhavan Venkataraman cache->cc_lcache = kmem_cache_create(s, sizeof (callout_list_t),
198487a18d3fSMadhavan Venkataraman CALLOUT_ALIGN, NULL, NULL, NULL, NULL, NULL, 0);
198587a18d3fSMadhavan Venkataraman cache->cc_next = callout_caches;
198687a18d3fSMadhavan Venkataraman callout_caches = cache;
198787a18d3fSMadhavan Venkataraman }
198887a18d3fSMadhavan Venkataraman
198987a18d3fSMadhavan Venkataraman seqid = cp->cpu_seqid;
199087a18d3fSMadhavan Venkataraman
199187a18d3fSMadhavan Venkataraman for (t = 0; t < CALLOUT_NTYPES; t++) {
199287a18d3fSMadhavan Venkataraman ct = &callout_table[CALLOUT_TABLE(t, seqid)];
199387a18d3fSMadhavan Venkataraman
199487a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex);
199587a18d3fSMadhavan Venkataraman /*
199687a18d3fSMadhavan Venkataraman * Store convinience pointers to the kmem caches
199787a18d3fSMadhavan Venkataraman * in the callout table. These assignments should always be
199887a18d3fSMadhavan Venkataraman * done as callout tables can map to different physical
199987a18d3fSMadhavan Venkataraman * CPUs each time.
200087a18d3fSMadhavan Venkataraman */
200187a18d3fSMadhavan Venkataraman ct->ct_cache = cache->cc_cache;
200287a18d3fSMadhavan Venkataraman ct->ct_lcache = cache->cc_lcache;
200387a18d3fSMadhavan Venkataraman
200487a18d3fSMadhavan Venkataraman /*
200587a18d3fSMadhavan Venkataraman * We use the heap pointer to check if stuff has been
200687a18d3fSMadhavan Venkataraman * initialized for this callout table.
200787a18d3fSMadhavan Venkataraman */
200887a18d3fSMadhavan Venkataraman if (ct->ct_heap == NULL) {
200987a18d3fSMadhavan Venkataraman callout_heap_init(ct);
201087a18d3fSMadhavan Venkataraman callout_hash_init(ct);
201187a18d3fSMadhavan Venkataraman callout_kstat_init(ct);
201287a18d3fSMadhavan Venkataraman callout_cyclic_init(ct);
201387a18d3fSMadhavan Venkataraman }
201487a18d3fSMadhavan Venkataraman
201587a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex);
201687a18d3fSMadhavan Venkataraman
201787a18d3fSMadhavan Venkataraman /*
2018060cedfbSMadhavan Venkataraman * Move the cyclics to this CPU by doing a bind.
201987a18d3fSMadhavan Venkataraman */
202087a18d3fSMadhavan Venkataraman cyclic_bind(ct->ct_cyclic, cp, NULL);
2021060cedfbSMadhavan Venkataraman cyclic_bind(ct->ct_qcyclic, cp, NULL);
2022454ab202SMadhavan Venkataraman }
2023454ab202SMadhavan Venkataraman }
2024454ab202SMadhavan Venkataraman
2025454ab202SMadhavan Venkataraman void
callout_cpu_offline(cpu_t * cp)2026454ab202SMadhavan Venkataraman callout_cpu_offline(cpu_t *cp)
2027454ab202SMadhavan Venkataraman {
2028454ab202SMadhavan Venkataraman callout_table_t *ct;
2029454ab202SMadhavan Venkataraman processorid_t seqid;
2030454ab202SMadhavan Venkataraman int t;
2031454ab202SMadhavan Venkataraman
2032454ab202SMadhavan Venkataraman ASSERT(MUTEX_HELD(&cpu_lock));
2033454ab202SMadhavan Venkataraman
2034454ab202SMadhavan Venkataraman seqid = cp->cpu_seqid;
2035454ab202SMadhavan Venkataraman
2036454ab202SMadhavan Venkataraman for (t = 0; t < CALLOUT_NTYPES; t++) {
2037454ab202SMadhavan Venkataraman ct = &callout_table[CALLOUT_TABLE(t, seqid)];
2038454ab202SMadhavan Venkataraman
2039454ab202SMadhavan Venkataraman /*
2040060cedfbSMadhavan Venkataraman * Unbind the cyclics. This will allow the cyclic subsystem
2041060cedfbSMadhavan Venkataraman * to juggle the cyclics during CPU offline.
2042454ab202SMadhavan Venkataraman */
204387a18d3fSMadhavan Venkataraman cyclic_bind(ct->ct_cyclic, NULL, NULL);
2044060cedfbSMadhavan Venkataraman cyclic_bind(ct->ct_qcyclic, NULL, NULL);
20457c478bd9Sstevel@tonic-gate }
20467c478bd9Sstevel@tonic-gate }
204787a18d3fSMadhavan Venkataraman
204887a18d3fSMadhavan Venkataraman /*
204987a18d3fSMadhavan Venkataraman * This is called to perform per-CPU initialization for slave CPUs at
205087a18d3fSMadhavan Venkataraman * boot time.
205187a18d3fSMadhavan Venkataraman */
205287a18d3fSMadhavan Venkataraman void
callout_mp_init(void)205387a18d3fSMadhavan Venkataraman callout_mp_init(void)
205487a18d3fSMadhavan Venkataraman {
205587a18d3fSMadhavan Venkataraman cpu_t *cp;
2056060cedfbSMadhavan Venkataraman size_t min, max;
2057060cedfbSMadhavan Venkataraman
2058060cedfbSMadhavan Venkataraman if (callout_chunk == CALLOUT_CHUNK) {
2059060cedfbSMadhavan Venkataraman /*
2060060cedfbSMadhavan Venkataraman * No one has specified a chunk in /etc/system. We need to
2061060cedfbSMadhavan Venkataraman * compute it here based on the number of online CPUs and
2062060cedfbSMadhavan Venkataraman * available physical memory.
2063060cedfbSMadhavan Venkataraman */
2064060cedfbSMadhavan Venkataraman min = CALLOUT_MIN_HEAP_SIZE;
20651671524dSMadhavan Venkataraman max = ptob(physmem / CALLOUT_MEM_FRACTION);
2066060cedfbSMadhavan Venkataraman if (min > max)
2067060cedfbSMadhavan Venkataraman min = max;
2068060cedfbSMadhavan Venkataraman callout_chunk = min / sizeof (callout_heap_t);
2069060cedfbSMadhavan Venkataraman callout_chunk /= ncpus_online;
2070060cedfbSMadhavan Venkataraman callout_chunk = P2ROUNDUP(callout_chunk, CALLOUT_CHUNK);
2071060cedfbSMadhavan Venkataraman }
207287a18d3fSMadhavan Venkataraman
207387a18d3fSMadhavan Venkataraman mutex_enter(&cpu_lock);
207487a18d3fSMadhavan Venkataraman
207587a18d3fSMadhavan Venkataraman cp = cpu_active;
207687a18d3fSMadhavan Venkataraman do {
207787a18d3fSMadhavan Venkataraman callout_cpu_online(cp);
207887a18d3fSMadhavan Venkataraman } while ((cp = cp->cpu_next_onln) != cpu_active);
207987a18d3fSMadhavan Venkataraman
208087a18d3fSMadhavan Venkataraman mutex_exit(&cpu_lock);
208187a18d3fSMadhavan Venkataraman }
208287a18d3fSMadhavan Venkataraman
208387a18d3fSMadhavan Venkataraman /*
208487a18d3fSMadhavan Venkataraman * Initialize all callout tables. Called at boot time just before clkstart().
208587a18d3fSMadhavan Venkataraman */
208687a18d3fSMadhavan Venkataraman void
callout_init(void)208787a18d3fSMadhavan Venkataraman callout_init(void)
208887a18d3fSMadhavan Venkataraman {
208987a18d3fSMadhavan Venkataraman int f, t;
209087a18d3fSMadhavan Venkataraman size_t size;
209187a18d3fSMadhavan Venkataraman int table_id;
209287a18d3fSMadhavan Venkataraman callout_table_t *ct;
209387a18d3fSMadhavan Venkataraman long bits, fanout;
209487a18d3fSMadhavan Venkataraman uintptr_t buf;
209587a18d3fSMadhavan Venkataraman
209687a18d3fSMadhavan Venkataraman /*
209787a18d3fSMadhavan Venkataraman * Initialize callout globals.
209887a18d3fSMadhavan Venkataraman */
209987a18d3fSMadhavan Venkataraman bits = 0;
210087a18d3fSMadhavan Venkataraman for (fanout = 1; (fanout < max_ncpus); fanout <<= 1)
210187a18d3fSMadhavan Venkataraman bits++;
210287a18d3fSMadhavan Venkataraman callout_table_bits = CALLOUT_TYPE_BITS + bits;
210387a18d3fSMadhavan Venkataraman callout_table_mask = (1 << callout_table_bits) - 1;
210487a18d3fSMadhavan Venkataraman callout_counter_low = 1 << CALLOUT_COUNTER_SHIFT;
210587a18d3fSMadhavan Venkataraman callout_longterm = TICK_TO_NSEC(CALLOUT_LONGTERM_TICKS);
2106454ab202SMadhavan Venkataraman callout_max_ticks = CALLOUT_MAX_TICKS;
210751b32bddSMadhavan Venkataraman if (callout_min_reap == 0)
210851b32bddSMadhavan Venkataraman callout_min_reap = CALLOUT_MIN_REAP;
210987a18d3fSMadhavan Venkataraman
211051b32bddSMadhavan Venkataraman if (callout_tolerance <= 0)
211151b32bddSMadhavan Venkataraman callout_tolerance = CALLOUT_TOLERANCE;
211251b32bddSMadhavan Venkataraman if (callout_threads <= 0)
211351b32bddSMadhavan Venkataraman callout_threads = CALLOUT_THREADS;
2114060cedfbSMadhavan Venkataraman if (callout_chunk <= 0)
2115060cedfbSMadhavan Venkataraman callout_chunk = CALLOUT_CHUNK;
2116060cedfbSMadhavan Venkataraman else
2117060cedfbSMadhavan Venkataraman callout_chunk = P2ROUNDUP(callout_chunk, CALLOUT_CHUNK);
211887a18d3fSMadhavan Venkataraman
211987a18d3fSMadhavan Venkataraman /*
212087a18d3fSMadhavan Venkataraman * Allocate all the callout tables based on max_ncpus. We have chosen
212187a18d3fSMadhavan Venkataraman * to do boot-time allocation instead of dynamic allocation because:
212287a18d3fSMadhavan Venkataraman *
212387a18d3fSMadhavan Venkataraman * - the size of the callout tables is not too large.
212487a18d3fSMadhavan Venkataraman * - there are race conditions involved in making this dynamic.
212587a18d3fSMadhavan Venkataraman * - the hash tables that go with the callout tables consume
212687a18d3fSMadhavan Venkataraman * most of the memory and they are only allocated in
212787a18d3fSMadhavan Venkataraman * callout_cpu_online().
212887a18d3fSMadhavan Venkataraman *
212987a18d3fSMadhavan Venkataraman * Each CPU has two tables that are consecutive in the array. The first
213087a18d3fSMadhavan Venkataraman * one is for realtime callouts and the second one is for normal ones.
213187a18d3fSMadhavan Venkataraman *
213287a18d3fSMadhavan Venkataraman * We do this alignment dance to make sure that callout table
213387a18d3fSMadhavan Venkataraman * structures will always be on a cache line boundary.
213487a18d3fSMadhavan Venkataraman */
213587a18d3fSMadhavan Venkataraman size = sizeof (callout_table_t) * CALLOUT_NTYPES * max_ncpus;
213687a18d3fSMadhavan Venkataraman size += CALLOUT_ALIGN;
213787a18d3fSMadhavan Venkataraman buf = (uintptr_t)kmem_zalloc(size, KM_SLEEP);
213887a18d3fSMadhavan Venkataraman callout_table = (callout_table_t *)P2ROUNDUP(buf, CALLOUT_ALIGN);
213987a18d3fSMadhavan Venkataraman
214087a18d3fSMadhavan Venkataraman size = sizeof (kstat_named_t) * CALLOUT_NUM_STATS;
214187a18d3fSMadhavan Venkataraman /*
214287a18d3fSMadhavan Venkataraman * Now, initialize the tables for all the CPUs.
214387a18d3fSMadhavan Venkataraman */
214487a18d3fSMadhavan Venkataraman for (f = 0; f < max_ncpus; f++) {
214587a18d3fSMadhavan Venkataraman for (t = 0; t < CALLOUT_NTYPES; t++) {
214687a18d3fSMadhavan Venkataraman table_id = CALLOUT_TABLE(t, f);
214787a18d3fSMadhavan Venkataraman ct = &callout_table[table_id];
2148454ab202SMadhavan Venkataraman ct->ct_type = t;
214987a18d3fSMadhavan Venkataraman mutex_init(&ct->ct_mutex, NULL, MUTEX_DEFAULT, NULL);
215087a18d3fSMadhavan Venkataraman /*
215187a18d3fSMadhavan Venkataraman * Precompute the base IDs for long and short-term
215287a18d3fSMadhavan Venkataraman * legacy IDs. This makes ID generation during
215387a18d3fSMadhavan Venkataraman * timeout() fast.
215487a18d3fSMadhavan Venkataraman */
215587a18d3fSMadhavan Venkataraman ct->ct_short_id = CALLOUT_SHORT_ID(table_id);
215687a18d3fSMadhavan Venkataraman ct->ct_long_id = CALLOUT_LONG_ID(table_id);
215787a18d3fSMadhavan Venkataraman /*
215887a18d3fSMadhavan Venkataraman * Precompute the base ID for generation-based IDs.
215987a18d3fSMadhavan Venkataraman * Note that when the first ID gets allocated, the
216087a18d3fSMadhavan Venkataraman * ID will wrap. This will cause the generation
216187a18d3fSMadhavan Venkataraman * number to be incremented to 1.
216287a18d3fSMadhavan Venkataraman */
216387a18d3fSMadhavan Venkataraman ct->ct_gen_id = CALLOUT_SHORT_ID(table_id);
216487a18d3fSMadhavan Venkataraman /*
2165060cedfbSMadhavan Venkataraman * Initialize the cyclics as NONE. This will get set
216687a18d3fSMadhavan Venkataraman * during CPU online. This is so that partially
216787a18d3fSMadhavan Venkataraman * populated systems will only have the required
216887a18d3fSMadhavan Venkataraman * number of cyclics, not more.
216987a18d3fSMadhavan Venkataraman */
217087a18d3fSMadhavan Venkataraman ct->ct_cyclic = CYCLIC_NONE;
2171060cedfbSMadhavan Venkataraman ct->ct_qcyclic = CYCLIC_NONE;
217287a18d3fSMadhavan Venkataraman ct->ct_kstat_data = kmem_zalloc(size, KM_SLEEP);
217387a18d3fSMadhavan Venkataraman }
217487a18d3fSMadhavan Venkataraman }
217587a18d3fSMadhavan Venkataraman
217687a18d3fSMadhavan Venkataraman /*
217787a18d3fSMadhavan Venkataraman * Add the callback for CPR. This is called during checkpoint
217887a18d3fSMadhavan Venkataraman * resume to suspend and resume callouts.
217987a18d3fSMadhavan Venkataraman */
218087a18d3fSMadhavan Venkataraman (void) callb_add(callout_cpr_callb, 0, CB_CL_CPR_CALLOUT,
218187a18d3fSMadhavan Venkataraman "callout_cpr");
218287a18d3fSMadhavan Venkataraman (void) callb_add(callout_debug_callb, 0, CB_CL_ENTER_DEBUGGER,
218387a18d3fSMadhavan Venkataraman "callout_debug");
218487a18d3fSMadhavan Venkataraman
218587a18d3fSMadhavan Venkataraman /*
218687a18d3fSMadhavan Venkataraman * Call the per-CPU initialization function for the boot CPU. This
218787a18d3fSMadhavan Venkataraman * is done here because the function is not called automatically for
218887a18d3fSMadhavan Venkataraman * the boot CPU from the CPU online/offline hooks. Note that the
218987a18d3fSMadhavan Venkataraman * CPU lock is taken here because of convention.
219087a18d3fSMadhavan Venkataraman */
219187a18d3fSMadhavan Venkataraman mutex_enter(&cpu_lock);
219287a18d3fSMadhavan Venkataraman callout_boot_ct = &callout_table[CALLOUT_TABLE(0, CPU->cpu_seqid)];
219387a18d3fSMadhavan Venkataraman callout_cpu_online(CPU);
219487a18d3fSMadhavan Venkataraman mutex_exit(&cpu_lock);
21956e00b116SPeter Telford
21966e00b116SPeter Telford /* heads-up to boot-time clients that timeouts now available */
21976e00b116SPeter Telford callout_init_done = 1;
21987c478bd9Sstevel@tonic-gate }
2199