17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5f635d46aSqiao * Common Development and Distribution License (the "License"). 6f635d46aSqiao * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22*6e00b116SPeter Telford * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 237c478bd9Sstevel@tonic-gate */ 247c478bd9Sstevel@tonic-gate 257c478bd9Sstevel@tonic-gate #include <sys/callo.h> 267c478bd9Sstevel@tonic-gate #include <sys/param.h> 277c478bd9Sstevel@tonic-gate #include <sys/types.h> 287c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 297c478bd9Sstevel@tonic-gate #include <sys/thread.h> 307c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 3187a18d3fSMadhavan Venkataraman #include <sys/kmem_impl.h> 327c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 337c478bd9Sstevel@tonic-gate #include <sys/callb.h> 347c478bd9Sstevel@tonic-gate #include <sys/debug.h> 357c478bd9Sstevel@tonic-gate #include <sys/vtrace.h> 367c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 377c478bd9Sstevel@tonic-gate #include <sys/sdt.h> 387c478bd9Sstevel@tonic-gate 39*6e00b116SPeter Telford int callout_init_done; /* useful during boot */ 40*6e00b116SPeter Telford 417c478bd9Sstevel@tonic-gate /* 427c478bd9Sstevel@tonic-gate * Callout tables. See timeout(9F) for details. 437c478bd9Sstevel@tonic-gate */ 4451b32bddSMadhavan Venkataraman static int callout_threads; /* callout normal threads */ 4587a18d3fSMadhavan Venkataraman static hrtime_t callout_debug_hrtime; /* debugger entry time */ 46060cedfbSMadhavan Venkataraman static int callout_chunk; /* callout heap chunk size */ 4751b32bddSMadhavan Venkataraman static int callout_min_reap; /* callout minimum reap count */ 4851b32bddSMadhavan Venkataraman static int callout_tolerance; /* callout hires tolerance */ 4987a18d3fSMadhavan Venkataraman static callout_table_t *callout_boot_ct; /* Boot CPU's callout tables */ 50454ab202SMadhavan Venkataraman static clock_t callout_max_ticks; /* max interval */ 5187a18d3fSMadhavan Venkataraman static hrtime_t callout_longterm; /* longterm nanoseconds */ 5287a18d3fSMadhavan Venkataraman static ulong_t callout_counter_low; /* callout ID increment */ 5387a18d3fSMadhavan Venkataraman static ulong_t callout_table_bits; /* number of table bits in ID */ 5487a18d3fSMadhavan Venkataraman static ulong_t callout_table_mask; /* mask for the table bits */ 5587a18d3fSMadhavan Venkataraman static callout_cache_t *callout_caches; /* linked list of caches */ 5687a18d3fSMadhavan Venkataraman #pragma align 64(callout_table) 5787a18d3fSMadhavan Venkataraman static callout_table_t *callout_table; /* global callout table array */ 587c478bd9Sstevel@tonic-gate 5907247649SMadhavan Venkataraman /* 604c06356bSdh142964 * We run 'realtime' callouts at PIL 1 (CY_LOW_LEVEL). For 'normal' 614c06356bSdh142964 * callouts, from PIL 10 (CY_LOCK_LEVEL) we dispatch the callout, 624c06356bSdh142964 * via taskq, to a thread that executes at PIL 0 - so we end up running 634c06356bSdh142964 * 'normal' callouts at PIL 0. 6407247649SMadhavan Venkataraman */ 6551b32bddSMadhavan Venkataraman static volatile int callout_realtime_level = CY_LOW_LEVEL; 6651b32bddSMadhavan Venkataraman static volatile int callout_normal_level = CY_LOCK_LEVEL; 6707247649SMadhavan Venkataraman 6887a18d3fSMadhavan Venkataraman static char *callout_kstat_names[] = { 6987a18d3fSMadhavan Venkataraman "callout_timeouts", 7087a18d3fSMadhavan Venkataraman "callout_timeouts_pending", 7187a18d3fSMadhavan Venkataraman "callout_untimeouts_unexpired", 7287a18d3fSMadhavan Venkataraman "callout_untimeouts_executing", 7387a18d3fSMadhavan Venkataraman "callout_untimeouts_expired", 7487a18d3fSMadhavan Venkataraman "callout_expirations", 7587a18d3fSMadhavan Venkataraman "callout_allocations", 7651b32bddSMadhavan Venkataraman "callout_cleanups", 7787a18d3fSMadhavan Venkataraman }; 7887a18d3fSMadhavan Venkataraman 7951b32bddSMadhavan Venkataraman static hrtime_t callout_heap_process(callout_table_t *, hrtime_t, int); 8051b32bddSMadhavan Venkataraman 8187a18d3fSMadhavan Venkataraman #define CALLOUT_HASH_INSERT(hash, cp, cnext, cprev) \ 827c478bd9Sstevel@tonic-gate { \ 8387a18d3fSMadhavan Venkataraman callout_hash_t *hashp = &(hash); \ 8487a18d3fSMadhavan Venkataraman \ 857c478bd9Sstevel@tonic-gate cp->cprev = NULL; \ 8687a18d3fSMadhavan Venkataraman cp->cnext = hashp->ch_head; \ 8787a18d3fSMadhavan Venkataraman if (hashp->ch_head == NULL) \ 8887a18d3fSMadhavan Venkataraman hashp->ch_tail = cp; \ 897c478bd9Sstevel@tonic-gate else \ 9087a18d3fSMadhavan Venkataraman cp->cnext->cprev = cp; \ 9187a18d3fSMadhavan Venkataraman hashp->ch_head = cp; \ 927c478bd9Sstevel@tonic-gate } 937c478bd9Sstevel@tonic-gate 9487a18d3fSMadhavan Venkataraman #define CALLOUT_HASH_APPEND(hash, cp, cnext, cprev) \ 9587a18d3fSMadhavan Venkataraman { \ 9687a18d3fSMadhavan Venkataraman callout_hash_t *hashp = &(hash); \ 9787a18d3fSMadhavan Venkataraman \ 9887a18d3fSMadhavan Venkataraman cp->cnext = NULL; \ 9987a18d3fSMadhavan Venkataraman cp->cprev = hashp->ch_tail; \ 10087a18d3fSMadhavan Venkataraman if (hashp->ch_tail == NULL) \ 10187a18d3fSMadhavan Venkataraman hashp->ch_head = cp; \ 10287a18d3fSMadhavan Venkataraman else \ 10387a18d3fSMadhavan Venkataraman cp->cprev->cnext = cp; \ 10487a18d3fSMadhavan Venkataraman hashp->ch_tail = cp; \ 10587a18d3fSMadhavan Venkataraman } 10687a18d3fSMadhavan Venkataraman 10787a18d3fSMadhavan Venkataraman #define CALLOUT_HASH_DELETE(hash, cp, cnext, cprev) \ 10887a18d3fSMadhavan Venkataraman { \ 10987a18d3fSMadhavan Venkataraman callout_hash_t *hashp = &(hash); \ 11087a18d3fSMadhavan Venkataraman \ 11187a18d3fSMadhavan Venkataraman if (cp->cnext == NULL) \ 11287a18d3fSMadhavan Venkataraman hashp->ch_tail = cp->cprev; \ 11387a18d3fSMadhavan Venkataraman else \ 11487a18d3fSMadhavan Venkataraman cp->cnext->cprev = cp->cprev; \ 11587a18d3fSMadhavan Venkataraman if (cp->cprev == NULL) \ 11687a18d3fSMadhavan Venkataraman hashp->ch_head = cp->cnext; \ 11787a18d3fSMadhavan Venkataraman else \ 11887a18d3fSMadhavan Venkataraman cp->cprev->cnext = cp->cnext; \ 11987a18d3fSMadhavan Venkataraman } 12087a18d3fSMadhavan Venkataraman 12187a18d3fSMadhavan Venkataraman /* 12287a18d3fSMadhavan Venkataraman * These definitions help us queue callouts and callout lists. Here is 12387a18d3fSMadhavan Venkataraman * the queueing rationale: 12487a18d3fSMadhavan Venkataraman * 12587a18d3fSMadhavan Venkataraman * - callouts are queued in a FIFO manner in the ID hash table. 12687a18d3fSMadhavan Venkataraman * TCP timers are typically cancelled in the same order that they 12787a18d3fSMadhavan Venkataraman * were issued. The FIFO queueing shortens the search for a callout 12887a18d3fSMadhavan Venkataraman * during untimeout(). 12987a18d3fSMadhavan Venkataraman * 13087a18d3fSMadhavan Venkataraman * - callouts are queued in a FIFO manner in their callout lists. 13187a18d3fSMadhavan Venkataraman * This ensures that the callouts are executed in the same order that 13287a18d3fSMadhavan Venkataraman * they were queued. This is fair. Plus, it helps to make each 13387a18d3fSMadhavan Venkataraman * callout expiration timely. It also favors cancellations. 13487a18d3fSMadhavan Venkataraman * 13551b32bddSMadhavan Venkataraman * - callout lists are queued in the following manner in the callout 13651b32bddSMadhavan Venkataraman * hash table buckets: 13751b32bddSMadhavan Venkataraman * 13851b32bddSMadhavan Venkataraman * - appended, if the callout list is a 1-nanosecond resolution 13951b32bddSMadhavan Venkataraman * callout list. When a callout is created, we first look for 14051b32bddSMadhavan Venkataraman * a callout list that has the same expiration so we can avoid 14151b32bddSMadhavan Venkataraman * allocating a callout list and inserting the expiration into 14251b32bddSMadhavan Venkataraman * the heap. However, we do not want to look at 1-nanosecond 14351b32bddSMadhavan Venkataraman * resolution callout lists as we will seldom find a match in 14451b32bddSMadhavan Venkataraman * them. Keeping these callout lists in the rear of the hash 14551b32bddSMadhavan Venkataraman * buckets allows us to skip these during the lookup. 14651b32bddSMadhavan Venkataraman * 14751b32bddSMadhavan Venkataraman * - inserted at the beginning, if the callout list is not a 14851b32bddSMadhavan Venkataraman * 1-nanosecond resolution callout list. This also has the 14951b32bddSMadhavan Venkataraman * side-effect of keeping the long term timers away from the 15051b32bddSMadhavan Venkataraman * front of the buckets. 15187a18d3fSMadhavan Venkataraman * 15287a18d3fSMadhavan Venkataraman * - callout lists are queued in a FIFO manner in the expired callouts 15387a18d3fSMadhavan Venkataraman * list. This ensures that callout lists are executed in the order 15487a18d3fSMadhavan Venkataraman * of expiration. 15587a18d3fSMadhavan Venkataraman */ 15687a18d3fSMadhavan Venkataraman #define CALLOUT_APPEND(ct, cp) \ 15787a18d3fSMadhavan Venkataraman CALLOUT_HASH_APPEND(ct->ct_idhash[CALLOUT_IDHASH(cp->c_xid)], \ 15887a18d3fSMadhavan Venkataraman cp, c_idnext, c_idprev); \ 15987a18d3fSMadhavan Venkataraman CALLOUT_HASH_APPEND(cp->c_list->cl_callouts, cp, c_clnext, c_clprev) 16087a18d3fSMadhavan Venkataraman 16187a18d3fSMadhavan Venkataraman #define CALLOUT_DELETE(ct, cp) \ 16287a18d3fSMadhavan Venkataraman CALLOUT_HASH_DELETE(ct->ct_idhash[CALLOUT_IDHASH(cp->c_xid)], \ 16387a18d3fSMadhavan Venkataraman cp, c_idnext, c_idprev); \ 16487a18d3fSMadhavan Venkataraman CALLOUT_HASH_DELETE(cp->c_list->cl_callouts, cp, c_clnext, c_clprev) 16587a18d3fSMadhavan Venkataraman 16687a18d3fSMadhavan Venkataraman #define CALLOUT_LIST_INSERT(hash, cl) \ 16787a18d3fSMadhavan Venkataraman CALLOUT_HASH_INSERT(hash, cl, cl_next, cl_prev) 16887a18d3fSMadhavan Venkataraman 16987a18d3fSMadhavan Venkataraman #define CALLOUT_LIST_APPEND(hash, cl) \ 17087a18d3fSMadhavan Venkataraman CALLOUT_HASH_APPEND(hash, cl, cl_next, cl_prev) 17187a18d3fSMadhavan Venkataraman 17287a18d3fSMadhavan Venkataraman #define CALLOUT_LIST_DELETE(hash, cl) \ 17387a18d3fSMadhavan Venkataraman CALLOUT_HASH_DELETE(hash, cl, cl_next, cl_prev) 1747c478bd9Sstevel@tonic-gate 175060cedfbSMadhavan Venkataraman #define CALLOUT_LIST_BEFORE(cl, nextcl) \ 176060cedfbSMadhavan Venkataraman { \ 177060cedfbSMadhavan Venkataraman (cl)->cl_prev = (nextcl)->cl_prev; \ 178060cedfbSMadhavan Venkataraman (cl)->cl_next = (nextcl); \ 179060cedfbSMadhavan Venkataraman (nextcl)->cl_prev = (cl); \ 180060cedfbSMadhavan Venkataraman if (cl->cl_prev != NULL) \ 181060cedfbSMadhavan Venkataraman cl->cl_prev->cl_next = cl; \ 182060cedfbSMadhavan Venkataraman } 183060cedfbSMadhavan Venkataraman 1847c478bd9Sstevel@tonic-gate /* 18507247649SMadhavan Venkataraman * For normal callouts, there is a deadlock scenario if two callouts that 18607247649SMadhavan Venkataraman * have an inter-dependency end up on the same callout list. To break the 18707247649SMadhavan Venkataraman * deadlock, you need two taskq threads running in parallel. We compute 18807247649SMadhavan Venkataraman * the number of taskq threads here using a bunch of conditions to make 18907247649SMadhavan Venkataraman * it optimal for the common case. This is an ugly hack, but one that is 19007247649SMadhavan Venkataraman * necessary (sigh). 19107247649SMadhavan Venkataraman */ 19207247649SMadhavan Venkataraman #define CALLOUT_THRESHOLD 100000000 193060cedfbSMadhavan Venkataraman #define CALLOUT_EXEC_COMPUTE(ct, nextexp, exec) \ 19407247649SMadhavan Venkataraman { \ 19507247649SMadhavan Venkataraman callout_list_t *cl; \ 19607247649SMadhavan Venkataraman \ 19707247649SMadhavan Venkataraman cl = ct->ct_expired.ch_head; \ 19807247649SMadhavan Venkataraman if (cl == NULL) { \ 19907247649SMadhavan Venkataraman /* \ 20007247649SMadhavan Venkataraman * If the expired list is NULL, there is nothing to \ 20107247649SMadhavan Venkataraman * process. \ 20207247649SMadhavan Venkataraman */ \ 20307247649SMadhavan Venkataraman exec = 0; \ 20407247649SMadhavan Venkataraman } else if ((cl->cl_next == NULL) && \ 20507247649SMadhavan Venkataraman (cl->cl_callouts.ch_head == cl->cl_callouts.ch_tail)) { \ 20607247649SMadhavan Venkataraman /* \ 20707247649SMadhavan Venkataraman * If there is only one callout list and it contains \ 20807247649SMadhavan Venkataraman * only one callout, there is no need for two threads. \ 20907247649SMadhavan Venkataraman */ \ 21007247649SMadhavan Venkataraman exec = 1; \ 211060cedfbSMadhavan Venkataraman } else if ((nextexp) > (gethrtime() + CALLOUT_THRESHOLD)) { \ 21207247649SMadhavan Venkataraman /* \ 213060cedfbSMadhavan Venkataraman * If the next expiration of the cyclic is way out into \ 214060cedfbSMadhavan Venkataraman * the future, we need two threads. \ 21507247649SMadhavan Venkataraman */ \ 21607247649SMadhavan Venkataraman exec = 2; \ 21707247649SMadhavan Venkataraman } else { \ 21807247649SMadhavan Venkataraman /* \ 21907247649SMadhavan Venkataraman * We have multiple callouts to process. But the cyclic \ 22007247649SMadhavan Venkataraman * will fire in the near future. So, we only need one \ 22107247649SMadhavan Venkataraman * thread for now. \ 22207247649SMadhavan Venkataraman */ \ 22307247649SMadhavan Venkataraman exec = 1; \ 22407247649SMadhavan Venkataraman } \ 22507247649SMadhavan Venkataraman } 22607247649SMadhavan Venkataraman 22707247649SMadhavan Venkataraman /* 22851b32bddSMadhavan Venkataraman * Macro to swap two heap items. 22951b32bddSMadhavan Venkataraman */ 23051b32bddSMadhavan Venkataraman #define CALLOUT_SWAP(h1, h2) \ 23151b32bddSMadhavan Venkataraman { \ 23251b32bddSMadhavan Venkataraman callout_heap_t tmp; \ 23351b32bddSMadhavan Venkataraman \ 23451b32bddSMadhavan Venkataraman tmp = *h1; \ 23551b32bddSMadhavan Venkataraman *h1 = *h2; \ 23651b32bddSMadhavan Venkataraman *h2 = tmp; \ 23751b32bddSMadhavan Venkataraman } 23851b32bddSMadhavan Venkataraman 23951b32bddSMadhavan Venkataraman /* 24051b32bddSMadhavan Venkataraman * Macro to free a callout list. 24151b32bddSMadhavan Venkataraman */ 24251b32bddSMadhavan Venkataraman #define CALLOUT_LIST_FREE(ct, cl) \ 24351b32bddSMadhavan Venkataraman { \ 24451b32bddSMadhavan Venkataraman cl->cl_next = ct->ct_lfree; \ 24551b32bddSMadhavan Venkataraman ct->ct_lfree = cl; \ 24651b32bddSMadhavan Venkataraman cl->cl_flags |= CALLOUT_LIST_FLAG_FREE; \ 24751b32bddSMadhavan Venkataraman } 24851b32bddSMadhavan Venkataraman 24951b32bddSMadhavan Venkataraman /* 250060cedfbSMadhavan Venkataraman * Macro to free a callout. 251060cedfbSMadhavan Venkataraman */ 252060cedfbSMadhavan Venkataraman #define CALLOUT_FREE(ct, cl) \ 253060cedfbSMadhavan Venkataraman { \ 254060cedfbSMadhavan Venkataraman cp->c_idnext = ct->ct_free; \ 255060cedfbSMadhavan Venkataraman ct->ct_free = cp; \ 256060cedfbSMadhavan Venkataraman cp->c_xid |= CALLOUT_ID_FREE; \ 257060cedfbSMadhavan Venkataraman } 258060cedfbSMadhavan Venkataraman 259060cedfbSMadhavan Venkataraman /* 2607c478bd9Sstevel@tonic-gate * Allocate a callout structure. We try quite hard because we 2617c478bd9Sstevel@tonic-gate * can't sleep, and if we can't do the allocation, we're toast. 26287a18d3fSMadhavan Venkataraman * Failing all, we try a KM_PANIC allocation. Note that we never 26387a18d3fSMadhavan Venkataraman * deallocate a callout. See untimeout() for the reasoning. 2647c478bd9Sstevel@tonic-gate */ 2657c478bd9Sstevel@tonic-gate static callout_t * 2667c478bd9Sstevel@tonic-gate callout_alloc(callout_table_t *ct) 2677c478bd9Sstevel@tonic-gate { 26887a18d3fSMadhavan Venkataraman size_t size; 26987a18d3fSMadhavan Venkataraman callout_t *cp; 2707c478bd9Sstevel@tonic-gate 27187a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 27287a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 27387a18d3fSMadhavan Venkataraman 27487a18d3fSMadhavan Venkataraman cp = kmem_cache_alloc(ct->ct_cache, KM_NOSLEEP); 27587a18d3fSMadhavan Venkataraman if (cp == NULL) { 27687a18d3fSMadhavan Venkataraman size = sizeof (callout_t); 27787a18d3fSMadhavan Venkataraman cp = kmem_alloc_tryhard(size, &size, KM_NOSLEEP | KM_PANIC); 27887a18d3fSMadhavan Venkataraman } 27987a18d3fSMadhavan Venkataraman cp->c_xid = 0; 28007247649SMadhavan Venkataraman cp->c_executor = NULL; 28107247649SMadhavan Venkataraman cv_init(&cp->c_done, NULL, CV_DEFAULT, NULL); 28207247649SMadhavan Venkataraman cp->c_waiting = 0; 28387a18d3fSMadhavan Venkataraman 28487a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 28587a18d3fSMadhavan Venkataraman ct->ct_allocations++; 2867c478bd9Sstevel@tonic-gate return (cp); 2877c478bd9Sstevel@tonic-gate } 2887c478bd9Sstevel@tonic-gate 2897c478bd9Sstevel@tonic-gate /* 29087a18d3fSMadhavan Venkataraman * Allocate a callout list structure. We try quite hard because we 29187a18d3fSMadhavan Venkataraman * can't sleep, and if we can't do the allocation, we're toast. 29287a18d3fSMadhavan Venkataraman * Failing all, we try a KM_PANIC allocation. Note that we never 29387a18d3fSMadhavan Venkataraman * deallocate a callout list. 2947c478bd9Sstevel@tonic-gate */ 29587a18d3fSMadhavan Venkataraman static void 29687a18d3fSMadhavan Venkataraman callout_list_alloc(callout_table_t *ct) 2977c478bd9Sstevel@tonic-gate { 29887a18d3fSMadhavan Venkataraman size_t size; 29987a18d3fSMadhavan Venkataraman callout_list_t *cl; 30087a18d3fSMadhavan Venkataraman 30187a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 30287a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 30387a18d3fSMadhavan Venkataraman 30487a18d3fSMadhavan Venkataraman cl = kmem_cache_alloc(ct->ct_lcache, KM_NOSLEEP); 30587a18d3fSMadhavan Venkataraman if (cl == NULL) { 30687a18d3fSMadhavan Venkataraman size = sizeof (callout_list_t); 30787a18d3fSMadhavan Venkataraman cl = kmem_alloc_tryhard(size, &size, KM_NOSLEEP | KM_PANIC); 30887a18d3fSMadhavan Venkataraman } 30987a18d3fSMadhavan Venkataraman bzero(cl, sizeof (callout_list_t)); 31087a18d3fSMadhavan Venkataraman 31187a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 31251b32bddSMadhavan Venkataraman CALLOUT_LIST_FREE(ct, cl); 31387a18d3fSMadhavan Venkataraman } 31487a18d3fSMadhavan Venkataraman 31587a18d3fSMadhavan Venkataraman /* 31651b32bddSMadhavan Venkataraman * Find a callout list that corresponds to an expiration and matching flags. 31787a18d3fSMadhavan Venkataraman */ 31887a18d3fSMadhavan Venkataraman static callout_list_t * 31907247649SMadhavan Venkataraman callout_list_get(callout_table_t *ct, hrtime_t expiration, int flags, int hash) 32087a18d3fSMadhavan Venkataraman { 32187a18d3fSMadhavan Venkataraman callout_list_t *cl; 32251b32bddSMadhavan Venkataraman int clflags; 32387a18d3fSMadhavan Venkataraman 32487a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 32587a18d3fSMadhavan Venkataraman 32651b32bddSMadhavan Venkataraman if (flags & CALLOUT_LIST_FLAG_NANO) { 32751b32bddSMadhavan Venkataraman /* 32851b32bddSMadhavan Venkataraman * This is a 1-nanosecond resolution callout. We will rarely 32951b32bddSMadhavan Venkataraman * find a match for this. So, bail out. 33051b32bddSMadhavan Venkataraman */ 33151b32bddSMadhavan Venkataraman return (NULL); 33251b32bddSMadhavan Venkataraman } 33351b32bddSMadhavan Venkataraman 33451b32bddSMadhavan Venkataraman clflags = (CALLOUT_LIST_FLAG_ABSOLUTE | CALLOUT_LIST_FLAG_HRESTIME); 33587a18d3fSMadhavan Venkataraman for (cl = ct->ct_clhash[hash].ch_head; (cl != NULL); cl = cl->cl_next) { 33651b32bddSMadhavan Venkataraman /* 33751b32bddSMadhavan Venkataraman * If we have reached a 1-nanosecond resolution callout list, 33851b32bddSMadhavan Venkataraman * we don't have much hope of finding a match in this hash 33951b32bddSMadhavan Venkataraman * bucket. So, just bail out. 34051b32bddSMadhavan Venkataraman */ 34151b32bddSMadhavan Venkataraman if (cl->cl_flags & CALLOUT_LIST_FLAG_NANO) 34251b32bddSMadhavan Venkataraman return (NULL); 34351b32bddSMadhavan Venkataraman 34407247649SMadhavan Venkataraman if ((cl->cl_expiration == expiration) && 34551b32bddSMadhavan Venkataraman ((cl->cl_flags & clflags) == (flags & clflags))) 34687a18d3fSMadhavan Venkataraman return (cl); 34787a18d3fSMadhavan Venkataraman } 34887a18d3fSMadhavan Venkataraman 34987a18d3fSMadhavan Venkataraman return (NULL); 35087a18d3fSMadhavan Venkataraman } 35187a18d3fSMadhavan Venkataraman 35287a18d3fSMadhavan Venkataraman /* 353060cedfbSMadhavan Venkataraman * Add a new callout list into a callout table's queue in sorted order by 354060cedfbSMadhavan Venkataraman * expiration. 355060cedfbSMadhavan Venkataraman */ 356060cedfbSMadhavan Venkataraman static int 357060cedfbSMadhavan Venkataraman callout_queue_add(callout_table_t *ct, callout_list_t *cl) 358060cedfbSMadhavan Venkataraman { 359060cedfbSMadhavan Venkataraman callout_list_t *nextcl; 360060cedfbSMadhavan Venkataraman hrtime_t expiration; 361060cedfbSMadhavan Venkataraman 362060cedfbSMadhavan Venkataraman expiration = cl->cl_expiration; 363060cedfbSMadhavan Venkataraman nextcl = ct->ct_queue.ch_head; 364060cedfbSMadhavan Venkataraman if ((nextcl == NULL) || (expiration < nextcl->cl_expiration)) { 365060cedfbSMadhavan Venkataraman CALLOUT_LIST_INSERT(ct->ct_queue, cl); 366060cedfbSMadhavan Venkataraman return (1); 367060cedfbSMadhavan Venkataraman } 368060cedfbSMadhavan Venkataraman 369060cedfbSMadhavan Venkataraman while (nextcl != NULL) { 370060cedfbSMadhavan Venkataraman if (expiration < nextcl->cl_expiration) { 371060cedfbSMadhavan Venkataraman CALLOUT_LIST_BEFORE(cl, nextcl); 372060cedfbSMadhavan Venkataraman return (0); 373060cedfbSMadhavan Venkataraman } 374060cedfbSMadhavan Venkataraman nextcl = nextcl->cl_next; 375060cedfbSMadhavan Venkataraman } 376060cedfbSMadhavan Venkataraman CALLOUT_LIST_APPEND(ct->ct_queue, cl); 377060cedfbSMadhavan Venkataraman 378060cedfbSMadhavan Venkataraman return (0); 379060cedfbSMadhavan Venkataraman } 380060cedfbSMadhavan Venkataraman 381060cedfbSMadhavan Venkataraman /* 382060cedfbSMadhavan Venkataraman * Insert a callout list into a callout table's queue and reprogram the queue 383060cedfbSMadhavan Venkataraman * cyclic if needed. 384060cedfbSMadhavan Venkataraman */ 385060cedfbSMadhavan Venkataraman static void 386060cedfbSMadhavan Venkataraman callout_queue_insert(callout_table_t *ct, callout_list_t *cl) 387060cedfbSMadhavan Venkataraman { 388060cedfbSMadhavan Venkataraman cl->cl_flags |= CALLOUT_LIST_FLAG_QUEUED; 389060cedfbSMadhavan Venkataraman 390060cedfbSMadhavan Venkataraman /* 391060cedfbSMadhavan Venkataraman * Add the callout to the callout queue. If it ends up at the head, 392060cedfbSMadhavan Venkataraman * the cyclic needs to be reprogrammed as we have an earlier 393060cedfbSMadhavan Venkataraman * expiration. 394060cedfbSMadhavan Venkataraman * 395060cedfbSMadhavan Venkataraman * Also, during the CPR suspend phase, do not reprogram the cyclic. 396060cedfbSMadhavan Venkataraman * We don't want any callout activity. When the CPR resume phase is 397060cedfbSMadhavan Venkataraman * entered, the cyclic will be programmed for the earliest expiration 398060cedfbSMadhavan Venkataraman * in the queue. 399060cedfbSMadhavan Venkataraman */ 400060cedfbSMadhavan Venkataraman if (callout_queue_add(ct, cl) && (ct->ct_suspend == 0)) 401060cedfbSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_qcyclic, cl->cl_expiration); 402060cedfbSMadhavan Venkataraman } 403060cedfbSMadhavan Venkataraman 404060cedfbSMadhavan Venkataraman /* 405060cedfbSMadhavan Venkataraman * Delete and handle all past expirations in a callout table's queue. 406060cedfbSMadhavan Venkataraman */ 407060cedfbSMadhavan Venkataraman static hrtime_t 408060cedfbSMadhavan Venkataraman callout_queue_delete(callout_table_t *ct) 409060cedfbSMadhavan Venkataraman { 410060cedfbSMadhavan Venkataraman callout_list_t *cl; 411060cedfbSMadhavan Venkataraman hrtime_t now; 412060cedfbSMadhavan Venkataraman 413060cedfbSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 414060cedfbSMadhavan Venkataraman 415060cedfbSMadhavan Venkataraman now = gethrtime(); 416060cedfbSMadhavan Venkataraman while ((cl = ct->ct_queue.ch_head) != NULL) { 417060cedfbSMadhavan Venkataraman if (cl->cl_expiration > now) 418060cedfbSMadhavan Venkataraman break; 419060cedfbSMadhavan Venkataraman cl->cl_flags &= ~CALLOUT_LIST_FLAG_QUEUED; 420060cedfbSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_queue, cl); 421060cedfbSMadhavan Venkataraman CALLOUT_LIST_APPEND(ct->ct_expired, cl); 422060cedfbSMadhavan Venkataraman } 423060cedfbSMadhavan Venkataraman 424060cedfbSMadhavan Venkataraman /* 425060cedfbSMadhavan Venkataraman * If this callout queue is empty or callouts have been suspended, 426060cedfbSMadhavan Venkataraman * just return. 427060cedfbSMadhavan Venkataraman */ 428060cedfbSMadhavan Venkataraman if ((cl == NULL) || (ct->ct_suspend > 0)) 429060cedfbSMadhavan Venkataraman return (CY_INFINITY); 430060cedfbSMadhavan Venkataraman 431060cedfbSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_qcyclic, cl->cl_expiration); 432060cedfbSMadhavan Venkataraman 433060cedfbSMadhavan Venkataraman return (cl->cl_expiration); 434060cedfbSMadhavan Venkataraman } 435060cedfbSMadhavan Venkataraman 436060cedfbSMadhavan Venkataraman static hrtime_t 437060cedfbSMadhavan Venkataraman callout_queue_process(callout_table_t *ct, hrtime_t delta, int timechange) 438060cedfbSMadhavan Venkataraman { 439060cedfbSMadhavan Venkataraman callout_list_t *firstcl, *cl; 440060cedfbSMadhavan Venkataraman hrtime_t expiration, now; 441060cedfbSMadhavan Venkataraman int clflags; 442060cedfbSMadhavan Venkataraman callout_hash_t temp; 443060cedfbSMadhavan Venkataraman 444060cedfbSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 445060cedfbSMadhavan Venkataraman 446060cedfbSMadhavan Venkataraman firstcl = ct->ct_queue.ch_head; 447060cedfbSMadhavan Venkataraman if (firstcl == NULL) 448060cedfbSMadhavan Venkataraman return (CY_INFINITY); 449060cedfbSMadhavan Venkataraman 450060cedfbSMadhavan Venkataraman /* 451060cedfbSMadhavan Venkataraman * We walk the callout queue. If we encounter a hrestime entry that 452060cedfbSMadhavan Venkataraman * must be removed, we clean it out. Otherwise, we apply any 453060cedfbSMadhavan Venkataraman * adjustments needed to it. Because of the latter, we need to 454060cedfbSMadhavan Venkataraman * recreate the list as we go along. 455060cedfbSMadhavan Venkataraman */ 456060cedfbSMadhavan Venkataraman temp = ct->ct_queue; 457060cedfbSMadhavan Venkataraman ct->ct_queue.ch_head = NULL; 458060cedfbSMadhavan Venkataraman ct->ct_queue.ch_tail = NULL; 459060cedfbSMadhavan Venkataraman 460060cedfbSMadhavan Venkataraman clflags = (CALLOUT_LIST_FLAG_HRESTIME | CALLOUT_LIST_FLAG_ABSOLUTE); 461060cedfbSMadhavan Venkataraman now = gethrtime(); 462060cedfbSMadhavan Venkataraman while ((cl = temp.ch_head) != NULL) { 463060cedfbSMadhavan Venkataraman CALLOUT_LIST_DELETE(temp, cl); 464060cedfbSMadhavan Venkataraman 465060cedfbSMadhavan Venkataraman /* 466060cedfbSMadhavan Venkataraman * Delete the callout and expire it, if one of the following 467060cedfbSMadhavan Venkataraman * is true: 468060cedfbSMadhavan Venkataraman * - the callout has expired 469060cedfbSMadhavan Venkataraman * - the callout is an absolute hrestime one and 470060cedfbSMadhavan Venkataraman * there has been a system time change 471060cedfbSMadhavan Venkataraman */ 472060cedfbSMadhavan Venkataraman if ((cl->cl_expiration <= now) || 473060cedfbSMadhavan Venkataraman (timechange && ((cl->cl_flags & clflags) == clflags))) { 474060cedfbSMadhavan Venkataraman cl->cl_flags &= ~CALLOUT_LIST_FLAG_QUEUED; 475060cedfbSMadhavan Venkataraman CALLOUT_LIST_APPEND(ct->ct_expired, cl); 476060cedfbSMadhavan Venkataraman continue; 477060cedfbSMadhavan Venkataraman } 478060cedfbSMadhavan Venkataraman 479060cedfbSMadhavan Venkataraman /* 480060cedfbSMadhavan Venkataraman * Apply adjustments, if any. Adjustments are applied after 481060cedfbSMadhavan Venkataraman * the system returns from KMDB or OBP. They are only applied 482060cedfbSMadhavan Venkataraman * to relative callout lists. 483060cedfbSMadhavan Venkataraman */ 484060cedfbSMadhavan Venkataraman if (delta && !(cl->cl_flags & CALLOUT_LIST_FLAG_ABSOLUTE)) { 485060cedfbSMadhavan Venkataraman expiration = cl->cl_expiration + delta; 486060cedfbSMadhavan Venkataraman if (expiration <= 0) 487060cedfbSMadhavan Venkataraman expiration = CY_INFINITY; 488060cedfbSMadhavan Venkataraman cl->cl_expiration = expiration; 489060cedfbSMadhavan Venkataraman } 490060cedfbSMadhavan Venkataraman 491060cedfbSMadhavan Venkataraman (void) callout_queue_add(ct, cl); 492060cedfbSMadhavan Venkataraman } 493060cedfbSMadhavan Venkataraman 494060cedfbSMadhavan Venkataraman /* 495060cedfbSMadhavan Venkataraman * We need to return the expiration to help program the cyclic. 496060cedfbSMadhavan Venkataraman * If there are expired callouts, the cyclic needs to go off 497060cedfbSMadhavan Venkataraman * immediately. If the queue has become empty, then we return infinity. 498060cedfbSMadhavan Venkataraman * Else, we return the expiration of the earliest callout in the queue. 499060cedfbSMadhavan Venkataraman */ 500060cedfbSMadhavan Venkataraman if (ct->ct_expired.ch_head != NULL) 501060cedfbSMadhavan Venkataraman return (gethrtime()); 502060cedfbSMadhavan Venkataraman 503060cedfbSMadhavan Venkataraman cl = ct->ct_queue.ch_head; 504060cedfbSMadhavan Venkataraman if (cl == NULL) 505060cedfbSMadhavan Venkataraman return (CY_INFINITY); 506060cedfbSMadhavan Venkataraman 507060cedfbSMadhavan Venkataraman return (cl->cl_expiration); 508060cedfbSMadhavan Venkataraman } 509060cedfbSMadhavan Venkataraman 510060cedfbSMadhavan Venkataraman /* 51187a18d3fSMadhavan Venkataraman * Initialize a callout table's heap, if necessary. Preallocate some free 51287a18d3fSMadhavan Venkataraman * entries so we don't have to check for NULL elsewhere. 51387a18d3fSMadhavan Venkataraman */ 51487a18d3fSMadhavan Venkataraman static void 51587a18d3fSMadhavan Venkataraman callout_heap_init(callout_table_t *ct) 51687a18d3fSMadhavan Venkataraman { 51787a18d3fSMadhavan Venkataraman size_t size; 51887a18d3fSMadhavan Venkataraman 51987a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 52087a18d3fSMadhavan Venkataraman ASSERT(ct->ct_heap == NULL); 52187a18d3fSMadhavan Venkataraman 52287a18d3fSMadhavan Venkataraman ct->ct_heap_num = 0; 523060cedfbSMadhavan Venkataraman ct->ct_heap_max = callout_chunk; 524060cedfbSMadhavan Venkataraman size = sizeof (callout_heap_t) * callout_chunk; 52587a18d3fSMadhavan Venkataraman ct->ct_heap = kmem_alloc(size, KM_SLEEP); 52687a18d3fSMadhavan Venkataraman } 52787a18d3fSMadhavan Venkataraman 52887a18d3fSMadhavan Venkataraman /* 529060cedfbSMadhavan Venkataraman * Reallocate the heap. Return 0 if the heap is still full at the end of it. 530060cedfbSMadhavan Venkataraman * Return 1 otherwise. Note that the heap only expands, it never contracts. 53187a18d3fSMadhavan Venkataraman */ 532060cedfbSMadhavan Venkataraman static int 53387a18d3fSMadhavan Venkataraman callout_heap_expand(callout_table_t *ct) 53487a18d3fSMadhavan Venkataraman { 53587a18d3fSMadhavan Venkataraman size_t max, size, osize; 53651b32bddSMadhavan Venkataraman callout_heap_t *heap; 53787a18d3fSMadhavan Venkataraman 53887a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 53987a18d3fSMadhavan Venkataraman ASSERT(ct->ct_heap_num <= ct->ct_heap_max); 54087a18d3fSMadhavan Venkataraman 54187a18d3fSMadhavan Venkataraman while (ct->ct_heap_num == ct->ct_heap_max) { 54287a18d3fSMadhavan Venkataraman max = ct->ct_heap_max; 54387a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 54487a18d3fSMadhavan Venkataraman 54551b32bddSMadhavan Venkataraman osize = sizeof (callout_heap_t) * max; 546060cedfbSMadhavan Venkataraman size = sizeof (callout_heap_t) * (max + callout_chunk); 547060cedfbSMadhavan Venkataraman heap = kmem_alloc(size, KM_NOSLEEP); 54887a18d3fSMadhavan Venkataraman 54987a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 550060cedfbSMadhavan Venkataraman if (heap == NULL) { 551060cedfbSMadhavan Venkataraman /* 552060cedfbSMadhavan Venkataraman * We could not allocate memory. If we can free up 553060cedfbSMadhavan Venkataraman * some entries, that would be great. 554060cedfbSMadhavan Venkataraman */ 555060cedfbSMadhavan Venkataraman if (ct->ct_nreap > 0) 556060cedfbSMadhavan Venkataraman (void) callout_heap_process(ct, 0, 0); 557060cedfbSMadhavan Venkataraman /* 558060cedfbSMadhavan Venkataraman * If we still have no space in the heap, inform the 559060cedfbSMadhavan Venkataraman * caller. 560060cedfbSMadhavan Venkataraman */ 561060cedfbSMadhavan Venkataraman if (ct->ct_heap_num == ct->ct_heap_max) 562060cedfbSMadhavan Venkataraman return (0); 563060cedfbSMadhavan Venkataraman return (1); 564060cedfbSMadhavan Venkataraman } 56587a18d3fSMadhavan Venkataraman if (max < ct->ct_heap_max) { 56687a18d3fSMadhavan Venkataraman /* 56787a18d3fSMadhavan Venkataraman * Someone beat us to the allocation. Free what we 56887a18d3fSMadhavan Venkataraman * just allocated and proceed. 56987a18d3fSMadhavan Venkataraman */ 57087a18d3fSMadhavan Venkataraman kmem_free(heap, size); 57187a18d3fSMadhavan Venkataraman continue; 57287a18d3fSMadhavan Venkataraman } 57387a18d3fSMadhavan Venkataraman 57487a18d3fSMadhavan Venkataraman bcopy(ct->ct_heap, heap, osize); 57587a18d3fSMadhavan Venkataraman kmem_free(ct->ct_heap, osize); 57687a18d3fSMadhavan Venkataraman ct->ct_heap = heap; 57751b32bddSMadhavan Venkataraman ct->ct_heap_max = size / sizeof (callout_heap_t); 57887a18d3fSMadhavan Venkataraman } 579060cedfbSMadhavan Venkataraman 580060cedfbSMadhavan Venkataraman return (1); 58187a18d3fSMadhavan Venkataraman } 58287a18d3fSMadhavan Venkataraman 58387a18d3fSMadhavan Venkataraman /* 58487a18d3fSMadhavan Venkataraman * Move an expiration from the bottom of the heap to its correct place 58587a18d3fSMadhavan Venkataraman * in the heap. If we reached the root doing this, return 1. Else, 58687a18d3fSMadhavan Venkataraman * return 0. 58787a18d3fSMadhavan Venkataraman */ 58887a18d3fSMadhavan Venkataraman static int 58987a18d3fSMadhavan Venkataraman callout_upheap(callout_table_t *ct) 59087a18d3fSMadhavan Venkataraman { 59187a18d3fSMadhavan Venkataraman int current, parent; 59251b32bddSMadhavan Venkataraman callout_heap_t *heap, *hcurrent, *hparent; 59387a18d3fSMadhavan Venkataraman 59487a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 59587a18d3fSMadhavan Venkataraman ASSERT(ct->ct_heap_num >= 1); 59687a18d3fSMadhavan Venkataraman 59787a18d3fSMadhavan Venkataraman if (ct->ct_heap_num == 1) { 59887a18d3fSMadhavan Venkataraman return (1); 59987a18d3fSMadhavan Venkataraman } 60087a18d3fSMadhavan Venkataraman 60187a18d3fSMadhavan Venkataraman heap = ct->ct_heap; 60287a18d3fSMadhavan Venkataraman current = ct->ct_heap_num - 1; 60387a18d3fSMadhavan Venkataraman 60487a18d3fSMadhavan Venkataraman for (;;) { 60587a18d3fSMadhavan Venkataraman parent = CALLOUT_HEAP_PARENT(current); 60651b32bddSMadhavan Venkataraman hparent = &heap[parent]; 60751b32bddSMadhavan Venkataraman hcurrent = &heap[current]; 60887a18d3fSMadhavan Venkataraman 60987a18d3fSMadhavan Venkataraman /* 61087a18d3fSMadhavan Venkataraman * We have an expiration later than our parent; we're done. 61187a18d3fSMadhavan Venkataraman */ 61251b32bddSMadhavan Venkataraman if (hcurrent->ch_expiration >= hparent->ch_expiration) { 61387a18d3fSMadhavan Venkataraman return (0); 61487a18d3fSMadhavan Venkataraman } 61587a18d3fSMadhavan Venkataraman 61687a18d3fSMadhavan Venkataraman /* 61787a18d3fSMadhavan Venkataraman * We need to swap with our parent, and continue up the heap. 61887a18d3fSMadhavan Venkataraman */ 61951b32bddSMadhavan Venkataraman CALLOUT_SWAP(hparent, hcurrent); 62087a18d3fSMadhavan Venkataraman 62187a18d3fSMadhavan Venkataraman /* 62287a18d3fSMadhavan Venkataraman * If we just reached the root, we're done. 62387a18d3fSMadhavan Venkataraman */ 62487a18d3fSMadhavan Venkataraman if (parent == 0) { 62587a18d3fSMadhavan Venkataraman return (1); 62687a18d3fSMadhavan Venkataraman } 62787a18d3fSMadhavan Venkataraman 62887a18d3fSMadhavan Venkataraman current = parent; 62987a18d3fSMadhavan Venkataraman } 63087a18d3fSMadhavan Venkataraman /*NOTREACHED*/ 63187a18d3fSMadhavan Venkataraman } 63287a18d3fSMadhavan Venkataraman 63387a18d3fSMadhavan Venkataraman /* 63451b32bddSMadhavan Venkataraman * Insert a new heap item into a callout table's heap. 63587a18d3fSMadhavan Venkataraman */ 63687a18d3fSMadhavan Venkataraman static void 63751b32bddSMadhavan Venkataraman callout_heap_insert(callout_table_t *ct, callout_list_t *cl) 63887a18d3fSMadhavan Venkataraman { 63987a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 64087a18d3fSMadhavan Venkataraman ASSERT(ct->ct_heap_num < ct->ct_heap_max); 64187a18d3fSMadhavan Venkataraman 642060cedfbSMadhavan Venkataraman cl->cl_flags |= CALLOUT_LIST_FLAG_HEAPED; 64387a18d3fSMadhavan Venkataraman /* 64451b32bddSMadhavan Venkataraman * First, copy the expiration and callout list pointer to the bottom 64551b32bddSMadhavan Venkataraman * of the heap. 64687a18d3fSMadhavan Venkataraman */ 64751b32bddSMadhavan Venkataraman ct->ct_heap[ct->ct_heap_num].ch_expiration = cl->cl_expiration; 64851b32bddSMadhavan Venkataraman ct->ct_heap[ct->ct_heap_num].ch_list = cl; 64987a18d3fSMadhavan Venkataraman ct->ct_heap_num++; 65087a18d3fSMadhavan Venkataraman 65187a18d3fSMadhavan Venkataraman /* 65287a18d3fSMadhavan Venkataraman * Now, perform an upheap operation. If we reached the root, then 65387a18d3fSMadhavan Venkataraman * the cyclic needs to be reprogrammed as we have an earlier 65487a18d3fSMadhavan Venkataraman * expiration. 65587a18d3fSMadhavan Venkataraman * 65687a18d3fSMadhavan Venkataraman * Also, during the CPR suspend phase, do not reprogram the cyclic. 65787a18d3fSMadhavan Venkataraman * We don't want any callout activity. When the CPR resume phase is 65887a18d3fSMadhavan Venkataraman * entered, the cyclic will be programmed for the earliest expiration 65987a18d3fSMadhavan Venkataraman * in the heap. 66087a18d3fSMadhavan Venkataraman */ 661454ab202SMadhavan Venkataraman if (callout_upheap(ct) && (ct->ct_suspend == 0)) 66251b32bddSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_cyclic, cl->cl_expiration); 66387a18d3fSMadhavan Venkataraman } 66487a18d3fSMadhavan Venkataraman 66587a18d3fSMadhavan Venkataraman /* 66687a18d3fSMadhavan Venkataraman * Move an expiration from the top of the heap to its correct place 66787a18d3fSMadhavan Venkataraman * in the heap. 66887a18d3fSMadhavan Venkataraman */ 66987a18d3fSMadhavan Venkataraman static void 67087a18d3fSMadhavan Venkataraman callout_downheap(callout_table_t *ct) 67187a18d3fSMadhavan Venkataraman { 67251b32bddSMadhavan Venkataraman int current, left, right, nelems; 67351b32bddSMadhavan Venkataraman callout_heap_t *heap, *hleft, *hright, *hcurrent; 67487a18d3fSMadhavan Venkataraman 67587a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 67687a18d3fSMadhavan Venkataraman ASSERT(ct->ct_heap_num >= 1); 67787a18d3fSMadhavan Venkataraman 67887a18d3fSMadhavan Venkataraman heap = ct->ct_heap; 67987a18d3fSMadhavan Venkataraman current = 0; 68087a18d3fSMadhavan Venkataraman nelems = ct->ct_heap_num; 68187a18d3fSMadhavan Venkataraman 68287a18d3fSMadhavan Venkataraman for (;;) { 68387a18d3fSMadhavan Venkataraman /* 68487a18d3fSMadhavan Venkataraman * If we don't have a left child (i.e., we're a leaf), we're 68587a18d3fSMadhavan Venkataraman * done. 68687a18d3fSMadhavan Venkataraman */ 68787a18d3fSMadhavan Venkataraman if ((left = CALLOUT_HEAP_LEFT(current)) >= nelems) 68887a18d3fSMadhavan Venkataraman return; 68987a18d3fSMadhavan Venkataraman 69051b32bddSMadhavan Venkataraman hleft = &heap[left]; 69151b32bddSMadhavan Venkataraman hcurrent = &heap[current]; 69287a18d3fSMadhavan Venkataraman 69387a18d3fSMadhavan Venkataraman right = CALLOUT_HEAP_RIGHT(current); 69487a18d3fSMadhavan Venkataraman 69587a18d3fSMadhavan Venkataraman /* 69687a18d3fSMadhavan Venkataraman * Even if we don't have a right child, we still need to compare 69787a18d3fSMadhavan Venkataraman * our expiration against that of our left child. 69887a18d3fSMadhavan Venkataraman */ 69987a18d3fSMadhavan Venkataraman if (right >= nelems) 70087a18d3fSMadhavan Venkataraman goto comp_left; 70187a18d3fSMadhavan Venkataraman 70251b32bddSMadhavan Venkataraman hright = &heap[right]; 70387a18d3fSMadhavan Venkataraman 70487a18d3fSMadhavan Venkataraman /* 70587a18d3fSMadhavan Venkataraman * We have both a left and a right child. We need to compare 70687a18d3fSMadhavan Venkataraman * the expiration of the children to determine which 70787a18d3fSMadhavan Venkataraman * expires earlier. 70887a18d3fSMadhavan Venkataraman */ 70951b32bddSMadhavan Venkataraman if (hright->ch_expiration < hleft->ch_expiration) { 71087a18d3fSMadhavan Venkataraman /* 71187a18d3fSMadhavan Venkataraman * Our right child is the earlier of our children. 71287a18d3fSMadhavan Venkataraman * We'll now compare our expiration to its expiration. 71387a18d3fSMadhavan Venkataraman * If ours is the earlier one, we're done. 71487a18d3fSMadhavan Venkataraman */ 71551b32bddSMadhavan Venkataraman if (hcurrent->ch_expiration <= hright->ch_expiration) 71687a18d3fSMadhavan Venkataraman return; 71787a18d3fSMadhavan Venkataraman 71887a18d3fSMadhavan Venkataraman /* 71987a18d3fSMadhavan Venkataraman * Our right child expires earlier than we do; swap 72087a18d3fSMadhavan Venkataraman * with our right child, and descend right. 72187a18d3fSMadhavan Venkataraman */ 72251b32bddSMadhavan Venkataraman CALLOUT_SWAP(hright, hcurrent); 72387a18d3fSMadhavan Venkataraman current = right; 72487a18d3fSMadhavan Venkataraman continue; 72587a18d3fSMadhavan Venkataraman } 72687a18d3fSMadhavan Venkataraman 72787a18d3fSMadhavan Venkataraman comp_left: 72887a18d3fSMadhavan Venkataraman /* 72987a18d3fSMadhavan Venkataraman * Our left child is the earlier of our children (or we have 73087a18d3fSMadhavan Venkataraman * no right child). We'll now compare our expiration 73187a18d3fSMadhavan Venkataraman * to its expiration. If ours is the earlier one, we're done. 73287a18d3fSMadhavan Venkataraman */ 73351b32bddSMadhavan Venkataraman if (hcurrent->ch_expiration <= hleft->ch_expiration) 73487a18d3fSMadhavan Venkataraman return; 73587a18d3fSMadhavan Venkataraman 73687a18d3fSMadhavan Venkataraman /* 73787a18d3fSMadhavan Venkataraman * Our left child expires earlier than we do; swap with our 73887a18d3fSMadhavan Venkataraman * left child, and descend left. 73987a18d3fSMadhavan Venkataraman */ 74051b32bddSMadhavan Venkataraman CALLOUT_SWAP(hleft, hcurrent); 74187a18d3fSMadhavan Venkataraman current = left; 74287a18d3fSMadhavan Venkataraman } 74387a18d3fSMadhavan Venkataraman } 74487a18d3fSMadhavan Venkataraman 74587a18d3fSMadhavan Venkataraman /* 74687a18d3fSMadhavan Venkataraman * Delete and handle all past expirations in a callout table's heap. 74787a18d3fSMadhavan Venkataraman */ 748060cedfbSMadhavan Venkataraman static hrtime_t 74987a18d3fSMadhavan Venkataraman callout_heap_delete(callout_table_t *ct) 75087a18d3fSMadhavan Venkataraman { 75151b32bddSMadhavan Venkataraman hrtime_t now, expiration, next; 75287a18d3fSMadhavan Venkataraman callout_list_t *cl; 75351b32bddSMadhavan Venkataraman callout_heap_t *heap; 75487a18d3fSMadhavan Venkataraman int hash; 75587a18d3fSMadhavan Venkataraman 75687a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 75787a18d3fSMadhavan Venkataraman 75851b32bddSMadhavan Venkataraman if (CALLOUT_CLEANUP(ct)) { 75951b32bddSMadhavan Venkataraman /* 76051b32bddSMadhavan Venkataraman * There are too many heap elements pointing to empty callout 76151b32bddSMadhavan Venkataraman * lists. Clean them out. 76251b32bddSMadhavan Venkataraman */ 76351b32bddSMadhavan Venkataraman (void) callout_heap_process(ct, 0, 0); 76451b32bddSMadhavan Venkataraman } 76551b32bddSMadhavan Venkataraman 76687a18d3fSMadhavan Venkataraman now = gethrtime(); 76751b32bddSMadhavan Venkataraman heap = ct->ct_heap; 76887a18d3fSMadhavan Venkataraman 76987a18d3fSMadhavan Venkataraman while (ct->ct_heap_num > 0) { 77051b32bddSMadhavan Venkataraman expiration = heap->ch_expiration; 77187a18d3fSMadhavan Venkataraman hash = CALLOUT_CLHASH(expiration); 77251b32bddSMadhavan Venkataraman cl = heap->ch_list; 77351b32bddSMadhavan Venkataraman ASSERT(expiration == cl->cl_expiration); 77451b32bddSMadhavan Venkataraman 77551b32bddSMadhavan Venkataraman if (cl->cl_callouts.ch_head == NULL) { 77687a18d3fSMadhavan Venkataraman /* 77751b32bddSMadhavan Venkataraman * If the callout list is empty, reap it. 77851b32bddSMadhavan Venkataraman * Decrement the reap count. 77951b32bddSMadhavan Venkataraman */ 78051b32bddSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_clhash[hash], cl); 78151b32bddSMadhavan Venkataraman CALLOUT_LIST_FREE(ct, cl); 78251b32bddSMadhavan Venkataraman ct->ct_nreap--; 78351b32bddSMadhavan Venkataraman } else { 78451b32bddSMadhavan Venkataraman /* 78551b32bddSMadhavan Venkataraman * If the root of the heap expires in the future, 78651b32bddSMadhavan Venkataraman * bail out. 78787a18d3fSMadhavan Venkataraman */ 78887a18d3fSMadhavan Venkataraman if (expiration > now) 78987a18d3fSMadhavan Venkataraman break; 79087a18d3fSMadhavan Venkataraman 79187a18d3fSMadhavan Venkataraman /* 79287a18d3fSMadhavan Venkataraman * Move the callout list for this expiration to the 79387a18d3fSMadhavan Venkataraman * list of expired callout lists. It will be processed 79487a18d3fSMadhavan Venkataraman * by the callout executor. 79587a18d3fSMadhavan Venkataraman */ 796060cedfbSMadhavan Venkataraman cl->cl_flags &= ~CALLOUT_LIST_FLAG_HEAPED; 79787a18d3fSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_clhash[hash], cl); 79887a18d3fSMadhavan Venkataraman CALLOUT_LIST_APPEND(ct->ct_expired, cl); 79987a18d3fSMadhavan Venkataraman } 80087a18d3fSMadhavan Venkataraman 80187a18d3fSMadhavan Venkataraman /* 80287a18d3fSMadhavan Venkataraman * Now delete the root. This is done by swapping the root with 80387a18d3fSMadhavan Venkataraman * the last item in the heap and downheaping the item. 80487a18d3fSMadhavan Venkataraman */ 80587a18d3fSMadhavan Venkataraman ct->ct_heap_num--; 80687a18d3fSMadhavan Venkataraman if (ct->ct_heap_num > 0) { 80751b32bddSMadhavan Venkataraman heap[0] = heap[ct->ct_heap_num]; 80887a18d3fSMadhavan Venkataraman callout_downheap(ct); 80987a18d3fSMadhavan Venkataraman } 81087a18d3fSMadhavan Venkataraman } 81187a18d3fSMadhavan Venkataraman 81287a18d3fSMadhavan Venkataraman /* 81351b32bddSMadhavan Venkataraman * If this callout table is empty or callouts have been suspended, 81451b32bddSMadhavan Venkataraman * just return. The cyclic has already been programmed to 81587a18d3fSMadhavan Venkataraman * infinity by the cyclic subsystem. 81687a18d3fSMadhavan Venkataraman */ 817454ab202SMadhavan Venkataraman if ((ct->ct_heap_num == 0) || (ct->ct_suspend > 0)) 818060cedfbSMadhavan Venkataraman return (CY_INFINITY); 81987a18d3fSMadhavan Venkataraman 82051b32bddSMadhavan Venkataraman /* 82151b32bddSMadhavan Venkataraman * If the top expirations are within callout_tolerance of each other, 82251b32bddSMadhavan Venkataraman * delay the cyclic expire so that they can be processed together. 82351b32bddSMadhavan Venkataraman * This is to prevent high resolution timers from swamping the system 82451b32bddSMadhavan Venkataraman * with cyclic activity. 82551b32bddSMadhavan Venkataraman */ 82651b32bddSMadhavan Venkataraman if (ct->ct_heap_num > 2) { 82751b32bddSMadhavan Venkataraman next = expiration + callout_tolerance; 82851b32bddSMadhavan Venkataraman if ((heap[1].ch_expiration < next) || 82951b32bddSMadhavan Venkataraman (heap[2].ch_expiration < next)) 83051b32bddSMadhavan Venkataraman expiration = next; 83151b32bddSMadhavan Venkataraman } 83251b32bddSMadhavan Venkataraman 83387a18d3fSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_cyclic, expiration); 834060cedfbSMadhavan Venkataraman 835060cedfbSMadhavan Venkataraman return (expiration); 83687a18d3fSMadhavan Venkataraman } 83787a18d3fSMadhavan Venkataraman 838454ab202SMadhavan Venkataraman /* 83951b32bddSMadhavan Venkataraman * There are some situations when the entire heap is walked and processed. 84051b32bddSMadhavan Venkataraman * This function is called to do the processing. These are the situations: 84151b32bddSMadhavan Venkataraman * 84251b32bddSMadhavan Venkataraman * 1. When the reap count reaches its threshold, the heap has to be cleared 84351b32bddSMadhavan Venkataraman * of all empty callout lists. 84451b32bddSMadhavan Venkataraman * 84551b32bddSMadhavan Venkataraman * 2. When the system enters and exits KMDB/OBP, all entries in the heap 84651b32bddSMadhavan Venkataraman * need to be adjusted by the interval spent in KMDB/OBP. 84751b32bddSMadhavan Venkataraman * 84851b32bddSMadhavan Venkataraman * 3. When system time is changed, the heap has to be scanned for 84951b32bddSMadhavan Venkataraman * absolute hrestime timers. These need to be removed from the heap 85051b32bddSMadhavan Venkataraman * and expired immediately. 85151b32bddSMadhavan Venkataraman * 85251b32bddSMadhavan Venkataraman * In cases 2 and 3, it is a good idea to do 1 as well since we are 85351b32bddSMadhavan Venkataraman * scanning the heap anyway. 85451b32bddSMadhavan Venkataraman * 85551b32bddSMadhavan Venkataraman * If the root gets changed and/or callout lists are expired, return the 85651b32bddSMadhavan Venkataraman * new expiration to the caller so he can reprogram the cyclic accordingly. 85751b32bddSMadhavan Venkataraman */ 85851b32bddSMadhavan Venkataraman static hrtime_t 85951b32bddSMadhavan Venkataraman callout_heap_process(callout_table_t *ct, hrtime_t delta, int timechange) 86051b32bddSMadhavan Venkataraman { 86151b32bddSMadhavan Venkataraman callout_heap_t *heap; 862060cedfbSMadhavan Venkataraman callout_list_t *cl; 86351b32bddSMadhavan Venkataraman hrtime_t expiration, now; 864060cedfbSMadhavan Venkataraman int i, hash, clflags; 86551b32bddSMadhavan Venkataraman ulong_t num; 86651b32bddSMadhavan Venkataraman 86751b32bddSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 86851b32bddSMadhavan Venkataraman 86951b32bddSMadhavan Venkataraman if (ct->ct_heap_num == 0) 870060cedfbSMadhavan Venkataraman return (CY_INFINITY); 87151b32bddSMadhavan Venkataraman 87251b32bddSMadhavan Venkataraman if (ct->ct_nreap > 0) 87351b32bddSMadhavan Venkataraman ct->ct_cleanups++; 87451b32bddSMadhavan Venkataraman 87551b32bddSMadhavan Venkataraman heap = ct->ct_heap; 87651b32bddSMadhavan Venkataraman 87751b32bddSMadhavan Venkataraman /* 87851b32bddSMadhavan Venkataraman * We walk the heap from the top to the bottom. If we encounter 87951b32bddSMadhavan Venkataraman * a heap item that points to an empty callout list, we clean 88051b32bddSMadhavan Venkataraman * it out. If we encounter a hrestime entry that must be removed, 88151b32bddSMadhavan Venkataraman * again we clean it out. Otherwise, we apply any adjustments needed 88251b32bddSMadhavan Venkataraman * to an element. 88351b32bddSMadhavan Venkataraman * 88451b32bddSMadhavan Venkataraman * During the walk, we also compact the heap from the bottom and 88551b32bddSMadhavan Venkataraman * reconstruct the heap using upheap operations. This is very 88651b32bddSMadhavan Venkataraman * efficient if the number of elements to be cleaned is greater than 88751b32bddSMadhavan Venkataraman * or equal to half the heap. This is the common case. 88851b32bddSMadhavan Venkataraman * 88951b32bddSMadhavan Venkataraman * Even in the non-common case, the upheap operations should be short 89051b32bddSMadhavan Venkataraman * as the entries below generally tend to be bigger than the entries 89151b32bddSMadhavan Venkataraman * above. 89251b32bddSMadhavan Venkataraman */ 89351b32bddSMadhavan Venkataraman num = ct->ct_heap_num; 89451b32bddSMadhavan Venkataraman ct->ct_heap_num = 0; 89551b32bddSMadhavan Venkataraman clflags = (CALLOUT_LIST_FLAG_HRESTIME | CALLOUT_LIST_FLAG_ABSOLUTE); 89651b32bddSMadhavan Venkataraman now = gethrtime(); 89751b32bddSMadhavan Venkataraman for (i = 0; i < num; i++) { 89851b32bddSMadhavan Venkataraman cl = heap[i].ch_list; 89951b32bddSMadhavan Venkataraman /* 90051b32bddSMadhavan Venkataraman * If the callout list is empty, delete the heap element and 90151b32bddSMadhavan Venkataraman * free the callout list. 90251b32bddSMadhavan Venkataraman */ 90351b32bddSMadhavan Venkataraman if (cl->cl_callouts.ch_head == NULL) { 90451b32bddSMadhavan Venkataraman hash = CALLOUT_CLHASH(cl->cl_expiration); 90551b32bddSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_clhash[hash], cl); 90651b32bddSMadhavan Venkataraman CALLOUT_LIST_FREE(ct, cl); 90751b32bddSMadhavan Venkataraman continue; 90851b32bddSMadhavan Venkataraman } 90951b32bddSMadhavan Venkataraman 91051b32bddSMadhavan Venkataraman /* 91151b32bddSMadhavan Venkataraman * Delete the heap element and expire the callout list, if 91251b32bddSMadhavan Venkataraman * one of the following is true: 91351b32bddSMadhavan Venkataraman * - the callout list has expired 91451b32bddSMadhavan Venkataraman * - the callout list is an absolute hrestime one and 91551b32bddSMadhavan Venkataraman * there has been a system time change 91651b32bddSMadhavan Venkataraman */ 91751b32bddSMadhavan Venkataraman if ((cl->cl_expiration <= now) || 91851b32bddSMadhavan Venkataraman (timechange && ((cl->cl_flags & clflags) == clflags))) { 91951b32bddSMadhavan Venkataraman hash = CALLOUT_CLHASH(cl->cl_expiration); 920060cedfbSMadhavan Venkataraman cl->cl_flags &= ~CALLOUT_LIST_FLAG_HEAPED; 92151b32bddSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_clhash[hash], cl); 92251b32bddSMadhavan Venkataraman CALLOUT_LIST_APPEND(ct->ct_expired, cl); 92351b32bddSMadhavan Venkataraman continue; 92451b32bddSMadhavan Venkataraman } 92551b32bddSMadhavan Venkataraman 92651b32bddSMadhavan Venkataraman /* 92751b32bddSMadhavan Venkataraman * Apply adjustments, if any. Adjustments are applied after 92851b32bddSMadhavan Venkataraman * the system returns from KMDB or OBP. They are only applied 92951b32bddSMadhavan Venkataraman * to relative callout lists. 93051b32bddSMadhavan Venkataraman */ 93151b32bddSMadhavan Venkataraman if (delta && !(cl->cl_flags & CALLOUT_LIST_FLAG_ABSOLUTE)) { 93251b32bddSMadhavan Venkataraman hash = CALLOUT_CLHASH(cl->cl_expiration); 93351b32bddSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_clhash[hash], cl); 93451b32bddSMadhavan Venkataraman expiration = cl->cl_expiration + delta; 93551b32bddSMadhavan Venkataraman if (expiration <= 0) 93651b32bddSMadhavan Venkataraman expiration = CY_INFINITY; 93751b32bddSMadhavan Venkataraman heap[i].ch_expiration = expiration; 93851b32bddSMadhavan Venkataraman cl->cl_expiration = expiration; 93951b32bddSMadhavan Venkataraman hash = CALLOUT_CLHASH(cl->cl_expiration); 94051b32bddSMadhavan Venkataraman if (cl->cl_flags & CALLOUT_LIST_FLAG_NANO) { 94151b32bddSMadhavan Venkataraman CALLOUT_LIST_APPEND(ct->ct_clhash[hash], cl); 94251b32bddSMadhavan Venkataraman } else { 94351b32bddSMadhavan Venkataraman CALLOUT_LIST_INSERT(ct->ct_clhash[hash], cl); 94451b32bddSMadhavan Venkataraman } 94551b32bddSMadhavan Venkataraman } 94651b32bddSMadhavan Venkataraman 94751b32bddSMadhavan Venkataraman heap[ct->ct_heap_num] = heap[i]; 94851b32bddSMadhavan Venkataraman ct->ct_heap_num++; 94951b32bddSMadhavan Venkataraman (void) callout_upheap(ct); 95051b32bddSMadhavan Venkataraman } 95151b32bddSMadhavan Venkataraman 95251b32bddSMadhavan Venkataraman ct->ct_nreap = 0; 95351b32bddSMadhavan Venkataraman 954060cedfbSMadhavan Venkataraman /* 955060cedfbSMadhavan Venkataraman * We need to return the expiration to help program the cyclic. 956060cedfbSMadhavan Venkataraman * If there are expired callouts, the cyclic needs to go off 957060cedfbSMadhavan Venkataraman * immediately. If the heap has become empty, then we return infinity. 958060cedfbSMadhavan Venkataraman * Else, return the expiration of the earliest callout in the heap. 959060cedfbSMadhavan Venkataraman */ 960060cedfbSMadhavan Venkataraman if (ct->ct_expired.ch_head != NULL) 961060cedfbSMadhavan Venkataraman return (gethrtime()); 96251b32bddSMadhavan Venkataraman 963060cedfbSMadhavan Venkataraman if (ct->ct_heap_num == 0) 964060cedfbSMadhavan Venkataraman return (CY_INFINITY); 965060cedfbSMadhavan Venkataraman 966060cedfbSMadhavan Venkataraman return (heap->ch_expiration); 96751b32bddSMadhavan Venkataraman } 96851b32bddSMadhavan Venkataraman 96951b32bddSMadhavan Venkataraman /* 970454ab202SMadhavan Venkataraman * Common function used to create normal and realtime callouts. 971454ab202SMadhavan Venkataraman * 972454ab202SMadhavan Venkataraman * Realtime callouts are handled at CY_LOW_PIL by a cyclic handler. So, 973454ab202SMadhavan Venkataraman * there is one restriction on a realtime callout handler - it should not 974454ab202SMadhavan Venkataraman * directly or indirectly acquire cpu_lock. CPU offline waits for pending 975454ab202SMadhavan Venkataraman * cyclic handlers to complete while holding cpu_lock. So, if a realtime 976454ab202SMadhavan Venkataraman * callout handler were to try to get cpu_lock, there would be a deadlock 977454ab202SMadhavan Venkataraman * during CPU offline. 978454ab202SMadhavan Venkataraman */ 97987a18d3fSMadhavan Venkataraman callout_id_t 98087a18d3fSMadhavan Venkataraman timeout_generic(int type, void (*func)(void *), void *arg, 98187a18d3fSMadhavan Venkataraman hrtime_t expiration, hrtime_t resolution, int flags) 98287a18d3fSMadhavan Venkataraman { 98387a18d3fSMadhavan Venkataraman callout_table_t *ct; 9847c478bd9Sstevel@tonic-gate callout_t *cp; 9857c478bd9Sstevel@tonic-gate callout_id_t id; 98687a18d3fSMadhavan Venkataraman callout_list_t *cl; 987060cedfbSMadhavan Venkataraman hrtime_t now, interval; 98851b32bddSMadhavan Venkataraman int hash, clflags; 989f635d46aSqiao 99087a18d3fSMadhavan Venkataraman ASSERT(resolution > 0); 99187a18d3fSMadhavan Venkataraman ASSERT(func != NULL); 9927c478bd9Sstevel@tonic-gate 99387a18d3fSMadhavan Venkataraman /* 99451b32bddSMadhavan Venkataraman * We get the current hrtime right upfront so that latencies in 99551b32bddSMadhavan Venkataraman * this function do not affect the accuracy of the callout. 99687a18d3fSMadhavan Venkataraman */ 99751b32bddSMadhavan Venkataraman now = gethrtime(); 9987c478bd9Sstevel@tonic-gate 99987a18d3fSMadhavan Venkataraman /* 100087a18d3fSMadhavan Venkataraman * We disable kernel preemption so that we remain on the same CPU 100187a18d3fSMadhavan Venkataraman * throughout. If we needed to reprogram the callout table's cyclic, 100287a18d3fSMadhavan Venkataraman * we can avoid X-calls if we are on the same CPU. 100387a18d3fSMadhavan Venkataraman * 100487a18d3fSMadhavan Venkataraman * Note that callout_alloc() releases and reacquires the callout 100587a18d3fSMadhavan Venkataraman * table mutex. While reacquiring the mutex, it is possible for us 100687a18d3fSMadhavan Venkataraman * to go to sleep and later migrate to another CPU. This should be 100787a18d3fSMadhavan Venkataraman * pretty rare, though. 100887a18d3fSMadhavan Venkataraman */ 100987a18d3fSMadhavan Venkataraman kpreempt_disable(); 101087a18d3fSMadhavan Venkataraman 101187a18d3fSMadhavan Venkataraman ct = &callout_table[CALLOUT_TABLE(type, CPU->cpu_seqid)]; 101287a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 101387a18d3fSMadhavan Venkataraman 101487a18d3fSMadhavan Venkataraman if (ct->ct_cyclic == CYCLIC_NONE) { 101587a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 101687a18d3fSMadhavan Venkataraman /* 101787a18d3fSMadhavan Venkataraman * The callout table has not yet been initialized fully. 101887a18d3fSMadhavan Venkataraman * So, put this one on the boot callout table which is 101987a18d3fSMadhavan Venkataraman * always initialized. 102087a18d3fSMadhavan Venkataraman */ 102187a18d3fSMadhavan Venkataraman ct = &callout_boot_ct[type]; 102287a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 102387a18d3fSMadhavan Venkataraman } 102487a18d3fSMadhavan Venkataraman 102551b32bddSMadhavan Venkataraman if (CALLOUT_CLEANUP(ct)) { 102651b32bddSMadhavan Venkataraman /* 102751b32bddSMadhavan Venkataraman * There are too many heap elements pointing to empty callout 1028060cedfbSMadhavan Venkataraman * lists. Clean them out. Since cleanup is only done once 1029060cedfbSMadhavan Venkataraman * in a while, no need to reprogram the cyclic if the root 1030060cedfbSMadhavan Venkataraman * of the heap gets cleaned out. 103151b32bddSMadhavan Venkataraman */ 1032060cedfbSMadhavan Venkataraman (void) callout_heap_process(ct, 0, 0); 103351b32bddSMadhavan Venkataraman } 103451b32bddSMadhavan Venkataraman 103587a18d3fSMadhavan Venkataraman if ((cp = ct->ct_free) == NULL) 10367c478bd9Sstevel@tonic-gate cp = callout_alloc(ct); 10377c478bd9Sstevel@tonic-gate else 103887a18d3fSMadhavan Venkataraman ct->ct_free = cp->c_idnext; 10397c478bd9Sstevel@tonic-gate 10407c478bd9Sstevel@tonic-gate cp->c_func = func; 10417c478bd9Sstevel@tonic-gate cp->c_arg = arg; 10427c478bd9Sstevel@tonic-gate 10437c478bd9Sstevel@tonic-gate /* 104487a18d3fSMadhavan Venkataraman * Compute the expiration hrtime. 104587a18d3fSMadhavan Venkataraman */ 104687a18d3fSMadhavan Venkataraman if (flags & CALLOUT_FLAG_ABSOLUTE) { 104787a18d3fSMadhavan Venkataraman interval = expiration - now; 104887a18d3fSMadhavan Venkataraman } else { 104987a18d3fSMadhavan Venkataraman interval = expiration; 105087a18d3fSMadhavan Venkataraman expiration += now; 105187a18d3fSMadhavan Venkataraman } 105251b32bddSMadhavan Venkataraman 105351b32bddSMadhavan Venkataraman if (resolution > 1) { 105451b32bddSMadhavan Venkataraman /* 105551b32bddSMadhavan Venkataraman * Align expiration to the specified resolution. 105651b32bddSMadhavan Venkataraman */ 105787a18d3fSMadhavan Venkataraman if (flags & CALLOUT_FLAG_ROUNDUP) 105887a18d3fSMadhavan Venkataraman expiration += resolution - 1; 105987a18d3fSMadhavan Venkataraman expiration = (expiration / resolution) * resolution; 106051b32bddSMadhavan Venkataraman } 106151b32bddSMadhavan Venkataraman 1062454ab202SMadhavan Venkataraman if (expiration <= 0) { 1063454ab202SMadhavan Venkataraman /* 1064454ab202SMadhavan Venkataraman * expiration hrtime overflow has occurred. Just set the 1065454ab202SMadhavan Venkataraman * expiration to infinity. 1066454ab202SMadhavan Venkataraman */ 1067454ab202SMadhavan Venkataraman expiration = CY_INFINITY; 1068454ab202SMadhavan Venkataraman } 106987a18d3fSMadhavan Venkataraman 107087a18d3fSMadhavan Venkataraman /* 107187a18d3fSMadhavan Venkataraman * Assign an ID to this callout 107287a18d3fSMadhavan Venkataraman */ 107387a18d3fSMadhavan Venkataraman if (flags & CALLOUT_FLAG_32BIT) { 107487a18d3fSMadhavan Venkataraman if (interval > callout_longterm) { 107587a18d3fSMadhavan Venkataraman id = (ct->ct_long_id - callout_counter_low); 107687a18d3fSMadhavan Venkataraman id |= CALLOUT_COUNTER_HIGH; 107787a18d3fSMadhavan Venkataraman ct->ct_long_id = id; 107887a18d3fSMadhavan Venkataraman } else { 107987a18d3fSMadhavan Venkataraman id = (ct->ct_short_id - callout_counter_low); 108087a18d3fSMadhavan Venkataraman id |= CALLOUT_COUNTER_HIGH; 108187a18d3fSMadhavan Venkataraman ct->ct_short_id = id; 108287a18d3fSMadhavan Venkataraman } 108387a18d3fSMadhavan Venkataraman } else { 108487a18d3fSMadhavan Venkataraman id = (ct->ct_gen_id - callout_counter_low); 108587a18d3fSMadhavan Venkataraman if ((id & CALLOUT_COUNTER_HIGH) == 0) { 108687a18d3fSMadhavan Venkataraman id |= CALLOUT_COUNTER_HIGH; 108787a18d3fSMadhavan Venkataraman id += CALLOUT_GENERATION_LOW; 108887a18d3fSMadhavan Venkataraman } 108987a18d3fSMadhavan Venkataraman ct->ct_gen_id = id; 109087a18d3fSMadhavan Venkataraman } 109187a18d3fSMadhavan Venkataraman 109287a18d3fSMadhavan Venkataraman cp->c_xid = id; 109387a18d3fSMadhavan Venkataraman 109451b32bddSMadhavan Venkataraman clflags = 0; 109551b32bddSMadhavan Venkataraman if (flags & CALLOUT_FLAG_ABSOLUTE) 109651b32bddSMadhavan Venkataraman clflags |= CALLOUT_LIST_FLAG_ABSOLUTE; 109751b32bddSMadhavan Venkataraman if (flags & CALLOUT_FLAG_HRESTIME) 109851b32bddSMadhavan Venkataraman clflags |= CALLOUT_LIST_FLAG_HRESTIME; 109951b32bddSMadhavan Venkataraman if (resolution == 1) 110051b32bddSMadhavan Venkataraman clflags |= CALLOUT_LIST_FLAG_NANO; 110187a18d3fSMadhavan Venkataraman hash = CALLOUT_CLHASH(expiration); 110287a18d3fSMadhavan Venkataraman 110387a18d3fSMadhavan Venkataraman again: 110487a18d3fSMadhavan Venkataraman /* 110587a18d3fSMadhavan Venkataraman * Try to see if a callout list already exists for this expiration. 110687a18d3fSMadhavan Venkataraman */ 110751b32bddSMadhavan Venkataraman cl = callout_list_get(ct, expiration, clflags, hash); 110887a18d3fSMadhavan Venkataraman if (cl == NULL) { 110987a18d3fSMadhavan Venkataraman /* 111087a18d3fSMadhavan Venkataraman * Check the free list. If we don't find one, we have to 111187a18d3fSMadhavan Venkataraman * take the slow path and allocate from kmem. 111287a18d3fSMadhavan Venkataraman */ 111387a18d3fSMadhavan Venkataraman if ((cl = ct->ct_lfree) == NULL) { 111487a18d3fSMadhavan Venkataraman callout_list_alloc(ct); 111587a18d3fSMadhavan Venkataraman /* 111687a18d3fSMadhavan Venkataraman * In the above call, we drop the lock, allocate and 111787a18d3fSMadhavan Venkataraman * reacquire the lock. So, we could have been away 111887a18d3fSMadhavan Venkataraman * for a while. In the meantime, someone could have 111987a18d3fSMadhavan Venkataraman * inserted a callout list with the same expiration. 112087a18d3fSMadhavan Venkataraman * Plus, the heap could have become full. So, the best 112187a18d3fSMadhavan Venkataraman * course is to repeat the steps. This should be an 112287a18d3fSMadhavan Venkataraman * infrequent event. 112387a18d3fSMadhavan Venkataraman */ 112487a18d3fSMadhavan Venkataraman goto again; 112587a18d3fSMadhavan Venkataraman } 112687a18d3fSMadhavan Venkataraman ct->ct_lfree = cl->cl_next; 112787a18d3fSMadhavan Venkataraman cl->cl_expiration = expiration; 112851b32bddSMadhavan Venkataraman cl->cl_flags = clflags; 112987a18d3fSMadhavan Venkataraman 1130060cedfbSMadhavan Venkataraman /* 1131060cedfbSMadhavan Venkataraman * Check if we have enough space in the heap to insert one 1132060cedfbSMadhavan Venkataraman * expiration. If not, expand the heap. 1133060cedfbSMadhavan Venkataraman */ 1134060cedfbSMadhavan Venkataraman if (ct->ct_heap_num == ct->ct_heap_max) { 1135060cedfbSMadhavan Venkataraman if (callout_heap_expand(ct) == 0) { 1136060cedfbSMadhavan Venkataraman /* 1137060cedfbSMadhavan Venkataraman * Could not expand the heap. Just queue it. 1138060cedfbSMadhavan Venkataraman */ 1139060cedfbSMadhavan Venkataraman callout_queue_insert(ct, cl); 1140060cedfbSMadhavan Venkataraman goto out; 1141060cedfbSMadhavan Venkataraman } 1142060cedfbSMadhavan Venkataraman 1143060cedfbSMadhavan Venkataraman /* 1144060cedfbSMadhavan Venkataraman * In the above call, we drop the lock, allocate and 1145060cedfbSMadhavan Venkataraman * reacquire the lock. So, we could have been away 1146060cedfbSMadhavan Venkataraman * for a while. In the meantime, someone could have 1147060cedfbSMadhavan Venkataraman * inserted a callout list with the same expiration. 1148060cedfbSMadhavan Venkataraman * But we will not go back and check for it as this 1149060cedfbSMadhavan Venkataraman * should be a really infrequent event. There is no 1150060cedfbSMadhavan Venkataraman * point. 1151060cedfbSMadhavan Venkataraman */ 1152060cedfbSMadhavan Venkataraman } 1153060cedfbSMadhavan Venkataraman 115451b32bddSMadhavan Venkataraman if (clflags & CALLOUT_LIST_FLAG_NANO) { 115551b32bddSMadhavan Venkataraman CALLOUT_LIST_APPEND(ct->ct_clhash[hash], cl); 115651b32bddSMadhavan Venkataraman } else { 115787a18d3fSMadhavan Venkataraman CALLOUT_LIST_INSERT(ct->ct_clhash[hash], cl); 115851b32bddSMadhavan Venkataraman } 115987a18d3fSMadhavan Venkataraman 116087a18d3fSMadhavan Venkataraman /* 116187a18d3fSMadhavan Venkataraman * This is a new expiration. So, insert it into the heap. 116287a18d3fSMadhavan Venkataraman * This will also reprogram the cyclic, if the expiration 116387a18d3fSMadhavan Venkataraman * propagated to the root of the heap. 116487a18d3fSMadhavan Venkataraman */ 116551b32bddSMadhavan Venkataraman callout_heap_insert(ct, cl); 116651b32bddSMadhavan Venkataraman } else { 116751b32bddSMadhavan Venkataraman /* 116851b32bddSMadhavan Venkataraman * If the callout list was empty, untimeout_generic() would 116951b32bddSMadhavan Venkataraman * have incremented a reap count. Decrement the reap count 117051b32bddSMadhavan Venkataraman * as we are going to insert a callout into this list. 117151b32bddSMadhavan Venkataraman */ 117251b32bddSMadhavan Venkataraman if (cl->cl_callouts.ch_head == NULL) 117351b32bddSMadhavan Venkataraman ct->ct_nreap--; 117487a18d3fSMadhavan Venkataraman } 1175060cedfbSMadhavan Venkataraman out: 117687a18d3fSMadhavan Venkataraman cp->c_list = cl; 117787a18d3fSMadhavan Venkataraman CALLOUT_APPEND(ct, cp); 117887a18d3fSMadhavan Venkataraman 117987a18d3fSMadhavan Venkataraman ct->ct_timeouts++; 118087a18d3fSMadhavan Venkataraman ct->ct_timeouts_pending++; 118187a18d3fSMadhavan Venkataraman 118287a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 118387a18d3fSMadhavan Venkataraman 118487a18d3fSMadhavan Venkataraman kpreempt_enable(); 118587a18d3fSMadhavan Venkataraman 118687a18d3fSMadhavan Venkataraman TRACE_4(TR_FAC_CALLOUT, TR_TIMEOUT, 118787a18d3fSMadhavan Venkataraman "timeout:%K(%p) in %llx expiration, cp %p", func, arg, expiration, 118887a18d3fSMadhavan Venkataraman cp); 118987a18d3fSMadhavan Venkataraman 119087a18d3fSMadhavan Venkataraman return (id); 119187a18d3fSMadhavan Venkataraman } 119287a18d3fSMadhavan Venkataraman 119387a18d3fSMadhavan Venkataraman timeout_id_t 119487a18d3fSMadhavan Venkataraman timeout(void (*func)(void *), void *arg, clock_t delta) 119587a18d3fSMadhavan Venkataraman { 119687a18d3fSMadhavan Venkataraman ulong_t id; 119787a18d3fSMadhavan Venkataraman 119887a18d3fSMadhavan Venkataraman /* 11997c478bd9Sstevel@tonic-gate * Make sure the callout runs at least 1 tick in the future. 12007c478bd9Sstevel@tonic-gate */ 12017c478bd9Sstevel@tonic-gate if (delta <= 0) 12027c478bd9Sstevel@tonic-gate delta = 1; 1203454ab202SMadhavan Venkataraman else if (delta > callout_max_ticks) 1204454ab202SMadhavan Venkataraman delta = callout_max_ticks; 12057c478bd9Sstevel@tonic-gate 120687a18d3fSMadhavan Venkataraman id = (ulong_t)timeout_generic(CALLOUT_NORMAL, func, arg, 120787a18d3fSMadhavan Venkataraman TICK_TO_NSEC(delta), nsec_per_tick, CALLOUT_LEGACY); 12087c478bd9Sstevel@tonic-gate 12097c478bd9Sstevel@tonic-gate return ((timeout_id_t)id); 12107c478bd9Sstevel@tonic-gate } 12117c478bd9Sstevel@tonic-gate 121287a18d3fSMadhavan Venkataraman /* 121387a18d3fSMadhavan Venkataraman * Convenience function that creates a normal callout with default parameters 121487a18d3fSMadhavan Venkataraman * and returns a full ID. 121587a18d3fSMadhavan Venkataraman */ 121687a18d3fSMadhavan Venkataraman callout_id_t 121787a18d3fSMadhavan Venkataraman timeout_default(void (*func)(void *), void *arg, clock_t delta) 12187c478bd9Sstevel@tonic-gate { 121987a18d3fSMadhavan Venkataraman callout_id_t id; 12207c478bd9Sstevel@tonic-gate 122187a18d3fSMadhavan Venkataraman /* 122287a18d3fSMadhavan Venkataraman * Make sure the callout runs at least 1 tick in the future. 122387a18d3fSMadhavan Venkataraman */ 122487a18d3fSMadhavan Venkataraman if (delta <= 0) 122587a18d3fSMadhavan Venkataraman delta = 1; 1226454ab202SMadhavan Venkataraman else if (delta > callout_max_ticks) 1227454ab202SMadhavan Venkataraman delta = callout_max_ticks; 122887a18d3fSMadhavan Venkataraman 122987a18d3fSMadhavan Venkataraman id = timeout_generic(CALLOUT_NORMAL, func, arg, TICK_TO_NSEC(delta), 123087a18d3fSMadhavan Venkataraman nsec_per_tick, 0); 123187a18d3fSMadhavan Venkataraman 123287a18d3fSMadhavan Venkataraman return (id); 12337c478bd9Sstevel@tonic-gate } 12347c478bd9Sstevel@tonic-gate 12357c478bd9Sstevel@tonic-gate timeout_id_t 12367c478bd9Sstevel@tonic-gate realtime_timeout(void (*func)(void *), void *arg, clock_t delta) 12377c478bd9Sstevel@tonic-gate { 123887a18d3fSMadhavan Venkataraman ulong_t id; 123987a18d3fSMadhavan Venkataraman 124087a18d3fSMadhavan Venkataraman /* 124187a18d3fSMadhavan Venkataraman * Make sure the callout runs at least 1 tick in the future. 124287a18d3fSMadhavan Venkataraman */ 124387a18d3fSMadhavan Venkataraman if (delta <= 0) 124487a18d3fSMadhavan Venkataraman delta = 1; 1245454ab202SMadhavan Venkataraman else if (delta > callout_max_ticks) 1246454ab202SMadhavan Venkataraman delta = callout_max_ticks; 124787a18d3fSMadhavan Venkataraman 124887a18d3fSMadhavan Venkataraman id = (ulong_t)timeout_generic(CALLOUT_REALTIME, func, arg, 124987a18d3fSMadhavan Venkataraman TICK_TO_NSEC(delta), nsec_per_tick, CALLOUT_LEGACY); 125087a18d3fSMadhavan Venkataraman 125187a18d3fSMadhavan Venkataraman return ((timeout_id_t)id); 12527c478bd9Sstevel@tonic-gate } 12537c478bd9Sstevel@tonic-gate 125487a18d3fSMadhavan Venkataraman /* 125587a18d3fSMadhavan Venkataraman * Convenience function that creates a realtime callout with default parameters 125687a18d3fSMadhavan Venkataraman * and returns a full ID. 125787a18d3fSMadhavan Venkataraman */ 125887a18d3fSMadhavan Venkataraman callout_id_t 125987a18d3fSMadhavan Venkataraman realtime_timeout_default(void (*func)(void *), void *arg, clock_t delta) 12607c478bd9Sstevel@tonic-gate { 126187a18d3fSMadhavan Venkataraman callout_id_t id; 126287a18d3fSMadhavan Venkataraman 126387a18d3fSMadhavan Venkataraman /* 126487a18d3fSMadhavan Venkataraman * Make sure the callout runs at least 1 tick in the future. 126587a18d3fSMadhavan Venkataraman */ 126687a18d3fSMadhavan Venkataraman if (delta <= 0) 126787a18d3fSMadhavan Venkataraman delta = 1; 1268454ab202SMadhavan Venkataraman else if (delta > callout_max_ticks) 1269454ab202SMadhavan Venkataraman delta = callout_max_ticks; 127087a18d3fSMadhavan Venkataraman 127187a18d3fSMadhavan Venkataraman id = timeout_generic(CALLOUT_REALTIME, func, arg, TICK_TO_NSEC(delta), 127287a18d3fSMadhavan Venkataraman nsec_per_tick, 0); 127387a18d3fSMadhavan Venkataraman 127487a18d3fSMadhavan Venkataraman return (id); 127587a18d3fSMadhavan Venkataraman } 127687a18d3fSMadhavan Venkataraman 127787a18d3fSMadhavan Venkataraman hrtime_t 127887a18d3fSMadhavan Venkataraman untimeout_generic(callout_id_t id, int nowait) 127987a18d3fSMadhavan Venkataraman { 12807c478bd9Sstevel@tonic-gate callout_table_t *ct; 12817c478bd9Sstevel@tonic-gate callout_t *cp; 12827c478bd9Sstevel@tonic-gate callout_id_t xid; 128351b32bddSMadhavan Venkataraman callout_list_t *cl; 1284060cedfbSMadhavan Venkataraman int hash, flags; 128587a18d3fSMadhavan Venkataraman callout_id_t bogus; 12867c478bd9Sstevel@tonic-gate 128787a18d3fSMadhavan Venkataraman ct = &callout_table[CALLOUT_ID_TO_TABLE(id)]; 128887a18d3fSMadhavan Venkataraman hash = CALLOUT_IDHASH(id); 12897c478bd9Sstevel@tonic-gate 129087a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 12917c478bd9Sstevel@tonic-gate 129287a18d3fSMadhavan Venkataraman /* 129387a18d3fSMadhavan Venkataraman * Search the ID hash table for the callout. 129487a18d3fSMadhavan Venkataraman */ 129587a18d3fSMadhavan Venkataraman for (cp = ct->ct_idhash[hash].ch_head; cp; cp = cp->c_idnext) { 12967c478bd9Sstevel@tonic-gate 129787a18d3fSMadhavan Venkataraman xid = cp->c_xid; 12987c478bd9Sstevel@tonic-gate 129987a18d3fSMadhavan Venkataraman /* 130087a18d3fSMadhavan Venkataraman * Match the ID and generation number. 130187a18d3fSMadhavan Venkataraman */ 130287a18d3fSMadhavan Venkataraman if ((xid & CALLOUT_ID_MASK) != id) 13037c478bd9Sstevel@tonic-gate continue; 13047c478bd9Sstevel@tonic-gate 130587a18d3fSMadhavan Venkataraman if ((xid & CALLOUT_EXECUTING) == 0) { 130687a18d3fSMadhavan Venkataraman hrtime_t expiration; 130787a18d3fSMadhavan Venkataraman 130887a18d3fSMadhavan Venkataraman /* 130987a18d3fSMadhavan Venkataraman * Delete the callout. If the callout list becomes 131087a18d3fSMadhavan Venkataraman * NULL, we don't remove it from the table. This is 131187a18d3fSMadhavan Venkataraman * so it can be reused. If the empty callout list 131287a18d3fSMadhavan Venkataraman * corresponds to the top of the the callout heap, we 131387a18d3fSMadhavan Venkataraman * don't reprogram the table cyclic here. This is in 131487a18d3fSMadhavan Venkataraman * order to avoid lots of X-calls to the CPU associated 131587a18d3fSMadhavan Venkataraman * with the callout table. 131687a18d3fSMadhavan Venkataraman */ 131751b32bddSMadhavan Venkataraman cl = cp->c_list; 131851b32bddSMadhavan Venkataraman expiration = cl->cl_expiration; 131987a18d3fSMadhavan Venkataraman CALLOUT_DELETE(ct, cp); 1320060cedfbSMadhavan Venkataraman CALLOUT_FREE(ct, cp); 132187a18d3fSMadhavan Venkataraman ct->ct_untimeouts_unexpired++; 132287a18d3fSMadhavan Venkataraman ct->ct_timeouts_pending--; 132351b32bddSMadhavan Venkataraman 132451b32bddSMadhavan Venkataraman /* 1325060cedfbSMadhavan Venkataraman * If the callout list has become empty, there are 3 1326060cedfbSMadhavan Venkataraman * possibilities. If it is present: 1327060cedfbSMadhavan Venkataraman * - in the heap, it needs to be cleaned along 1328060cedfbSMadhavan Venkataraman * with its heap entry. Increment a reap count. 1329060cedfbSMadhavan Venkataraman * - in the callout queue, free it. 1330060cedfbSMadhavan Venkataraman * - in the expired list, free it. 133151b32bddSMadhavan Venkataraman */ 1332060cedfbSMadhavan Venkataraman if (cl->cl_callouts.ch_head == NULL) { 1333060cedfbSMadhavan Venkataraman flags = cl->cl_flags; 1334060cedfbSMadhavan Venkataraman if (flags & CALLOUT_LIST_FLAG_HEAPED) { 133551b32bddSMadhavan Venkataraman ct->ct_nreap++; 1336060cedfbSMadhavan Venkataraman } else if (flags & CALLOUT_LIST_FLAG_QUEUED) { 1337060cedfbSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_queue, cl); 1338060cedfbSMadhavan Venkataraman CALLOUT_LIST_FREE(ct, cl); 1339060cedfbSMadhavan Venkataraman } else { 1340060cedfbSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_expired, cl); 1341060cedfbSMadhavan Venkataraman CALLOUT_LIST_FREE(ct, cl); 1342060cedfbSMadhavan Venkataraman } 1343060cedfbSMadhavan Venkataraman } 134487a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 134587a18d3fSMadhavan Venkataraman 134687a18d3fSMadhavan Venkataraman expiration -= gethrtime(); 134787a18d3fSMadhavan Venkataraman TRACE_2(TR_FAC_CALLOUT, TR_UNTIMEOUT, 134887a18d3fSMadhavan Venkataraman "untimeout:ID %lx hrtime left %llx", id, 134987a18d3fSMadhavan Venkataraman expiration); 135087a18d3fSMadhavan Venkataraman return (expiration < 0 ? 0 : expiration); 135187a18d3fSMadhavan Venkataraman } 135287a18d3fSMadhavan Venkataraman 135387a18d3fSMadhavan Venkataraman ct->ct_untimeouts_executing++; 13547c478bd9Sstevel@tonic-gate /* 13557c478bd9Sstevel@tonic-gate * The callout we want to delete is currently executing. 13567c478bd9Sstevel@tonic-gate * The DDI states that we must wait until the callout 135707247649SMadhavan Venkataraman * completes before returning, so we block on c_done until the 135887a18d3fSMadhavan Venkataraman * callout ID changes (to the old ID if it's on the freelist, 13597c478bd9Sstevel@tonic-gate * or to a new callout ID if it's in use). This implicitly 13607c478bd9Sstevel@tonic-gate * assumes that callout structures are persistent (they are). 13617c478bd9Sstevel@tonic-gate */ 136207247649SMadhavan Venkataraman if (cp->c_executor == curthread) { 13637c478bd9Sstevel@tonic-gate /* 13647c478bd9Sstevel@tonic-gate * The timeout handler called untimeout() on itself. 13657c478bd9Sstevel@tonic-gate * Stupid, but legal. We can't wait for the timeout 13667c478bd9Sstevel@tonic-gate * to complete without deadlocking, so we just return. 13677c478bd9Sstevel@tonic-gate */ 136887a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 13697c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_CALLOUT, TR_UNTIMEOUT_SELF, 13707c478bd9Sstevel@tonic-gate "untimeout_self:ID %x", id); 13717c478bd9Sstevel@tonic-gate return (-1); 13727c478bd9Sstevel@tonic-gate } 137387a18d3fSMadhavan Venkataraman if (nowait == 0) { 137487a18d3fSMadhavan Venkataraman /* 137587a18d3fSMadhavan Venkataraman * We need to wait. Indicate that we are waiting by 137607247649SMadhavan Venkataraman * incrementing c_waiting. This prevents the executor 137707247649SMadhavan Venkataraman * from doing a wakeup on c_done if there are no 137887a18d3fSMadhavan Venkataraman * waiters. 137987a18d3fSMadhavan Venkataraman */ 138087a18d3fSMadhavan Venkataraman while (cp->c_xid == xid) { 138107247649SMadhavan Venkataraman cp->c_waiting = 1; 138207247649SMadhavan Venkataraman cv_wait(&cp->c_done, &ct->ct_mutex); 138387a18d3fSMadhavan Venkataraman } 138487a18d3fSMadhavan Venkataraman } 138587a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 13867c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_CALLOUT, TR_UNTIMEOUT_EXECUTING, 13877c478bd9Sstevel@tonic-gate "untimeout_executing:ID %lx", id); 13887c478bd9Sstevel@tonic-gate return (-1); 13897c478bd9Sstevel@tonic-gate } 139087a18d3fSMadhavan Venkataraman ct->ct_untimeouts_expired++; 13917c478bd9Sstevel@tonic-gate 139287a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 13937c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_CALLOUT, TR_UNTIMEOUT_BOGUS_ID, 13947c478bd9Sstevel@tonic-gate "untimeout_bogus_id:ID %lx", id); 13957c478bd9Sstevel@tonic-gate 13967c478bd9Sstevel@tonic-gate /* 13977c478bd9Sstevel@tonic-gate * We didn't find the specified callout ID. This means either 13987c478bd9Sstevel@tonic-gate * (1) the callout already fired, or (2) the caller passed us 13997c478bd9Sstevel@tonic-gate * a bogus value. Perform a sanity check to detect case (2). 14007c478bd9Sstevel@tonic-gate */ 140151b32bddSMadhavan Venkataraman bogus = (CALLOUT_ID_FLAGS | CALLOUT_COUNTER_HIGH); 140287a18d3fSMadhavan Venkataraman if (((id & bogus) != CALLOUT_COUNTER_HIGH) && (id != 0)) 140387a18d3fSMadhavan Venkataraman panic("untimeout: impossible timeout id %llx", 140487a18d3fSMadhavan Venkataraman (unsigned long long)id); 14057c478bd9Sstevel@tonic-gate 14067c478bd9Sstevel@tonic-gate return (-1); 14077c478bd9Sstevel@tonic-gate } 14087c478bd9Sstevel@tonic-gate 140987a18d3fSMadhavan Venkataraman clock_t 141087a18d3fSMadhavan Venkataraman untimeout(timeout_id_t id_arg) 141187a18d3fSMadhavan Venkataraman { 141287a18d3fSMadhavan Venkataraman hrtime_t hleft; 141387a18d3fSMadhavan Venkataraman clock_t tleft; 141487a18d3fSMadhavan Venkataraman callout_id_t id; 141587a18d3fSMadhavan Venkataraman 141687a18d3fSMadhavan Venkataraman id = (ulong_t)id_arg; 141787a18d3fSMadhavan Venkataraman hleft = untimeout_generic(id, 0); 141887a18d3fSMadhavan Venkataraman if (hleft < 0) 141987a18d3fSMadhavan Venkataraman tleft = -1; 142087a18d3fSMadhavan Venkataraman else if (hleft == 0) 142187a18d3fSMadhavan Venkataraman tleft = 0; 142287a18d3fSMadhavan Venkataraman else 142387a18d3fSMadhavan Venkataraman tleft = NSEC_TO_TICK(hleft); 142487a18d3fSMadhavan Venkataraman 142587a18d3fSMadhavan Venkataraman return (tleft); 142687a18d3fSMadhavan Venkataraman } 142787a18d3fSMadhavan Venkataraman 14287c478bd9Sstevel@tonic-gate /* 142987a18d3fSMadhavan Venkataraman * Convenience function to untimeout a timeout with a full ID with default 143087a18d3fSMadhavan Venkataraman * parameters. 143187a18d3fSMadhavan Venkataraman */ 143287a18d3fSMadhavan Venkataraman clock_t 143387a18d3fSMadhavan Venkataraman untimeout_default(callout_id_t id, int nowait) 143487a18d3fSMadhavan Venkataraman { 143587a18d3fSMadhavan Venkataraman hrtime_t hleft; 143687a18d3fSMadhavan Venkataraman clock_t tleft; 143787a18d3fSMadhavan Venkataraman 143887a18d3fSMadhavan Venkataraman hleft = untimeout_generic(id, nowait); 143987a18d3fSMadhavan Venkataraman if (hleft < 0) 144087a18d3fSMadhavan Venkataraman tleft = -1; 144187a18d3fSMadhavan Venkataraman else if (hleft == 0) 144287a18d3fSMadhavan Venkataraman tleft = 0; 144387a18d3fSMadhavan Venkataraman else 144487a18d3fSMadhavan Venkataraman tleft = NSEC_TO_TICK(hleft); 144587a18d3fSMadhavan Venkataraman 144687a18d3fSMadhavan Venkataraman return (tleft); 144787a18d3fSMadhavan Venkataraman } 144887a18d3fSMadhavan Venkataraman 144987a18d3fSMadhavan Venkataraman /* 145087a18d3fSMadhavan Venkataraman * Expire all the callouts queued in the specified callout list. 14517c478bd9Sstevel@tonic-gate */ 14527c478bd9Sstevel@tonic-gate static void 145387a18d3fSMadhavan Venkataraman callout_list_expire(callout_table_t *ct, callout_list_t *cl) 14547c478bd9Sstevel@tonic-gate { 145507247649SMadhavan Venkataraman callout_t *cp, *cnext; 14567c478bd9Sstevel@tonic-gate 145787a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 145887a18d3fSMadhavan Venkataraman ASSERT(cl != NULL); 14597c478bd9Sstevel@tonic-gate 146007247649SMadhavan Venkataraman for (cp = cl->cl_callouts.ch_head; cp != NULL; cp = cnext) { 146107247649SMadhavan Venkataraman /* 146207247649SMadhavan Venkataraman * Multiple executor threads could be running at the same 146307247649SMadhavan Venkataraman * time. If this callout is already being executed, 146407247649SMadhavan Venkataraman * go on to the next one. 146507247649SMadhavan Venkataraman */ 146607247649SMadhavan Venkataraman if (cp->c_xid & CALLOUT_EXECUTING) { 146707247649SMadhavan Venkataraman cnext = cp->c_clnext; 146807247649SMadhavan Venkataraman continue; 146907247649SMadhavan Venkataraman } 147087a18d3fSMadhavan Venkataraman 1471f635d46aSqiao /* 147287a18d3fSMadhavan Venkataraman * Indicate to untimeout() that a callout is 147387a18d3fSMadhavan Venkataraman * being expired by the executor. 1474f635d46aSqiao */ 147587a18d3fSMadhavan Venkataraman cp->c_xid |= CALLOUT_EXECUTING; 147607247649SMadhavan Venkataraman cp->c_executor = curthread; 147787a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 147887a18d3fSMadhavan Venkataraman 14797c478bd9Sstevel@tonic-gate DTRACE_PROBE1(callout__start, callout_t *, cp); 14807c478bd9Sstevel@tonic-gate (*cp->c_func)(cp->c_arg); 14817c478bd9Sstevel@tonic-gate DTRACE_PROBE1(callout__end, callout_t *, cp); 14827c478bd9Sstevel@tonic-gate 148387a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 148487a18d3fSMadhavan Venkataraman 148587a18d3fSMadhavan Venkataraman ct->ct_expirations++; 148687a18d3fSMadhavan Venkataraman ct->ct_timeouts_pending--; 14877c478bd9Sstevel@tonic-gate /* 148807247649SMadhavan Venkataraman * Indicate completion for c_done. 14897c478bd9Sstevel@tonic-gate */ 149087a18d3fSMadhavan Venkataraman cp->c_xid &= ~CALLOUT_EXECUTING; 149107247649SMadhavan Venkataraman cp->c_executor = NULL; 149207247649SMadhavan Venkataraman cnext = cp->c_clnext; 1493f635d46aSqiao 14947c478bd9Sstevel@tonic-gate /* 149587a18d3fSMadhavan Venkataraman * Delete callout from ID hash table and the callout 149687a18d3fSMadhavan Venkataraman * list, return to freelist, and tell any untimeout() that 149787a18d3fSMadhavan Venkataraman * cares that we're done. 14987c478bd9Sstevel@tonic-gate */ 149987a18d3fSMadhavan Venkataraman CALLOUT_DELETE(ct, cp); 1500060cedfbSMadhavan Venkataraman CALLOUT_FREE(ct, cp); 150187a18d3fSMadhavan Venkataraman 150207247649SMadhavan Venkataraman if (cp->c_waiting) { 150307247649SMadhavan Venkataraman cp->c_waiting = 0; 150407247649SMadhavan Venkataraman cv_broadcast(&cp->c_done); 15057c478bd9Sstevel@tonic-gate } 150687a18d3fSMadhavan Venkataraman } 15077c478bd9Sstevel@tonic-gate } 15087c478bd9Sstevel@tonic-gate 15097c478bd9Sstevel@tonic-gate /* 151087a18d3fSMadhavan Venkataraman * Execute all expired callout lists for a callout table. 15117c478bd9Sstevel@tonic-gate */ 15127c478bd9Sstevel@tonic-gate static void 151387a18d3fSMadhavan Venkataraman callout_expire(callout_table_t *ct) 15147c478bd9Sstevel@tonic-gate { 151587a18d3fSMadhavan Venkataraman callout_list_t *cl, *clnext; 1516f635d46aSqiao 151787a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 15187c478bd9Sstevel@tonic-gate 151987a18d3fSMadhavan Venkataraman for (cl = ct->ct_expired.ch_head; (cl != NULL); cl = clnext) { 1520f635d46aSqiao /* 152187a18d3fSMadhavan Venkataraman * Expire all the callouts in this callout list. 152287a18d3fSMadhavan Venkataraman */ 152387a18d3fSMadhavan Venkataraman callout_list_expire(ct, cl); 152487a18d3fSMadhavan Venkataraman 152507247649SMadhavan Venkataraman clnext = cl->cl_next; 152607247649SMadhavan Venkataraman if (cl->cl_callouts.ch_head == NULL) { 152787a18d3fSMadhavan Venkataraman /* 152887a18d3fSMadhavan Venkataraman * Free the callout list. 152987a18d3fSMadhavan Venkataraman */ 153087a18d3fSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_expired, cl); 153151b32bddSMadhavan Venkataraman CALLOUT_LIST_FREE(ct, cl); 153287a18d3fSMadhavan Venkataraman } 153387a18d3fSMadhavan Venkataraman } 153407247649SMadhavan Venkataraman } 153587a18d3fSMadhavan Venkataraman 153687a18d3fSMadhavan Venkataraman /* 153787a18d3fSMadhavan Venkataraman * The cyclic handlers below process callouts in two steps: 153887a18d3fSMadhavan Venkataraman * 153987a18d3fSMadhavan Venkataraman * 1. Find all expired callout lists and queue them in a separate 154087a18d3fSMadhavan Venkataraman * list of expired callouts. 154187a18d3fSMadhavan Venkataraman * 2. Execute the expired callout lists. 154287a18d3fSMadhavan Venkataraman * 154387a18d3fSMadhavan Venkataraman * This is done for two reasons: 154487a18d3fSMadhavan Venkataraman * 154587a18d3fSMadhavan Venkataraman * 1. We want to quickly find the next earliest expiration to program 154687a18d3fSMadhavan Venkataraman * the cyclic to and reprogram it. We can do this right at the end 154787a18d3fSMadhavan Venkataraman * of step 1. 154887a18d3fSMadhavan Venkataraman * 2. The realtime cyclic handler expires callouts in place. However, 154987a18d3fSMadhavan Venkataraman * for normal callouts, callouts are expired by a taskq thread. 155087a18d3fSMadhavan Venkataraman * So, it is simpler and more robust to have the taskq thread just 155187a18d3fSMadhavan Venkataraman * do step 2. 155287a18d3fSMadhavan Venkataraman */ 155387a18d3fSMadhavan Venkataraman 155487a18d3fSMadhavan Venkataraman /* 1555060cedfbSMadhavan Venkataraman * Realtime callout cyclic handlers. 15567c478bd9Sstevel@tonic-gate */ 15577c478bd9Sstevel@tonic-gate void 155887a18d3fSMadhavan Venkataraman callout_realtime(callout_table_t *ct) 15597c478bd9Sstevel@tonic-gate { 156087a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 1561060cedfbSMadhavan Venkataraman (void) callout_heap_delete(ct); 1562060cedfbSMadhavan Venkataraman callout_expire(ct); 1563060cedfbSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 1564060cedfbSMadhavan Venkataraman } 1565060cedfbSMadhavan Venkataraman 1566060cedfbSMadhavan Venkataraman void 1567060cedfbSMadhavan Venkataraman callout_queue_realtime(callout_table_t *ct) 1568060cedfbSMadhavan Venkataraman { 1569060cedfbSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 1570060cedfbSMadhavan Venkataraman (void) callout_queue_delete(ct); 157187a18d3fSMadhavan Venkataraman callout_expire(ct); 157287a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 157387a18d3fSMadhavan Venkataraman } 15747c478bd9Sstevel@tonic-gate 157587a18d3fSMadhavan Venkataraman void 157687a18d3fSMadhavan Venkataraman callout_execute(callout_table_t *ct) 157787a18d3fSMadhavan Venkataraman { 157887a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 157987a18d3fSMadhavan Venkataraman callout_expire(ct); 158087a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 158187a18d3fSMadhavan Venkataraman } 15827c478bd9Sstevel@tonic-gate 158387a18d3fSMadhavan Venkataraman /* 1584060cedfbSMadhavan Venkataraman * Normal callout cyclic handlers. 158587a18d3fSMadhavan Venkataraman */ 158687a18d3fSMadhavan Venkataraman void 158787a18d3fSMadhavan Venkataraman callout_normal(callout_table_t *ct) 158887a18d3fSMadhavan Venkataraman { 158907247649SMadhavan Venkataraman int i, exec; 1590060cedfbSMadhavan Venkataraman hrtime_t exp; 159187a18d3fSMadhavan Venkataraman 159287a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 1593060cedfbSMadhavan Venkataraman exp = callout_heap_delete(ct); 1594060cedfbSMadhavan Venkataraman CALLOUT_EXEC_COMPUTE(ct, exp, exec); 1595060cedfbSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 1596060cedfbSMadhavan Venkataraman 1597060cedfbSMadhavan Venkataraman for (i = 0; i < exec; i++) { 1598060cedfbSMadhavan Venkataraman ASSERT(ct->ct_taskq != NULL); 1599060cedfbSMadhavan Venkataraman (void) taskq_dispatch(ct->ct_taskq, 1600060cedfbSMadhavan Venkataraman (task_func_t *)callout_execute, ct, TQ_NOSLEEP); 1601060cedfbSMadhavan Venkataraman } 1602060cedfbSMadhavan Venkataraman } 1603060cedfbSMadhavan Venkataraman 1604060cedfbSMadhavan Venkataraman void 1605060cedfbSMadhavan Venkataraman callout_queue_normal(callout_table_t *ct) 1606060cedfbSMadhavan Venkataraman { 1607060cedfbSMadhavan Venkataraman int i, exec; 1608060cedfbSMadhavan Venkataraman hrtime_t exp; 1609060cedfbSMadhavan Venkataraman 1610060cedfbSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 1611060cedfbSMadhavan Venkataraman exp = callout_queue_delete(ct); 1612060cedfbSMadhavan Venkataraman CALLOUT_EXEC_COMPUTE(ct, exp, exec); 161387a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 161487a18d3fSMadhavan Venkataraman 161507247649SMadhavan Venkataraman for (i = 0; i < exec; i++) { 161687a18d3fSMadhavan Venkataraman ASSERT(ct->ct_taskq != NULL); 161787a18d3fSMadhavan Venkataraman (void) taskq_dispatch(ct->ct_taskq, 161887a18d3fSMadhavan Venkataraman (task_func_t *)callout_execute, ct, TQ_NOSLEEP); 161987a18d3fSMadhavan Venkataraman } 162087a18d3fSMadhavan Venkataraman } 162187a18d3fSMadhavan Venkataraman 162287a18d3fSMadhavan Venkataraman /* 162387a18d3fSMadhavan Venkataraman * Suspend callout processing. 162487a18d3fSMadhavan Venkataraman */ 162587a18d3fSMadhavan Venkataraman static void 162687a18d3fSMadhavan Venkataraman callout_suspend(void) 162787a18d3fSMadhavan Venkataraman { 162887a18d3fSMadhavan Venkataraman int t, f; 162987a18d3fSMadhavan Venkataraman callout_table_t *ct; 163087a18d3fSMadhavan Venkataraman 163187a18d3fSMadhavan Venkataraman /* 163287a18d3fSMadhavan Venkataraman * Traverse every callout table in the system and suspend callout 163387a18d3fSMadhavan Venkataraman * processing. 163487a18d3fSMadhavan Venkataraman * 163587a18d3fSMadhavan Venkataraman * We need to suspend all the tables (including the inactive ones) 163687a18d3fSMadhavan Venkataraman * so that if a table is made active while the suspend is still on, 163787a18d3fSMadhavan Venkataraman * the table remains suspended. 163887a18d3fSMadhavan Venkataraman */ 163987a18d3fSMadhavan Venkataraman for (f = 0; f < max_ncpus; f++) { 164087a18d3fSMadhavan Venkataraman for (t = 0; t < CALLOUT_NTYPES; t++) { 164187a18d3fSMadhavan Venkataraman ct = &callout_table[CALLOUT_TABLE(t, f)]; 164287a18d3fSMadhavan Venkataraman 164387a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 1644454ab202SMadhavan Venkataraman ct->ct_suspend++; 164587a18d3fSMadhavan Venkataraman if (ct->ct_cyclic == CYCLIC_NONE) { 164687a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 164787a18d3fSMadhavan Venkataraman continue; 164887a18d3fSMadhavan Venkataraman } 1649060cedfbSMadhavan Venkataraman if (ct->ct_suspend == 1) { 1650454ab202SMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_cyclic, 1651454ab202SMadhavan Venkataraman CY_INFINITY); 1652060cedfbSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_qcyclic, 1653060cedfbSMadhavan Venkataraman CY_INFINITY); 1654060cedfbSMadhavan Venkataraman } 165587a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 165687a18d3fSMadhavan Venkataraman } 165787a18d3fSMadhavan Venkataraman } 165887a18d3fSMadhavan Venkataraman } 165987a18d3fSMadhavan Venkataraman 166087a18d3fSMadhavan Venkataraman /* 166187a18d3fSMadhavan Venkataraman * Resume callout processing. 166287a18d3fSMadhavan Venkataraman */ 166387a18d3fSMadhavan Venkataraman static void 166451b32bddSMadhavan Venkataraman callout_resume(hrtime_t delta, int timechange) 166587a18d3fSMadhavan Venkataraman { 1666060cedfbSMadhavan Venkataraman hrtime_t hexp, qexp; 166787a18d3fSMadhavan Venkataraman int t, f; 166887a18d3fSMadhavan Venkataraman callout_table_t *ct; 166987a18d3fSMadhavan Venkataraman 167087a18d3fSMadhavan Venkataraman /* 167187a18d3fSMadhavan Venkataraman * Traverse every callout table in the system and resume callout 167287a18d3fSMadhavan Venkataraman * processing. For active tables, perform any hrtime adjustments 167387a18d3fSMadhavan Venkataraman * necessary. 167487a18d3fSMadhavan Venkataraman */ 167587a18d3fSMadhavan Venkataraman for (f = 0; f < max_ncpus; f++) { 167687a18d3fSMadhavan Venkataraman for (t = 0; t < CALLOUT_NTYPES; t++) { 167787a18d3fSMadhavan Venkataraman ct = &callout_table[CALLOUT_TABLE(t, f)]; 167887a18d3fSMadhavan Venkataraman 167987a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 168087a18d3fSMadhavan Venkataraman if (ct->ct_cyclic == CYCLIC_NONE) { 1681454ab202SMadhavan Venkataraman ct->ct_suspend--; 168287a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 168387a18d3fSMadhavan Venkataraman continue; 168487a18d3fSMadhavan Venkataraman } 168587a18d3fSMadhavan Venkataraman 168651b32bddSMadhavan Venkataraman /* 168751b32bddSMadhavan Venkataraman * If a delta is specified, adjust the expirations in 168851b32bddSMadhavan Venkataraman * the heap by delta. Also, if the caller indicates 168951b32bddSMadhavan Venkataraman * a timechange, process that. This step also cleans 169051b32bddSMadhavan Venkataraman * out any empty callout lists that might happen to 169151b32bddSMadhavan Venkataraman * be there. 169251b32bddSMadhavan Venkataraman */ 1693060cedfbSMadhavan Venkataraman hexp = callout_heap_process(ct, delta, timechange); 1694060cedfbSMadhavan Venkataraman qexp = callout_queue_process(ct, delta, timechange); 169587a18d3fSMadhavan Venkataraman 1696454ab202SMadhavan Venkataraman ct->ct_suspend--; 1697454ab202SMadhavan Venkataraman if (ct->ct_suspend == 0) { 1698060cedfbSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_cyclic, hexp); 1699060cedfbSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_qcyclic, qexp); 1700454ab202SMadhavan Venkataraman } 170151b32bddSMadhavan Venkataraman 170287a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 170387a18d3fSMadhavan Venkataraman } 170487a18d3fSMadhavan Venkataraman } 17057c478bd9Sstevel@tonic-gate } 17067c478bd9Sstevel@tonic-gate 17077c478bd9Sstevel@tonic-gate /* 17087c478bd9Sstevel@tonic-gate * Callback handler used by CPR to stop and resume callouts. 170951b32bddSMadhavan Venkataraman * The cyclic subsystem saves and restores hrtime during CPR. 171051b32bddSMadhavan Venkataraman * That is why callout_resume() is called with a 0 delta. 171151b32bddSMadhavan Venkataraman * Although hrtime is the same, hrestime (system time) has 171251b32bddSMadhavan Venkataraman * progressed during CPR. So, we have to indicate a time change 171351b32bddSMadhavan Venkataraman * to expire the absolute hrestime timers. 17147c478bd9Sstevel@tonic-gate */ 17157c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 17167c478bd9Sstevel@tonic-gate static boolean_t 17177c478bd9Sstevel@tonic-gate callout_cpr_callb(void *arg, int code) 17187c478bd9Sstevel@tonic-gate { 171987a18d3fSMadhavan Venkataraman if (code == CB_CODE_CPR_CHKPT) 172087a18d3fSMadhavan Venkataraman callout_suspend(); 172187a18d3fSMadhavan Venkataraman else 172251b32bddSMadhavan Venkataraman callout_resume(0, 1); 172387a18d3fSMadhavan Venkataraman 17247c478bd9Sstevel@tonic-gate return (B_TRUE); 17257c478bd9Sstevel@tonic-gate } 17267c478bd9Sstevel@tonic-gate 17277c478bd9Sstevel@tonic-gate /* 172887a18d3fSMadhavan Venkataraman * Callback handler invoked when the debugger is entered or exited. 17297c478bd9Sstevel@tonic-gate */ 173087a18d3fSMadhavan Venkataraman /*ARGSUSED*/ 173187a18d3fSMadhavan Venkataraman static boolean_t 173287a18d3fSMadhavan Venkataraman callout_debug_callb(void *arg, int code) 17337c478bd9Sstevel@tonic-gate { 173487a18d3fSMadhavan Venkataraman hrtime_t delta; 1735f635d46aSqiao 1736f635d46aSqiao /* 173787a18d3fSMadhavan Venkataraman * When the system enters the debugger. make a note of the hrtime. 173887a18d3fSMadhavan Venkataraman * When it is resumed, compute how long the system was in the 173987a18d3fSMadhavan Venkataraman * debugger. This interval should not be counted for callouts. 1740f635d46aSqiao */ 174187a18d3fSMadhavan Venkataraman if (code == 0) { 174287a18d3fSMadhavan Venkataraman callout_suspend(); 174387a18d3fSMadhavan Venkataraman callout_debug_hrtime = gethrtime(); 174487a18d3fSMadhavan Venkataraman } else { 174587a18d3fSMadhavan Venkataraman delta = gethrtime() - callout_debug_hrtime; 174651b32bddSMadhavan Venkataraman callout_resume(delta, 0); 174787a18d3fSMadhavan Venkataraman } 1748f635d46aSqiao 174987a18d3fSMadhavan Venkataraman return (B_TRUE); 175087a18d3fSMadhavan Venkataraman } 175187a18d3fSMadhavan Venkataraman 175287a18d3fSMadhavan Venkataraman /* 175307247649SMadhavan Venkataraman * Move the absolute hrestime callouts to the expired list. Then program the 175407247649SMadhavan Venkataraman * table's cyclic to expire immediately so that the callouts can be executed 175587a18d3fSMadhavan Venkataraman * immediately. 175687a18d3fSMadhavan Venkataraman */ 175787a18d3fSMadhavan Venkataraman static void 175887a18d3fSMadhavan Venkataraman callout_hrestime_one(callout_table_t *ct) 175987a18d3fSMadhavan Venkataraman { 1760060cedfbSMadhavan Venkataraman hrtime_t hexp, qexp; 176187a18d3fSMadhavan Venkataraman 176287a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 1763060cedfbSMadhavan Venkataraman if (ct->ct_cyclic == CYCLIC_NONE) { 176487a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 176587a18d3fSMadhavan Venkataraman return; 176687a18d3fSMadhavan Venkataraman } 176787a18d3fSMadhavan Venkataraman 176851b32bddSMadhavan Venkataraman /* 176951b32bddSMadhavan Venkataraman * Walk the heap and process all the absolute hrestime entries. 177051b32bddSMadhavan Venkataraman */ 1771060cedfbSMadhavan Venkataraman hexp = callout_heap_process(ct, 0, 1); 1772060cedfbSMadhavan Venkataraman qexp = callout_queue_process(ct, 0, 1); 177387a18d3fSMadhavan Venkataraman 1774060cedfbSMadhavan Venkataraman if (ct->ct_suspend == 0) { 1775060cedfbSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_cyclic, hexp); 1776060cedfbSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_qcyclic, qexp); 1777060cedfbSMadhavan Venkataraman } 177807247649SMadhavan Venkataraman 177987a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 178087a18d3fSMadhavan Venkataraman } 178187a18d3fSMadhavan Venkataraman 178287a18d3fSMadhavan Venkataraman /* 178387a18d3fSMadhavan Venkataraman * This function is called whenever system time (hrestime) is changed 178487a18d3fSMadhavan Venkataraman * explicitly. All the HRESTIME callouts must be expired at once. 178587a18d3fSMadhavan Venkataraman */ 178687a18d3fSMadhavan Venkataraman /*ARGSUSED*/ 178787a18d3fSMadhavan Venkataraman void 178887a18d3fSMadhavan Venkataraman callout_hrestime(void) 178987a18d3fSMadhavan Venkataraman { 179087a18d3fSMadhavan Venkataraman int t, f; 179187a18d3fSMadhavan Venkataraman callout_table_t *ct; 179287a18d3fSMadhavan Venkataraman 179387a18d3fSMadhavan Venkataraman /* 179487a18d3fSMadhavan Venkataraman * Traverse every callout table in the system and process the hrestime 179587a18d3fSMadhavan Venkataraman * callouts therein. 179687a18d3fSMadhavan Venkataraman * 179787a18d3fSMadhavan Venkataraman * We look at all the tables because we don't know which ones were 179887a18d3fSMadhavan Venkataraman * onlined and offlined in the past. The offlined tables may still 179987a18d3fSMadhavan Venkataraman * have active cyclics processing timers somewhere. 180087a18d3fSMadhavan Venkataraman */ 180187a18d3fSMadhavan Venkataraman for (f = 0; f < max_ncpus; f++) { 180287a18d3fSMadhavan Venkataraman for (t = 0; t < CALLOUT_NTYPES; t++) { 180387a18d3fSMadhavan Venkataraman ct = &callout_table[CALLOUT_TABLE(t, f)]; 180487a18d3fSMadhavan Venkataraman callout_hrestime_one(ct); 180587a18d3fSMadhavan Venkataraman } 180687a18d3fSMadhavan Venkataraman } 180787a18d3fSMadhavan Venkataraman } 180887a18d3fSMadhavan Venkataraman 180987a18d3fSMadhavan Venkataraman /* 181087a18d3fSMadhavan Venkataraman * Create the hash tables for this callout table. 181187a18d3fSMadhavan Venkataraman */ 181287a18d3fSMadhavan Venkataraman static void 181387a18d3fSMadhavan Venkataraman callout_hash_init(callout_table_t *ct) 181487a18d3fSMadhavan Venkataraman { 181587a18d3fSMadhavan Venkataraman size_t size; 181687a18d3fSMadhavan Venkataraman 181787a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 181887a18d3fSMadhavan Venkataraman ASSERT((ct->ct_idhash == NULL) && (ct->ct_clhash == NULL)); 181987a18d3fSMadhavan Venkataraman 182087a18d3fSMadhavan Venkataraman size = sizeof (callout_hash_t) * CALLOUT_BUCKETS; 182187a18d3fSMadhavan Venkataraman ct->ct_idhash = kmem_zalloc(size, KM_SLEEP); 182287a18d3fSMadhavan Venkataraman ct->ct_clhash = kmem_zalloc(size, KM_SLEEP); 182387a18d3fSMadhavan Venkataraman } 182487a18d3fSMadhavan Venkataraman 182587a18d3fSMadhavan Venkataraman /* 182687a18d3fSMadhavan Venkataraman * Create per-callout table kstats. 182787a18d3fSMadhavan Venkataraman */ 182887a18d3fSMadhavan Venkataraman static void 182987a18d3fSMadhavan Venkataraman callout_kstat_init(callout_table_t *ct) 183087a18d3fSMadhavan Venkataraman { 183187a18d3fSMadhavan Venkataraman callout_stat_type_t stat; 183287a18d3fSMadhavan Venkataraman kstat_t *ct_kstats; 183387a18d3fSMadhavan Venkataraman int ndx; 183487a18d3fSMadhavan Venkataraman 183587a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 183687a18d3fSMadhavan Venkataraman ASSERT(ct->ct_kstats == NULL); 183787a18d3fSMadhavan Venkataraman 183887a18d3fSMadhavan Venkataraman ndx = ct - callout_table; 183987a18d3fSMadhavan Venkataraman ct_kstats = kstat_create("unix", ndx, "callout", 184087a18d3fSMadhavan Venkataraman "misc", KSTAT_TYPE_NAMED, CALLOUT_NUM_STATS, KSTAT_FLAG_VIRTUAL); 184187a18d3fSMadhavan Venkataraman 184287a18d3fSMadhavan Venkataraman if (ct_kstats == NULL) { 184387a18d3fSMadhavan Venkataraman cmn_err(CE_WARN, "kstat_create for callout table %p failed", 184487a18d3fSMadhavan Venkataraman (void *)ct); 184587a18d3fSMadhavan Venkataraman } else { 184687a18d3fSMadhavan Venkataraman ct_kstats->ks_data = ct->ct_kstat_data; 184787a18d3fSMadhavan Venkataraman for (stat = 0; stat < CALLOUT_NUM_STATS; stat++) 184887a18d3fSMadhavan Venkataraman kstat_named_init(&ct->ct_kstat_data[stat], 184987a18d3fSMadhavan Venkataraman callout_kstat_names[stat], KSTAT_DATA_INT64); 185087a18d3fSMadhavan Venkataraman ct->ct_kstats = ct_kstats; 185187a18d3fSMadhavan Venkataraman kstat_install(ct_kstats); 185287a18d3fSMadhavan Venkataraman } 185387a18d3fSMadhavan Venkataraman } 185487a18d3fSMadhavan Venkataraman 185587a18d3fSMadhavan Venkataraman static void 185687a18d3fSMadhavan Venkataraman callout_cyclic_init(callout_table_t *ct) 185787a18d3fSMadhavan Venkataraman { 185887a18d3fSMadhavan Venkataraman cyc_handler_t hdlr; 185987a18d3fSMadhavan Venkataraman cyc_time_t when; 186087a18d3fSMadhavan Venkataraman processorid_t seqid; 186187a18d3fSMadhavan Venkataraman int t; 1862060cedfbSMadhavan Venkataraman cyclic_id_t cyclic, qcyclic; 186387a18d3fSMadhavan Venkataraman 186487a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 186587a18d3fSMadhavan Venkataraman 1866060cedfbSMadhavan Venkataraman t = ct->ct_type; 186787a18d3fSMadhavan Venkataraman seqid = CALLOUT_TABLE_SEQID(ct); 186887a18d3fSMadhavan Venkataraman 186987a18d3fSMadhavan Venkataraman /* 187087a18d3fSMadhavan Venkataraman * Create the taskq thread if the table type is normal. 187187a18d3fSMadhavan Venkataraman * Realtime tables are handled at PIL1 by a softint 187287a18d3fSMadhavan Venkataraman * handler. 187387a18d3fSMadhavan Venkataraman */ 18747c478bd9Sstevel@tonic-gate if (t == CALLOUT_NORMAL) { 187587a18d3fSMadhavan Venkataraman ASSERT(ct->ct_taskq == NULL); 18767c478bd9Sstevel@tonic-gate /* 18777c478bd9Sstevel@tonic-gate * Each callout thread consumes exactly one 18787c478bd9Sstevel@tonic-gate * task structure while active. Therefore, 187951b32bddSMadhavan Venkataraman * prepopulating with 2 * callout_threads tasks 18807c478bd9Sstevel@tonic-gate * ensures that there's at least one task per 18817c478bd9Sstevel@tonic-gate * thread that's either scheduled or on the 18827c478bd9Sstevel@tonic-gate * freelist. In turn, this guarantees that 18837c478bd9Sstevel@tonic-gate * taskq_dispatch() will always either succeed 18847c478bd9Sstevel@tonic-gate * (because there's a free task structure) or 18857c478bd9Sstevel@tonic-gate * be unnecessary (because "callout_excute(ct)" 18867c478bd9Sstevel@tonic-gate * has already scheduled). 18877c478bd9Sstevel@tonic-gate */ 18887c478bd9Sstevel@tonic-gate ct->ct_taskq = 188987a18d3fSMadhavan Venkataraman taskq_create_instance("callout_taskq", seqid, 189051b32bddSMadhavan Venkataraman callout_threads, maxclsyspri, 189151b32bddSMadhavan Venkataraman 2 * callout_threads, 2 * callout_threads, 18927c478bd9Sstevel@tonic-gate TASKQ_PREPOPULATE | TASKQ_CPR_SAFE); 18937c478bd9Sstevel@tonic-gate } 189487a18d3fSMadhavan Venkataraman 189587a18d3fSMadhavan Venkataraman /* 189687a18d3fSMadhavan Venkataraman * callouts can only be created in a table whose 189787a18d3fSMadhavan Venkataraman * cyclic has been initialized. 189887a18d3fSMadhavan Venkataraman */ 189987a18d3fSMadhavan Venkataraman ASSERT(ct->ct_heap_num == 0); 190087a18d3fSMadhavan Venkataraman 190187a18d3fSMadhavan Venkataraman /* 1902113d3ed7SMadhavan Venkataraman * Drop the mutex before creating the callout cyclics. cyclic_add() 1903113d3ed7SMadhavan Venkataraman * could potentially expand the cyclic heap. We don't want to be 1904113d3ed7SMadhavan Venkataraman * holding the callout table mutex in that case. Note that this 1905113d3ed7SMadhavan Venkataraman * function is called during CPU online. cpu_lock is held at this 1906113d3ed7SMadhavan Venkataraman * point. So, only one thread can be executing the cyclic add logic 1907113d3ed7SMadhavan Venkataraman * below at any time. 1908113d3ed7SMadhavan Venkataraman */ 1909113d3ed7SMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 1910113d3ed7SMadhavan Venkataraman 1911113d3ed7SMadhavan Venkataraman /* 191287a18d3fSMadhavan Venkataraman * Create the callout table cyclics. 191307247649SMadhavan Venkataraman * 191407247649SMadhavan Venkataraman * The realtime cyclic handler executes at low PIL. The normal cyclic 191507247649SMadhavan Venkataraman * handler executes at lock PIL. This is because there are cases 191607247649SMadhavan Venkataraman * where code can block at PIL > 1 waiting for a normal callout handler 191707247649SMadhavan Venkataraman * to unblock it directly or indirectly. If the normal cyclic were to 191807247649SMadhavan Venkataraman * be executed at low PIL, it could get blocked out by the waiter 191907247649SMadhavan Venkataraman * and cause a deadlock. 192087a18d3fSMadhavan Venkataraman */ 192187a18d3fSMadhavan Venkataraman ASSERT(ct->ct_cyclic == CYCLIC_NONE); 192287a18d3fSMadhavan Venkataraman 1923060cedfbSMadhavan Venkataraman if (t == CALLOUT_REALTIME) { 192407247649SMadhavan Venkataraman hdlr.cyh_level = callout_realtime_level; 1925060cedfbSMadhavan Venkataraman hdlr.cyh_func = (cyc_func_t)callout_realtime; 1926060cedfbSMadhavan Venkataraman } else { 192707247649SMadhavan Venkataraman hdlr.cyh_level = callout_normal_level; 1928060cedfbSMadhavan Venkataraman hdlr.cyh_func = (cyc_func_t)callout_normal; 1929060cedfbSMadhavan Venkataraman } 193087a18d3fSMadhavan Venkataraman hdlr.cyh_arg = ct; 193187a18d3fSMadhavan Venkataraman when.cyt_when = CY_INFINITY; 193287a18d3fSMadhavan Venkataraman when.cyt_interval = CY_INFINITY; 193387a18d3fSMadhavan Venkataraman 1934113d3ed7SMadhavan Venkataraman cyclic = cyclic_add(&hdlr, &when); 1935113d3ed7SMadhavan Venkataraman 1936060cedfbSMadhavan Venkataraman if (t == CALLOUT_REALTIME) 1937060cedfbSMadhavan Venkataraman hdlr.cyh_func = (cyc_func_t)callout_queue_realtime; 1938060cedfbSMadhavan Venkataraman else 1939060cedfbSMadhavan Venkataraman hdlr.cyh_func = (cyc_func_t)callout_queue_normal; 1940060cedfbSMadhavan Venkataraman 1941060cedfbSMadhavan Venkataraman qcyclic = cyclic_add(&hdlr, &when); 1942060cedfbSMadhavan Venkataraman 1943113d3ed7SMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 1944113d3ed7SMadhavan Venkataraman ct->ct_cyclic = cyclic; 1945060cedfbSMadhavan Venkataraman ct->ct_qcyclic = qcyclic; 194687a18d3fSMadhavan Venkataraman } 194787a18d3fSMadhavan Venkataraman 194887a18d3fSMadhavan Venkataraman void 194987a18d3fSMadhavan Venkataraman callout_cpu_online(cpu_t *cp) 195087a18d3fSMadhavan Venkataraman { 195187a18d3fSMadhavan Venkataraman lgrp_handle_t hand; 195287a18d3fSMadhavan Venkataraman callout_cache_t *cache; 195387a18d3fSMadhavan Venkataraman char s[KMEM_CACHE_NAMELEN]; 195487a18d3fSMadhavan Venkataraman callout_table_t *ct; 195587a18d3fSMadhavan Venkataraman processorid_t seqid; 195687a18d3fSMadhavan Venkataraman int t; 195787a18d3fSMadhavan Venkataraman 195887a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&cpu_lock)); 195987a18d3fSMadhavan Venkataraman 196087a18d3fSMadhavan Venkataraman /* 196187a18d3fSMadhavan Venkataraman * Locate the cache corresponding to the onlined CPU's lgroup. 196287a18d3fSMadhavan Venkataraman * Note that access to callout_caches is protected by cpu_lock. 196387a18d3fSMadhavan Venkataraman */ 196487a18d3fSMadhavan Venkataraman hand = lgrp_plat_cpu_to_hand(cp->cpu_id); 196587a18d3fSMadhavan Venkataraman for (cache = callout_caches; cache != NULL; cache = cache->cc_next) { 196687a18d3fSMadhavan Venkataraman if (cache->cc_hand == hand) 196787a18d3fSMadhavan Venkataraman break; 196887a18d3fSMadhavan Venkataraman } 196987a18d3fSMadhavan Venkataraman 197087a18d3fSMadhavan Venkataraman /* 197187a18d3fSMadhavan Venkataraman * If not found, create one. The caches are never destroyed. 197287a18d3fSMadhavan Venkataraman */ 197387a18d3fSMadhavan Venkataraman if (cache == NULL) { 197487a18d3fSMadhavan Venkataraman cache = kmem_alloc(sizeof (callout_cache_t), KM_SLEEP); 197587a18d3fSMadhavan Venkataraman cache->cc_hand = hand; 197687a18d3fSMadhavan Venkataraman (void) snprintf(s, KMEM_CACHE_NAMELEN, "callout_cache%lx", 197787a18d3fSMadhavan Venkataraman (long)hand); 197887a18d3fSMadhavan Venkataraman cache->cc_cache = kmem_cache_create(s, sizeof (callout_t), 197987a18d3fSMadhavan Venkataraman CALLOUT_ALIGN, NULL, NULL, NULL, NULL, NULL, 0); 198087a18d3fSMadhavan Venkataraman (void) snprintf(s, KMEM_CACHE_NAMELEN, "callout_lcache%lx", 198187a18d3fSMadhavan Venkataraman (long)hand); 198287a18d3fSMadhavan Venkataraman cache->cc_lcache = kmem_cache_create(s, sizeof (callout_list_t), 198387a18d3fSMadhavan Venkataraman CALLOUT_ALIGN, NULL, NULL, NULL, NULL, NULL, 0); 198487a18d3fSMadhavan Venkataraman cache->cc_next = callout_caches; 198587a18d3fSMadhavan Venkataraman callout_caches = cache; 198687a18d3fSMadhavan Venkataraman } 198787a18d3fSMadhavan Venkataraman 198887a18d3fSMadhavan Venkataraman seqid = cp->cpu_seqid; 198987a18d3fSMadhavan Venkataraman 199087a18d3fSMadhavan Venkataraman for (t = 0; t < CALLOUT_NTYPES; t++) { 199187a18d3fSMadhavan Venkataraman ct = &callout_table[CALLOUT_TABLE(t, seqid)]; 199287a18d3fSMadhavan Venkataraman 199387a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 199487a18d3fSMadhavan Venkataraman /* 199587a18d3fSMadhavan Venkataraman * Store convinience pointers to the kmem caches 199687a18d3fSMadhavan Venkataraman * in the callout table. These assignments should always be 199787a18d3fSMadhavan Venkataraman * done as callout tables can map to different physical 199887a18d3fSMadhavan Venkataraman * CPUs each time. 199987a18d3fSMadhavan Venkataraman */ 200087a18d3fSMadhavan Venkataraman ct->ct_cache = cache->cc_cache; 200187a18d3fSMadhavan Venkataraman ct->ct_lcache = cache->cc_lcache; 200287a18d3fSMadhavan Venkataraman 200387a18d3fSMadhavan Venkataraman /* 200487a18d3fSMadhavan Venkataraman * We use the heap pointer to check if stuff has been 200587a18d3fSMadhavan Venkataraman * initialized for this callout table. 200687a18d3fSMadhavan Venkataraman */ 200787a18d3fSMadhavan Venkataraman if (ct->ct_heap == NULL) { 200887a18d3fSMadhavan Venkataraman callout_heap_init(ct); 200987a18d3fSMadhavan Venkataraman callout_hash_init(ct); 201087a18d3fSMadhavan Venkataraman callout_kstat_init(ct); 201187a18d3fSMadhavan Venkataraman callout_cyclic_init(ct); 201287a18d3fSMadhavan Venkataraman } 201387a18d3fSMadhavan Venkataraman 201487a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 201587a18d3fSMadhavan Venkataraman 201687a18d3fSMadhavan Venkataraman /* 2017060cedfbSMadhavan Venkataraman * Move the cyclics to this CPU by doing a bind. 201887a18d3fSMadhavan Venkataraman */ 201987a18d3fSMadhavan Venkataraman cyclic_bind(ct->ct_cyclic, cp, NULL); 2020060cedfbSMadhavan Venkataraman cyclic_bind(ct->ct_qcyclic, cp, NULL); 2021454ab202SMadhavan Venkataraman } 2022454ab202SMadhavan Venkataraman } 2023454ab202SMadhavan Venkataraman 2024454ab202SMadhavan Venkataraman void 2025454ab202SMadhavan Venkataraman callout_cpu_offline(cpu_t *cp) 2026454ab202SMadhavan Venkataraman { 2027454ab202SMadhavan Venkataraman callout_table_t *ct; 2028454ab202SMadhavan Venkataraman processorid_t seqid; 2029454ab202SMadhavan Venkataraman int t; 2030454ab202SMadhavan Venkataraman 2031454ab202SMadhavan Venkataraman ASSERT(MUTEX_HELD(&cpu_lock)); 2032454ab202SMadhavan Venkataraman 2033454ab202SMadhavan Venkataraman seqid = cp->cpu_seqid; 2034454ab202SMadhavan Venkataraman 2035454ab202SMadhavan Venkataraman for (t = 0; t < CALLOUT_NTYPES; t++) { 2036454ab202SMadhavan Venkataraman ct = &callout_table[CALLOUT_TABLE(t, seqid)]; 2037454ab202SMadhavan Venkataraman 2038454ab202SMadhavan Venkataraman /* 2039060cedfbSMadhavan Venkataraman * Unbind the cyclics. This will allow the cyclic subsystem 2040060cedfbSMadhavan Venkataraman * to juggle the cyclics during CPU offline. 2041454ab202SMadhavan Venkataraman */ 204287a18d3fSMadhavan Venkataraman cyclic_bind(ct->ct_cyclic, NULL, NULL); 2043060cedfbSMadhavan Venkataraman cyclic_bind(ct->ct_qcyclic, NULL, NULL); 20447c478bd9Sstevel@tonic-gate } 20457c478bd9Sstevel@tonic-gate } 204687a18d3fSMadhavan Venkataraman 204787a18d3fSMadhavan Venkataraman /* 204887a18d3fSMadhavan Venkataraman * This is called to perform per-CPU initialization for slave CPUs at 204987a18d3fSMadhavan Venkataraman * boot time. 205087a18d3fSMadhavan Venkataraman */ 205187a18d3fSMadhavan Venkataraman void 205287a18d3fSMadhavan Venkataraman callout_mp_init(void) 205387a18d3fSMadhavan Venkataraman { 205487a18d3fSMadhavan Venkataraman cpu_t *cp; 2055060cedfbSMadhavan Venkataraman size_t min, max; 2056060cedfbSMadhavan Venkataraman 2057060cedfbSMadhavan Venkataraman if (callout_chunk == CALLOUT_CHUNK) { 2058060cedfbSMadhavan Venkataraman /* 2059060cedfbSMadhavan Venkataraman * No one has specified a chunk in /etc/system. We need to 2060060cedfbSMadhavan Venkataraman * compute it here based on the number of online CPUs and 2061060cedfbSMadhavan Venkataraman * available physical memory. 2062060cedfbSMadhavan Venkataraman */ 2063060cedfbSMadhavan Venkataraman min = CALLOUT_MIN_HEAP_SIZE; 20641671524dSMadhavan Venkataraman max = ptob(physmem / CALLOUT_MEM_FRACTION); 2065060cedfbSMadhavan Venkataraman if (min > max) 2066060cedfbSMadhavan Venkataraman min = max; 2067060cedfbSMadhavan Venkataraman callout_chunk = min / sizeof (callout_heap_t); 2068060cedfbSMadhavan Venkataraman callout_chunk /= ncpus_online; 2069060cedfbSMadhavan Venkataraman callout_chunk = P2ROUNDUP(callout_chunk, CALLOUT_CHUNK); 2070060cedfbSMadhavan Venkataraman } 207187a18d3fSMadhavan Venkataraman 207287a18d3fSMadhavan Venkataraman mutex_enter(&cpu_lock); 207387a18d3fSMadhavan Venkataraman 207487a18d3fSMadhavan Venkataraman cp = cpu_active; 207587a18d3fSMadhavan Venkataraman do { 207687a18d3fSMadhavan Venkataraman callout_cpu_online(cp); 207787a18d3fSMadhavan Venkataraman } while ((cp = cp->cpu_next_onln) != cpu_active); 207887a18d3fSMadhavan Venkataraman 207987a18d3fSMadhavan Venkataraman mutex_exit(&cpu_lock); 208087a18d3fSMadhavan Venkataraman } 208187a18d3fSMadhavan Venkataraman 208287a18d3fSMadhavan Venkataraman /* 208387a18d3fSMadhavan Venkataraman * Initialize all callout tables. Called at boot time just before clkstart(). 208487a18d3fSMadhavan Venkataraman */ 208587a18d3fSMadhavan Venkataraman void 208687a18d3fSMadhavan Venkataraman callout_init(void) 208787a18d3fSMadhavan Venkataraman { 208887a18d3fSMadhavan Venkataraman int f, t; 208987a18d3fSMadhavan Venkataraman size_t size; 209087a18d3fSMadhavan Venkataraman int table_id; 209187a18d3fSMadhavan Venkataraman callout_table_t *ct; 209287a18d3fSMadhavan Venkataraman long bits, fanout; 209387a18d3fSMadhavan Venkataraman uintptr_t buf; 209487a18d3fSMadhavan Venkataraman 209587a18d3fSMadhavan Venkataraman /* 209687a18d3fSMadhavan Venkataraman * Initialize callout globals. 209787a18d3fSMadhavan Venkataraman */ 209887a18d3fSMadhavan Venkataraman bits = 0; 209987a18d3fSMadhavan Venkataraman for (fanout = 1; (fanout < max_ncpus); fanout <<= 1) 210087a18d3fSMadhavan Venkataraman bits++; 210187a18d3fSMadhavan Venkataraman callout_table_bits = CALLOUT_TYPE_BITS + bits; 210287a18d3fSMadhavan Venkataraman callout_table_mask = (1 << callout_table_bits) - 1; 210387a18d3fSMadhavan Venkataraman callout_counter_low = 1 << CALLOUT_COUNTER_SHIFT; 210487a18d3fSMadhavan Venkataraman callout_longterm = TICK_TO_NSEC(CALLOUT_LONGTERM_TICKS); 2105454ab202SMadhavan Venkataraman callout_max_ticks = CALLOUT_MAX_TICKS; 210651b32bddSMadhavan Venkataraman if (callout_min_reap == 0) 210751b32bddSMadhavan Venkataraman callout_min_reap = CALLOUT_MIN_REAP; 210887a18d3fSMadhavan Venkataraman 210951b32bddSMadhavan Venkataraman if (callout_tolerance <= 0) 211051b32bddSMadhavan Venkataraman callout_tolerance = CALLOUT_TOLERANCE; 211151b32bddSMadhavan Venkataraman if (callout_threads <= 0) 211251b32bddSMadhavan Venkataraman callout_threads = CALLOUT_THREADS; 2113060cedfbSMadhavan Venkataraman if (callout_chunk <= 0) 2114060cedfbSMadhavan Venkataraman callout_chunk = CALLOUT_CHUNK; 2115060cedfbSMadhavan Venkataraman else 2116060cedfbSMadhavan Venkataraman callout_chunk = P2ROUNDUP(callout_chunk, CALLOUT_CHUNK); 211787a18d3fSMadhavan Venkataraman 211887a18d3fSMadhavan Venkataraman /* 211987a18d3fSMadhavan Venkataraman * Allocate all the callout tables based on max_ncpus. We have chosen 212087a18d3fSMadhavan Venkataraman * to do boot-time allocation instead of dynamic allocation because: 212187a18d3fSMadhavan Venkataraman * 212287a18d3fSMadhavan Venkataraman * - the size of the callout tables is not too large. 212387a18d3fSMadhavan Venkataraman * - there are race conditions involved in making this dynamic. 212487a18d3fSMadhavan Venkataraman * - the hash tables that go with the callout tables consume 212587a18d3fSMadhavan Venkataraman * most of the memory and they are only allocated in 212687a18d3fSMadhavan Venkataraman * callout_cpu_online(). 212787a18d3fSMadhavan Venkataraman * 212887a18d3fSMadhavan Venkataraman * Each CPU has two tables that are consecutive in the array. The first 212987a18d3fSMadhavan Venkataraman * one is for realtime callouts and the second one is for normal ones. 213087a18d3fSMadhavan Venkataraman * 213187a18d3fSMadhavan Venkataraman * We do this alignment dance to make sure that callout table 213287a18d3fSMadhavan Venkataraman * structures will always be on a cache line boundary. 213387a18d3fSMadhavan Venkataraman */ 213487a18d3fSMadhavan Venkataraman size = sizeof (callout_table_t) * CALLOUT_NTYPES * max_ncpus; 213587a18d3fSMadhavan Venkataraman size += CALLOUT_ALIGN; 213687a18d3fSMadhavan Venkataraman buf = (uintptr_t)kmem_zalloc(size, KM_SLEEP); 213787a18d3fSMadhavan Venkataraman callout_table = (callout_table_t *)P2ROUNDUP(buf, CALLOUT_ALIGN); 213887a18d3fSMadhavan Venkataraman 213987a18d3fSMadhavan Venkataraman size = sizeof (kstat_named_t) * CALLOUT_NUM_STATS; 214087a18d3fSMadhavan Venkataraman /* 214187a18d3fSMadhavan Venkataraman * Now, initialize the tables for all the CPUs. 214287a18d3fSMadhavan Venkataraman */ 214387a18d3fSMadhavan Venkataraman for (f = 0; f < max_ncpus; f++) { 214487a18d3fSMadhavan Venkataraman for (t = 0; t < CALLOUT_NTYPES; t++) { 214587a18d3fSMadhavan Venkataraman table_id = CALLOUT_TABLE(t, f); 214687a18d3fSMadhavan Venkataraman ct = &callout_table[table_id]; 2147454ab202SMadhavan Venkataraman ct->ct_type = t; 214887a18d3fSMadhavan Venkataraman mutex_init(&ct->ct_mutex, NULL, MUTEX_DEFAULT, NULL); 214987a18d3fSMadhavan Venkataraman /* 215087a18d3fSMadhavan Venkataraman * Precompute the base IDs for long and short-term 215187a18d3fSMadhavan Venkataraman * legacy IDs. This makes ID generation during 215287a18d3fSMadhavan Venkataraman * timeout() fast. 215387a18d3fSMadhavan Venkataraman */ 215487a18d3fSMadhavan Venkataraman ct->ct_short_id = CALLOUT_SHORT_ID(table_id); 215587a18d3fSMadhavan Venkataraman ct->ct_long_id = CALLOUT_LONG_ID(table_id); 215687a18d3fSMadhavan Venkataraman /* 215787a18d3fSMadhavan Venkataraman * Precompute the base ID for generation-based IDs. 215887a18d3fSMadhavan Venkataraman * Note that when the first ID gets allocated, the 215987a18d3fSMadhavan Venkataraman * ID will wrap. This will cause the generation 216087a18d3fSMadhavan Venkataraman * number to be incremented to 1. 216187a18d3fSMadhavan Venkataraman */ 216287a18d3fSMadhavan Venkataraman ct->ct_gen_id = CALLOUT_SHORT_ID(table_id); 216387a18d3fSMadhavan Venkataraman /* 2164060cedfbSMadhavan Venkataraman * Initialize the cyclics as NONE. This will get set 216587a18d3fSMadhavan Venkataraman * during CPU online. This is so that partially 216687a18d3fSMadhavan Venkataraman * populated systems will only have the required 216787a18d3fSMadhavan Venkataraman * number of cyclics, not more. 216887a18d3fSMadhavan Venkataraman */ 216987a18d3fSMadhavan Venkataraman ct->ct_cyclic = CYCLIC_NONE; 2170060cedfbSMadhavan Venkataraman ct->ct_qcyclic = CYCLIC_NONE; 217187a18d3fSMadhavan Venkataraman ct->ct_kstat_data = kmem_zalloc(size, KM_SLEEP); 217287a18d3fSMadhavan Venkataraman } 217387a18d3fSMadhavan Venkataraman } 217487a18d3fSMadhavan Venkataraman 217587a18d3fSMadhavan Venkataraman /* 217687a18d3fSMadhavan Venkataraman * Add the callback for CPR. This is called during checkpoint 217787a18d3fSMadhavan Venkataraman * resume to suspend and resume callouts. 217887a18d3fSMadhavan Venkataraman */ 217987a18d3fSMadhavan Venkataraman (void) callb_add(callout_cpr_callb, 0, CB_CL_CPR_CALLOUT, 218087a18d3fSMadhavan Venkataraman "callout_cpr"); 218187a18d3fSMadhavan Venkataraman (void) callb_add(callout_debug_callb, 0, CB_CL_ENTER_DEBUGGER, 218287a18d3fSMadhavan Venkataraman "callout_debug"); 218387a18d3fSMadhavan Venkataraman 218487a18d3fSMadhavan Venkataraman /* 218587a18d3fSMadhavan Venkataraman * Call the per-CPU initialization function for the boot CPU. This 218687a18d3fSMadhavan Venkataraman * is done here because the function is not called automatically for 218787a18d3fSMadhavan Venkataraman * the boot CPU from the CPU online/offline hooks. Note that the 218887a18d3fSMadhavan Venkataraman * CPU lock is taken here because of convention. 218987a18d3fSMadhavan Venkataraman */ 219087a18d3fSMadhavan Venkataraman mutex_enter(&cpu_lock); 219187a18d3fSMadhavan Venkataraman callout_boot_ct = &callout_table[CALLOUT_TABLE(0, CPU->cpu_seqid)]; 219287a18d3fSMadhavan Venkataraman callout_cpu_online(CPU); 219387a18d3fSMadhavan Venkataraman mutex_exit(&cpu_lock); 2194*6e00b116SPeter Telford 2195*6e00b116SPeter Telford /* heads-up to boot-time clients that timeouts now available */ 2196*6e00b116SPeter Telford callout_init_done = 1; 21977c478bd9Sstevel@tonic-gate } 2198