17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5f635d46aSqiao * Common Development and Distribution License (the "License"). 6f635d46aSqiao * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22*454ab202SMadhavan Venkataraman * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate #include <sys/callo.h> 277c478bd9Sstevel@tonic-gate #include <sys/param.h> 287c478bd9Sstevel@tonic-gate #include <sys/types.h> 297c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 307c478bd9Sstevel@tonic-gate #include <sys/thread.h> 317c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 3287a18d3fSMadhavan Venkataraman #include <sys/kmem_impl.h> 337c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 347c478bd9Sstevel@tonic-gate #include <sys/callb.h> 357c478bd9Sstevel@tonic-gate #include <sys/debug.h> 367c478bd9Sstevel@tonic-gate #include <sys/vtrace.h> 377c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 387c478bd9Sstevel@tonic-gate #include <sys/sdt.h> 397c478bd9Sstevel@tonic-gate 407c478bd9Sstevel@tonic-gate /* 417c478bd9Sstevel@tonic-gate * Callout tables. See timeout(9F) for details. 427c478bd9Sstevel@tonic-gate */ 4387a18d3fSMadhavan Venkataraman static hrtime_t callout_debug_hrtime; /* debugger entry time */ 4487a18d3fSMadhavan Venkataraman static int callout_min_resolution; /* Minimum resolution */ 4587a18d3fSMadhavan Venkataraman static callout_table_t *callout_boot_ct; /* Boot CPU's callout tables */ 46*454ab202SMadhavan Venkataraman static clock_t callout_max_ticks; /* max interval */ 4787a18d3fSMadhavan Venkataraman static hrtime_t callout_longterm; /* longterm nanoseconds */ 4887a18d3fSMadhavan Venkataraman static ulong_t callout_counter_low; /* callout ID increment */ 4987a18d3fSMadhavan Venkataraman static ulong_t callout_table_bits; /* number of table bits in ID */ 5087a18d3fSMadhavan Venkataraman static ulong_t callout_table_mask; /* mask for the table bits */ 5187a18d3fSMadhavan Venkataraman static callout_cache_t *callout_caches; /* linked list of caches */ 5287a18d3fSMadhavan Venkataraman #pragma align 64(callout_table) 5387a18d3fSMadhavan Venkataraman static callout_table_t *callout_table; /* global callout table array */ 547c478bd9Sstevel@tonic-gate 5587a18d3fSMadhavan Venkataraman static char *callout_kstat_names[] = { 5687a18d3fSMadhavan Venkataraman "callout_timeouts", 5787a18d3fSMadhavan Venkataraman "callout_timeouts_pending", 5887a18d3fSMadhavan Venkataraman "callout_untimeouts_unexpired", 5987a18d3fSMadhavan Venkataraman "callout_untimeouts_executing", 6087a18d3fSMadhavan Venkataraman "callout_untimeouts_expired", 6187a18d3fSMadhavan Venkataraman "callout_expirations", 6287a18d3fSMadhavan Venkataraman "callout_allocations", 6387a18d3fSMadhavan Venkataraman }; 6487a18d3fSMadhavan Venkataraman 6587a18d3fSMadhavan Venkataraman #define CALLOUT_HASH_INSERT(hash, cp, cnext, cprev) \ 667c478bd9Sstevel@tonic-gate { \ 6787a18d3fSMadhavan Venkataraman callout_hash_t *hashp = &(hash); \ 6887a18d3fSMadhavan Venkataraman \ 697c478bd9Sstevel@tonic-gate cp->cprev = NULL; \ 7087a18d3fSMadhavan Venkataraman cp->cnext = hashp->ch_head; \ 7187a18d3fSMadhavan Venkataraman if (hashp->ch_head == NULL) \ 7287a18d3fSMadhavan Venkataraman hashp->ch_tail = cp; \ 737c478bd9Sstevel@tonic-gate else \ 7487a18d3fSMadhavan Venkataraman cp->cnext->cprev = cp; \ 7587a18d3fSMadhavan Venkataraman hashp->ch_head = cp; \ 767c478bd9Sstevel@tonic-gate } 777c478bd9Sstevel@tonic-gate 7887a18d3fSMadhavan Venkataraman #define CALLOUT_HASH_APPEND(hash, cp, cnext, cprev) \ 7987a18d3fSMadhavan Venkataraman { \ 8087a18d3fSMadhavan Venkataraman callout_hash_t *hashp = &(hash); \ 8187a18d3fSMadhavan Venkataraman \ 8287a18d3fSMadhavan Venkataraman cp->cnext = NULL; \ 8387a18d3fSMadhavan Venkataraman cp->cprev = hashp->ch_tail; \ 8487a18d3fSMadhavan Venkataraman if (hashp->ch_tail == NULL) \ 8587a18d3fSMadhavan Venkataraman hashp->ch_head = cp; \ 8687a18d3fSMadhavan Venkataraman else \ 8787a18d3fSMadhavan Venkataraman cp->cprev->cnext = cp; \ 8887a18d3fSMadhavan Venkataraman hashp->ch_tail = cp; \ 8987a18d3fSMadhavan Venkataraman } 9087a18d3fSMadhavan Venkataraman 9187a18d3fSMadhavan Venkataraman #define CALLOUT_HASH_DELETE(hash, cp, cnext, cprev) \ 9287a18d3fSMadhavan Venkataraman { \ 9387a18d3fSMadhavan Venkataraman callout_hash_t *hashp = &(hash); \ 9487a18d3fSMadhavan Venkataraman \ 9587a18d3fSMadhavan Venkataraman if (cp->cnext == NULL) \ 9687a18d3fSMadhavan Venkataraman hashp->ch_tail = cp->cprev; \ 9787a18d3fSMadhavan Venkataraman else \ 9887a18d3fSMadhavan Venkataraman cp->cnext->cprev = cp->cprev; \ 9987a18d3fSMadhavan Venkataraman if (cp->cprev == NULL) \ 10087a18d3fSMadhavan Venkataraman hashp->ch_head = cp->cnext; \ 10187a18d3fSMadhavan Venkataraman else \ 10287a18d3fSMadhavan Venkataraman cp->cprev->cnext = cp->cnext; \ 10387a18d3fSMadhavan Venkataraman } 10487a18d3fSMadhavan Venkataraman 10587a18d3fSMadhavan Venkataraman /* 10687a18d3fSMadhavan Venkataraman * These definitions help us queue callouts and callout lists. Here is 10787a18d3fSMadhavan Venkataraman * the queueing rationale: 10887a18d3fSMadhavan Venkataraman * 10987a18d3fSMadhavan Venkataraman * - callouts are queued in a FIFO manner in the ID hash table. 11087a18d3fSMadhavan Venkataraman * TCP timers are typically cancelled in the same order that they 11187a18d3fSMadhavan Venkataraman * were issued. The FIFO queueing shortens the search for a callout 11287a18d3fSMadhavan Venkataraman * during untimeout(). 11387a18d3fSMadhavan Venkataraman * 11487a18d3fSMadhavan Venkataraman * - callouts are queued in a FIFO manner in their callout lists. 11587a18d3fSMadhavan Venkataraman * This ensures that the callouts are executed in the same order that 11687a18d3fSMadhavan Venkataraman * they were queued. This is fair. Plus, it helps to make each 11787a18d3fSMadhavan Venkataraman * callout expiration timely. It also favors cancellations. 11887a18d3fSMadhavan Venkataraman * 11987a18d3fSMadhavan Venkataraman * - callout lists are queued in a LIFO manner in the callout list hash 12087a18d3fSMadhavan Venkataraman * table. This ensures that long term timers stay at the rear of the 12187a18d3fSMadhavan Venkataraman * hash lists. 12287a18d3fSMadhavan Venkataraman * 12387a18d3fSMadhavan Venkataraman * - callout lists are queued in a FIFO manner in the expired callouts 12487a18d3fSMadhavan Venkataraman * list. This ensures that callout lists are executed in the order 12587a18d3fSMadhavan Venkataraman * of expiration. 12687a18d3fSMadhavan Venkataraman */ 12787a18d3fSMadhavan Venkataraman #define CALLOUT_APPEND(ct, cp) \ 12887a18d3fSMadhavan Venkataraman CALLOUT_HASH_APPEND(ct->ct_idhash[CALLOUT_IDHASH(cp->c_xid)], \ 12987a18d3fSMadhavan Venkataraman cp, c_idnext, c_idprev); \ 13087a18d3fSMadhavan Venkataraman CALLOUT_HASH_APPEND(cp->c_list->cl_callouts, cp, c_clnext, c_clprev) 13187a18d3fSMadhavan Venkataraman 13287a18d3fSMadhavan Venkataraman #define CALLOUT_DELETE(ct, cp) \ 13387a18d3fSMadhavan Venkataraman CALLOUT_HASH_DELETE(ct->ct_idhash[CALLOUT_IDHASH(cp->c_xid)], \ 13487a18d3fSMadhavan Venkataraman cp, c_idnext, c_idprev); \ 13587a18d3fSMadhavan Venkataraman CALLOUT_HASH_DELETE(cp->c_list->cl_callouts, cp, c_clnext, c_clprev) 13687a18d3fSMadhavan Venkataraman 13787a18d3fSMadhavan Venkataraman #define CALLOUT_LIST_INSERT(hash, cl) \ 13887a18d3fSMadhavan Venkataraman CALLOUT_HASH_INSERT(hash, cl, cl_next, cl_prev) 13987a18d3fSMadhavan Venkataraman 14087a18d3fSMadhavan Venkataraman #define CALLOUT_LIST_APPEND(hash, cl) \ 14187a18d3fSMadhavan Venkataraman CALLOUT_HASH_APPEND(hash, cl, cl_next, cl_prev) 14287a18d3fSMadhavan Venkataraman 14387a18d3fSMadhavan Venkataraman #define CALLOUT_LIST_DELETE(hash, cl) \ 14487a18d3fSMadhavan Venkataraman CALLOUT_HASH_DELETE(hash, cl, cl_next, cl_prev) 1457c478bd9Sstevel@tonic-gate 1467c478bd9Sstevel@tonic-gate /* 1477c478bd9Sstevel@tonic-gate * Allocate a callout structure. We try quite hard because we 1487c478bd9Sstevel@tonic-gate * can't sleep, and if we can't do the allocation, we're toast. 14987a18d3fSMadhavan Venkataraman * Failing all, we try a KM_PANIC allocation. Note that we never 15087a18d3fSMadhavan Venkataraman * deallocate a callout. See untimeout() for the reasoning. 1517c478bd9Sstevel@tonic-gate */ 1527c478bd9Sstevel@tonic-gate static callout_t * 1537c478bd9Sstevel@tonic-gate callout_alloc(callout_table_t *ct) 1547c478bd9Sstevel@tonic-gate { 15587a18d3fSMadhavan Venkataraman size_t size; 15687a18d3fSMadhavan Venkataraman callout_t *cp; 1577c478bd9Sstevel@tonic-gate 15887a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 15987a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 16087a18d3fSMadhavan Venkataraman 16187a18d3fSMadhavan Venkataraman cp = kmem_cache_alloc(ct->ct_cache, KM_NOSLEEP); 16287a18d3fSMadhavan Venkataraman if (cp == NULL) { 16387a18d3fSMadhavan Venkataraman size = sizeof (callout_t); 16487a18d3fSMadhavan Venkataraman cp = kmem_alloc_tryhard(size, &size, KM_NOSLEEP | KM_PANIC); 16587a18d3fSMadhavan Venkataraman } 16687a18d3fSMadhavan Venkataraman cp->c_xid = 0; 16787a18d3fSMadhavan Venkataraman 16887a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 16987a18d3fSMadhavan Venkataraman ct->ct_allocations++; 1707c478bd9Sstevel@tonic-gate return (cp); 1717c478bd9Sstevel@tonic-gate } 1727c478bd9Sstevel@tonic-gate 1737c478bd9Sstevel@tonic-gate /* 17487a18d3fSMadhavan Venkataraman * Allocate a callout list structure. We try quite hard because we 17587a18d3fSMadhavan Venkataraman * can't sleep, and if we can't do the allocation, we're toast. 17687a18d3fSMadhavan Venkataraman * Failing all, we try a KM_PANIC allocation. Note that we never 17787a18d3fSMadhavan Venkataraman * deallocate a callout list. 1787c478bd9Sstevel@tonic-gate */ 17987a18d3fSMadhavan Venkataraman static void 18087a18d3fSMadhavan Venkataraman callout_list_alloc(callout_table_t *ct) 1817c478bd9Sstevel@tonic-gate { 18287a18d3fSMadhavan Venkataraman size_t size; 18387a18d3fSMadhavan Venkataraman callout_list_t *cl; 18487a18d3fSMadhavan Venkataraman 18587a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 18687a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 18787a18d3fSMadhavan Venkataraman 18887a18d3fSMadhavan Venkataraman cl = kmem_cache_alloc(ct->ct_lcache, KM_NOSLEEP); 18987a18d3fSMadhavan Venkataraman if (cl == NULL) { 19087a18d3fSMadhavan Venkataraman size = sizeof (callout_list_t); 19187a18d3fSMadhavan Venkataraman cl = kmem_alloc_tryhard(size, &size, KM_NOSLEEP | KM_PANIC); 19287a18d3fSMadhavan Venkataraman } 19387a18d3fSMadhavan Venkataraman bzero(cl, sizeof (callout_list_t)); 19487a18d3fSMadhavan Venkataraman 19587a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 19687a18d3fSMadhavan Venkataraman cl->cl_next = ct->ct_lfree; 19787a18d3fSMadhavan Venkataraman ct->ct_lfree = cl; 19887a18d3fSMadhavan Venkataraman } 19987a18d3fSMadhavan Venkataraman 20087a18d3fSMadhavan Venkataraman /* 20187a18d3fSMadhavan Venkataraman * Find the callout list that corresponds to an expiration. There can 20287a18d3fSMadhavan Venkataraman * be only one. 20387a18d3fSMadhavan Venkataraman */ 20487a18d3fSMadhavan Venkataraman static callout_list_t * 20587a18d3fSMadhavan Venkataraman callout_list_get(callout_table_t *ct, hrtime_t expiration, int hash) 20687a18d3fSMadhavan Venkataraman { 20787a18d3fSMadhavan Venkataraman callout_list_t *cl; 20887a18d3fSMadhavan Venkataraman 20987a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 21087a18d3fSMadhavan Venkataraman 21187a18d3fSMadhavan Venkataraman for (cl = ct->ct_clhash[hash].ch_head; (cl != NULL); cl = cl->cl_next) { 21287a18d3fSMadhavan Venkataraman if (cl->cl_expiration == expiration) 21387a18d3fSMadhavan Venkataraman return (cl); 21487a18d3fSMadhavan Venkataraman } 21587a18d3fSMadhavan Venkataraman 21687a18d3fSMadhavan Venkataraman return (NULL); 21787a18d3fSMadhavan Venkataraman } 21887a18d3fSMadhavan Venkataraman 21987a18d3fSMadhavan Venkataraman /* 22087a18d3fSMadhavan Venkataraman * Find the callout list that corresponds to an expiration. There can 22187a18d3fSMadhavan Venkataraman * be only one. If the callout list is null, free it. Else, return it. 22287a18d3fSMadhavan Venkataraman */ 22387a18d3fSMadhavan Venkataraman static callout_list_t * 22487a18d3fSMadhavan Venkataraman callout_list_check(callout_table_t *ct, hrtime_t expiration, int hash) 22587a18d3fSMadhavan Venkataraman { 22687a18d3fSMadhavan Venkataraman callout_list_t *cl; 22787a18d3fSMadhavan Venkataraman 22887a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 22987a18d3fSMadhavan Venkataraman 23087a18d3fSMadhavan Venkataraman cl = callout_list_get(ct, expiration, hash); 23187a18d3fSMadhavan Venkataraman if (cl != NULL) { 23287a18d3fSMadhavan Venkataraman if (cl->cl_callouts.ch_head != NULL) { 23387a18d3fSMadhavan Venkataraman /* 23487a18d3fSMadhavan Venkataraman * There is exactly one callout list for every 23587a18d3fSMadhavan Venkataraman * unique expiration. So, we are done. 23687a18d3fSMadhavan Venkataraman */ 23787a18d3fSMadhavan Venkataraman return (cl); 23887a18d3fSMadhavan Venkataraman } 23987a18d3fSMadhavan Venkataraman 24087a18d3fSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_clhash[hash], cl); 24187a18d3fSMadhavan Venkataraman cl->cl_next = ct->ct_lfree; 24287a18d3fSMadhavan Venkataraman ct->ct_lfree = cl; 24387a18d3fSMadhavan Venkataraman } 24487a18d3fSMadhavan Venkataraman 24587a18d3fSMadhavan Venkataraman return (NULL); 24687a18d3fSMadhavan Venkataraman } 24787a18d3fSMadhavan Venkataraman 24887a18d3fSMadhavan Venkataraman /* 24987a18d3fSMadhavan Venkataraman * Initialize a callout table's heap, if necessary. Preallocate some free 25087a18d3fSMadhavan Venkataraman * entries so we don't have to check for NULL elsewhere. 25187a18d3fSMadhavan Venkataraman */ 25287a18d3fSMadhavan Venkataraman static void 25387a18d3fSMadhavan Venkataraman callout_heap_init(callout_table_t *ct) 25487a18d3fSMadhavan Venkataraman { 25587a18d3fSMadhavan Venkataraman size_t size; 25687a18d3fSMadhavan Venkataraman 25787a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 25887a18d3fSMadhavan Venkataraman ASSERT(ct->ct_heap == NULL); 25987a18d3fSMadhavan Venkataraman 26087a18d3fSMadhavan Venkataraman ct->ct_heap_num = 0; 26187a18d3fSMadhavan Venkataraman ct->ct_heap_max = CALLOUT_CHUNK; 26287a18d3fSMadhavan Venkataraman size = sizeof (hrtime_t) * CALLOUT_CHUNK; 26387a18d3fSMadhavan Venkataraman ct->ct_heap = kmem_alloc(size, KM_SLEEP); 26487a18d3fSMadhavan Venkataraman } 26587a18d3fSMadhavan Venkataraman 26687a18d3fSMadhavan Venkataraman /* 26787a18d3fSMadhavan Venkataraman * Reallocate the heap. We try quite hard because we can't sleep, and if 26887a18d3fSMadhavan Venkataraman * we can't do the allocation, we're toast. Failing all, we try a KM_PANIC 26987a18d3fSMadhavan Venkataraman * allocation. Note that the heap only expands, it never contracts. 27087a18d3fSMadhavan Venkataraman */ 27187a18d3fSMadhavan Venkataraman static void 27287a18d3fSMadhavan Venkataraman callout_heap_expand(callout_table_t *ct) 27387a18d3fSMadhavan Venkataraman { 27487a18d3fSMadhavan Venkataraman size_t max, size, osize; 27587a18d3fSMadhavan Venkataraman hrtime_t *heap; 27687a18d3fSMadhavan Venkataraman 27787a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 27887a18d3fSMadhavan Venkataraman ASSERT(ct->ct_heap_num <= ct->ct_heap_max); 27987a18d3fSMadhavan Venkataraman 28087a18d3fSMadhavan Venkataraman while (ct->ct_heap_num == ct->ct_heap_max) { 28187a18d3fSMadhavan Venkataraman max = ct->ct_heap_max; 28287a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 28387a18d3fSMadhavan Venkataraman 28487a18d3fSMadhavan Venkataraman osize = sizeof (hrtime_t) * max; 28587a18d3fSMadhavan Venkataraman size = sizeof (hrtime_t) * (max + CALLOUT_CHUNK); 28687a18d3fSMadhavan Venkataraman heap = kmem_alloc_tryhard(size, &size, KM_NOSLEEP | KM_PANIC); 28787a18d3fSMadhavan Venkataraman 28887a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 28987a18d3fSMadhavan Venkataraman if (max < ct->ct_heap_max) { 29087a18d3fSMadhavan Venkataraman /* 29187a18d3fSMadhavan Venkataraman * Someone beat us to the allocation. Free what we 29287a18d3fSMadhavan Venkataraman * just allocated and proceed. 29387a18d3fSMadhavan Venkataraman */ 29487a18d3fSMadhavan Venkataraman kmem_free(heap, size); 29587a18d3fSMadhavan Venkataraman continue; 29687a18d3fSMadhavan Venkataraman } 29787a18d3fSMadhavan Venkataraman 29887a18d3fSMadhavan Venkataraman bcopy(ct->ct_heap, heap, osize); 29987a18d3fSMadhavan Venkataraman kmem_free(ct->ct_heap, osize); 30087a18d3fSMadhavan Venkataraman ct->ct_heap = heap; 30187a18d3fSMadhavan Venkataraman ct->ct_heap_max = size / sizeof (hrtime_t); 30287a18d3fSMadhavan Venkataraman } 30387a18d3fSMadhavan Venkataraman } 30487a18d3fSMadhavan Venkataraman 30587a18d3fSMadhavan Venkataraman /* 30687a18d3fSMadhavan Venkataraman * Move an expiration from the bottom of the heap to its correct place 30787a18d3fSMadhavan Venkataraman * in the heap. If we reached the root doing this, return 1. Else, 30887a18d3fSMadhavan Venkataraman * return 0. 30987a18d3fSMadhavan Venkataraman */ 31087a18d3fSMadhavan Venkataraman static int 31187a18d3fSMadhavan Venkataraman callout_upheap(callout_table_t *ct) 31287a18d3fSMadhavan Venkataraman { 31387a18d3fSMadhavan Venkataraman int current, parent; 31487a18d3fSMadhavan Venkataraman hrtime_t *heap, current_expiration, parent_expiration; 31587a18d3fSMadhavan Venkataraman 31687a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 31787a18d3fSMadhavan Venkataraman ASSERT(ct->ct_heap_num >= 1); 31887a18d3fSMadhavan Venkataraman 31987a18d3fSMadhavan Venkataraman if (ct->ct_heap_num == 1) { 32087a18d3fSMadhavan Venkataraman return (1); 32187a18d3fSMadhavan Venkataraman } 32287a18d3fSMadhavan Venkataraman 32387a18d3fSMadhavan Venkataraman heap = ct->ct_heap; 32487a18d3fSMadhavan Venkataraman current = ct->ct_heap_num - 1; 32587a18d3fSMadhavan Venkataraman 32687a18d3fSMadhavan Venkataraman for (;;) { 32787a18d3fSMadhavan Venkataraman parent = CALLOUT_HEAP_PARENT(current); 32887a18d3fSMadhavan Venkataraman current_expiration = heap[current]; 32987a18d3fSMadhavan Venkataraman parent_expiration = heap[parent]; 33087a18d3fSMadhavan Venkataraman 33187a18d3fSMadhavan Venkataraman /* 33287a18d3fSMadhavan Venkataraman * We have an expiration later than our parent; we're done. 33387a18d3fSMadhavan Venkataraman */ 33487a18d3fSMadhavan Venkataraman if (current_expiration >= parent_expiration) { 33587a18d3fSMadhavan Venkataraman return (0); 33687a18d3fSMadhavan Venkataraman } 33787a18d3fSMadhavan Venkataraman 33887a18d3fSMadhavan Venkataraman /* 33987a18d3fSMadhavan Venkataraman * We need to swap with our parent, and continue up the heap. 34087a18d3fSMadhavan Venkataraman */ 34187a18d3fSMadhavan Venkataraman heap[parent] = current_expiration; 34287a18d3fSMadhavan Venkataraman heap[current] = parent_expiration; 34387a18d3fSMadhavan Venkataraman 34487a18d3fSMadhavan Venkataraman /* 34587a18d3fSMadhavan Venkataraman * If we just reached the root, we're done. 34687a18d3fSMadhavan Venkataraman */ 34787a18d3fSMadhavan Venkataraman if (parent == 0) { 34887a18d3fSMadhavan Venkataraman return (1); 34987a18d3fSMadhavan Venkataraman } 35087a18d3fSMadhavan Venkataraman 35187a18d3fSMadhavan Venkataraman current = parent; 35287a18d3fSMadhavan Venkataraman } 35387a18d3fSMadhavan Venkataraman /*NOTREACHED*/ 35487a18d3fSMadhavan Venkataraman } 35587a18d3fSMadhavan Venkataraman 35687a18d3fSMadhavan Venkataraman /* 35787a18d3fSMadhavan Venkataraman * Insert a new, unique expiration into a callout table's heap. 35887a18d3fSMadhavan Venkataraman */ 35987a18d3fSMadhavan Venkataraman static void 36087a18d3fSMadhavan Venkataraman callout_heap_insert(callout_table_t *ct, hrtime_t expiration) 36187a18d3fSMadhavan Venkataraman { 36287a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 36387a18d3fSMadhavan Venkataraman ASSERT(ct->ct_heap_num < ct->ct_heap_max); 36487a18d3fSMadhavan Venkataraman 36587a18d3fSMadhavan Venkataraman /* 36687a18d3fSMadhavan Venkataraman * First, copy the expiration to the bottom of the heap. 36787a18d3fSMadhavan Venkataraman */ 36887a18d3fSMadhavan Venkataraman ct->ct_heap[ct->ct_heap_num] = expiration; 36987a18d3fSMadhavan Venkataraman ct->ct_heap_num++; 37087a18d3fSMadhavan Venkataraman 37187a18d3fSMadhavan Venkataraman /* 37287a18d3fSMadhavan Venkataraman * Now, perform an upheap operation. If we reached the root, then 37387a18d3fSMadhavan Venkataraman * the cyclic needs to be reprogrammed as we have an earlier 37487a18d3fSMadhavan Venkataraman * expiration. 37587a18d3fSMadhavan Venkataraman * 37687a18d3fSMadhavan Venkataraman * Also, during the CPR suspend phase, do not reprogram the cyclic. 37787a18d3fSMadhavan Venkataraman * We don't want any callout activity. When the CPR resume phase is 37887a18d3fSMadhavan Venkataraman * entered, the cyclic will be programmed for the earliest expiration 37987a18d3fSMadhavan Venkataraman * in the heap. 38087a18d3fSMadhavan Venkataraman */ 381*454ab202SMadhavan Venkataraman if (callout_upheap(ct) && (ct->ct_suspend == 0)) 38287a18d3fSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_cyclic, expiration); 38387a18d3fSMadhavan Venkataraman } 38487a18d3fSMadhavan Venkataraman 38587a18d3fSMadhavan Venkataraman /* 38687a18d3fSMadhavan Venkataraman * Move an expiration from the top of the heap to its correct place 38787a18d3fSMadhavan Venkataraman * in the heap. 38887a18d3fSMadhavan Venkataraman */ 38987a18d3fSMadhavan Venkataraman static void 39087a18d3fSMadhavan Venkataraman callout_downheap(callout_table_t *ct) 39187a18d3fSMadhavan Venkataraman { 39287a18d3fSMadhavan Venkataraman int left, right, current, nelems; 39387a18d3fSMadhavan Venkataraman hrtime_t *heap, left_expiration, right_expiration, current_expiration; 39487a18d3fSMadhavan Venkataraman 39587a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 39687a18d3fSMadhavan Venkataraman ASSERT(ct->ct_heap_num >= 1); 39787a18d3fSMadhavan Venkataraman 39887a18d3fSMadhavan Venkataraman heap = ct->ct_heap; 39987a18d3fSMadhavan Venkataraman current = 0; 40087a18d3fSMadhavan Venkataraman nelems = ct->ct_heap_num; 40187a18d3fSMadhavan Venkataraman 40287a18d3fSMadhavan Venkataraman for (;;) { 40387a18d3fSMadhavan Venkataraman /* 40487a18d3fSMadhavan Venkataraman * If we don't have a left child (i.e., we're a leaf), we're 40587a18d3fSMadhavan Venkataraman * done. 40687a18d3fSMadhavan Venkataraman */ 40787a18d3fSMadhavan Venkataraman if ((left = CALLOUT_HEAP_LEFT(current)) >= nelems) 40887a18d3fSMadhavan Venkataraman return; 40987a18d3fSMadhavan Venkataraman 41087a18d3fSMadhavan Venkataraman left_expiration = heap[left]; 41187a18d3fSMadhavan Venkataraman current_expiration = heap[current]; 41287a18d3fSMadhavan Venkataraman 41387a18d3fSMadhavan Venkataraman right = CALLOUT_HEAP_RIGHT(current); 41487a18d3fSMadhavan Venkataraman 41587a18d3fSMadhavan Venkataraman /* 41687a18d3fSMadhavan Venkataraman * Even if we don't have a right child, we still need to compare 41787a18d3fSMadhavan Venkataraman * our expiration against that of our left child. 41887a18d3fSMadhavan Venkataraman */ 41987a18d3fSMadhavan Venkataraman if (right >= nelems) 42087a18d3fSMadhavan Venkataraman goto comp_left; 42187a18d3fSMadhavan Venkataraman 42287a18d3fSMadhavan Venkataraman right_expiration = heap[right]; 42387a18d3fSMadhavan Venkataraman 42487a18d3fSMadhavan Venkataraman /* 42587a18d3fSMadhavan Venkataraman * We have both a left and a right child. We need to compare 42687a18d3fSMadhavan Venkataraman * the expiration of the children to determine which 42787a18d3fSMadhavan Venkataraman * expires earlier. 42887a18d3fSMadhavan Venkataraman */ 42987a18d3fSMadhavan Venkataraman if (right_expiration < left_expiration) { 43087a18d3fSMadhavan Venkataraman /* 43187a18d3fSMadhavan Venkataraman * Our right child is the earlier of our children. 43287a18d3fSMadhavan Venkataraman * We'll now compare our expiration to its expiration. 43387a18d3fSMadhavan Venkataraman * If ours is the earlier one, we're done. 43487a18d3fSMadhavan Venkataraman */ 43587a18d3fSMadhavan Venkataraman if (current_expiration <= right_expiration) 43687a18d3fSMadhavan Venkataraman return; 43787a18d3fSMadhavan Venkataraman 43887a18d3fSMadhavan Venkataraman /* 43987a18d3fSMadhavan Venkataraman * Our right child expires earlier than we do; swap 44087a18d3fSMadhavan Venkataraman * with our right child, and descend right. 44187a18d3fSMadhavan Venkataraman */ 44287a18d3fSMadhavan Venkataraman heap[right] = current_expiration; 44387a18d3fSMadhavan Venkataraman heap[current] = right_expiration; 44487a18d3fSMadhavan Venkataraman current = right; 44587a18d3fSMadhavan Venkataraman continue; 44687a18d3fSMadhavan Venkataraman } 44787a18d3fSMadhavan Venkataraman 44887a18d3fSMadhavan Venkataraman comp_left: 44987a18d3fSMadhavan Venkataraman /* 45087a18d3fSMadhavan Venkataraman * Our left child is the earlier of our children (or we have 45187a18d3fSMadhavan Venkataraman * no right child). We'll now compare our expiration 45287a18d3fSMadhavan Venkataraman * to its expiration. If ours is the earlier one, we're done. 45387a18d3fSMadhavan Venkataraman */ 45487a18d3fSMadhavan Venkataraman if (current_expiration <= left_expiration) 45587a18d3fSMadhavan Venkataraman return; 45687a18d3fSMadhavan Venkataraman 45787a18d3fSMadhavan Venkataraman /* 45887a18d3fSMadhavan Venkataraman * Our left child expires earlier than we do; swap with our 45987a18d3fSMadhavan Venkataraman * left child, and descend left. 46087a18d3fSMadhavan Venkataraman */ 46187a18d3fSMadhavan Venkataraman heap[left] = current_expiration; 46287a18d3fSMadhavan Venkataraman heap[current] = left_expiration; 46387a18d3fSMadhavan Venkataraman current = left; 46487a18d3fSMadhavan Venkataraman } 46587a18d3fSMadhavan Venkataraman } 46687a18d3fSMadhavan Venkataraman 46787a18d3fSMadhavan Venkataraman /* 46887a18d3fSMadhavan Venkataraman * Delete and handle all past expirations in a callout table's heap. 46987a18d3fSMadhavan Venkataraman */ 47087a18d3fSMadhavan Venkataraman static void 47187a18d3fSMadhavan Venkataraman callout_heap_delete(callout_table_t *ct) 47287a18d3fSMadhavan Venkataraman { 47387a18d3fSMadhavan Venkataraman hrtime_t now, expiration; 47487a18d3fSMadhavan Venkataraman callout_list_t *cl; 47587a18d3fSMadhavan Venkataraman int hash; 47687a18d3fSMadhavan Venkataraman 47787a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 47887a18d3fSMadhavan Venkataraman 47987a18d3fSMadhavan Venkataraman now = gethrtime(); 48087a18d3fSMadhavan Venkataraman 48187a18d3fSMadhavan Venkataraman while (ct->ct_heap_num > 0) { 48287a18d3fSMadhavan Venkataraman expiration = ct->ct_heap[0]; 48387a18d3fSMadhavan Venkataraman /* 48487a18d3fSMadhavan Venkataraman * Find the callout list that corresponds to the expiration. 48587a18d3fSMadhavan Venkataraman * If the callout list is empty, callout_list_check() 48687a18d3fSMadhavan Venkataraman * will free the callout list and return NULL. 48787a18d3fSMadhavan Venkataraman */ 48887a18d3fSMadhavan Venkataraman hash = CALLOUT_CLHASH(expiration); 48987a18d3fSMadhavan Venkataraman cl = callout_list_check(ct, expiration, hash); 49087a18d3fSMadhavan Venkataraman if (cl != NULL) { 49187a18d3fSMadhavan Venkataraman /* 49287a18d3fSMadhavan Venkataraman * If the root of the heap expires in the future, we are 49387a18d3fSMadhavan Venkataraman * done. We are doing this check here instead of at the 49487a18d3fSMadhavan Venkataraman * beginning because we want to first free all the 49587a18d3fSMadhavan Venkataraman * empty callout lists at the top of the heap. 49687a18d3fSMadhavan Venkataraman */ 49787a18d3fSMadhavan Venkataraman if (expiration > now) 49887a18d3fSMadhavan Venkataraman break; 49987a18d3fSMadhavan Venkataraman 50087a18d3fSMadhavan Venkataraman /* 50187a18d3fSMadhavan Venkataraman * Move the callout list for this expiration to the 50287a18d3fSMadhavan Venkataraman * list of expired callout lists. It will be processed 50387a18d3fSMadhavan Venkataraman * by the callout executor. 50487a18d3fSMadhavan Venkataraman */ 50587a18d3fSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_clhash[hash], cl); 50687a18d3fSMadhavan Venkataraman CALLOUT_LIST_APPEND(ct->ct_expired, cl); 50787a18d3fSMadhavan Venkataraman } 50887a18d3fSMadhavan Venkataraman 50987a18d3fSMadhavan Venkataraman /* 51087a18d3fSMadhavan Venkataraman * Now delete the root. This is done by swapping the root with 51187a18d3fSMadhavan Venkataraman * the last item in the heap and downheaping the item. 51287a18d3fSMadhavan Venkataraman */ 51387a18d3fSMadhavan Venkataraman ct->ct_heap_num--; 51487a18d3fSMadhavan Venkataraman if (ct->ct_heap_num > 0) { 51587a18d3fSMadhavan Venkataraman ct->ct_heap[0] = ct->ct_heap[ct->ct_heap_num]; 51687a18d3fSMadhavan Venkataraman callout_downheap(ct); 51787a18d3fSMadhavan Venkataraman } 51887a18d3fSMadhavan Venkataraman } 51987a18d3fSMadhavan Venkataraman 52087a18d3fSMadhavan Venkataraman /* 52187a18d3fSMadhavan Venkataraman * If this callout table is empty or callouts have been suspended 52287a18d3fSMadhavan Venkataraman * by CPR, just return. The cyclic has already been programmed to 52387a18d3fSMadhavan Venkataraman * infinity by the cyclic subsystem. 52487a18d3fSMadhavan Venkataraman */ 525*454ab202SMadhavan Venkataraman if ((ct->ct_heap_num == 0) || (ct->ct_suspend > 0)) 52687a18d3fSMadhavan Venkataraman return; 52787a18d3fSMadhavan Venkataraman 52887a18d3fSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_cyclic, expiration); 52987a18d3fSMadhavan Venkataraman } 53087a18d3fSMadhavan Venkataraman 531*454ab202SMadhavan Venkataraman /* 532*454ab202SMadhavan Venkataraman * Common function used to create normal and realtime callouts. 533*454ab202SMadhavan Venkataraman * 534*454ab202SMadhavan Venkataraman * Realtime callouts are handled at CY_LOW_PIL by a cyclic handler. So, 535*454ab202SMadhavan Venkataraman * there is one restriction on a realtime callout handler - it should not 536*454ab202SMadhavan Venkataraman * directly or indirectly acquire cpu_lock. CPU offline waits for pending 537*454ab202SMadhavan Venkataraman * cyclic handlers to complete while holding cpu_lock. So, if a realtime 538*454ab202SMadhavan Venkataraman * callout handler were to try to get cpu_lock, there would be a deadlock 539*454ab202SMadhavan Venkataraman * during CPU offline. 540*454ab202SMadhavan Venkataraman */ 54187a18d3fSMadhavan Venkataraman callout_id_t 54287a18d3fSMadhavan Venkataraman timeout_generic(int type, void (*func)(void *), void *arg, 54387a18d3fSMadhavan Venkataraman hrtime_t expiration, hrtime_t resolution, int flags) 54487a18d3fSMadhavan Venkataraman { 54587a18d3fSMadhavan Venkataraman callout_table_t *ct; 5467c478bd9Sstevel@tonic-gate callout_t *cp; 5477c478bd9Sstevel@tonic-gate callout_id_t id; 54887a18d3fSMadhavan Venkataraman callout_list_t *cl; 54987a18d3fSMadhavan Venkataraman hrtime_t now, interval; 55087a18d3fSMadhavan Venkataraman int hash; 551f635d46aSqiao 55287a18d3fSMadhavan Venkataraman ASSERT(resolution > 0); 55387a18d3fSMadhavan Venkataraman ASSERT(func != NULL); 5547c478bd9Sstevel@tonic-gate 55587a18d3fSMadhavan Venkataraman /* 55687a18d3fSMadhavan Venkataraman * Please see comment about minimum resolution in callout_init(). 55787a18d3fSMadhavan Venkataraman */ 55887a18d3fSMadhavan Venkataraman if (resolution < callout_min_resolution) 55987a18d3fSMadhavan Venkataraman resolution = callout_min_resolution; 5607c478bd9Sstevel@tonic-gate 56187a18d3fSMadhavan Venkataraman /* 56287a18d3fSMadhavan Venkataraman * We disable kernel preemption so that we remain on the same CPU 56387a18d3fSMadhavan Venkataraman * throughout. If we needed to reprogram the callout table's cyclic, 56487a18d3fSMadhavan Venkataraman * we can avoid X-calls if we are on the same CPU. 56587a18d3fSMadhavan Venkataraman * 56687a18d3fSMadhavan Venkataraman * Note that callout_alloc() releases and reacquires the callout 56787a18d3fSMadhavan Venkataraman * table mutex. While reacquiring the mutex, it is possible for us 56887a18d3fSMadhavan Venkataraman * to go to sleep and later migrate to another CPU. This should be 56987a18d3fSMadhavan Venkataraman * pretty rare, though. 57087a18d3fSMadhavan Venkataraman */ 57187a18d3fSMadhavan Venkataraman kpreempt_disable(); 57287a18d3fSMadhavan Venkataraman 57387a18d3fSMadhavan Venkataraman ct = &callout_table[CALLOUT_TABLE(type, CPU->cpu_seqid)]; 57487a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 57587a18d3fSMadhavan Venkataraman 57687a18d3fSMadhavan Venkataraman if (ct->ct_cyclic == CYCLIC_NONE) { 57787a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 57887a18d3fSMadhavan Venkataraman /* 57987a18d3fSMadhavan Venkataraman * The callout table has not yet been initialized fully. 58087a18d3fSMadhavan Venkataraman * So, put this one on the boot callout table which is 58187a18d3fSMadhavan Venkataraman * always initialized. 58287a18d3fSMadhavan Venkataraman */ 58387a18d3fSMadhavan Venkataraman ct = &callout_boot_ct[type]; 58487a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 58587a18d3fSMadhavan Venkataraman } 58687a18d3fSMadhavan Venkataraman 58787a18d3fSMadhavan Venkataraman if ((cp = ct->ct_free) == NULL) 5887c478bd9Sstevel@tonic-gate cp = callout_alloc(ct); 5897c478bd9Sstevel@tonic-gate else 59087a18d3fSMadhavan Venkataraman ct->ct_free = cp->c_idnext; 5917c478bd9Sstevel@tonic-gate 5927c478bd9Sstevel@tonic-gate cp->c_func = func; 5937c478bd9Sstevel@tonic-gate cp->c_arg = arg; 5947c478bd9Sstevel@tonic-gate 5957c478bd9Sstevel@tonic-gate /* 59687a18d3fSMadhavan Venkataraman * Compute the expiration hrtime. 59787a18d3fSMadhavan Venkataraman */ 59887a18d3fSMadhavan Venkataraman now = gethrtime(); 59987a18d3fSMadhavan Venkataraman if (flags & CALLOUT_FLAG_ABSOLUTE) { 60087a18d3fSMadhavan Venkataraman ASSERT(expiration > 0); 60187a18d3fSMadhavan Venkataraman interval = expiration - now; 60287a18d3fSMadhavan Venkataraman } else { 60387a18d3fSMadhavan Venkataraman interval = expiration; 60487a18d3fSMadhavan Venkataraman expiration += now; 60587a18d3fSMadhavan Venkataraman ASSERT(expiration > 0); 60687a18d3fSMadhavan Venkataraman } 60787a18d3fSMadhavan Venkataraman if (flags & CALLOUT_FLAG_ROUNDUP) 60887a18d3fSMadhavan Venkataraman expiration += resolution - 1; 60987a18d3fSMadhavan Venkataraman expiration = (expiration / resolution) * resolution; 610*454ab202SMadhavan Venkataraman if (expiration <= 0) { 611*454ab202SMadhavan Venkataraman /* 612*454ab202SMadhavan Venkataraman * expiration hrtime overflow has occurred. Just set the 613*454ab202SMadhavan Venkataraman * expiration to infinity. 614*454ab202SMadhavan Venkataraman */ 615*454ab202SMadhavan Venkataraman expiration = CY_INFINITY; 616*454ab202SMadhavan Venkataraman } 61787a18d3fSMadhavan Venkataraman 61887a18d3fSMadhavan Venkataraman /* 61987a18d3fSMadhavan Venkataraman * Assign an ID to this callout 62087a18d3fSMadhavan Venkataraman */ 62187a18d3fSMadhavan Venkataraman if (flags & CALLOUT_FLAG_32BIT) { 62287a18d3fSMadhavan Venkataraman if (interval > callout_longterm) { 62387a18d3fSMadhavan Venkataraman id = (ct->ct_long_id - callout_counter_low); 62487a18d3fSMadhavan Venkataraman id |= CALLOUT_COUNTER_HIGH; 62587a18d3fSMadhavan Venkataraman ct->ct_long_id = id; 62687a18d3fSMadhavan Venkataraman } else { 62787a18d3fSMadhavan Venkataraman id = (ct->ct_short_id - callout_counter_low); 62887a18d3fSMadhavan Venkataraman id |= CALLOUT_COUNTER_HIGH; 62987a18d3fSMadhavan Venkataraman ct->ct_short_id = id; 63087a18d3fSMadhavan Venkataraman } 63187a18d3fSMadhavan Venkataraman } else { 63287a18d3fSMadhavan Venkataraman id = (ct->ct_gen_id - callout_counter_low); 63387a18d3fSMadhavan Venkataraman if ((id & CALLOUT_COUNTER_HIGH) == 0) { 63487a18d3fSMadhavan Venkataraman id |= CALLOUT_COUNTER_HIGH; 63587a18d3fSMadhavan Venkataraman id += CALLOUT_GENERATION_LOW; 63687a18d3fSMadhavan Venkataraman } 63787a18d3fSMadhavan Venkataraman ct->ct_gen_id = id; 63887a18d3fSMadhavan Venkataraman } 63987a18d3fSMadhavan Venkataraman 64087a18d3fSMadhavan Venkataraman cp->c_xid = id; 64187a18d3fSMadhavan Venkataraman if (flags & CALLOUT_FLAG_HRESTIME) 64287a18d3fSMadhavan Venkataraman cp->c_xid |= CALLOUT_HRESTIME; 64387a18d3fSMadhavan Venkataraman 64487a18d3fSMadhavan Venkataraman hash = CALLOUT_CLHASH(expiration); 64587a18d3fSMadhavan Venkataraman 64687a18d3fSMadhavan Venkataraman again: 64787a18d3fSMadhavan Venkataraman /* 64887a18d3fSMadhavan Venkataraman * Try to see if a callout list already exists for this expiration. 64987a18d3fSMadhavan Venkataraman * Most of the time, this will be the case. 65087a18d3fSMadhavan Venkataraman */ 65187a18d3fSMadhavan Venkataraman cl = callout_list_get(ct, expiration, hash); 65287a18d3fSMadhavan Venkataraman if (cl == NULL) { 65387a18d3fSMadhavan Venkataraman /* 65487a18d3fSMadhavan Venkataraman * Check if we have enough space in the heap to insert one 65587a18d3fSMadhavan Venkataraman * expiration. If not, expand the heap. 65687a18d3fSMadhavan Venkataraman */ 65787a18d3fSMadhavan Venkataraman if (ct->ct_heap_num == ct->ct_heap_max) { 65887a18d3fSMadhavan Venkataraman callout_heap_expand(ct); 65987a18d3fSMadhavan Venkataraman /* 66087a18d3fSMadhavan Venkataraman * In the above call, we drop the lock, allocate and 66187a18d3fSMadhavan Venkataraman * reacquire the lock. So, we could have been away 66287a18d3fSMadhavan Venkataraman * for a while. In the meantime, someone could have 66387a18d3fSMadhavan Venkataraman * inserted a callout list with the same expiration. 66487a18d3fSMadhavan Venkataraman * So, the best course is to repeat the steps. This 66587a18d3fSMadhavan Venkataraman * should be an infrequent event. 66687a18d3fSMadhavan Venkataraman */ 66787a18d3fSMadhavan Venkataraman goto again; 66887a18d3fSMadhavan Venkataraman } 66987a18d3fSMadhavan Venkataraman 67087a18d3fSMadhavan Venkataraman /* 67187a18d3fSMadhavan Venkataraman * Check the free list. If we don't find one, we have to 67287a18d3fSMadhavan Venkataraman * take the slow path and allocate from kmem. 67387a18d3fSMadhavan Venkataraman */ 67487a18d3fSMadhavan Venkataraman if ((cl = ct->ct_lfree) == NULL) { 67587a18d3fSMadhavan Venkataraman callout_list_alloc(ct); 67687a18d3fSMadhavan Venkataraman /* 67787a18d3fSMadhavan Venkataraman * In the above call, we drop the lock, allocate and 67887a18d3fSMadhavan Venkataraman * reacquire the lock. So, we could have been away 67987a18d3fSMadhavan Venkataraman * for a while. In the meantime, someone could have 68087a18d3fSMadhavan Venkataraman * inserted a callout list with the same expiration. 68187a18d3fSMadhavan Venkataraman * Plus, the heap could have become full. So, the best 68287a18d3fSMadhavan Venkataraman * course is to repeat the steps. This should be an 68387a18d3fSMadhavan Venkataraman * infrequent event. 68487a18d3fSMadhavan Venkataraman */ 68587a18d3fSMadhavan Venkataraman goto again; 68687a18d3fSMadhavan Venkataraman } 68787a18d3fSMadhavan Venkataraman ct->ct_lfree = cl->cl_next; 68887a18d3fSMadhavan Venkataraman cl->cl_expiration = expiration; 68987a18d3fSMadhavan Venkataraman 69087a18d3fSMadhavan Venkataraman CALLOUT_LIST_INSERT(ct->ct_clhash[hash], cl); 69187a18d3fSMadhavan Venkataraman 69287a18d3fSMadhavan Venkataraman /* 69387a18d3fSMadhavan Venkataraman * This is a new expiration. So, insert it into the heap. 69487a18d3fSMadhavan Venkataraman * This will also reprogram the cyclic, if the expiration 69587a18d3fSMadhavan Venkataraman * propagated to the root of the heap. 69687a18d3fSMadhavan Venkataraman */ 69787a18d3fSMadhavan Venkataraman callout_heap_insert(ct, expiration); 69887a18d3fSMadhavan Venkataraman } 69987a18d3fSMadhavan Venkataraman cp->c_list = cl; 70087a18d3fSMadhavan Venkataraman CALLOUT_APPEND(ct, cp); 70187a18d3fSMadhavan Venkataraman 70287a18d3fSMadhavan Venkataraman ct->ct_timeouts++; 70387a18d3fSMadhavan Venkataraman ct->ct_timeouts_pending++; 70487a18d3fSMadhavan Venkataraman 70587a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 70687a18d3fSMadhavan Venkataraman 70787a18d3fSMadhavan Venkataraman kpreempt_enable(); 70887a18d3fSMadhavan Venkataraman 70987a18d3fSMadhavan Venkataraman TRACE_4(TR_FAC_CALLOUT, TR_TIMEOUT, 71087a18d3fSMadhavan Venkataraman "timeout:%K(%p) in %llx expiration, cp %p", func, arg, expiration, 71187a18d3fSMadhavan Venkataraman cp); 71287a18d3fSMadhavan Venkataraman 71387a18d3fSMadhavan Venkataraman return (id); 71487a18d3fSMadhavan Venkataraman } 71587a18d3fSMadhavan Venkataraman 71687a18d3fSMadhavan Venkataraman timeout_id_t 71787a18d3fSMadhavan Venkataraman timeout(void (*func)(void *), void *arg, clock_t delta) 71887a18d3fSMadhavan Venkataraman { 71987a18d3fSMadhavan Venkataraman ulong_t id; 72087a18d3fSMadhavan Venkataraman 72187a18d3fSMadhavan Venkataraman /* 7227c478bd9Sstevel@tonic-gate * Make sure the callout runs at least 1 tick in the future. 7237c478bd9Sstevel@tonic-gate */ 7247c478bd9Sstevel@tonic-gate if (delta <= 0) 7257c478bd9Sstevel@tonic-gate delta = 1; 726*454ab202SMadhavan Venkataraman else if (delta > callout_max_ticks) 727*454ab202SMadhavan Venkataraman delta = callout_max_ticks; 7287c478bd9Sstevel@tonic-gate 72987a18d3fSMadhavan Venkataraman id = (ulong_t)timeout_generic(CALLOUT_NORMAL, func, arg, 73087a18d3fSMadhavan Venkataraman TICK_TO_NSEC(delta), nsec_per_tick, CALLOUT_LEGACY); 7317c478bd9Sstevel@tonic-gate 7327c478bd9Sstevel@tonic-gate return ((timeout_id_t)id); 7337c478bd9Sstevel@tonic-gate } 7347c478bd9Sstevel@tonic-gate 73587a18d3fSMadhavan Venkataraman /* 73687a18d3fSMadhavan Venkataraman * Convenience function that creates a normal callout with default parameters 73787a18d3fSMadhavan Venkataraman * and returns a full ID. 73887a18d3fSMadhavan Venkataraman */ 73987a18d3fSMadhavan Venkataraman callout_id_t 74087a18d3fSMadhavan Venkataraman timeout_default(void (*func)(void *), void *arg, clock_t delta) 7417c478bd9Sstevel@tonic-gate { 74287a18d3fSMadhavan Venkataraman callout_id_t id; 7437c478bd9Sstevel@tonic-gate 74487a18d3fSMadhavan Venkataraman /* 74587a18d3fSMadhavan Venkataraman * Make sure the callout runs at least 1 tick in the future. 74687a18d3fSMadhavan Venkataraman */ 74787a18d3fSMadhavan Venkataraman if (delta <= 0) 74887a18d3fSMadhavan Venkataraman delta = 1; 749*454ab202SMadhavan Venkataraman else if (delta > callout_max_ticks) 750*454ab202SMadhavan Venkataraman delta = callout_max_ticks; 75187a18d3fSMadhavan Venkataraman 75287a18d3fSMadhavan Venkataraman id = timeout_generic(CALLOUT_NORMAL, func, arg, TICK_TO_NSEC(delta), 75387a18d3fSMadhavan Venkataraman nsec_per_tick, 0); 75487a18d3fSMadhavan Venkataraman 75587a18d3fSMadhavan Venkataraman return (id); 7567c478bd9Sstevel@tonic-gate } 7577c478bd9Sstevel@tonic-gate 7587c478bd9Sstevel@tonic-gate timeout_id_t 7597c478bd9Sstevel@tonic-gate realtime_timeout(void (*func)(void *), void *arg, clock_t delta) 7607c478bd9Sstevel@tonic-gate { 76187a18d3fSMadhavan Venkataraman ulong_t id; 76287a18d3fSMadhavan Venkataraman 76387a18d3fSMadhavan Venkataraman /* 76487a18d3fSMadhavan Venkataraman * Make sure the callout runs at least 1 tick in the future. 76587a18d3fSMadhavan Venkataraman */ 76687a18d3fSMadhavan Venkataraman if (delta <= 0) 76787a18d3fSMadhavan Venkataraman delta = 1; 768*454ab202SMadhavan Venkataraman else if (delta > callout_max_ticks) 769*454ab202SMadhavan Venkataraman delta = callout_max_ticks; 77087a18d3fSMadhavan Venkataraman 77187a18d3fSMadhavan Venkataraman id = (ulong_t)timeout_generic(CALLOUT_REALTIME, func, arg, 77287a18d3fSMadhavan Venkataraman TICK_TO_NSEC(delta), nsec_per_tick, CALLOUT_LEGACY); 77387a18d3fSMadhavan Venkataraman 77487a18d3fSMadhavan Venkataraman return ((timeout_id_t)id); 7757c478bd9Sstevel@tonic-gate } 7767c478bd9Sstevel@tonic-gate 77787a18d3fSMadhavan Venkataraman /* 77887a18d3fSMadhavan Venkataraman * Convenience function that creates a realtime callout with default parameters 77987a18d3fSMadhavan Venkataraman * and returns a full ID. 78087a18d3fSMadhavan Venkataraman */ 78187a18d3fSMadhavan Venkataraman callout_id_t 78287a18d3fSMadhavan Venkataraman realtime_timeout_default(void (*func)(void *), void *arg, clock_t delta) 7837c478bd9Sstevel@tonic-gate { 78487a18d3fSMadhavan Venkataraman callout_id_t id; 78587a18d3fSMadhavan Venkataraman 78687a18d3fSMadhavan Venkataraman /* 78787a18d3fSMadhavan Venkataraman * Make sure the callout runs at least 1 tick in the future. 78887a18d3fSMadhavan Venkataraman */ 78987a18d3fSMadhavan Venkataraman if (delta <= 0) 79087a18d3fSMadhavan Venkataraman delta = 1; 791*454ab202SMadhavan Venkataraman else if (delta > callout_max_ticks) 792*454ab202SMadhavan Venkataraman delta = callout_max_ticks; 79387a18d3fSMadhavan Venkataraman 79487a18d3fSMadhavan Venkataraman id = timeout_generic(CALLOUT_REALTIME, func, arg, TICK_TO_NSEC(delta), 79587a18d3fSMadhavan Venkataraman nsec_per_tick, 0); 79687a18d3fSMadhavan Venkataraman 79787a18d3fSMadhavan Venkataraman return (id); 79887a18d3fSMadhavan Venkataraman } 79987a18d3fSMadhavan Venkataraman 80087a18d3fSMadhavan Venkataraman hrtime_t 80187a18d3fSMadhavan Venkataraman untimeout_generic(callout_id_t id, int nowait) 80287a18d3fSMadhavan Venkataraman { 8037c478bd9Sstevel@tonic-gate callout_table_t *ct; 8047c478bd9Sstevel@tonic-gate callout_t *cp; 8057c478bd9Sstevel@tonic-gate callout_id_t xid; 80687a18d3fSMadhavan Venkataraman callout_list_t *cl; 80787a18d3fSMadhavan Venkataraman int hash; 80887a18d3fSMadhavan Venkataraman callout_id_t bogus; 8097c478bd9Sstevel@tonic-gate 81087a18d3fSMadhavan Venkataraman ct = &callout_table[CALLOUT_ID_TO_TABLE(id)]; 81187a18d3fSMadhavan Venkataraman hash = CALLOUT_IDHASH(id); 8127c478bd9Sstevel@tonic-gate 81387a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 8147c478bd9Sstevel@tonic-gate 81587a18d3fSMadhavan Venkataraman /* 81687a18d3fSMadhavan Venkataraman * Search the ID hash table for the callout. 81787a18d3fSMadhavan Venkataraman */ 81887a18d3fSMadhavan Venkataraman for (cp = ct->ct_idhash[hash].ch_head; cp; cp = cp->c_idnext) { 8197c478bd9Sstevel@tonic-gate 82087a18d3fSMadhavan Venkataraman xid = cp->c_xid; 8217c478bd9Sstevel@tonic-gate 82287a18d3fSMadhavan Venkataraman /* 82387a18d3fSMadhavan Venkataraman * Match the ID and generation number. 82487a18d3fSMadhavan Venkataraman */ 82587a18d3fSMadhavan Venkataraman if ((xid & CALLOUT_ID_MASK) != id) 8267c478bd9Sstevel@tonic-gate continue; 8277c478bd9Sstevel@tonic-gate 82887a18d3fSMadhavan Venkataraman cl = cp->c_list; 82987a18d3fSMadhavan Venkataraman if ((xid & CALLOUT_EXECUTING) == 0) { 83087a18d3fSMadhavan Venkataraman hrtime_t expiration; 83187a18d3fSMadhavan Venkataraman 83287a18d3fSMadhavan Venkataraman /* 83387a18d3fSMadhavan Venkataraman * Delete the callout. If the callout list becomes 83487a18d3fSMadhavan Venkataraman * NULL, we don't remove it from the table. This is 83587a18d3fSMadhavan Venkataraman * so it can be reused. If the empty callout list 83687a18d3fSMadhavan Venkataraman * corresponds to the top of the the callout heap, we 83787a18d3fSMadhavan Venkataraman * don't reprogram the table cyclic here. This is in 83887a18d3fSMadhavan Venkataraman * order to avoid lots of X-calls to the CPU associated 83987a18d3fSMadhavan Venkataraman * with the callout table. 84087a18d3fSMadhavan Venkataraman */ 84187a18d3fSMadhavan Venkataraman expiration = cl->cl_expiration; 84287a18d3fSMadhavan Venkataraman CALLOUT_DELETE(ct, cp); 84387a18d3fSMadhavan Venkataraman cp->c_idnext = ct->ct_free; 84487a18d3fSMadhavan Venkataraman ct->ct_free = cp; 84587a18d3fSMadhavan Venkataraman ct->ct_untimeouts_unexpired++; 84687a18d3fSMadhavan Venkataraman ct->ct_timeouts_pending--; 84787a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 84887a18d3fSMadhavan Venkataraman 84987a18d3fSMadhavan Venkataraman expiration -= gethrtime(); 85087a18d3fSMadhavan Venkataraman TRACE_2(TR_FAC_CALLOUT, TR_UNTIMEOUT, 85187a18d3fSMadhavan Venkataraman "untimeout:ID %lx hrtime left %llx", id, 85287a18d3fSMadhavan Venkataraman expiration); 85387a18d3fSMadhavan Venkataraman return (expiration < 0 ? 0 : expiration); 85487a18d3fSMadhavan Venkataraman } 85587a18d3fSMadhavan Venkataraman 85687a18d3fSMadhavan Venkataraman ct->ct_untimeouts_executing++; 8577c478bd9Sstevel@tonic-gate /* 8587c478bd9Sstevel@tonic-gate * The callout we want to delete is currently executing. 8597c478bd9Sstevel@tonic-gate * The DDI states that we must wait until the callout 86087a18d3fSMadhavan Venkataraman * completes before returning, so we block on cl_done until the 86187a18d3fSMadhavan Venkataraman * callout ID changes (to the old ID if it's on the freelist, 8627c478bd9Sstevel@tonic-gate * or to a new callout ID if it's in use). This implicitly 8637c478bd9Sstevel@tonic-gate * assumes that callout structures are persistent (they are). 8647c478bd9Sstevel@tonic-gate */ 86587a18d3fSMadhavan Venkataraman if (cl->cl_executor == curthread) { 8667c478bd9Sstevel@tonic-gate /* 8677c478bd9Sstevel@tonic-gate * The timeout handler called untimeout() on itself. 8687c478bd9Sstevel@tonic-gate * Stupid, but legal. We can't wait for the timeout 8697c478bd9Sstevel@tonic-gate * to complete without deadlocking, so we just return. 8707c478bd9Sstevel@tonic-gate */ 87187a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 8727c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_CALLOUT, TR_UNTIMEOUT_SELF, 8737c478bd9Sstevel@tonic-gate "untimeout_self:ID %x", id); 8747c478bd9Sstevel@tonic-gate return (-1); 8757c478bd9Sstevel@tonic-gate } 87687a18d3fSMadhavan Venkataraman if (nowait == 0) { 87787a18d3fSMadhavan Venkataraman /* 87887a18d3fSMadhavan Venkataraman * We need to wait. Indicate that we are waiting by 87987a18d3fSMadhavan Venkataraman * incrementing cl_waiting. This prevents the executor 88087a18d3fSMadhavan Venkataraman * from doing a wakeup on cl_done if there are no 88187a18d3fSMadhavan Venkataraman * waiters. 88287a18d3fSMadhavan Venkataraman */ 88387a18d3fSMadhavan Venkataraman while (cp->c_xid == xid) { 88487a18d3fSMadhavan Venkataraman cl->cl_waiting = 1; 88587a18d3fSMadhavan Venkataraman cv_wait(&cl->cl_done, &ct->ct_mutex); 88687a18d3fSMadhavan Venkataraman } 88787a18d3fSMadhavan Venkataraman } 88887a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 8897c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_CALLOUT, TR_UNTIMEOUT_EXECUTING, 8907c478bd9Sstevel@tonic-gate "untimeout_executing:ID %lx", id); 8917c478bd9Sstevel@tonic-gate return (-1); 8927c478bd9Sstevel@tonic-gate } 89387a18d3fSMadhavan Venkataraman ct->ct_untimeouts_expired++; 8947c478bd9Sstevel@tonic-gate 89587a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 8967c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_CALLOUT, TR_UNTIMEOUT_BOGUS_ID, 8977c478bd9Sstevel@tonic-gate "untimeout_bogus_id:ID %lx", id); 8987c478bd9Sstevel@tonic-gate 8997c478bd9Sstevel@tonic-gate /* 9007c478bd9Sstevel@tonic-gate * We didn't find the specified callout ID. This means either 9017c478bd9Sstevel@tonic-gate * (1) the callout already fired, or (2) the caller passed us 9027c478bd9Sstevel@tonic-gate * a bogus value. Perform a sanity check to detect case (2). 9037c478bd9Sstevel@tonic-gate */ 90487a18d3fSMadhavan Venkataraman bogus = (CALLOUT_EXECUTING | CALLOUT_HRESTIME | CALLOUT_COUNTER_HIGH); 90587a18d3fSMadhavan Venkataraman if (((id & bogus) != CALLOUT_COUNTER_HIGH) && (id != 0)) 90687a18d3fSMadhavan Venkataraman panic("untimeout: impossible timeout id %llx", 90787a18d3fSMadhavan Venkataraman (unsigned long long)id); 9087c478bd9Sstevel@tonic-gate 9097c478bd9Sstevel@tonic-gate return (-1); 9107c478bd9Sstevel@tonic-gate } 9117c478bd9Sstevel@tonic-gate 91287a18d3fSMadhavan Venkataraman clock_t 91387a18d3fSMadhavan Venkataraman untimeout(timeout_id_t id_arg) 91487a18d3fSMadhavan Venkataraman { 91587a18d3fSMadhavan Venkataraman hrtime_t hleft; 91687a18d3fSMadhavan Venkataraman clock_t tleft; 91787a18d3fSMadhavan Venkataraman callout_id_t id; 91887a18d3fSMadhavan Venkataraman 91987a18d3fSMadhavan Venkataraman id = (ulong_t)id_arg; 92087a18d3fSMadhavan Venkataraman hleft = untimeout_generic(id, 0); 92187a18d3fSMadhavan Venkataraman if (hleft < 0) 92287a18d3fSMadhavan Venkataraman tleft = -1; 92387a18d3fSMadhavan Venkataraman else if (hleft == 0) 92487a18d3fSMadhavan Venkataraman tleft = 0; 92587a18d3fSMadhavan Venkataraman else 92687a18d3fSMadhavan Venkataraman tleft = NSEC_TO_TICK(hleft); 92787a18d3fSMadhavan Venkataraman 92887a18d3fSMadhavan Venkataraman return (tleft); 92987a18d3fSMadhavan Venkataraman } 93087a18d3fSMadhavan Venkataraman 9317c478bd9Sstevel@tonic-gate /* 93287a18d3fSMadhavan Venkataraman * Convenience function to untimeout a timeout with a full ID with default 93387a18d3fSMadhavan Venkataraman * parameters. 93487a18d3fSMadhavan Venkataraman */ 93587a18d3fSMadhavan Venkataraman clock_t 93687a18d3fSMadhavan Venkataraman untimeout_default(callout_id_t id, int nowait) 93787a18d3fSMadhavan Venkataraman { 93887a18d3fSMadhavan Venkataraman hrtime_t hleft; 93987a18d3fSMadhavan Venkataraman clock_t tleft; 94087a18d3fSMadhavan Venkataraman 94187a18d3fSMadhavan Venkataraman hleft = untimeout_generic(id, nowait); 94287a18d3fSMadhavan Venkataraman if (hleft < 0) 94387a18d3fSMadhavan Venkataraman tleft = -1; 94487a18d3fSMadhavan Venkataraman else if (hleft == 0) 94587a18d3fSMadhavan Venkataraman tleft = 0; 94687a18d3fSMadhavan Venkataraman else 94787a18d3fSMadhavan Venkataraman tleft = NSEC_TO_TICK(hleft); 94887a18d3fSMadhavan Venkataraman 94987a18d3fSMadhavan Venkataraman return (tleft); 95087a18d3fSMadhavan Venkataraman } 95187a18d3fSMadhavan Venkataraman 95287a18d3fSMadhavan Venkataraman /* 95387a18d3fSMadhavan Venkataraman * Expire all the callouts queued in the specified callout list. 9547c478bd9Sstevel@tonic-gate */ 9557c478bd9Sstevel@tonic-gate static void 95687a18d3fSMadhavan Venkataraman callout_list_expire(callout_table_t *ct, callout_list_t *cl) 9577c478bd9Sstevel@tonic-gate { 9587c478bd9Sstevel@tonic-gate callout_t *cp; 9597c478bd9Sstevel@tonic-gate 96087a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 96187a18d3fSMadhavan Venkataraman ASSERT(cl != NULL); 9627c478bd9Sstevel@tonic-gate 96387a18d3fSMadhavan Venkataraman cl->cl_executor = curthread; 96487a18d3fSMadhavan Venkataraman 96587a18d3fSMadhavan Venkataraman while ((cp = cl->cl_callouts.ch_head) != NULL) { 966f635d46aSqiao /* 96787a18d3fSMadhavan Venkataraman * Indicate to untimeout() that a callout is 96887a18d3fSMadhavan Venkataraman * being expired by the executor. 969f635d46aSqiao */ 97087a18d3fSMadhavan Venkataraman cp->c_xid |= CALLOUT_EXECUTING; 97187a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 97287a18d3fSMadhavan Venkataraman 9737c478bd9Sstevel@tonic-gate DTRACE_PROBE1(callout__start, callout_t *, cp); 9747c478bd9Sstevel@tonic-gate (*cp->c_func)(cp->c_arg); 9757c478bd9Sstevel@tonic-gate DTRACE_PROBE1(callout__end, callout_t *, cp); 9767c478bd9Sstevel@tonic-gate 97787a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 97887a18d3fSMadhavan Venkataraman 97987a18d3fSMadhavan Venkataraman ct->ct_expirations++; 98087a18d3fSMadhavan Venkataraman ct->ct_timeouts_pending--; 9817c478bd9Sstevel@tonic-gate /* 98287a18d3fSMadhavan Venkataraman * Indicate completion for cl_done. 9837c478bd9Sstevel@tonic-gate */ 98487a18d3fSMadhavan Venkataraman cp->c_xid &= ~CALLOUT_EXECUTING; 985f635d46aSqiao 9867c478bd9Sstevel@tonic-gate /* 98787a18d3fSMadhavan Venkataraman * Delete callout from ID hash table and the callout 98887a18d3fSMadhavan Venkataraman * list, return to freelist, and tell any untimeout() that 98987a18d3fSMadhavan Venkataraman * cares that we're done. 9907c478bd9Sstevel@tonic-gate */ 99187a18d3fSMadhavan Venkataraman CALLOUT_DELETE(ct, cp); 99287a18d3fSMadhavan Venkataraman cp->c_idnext = ct->ct_free; 99387a18d3fSMadhavan Venkataraman ct->ct_free = cp; 99487a18d3fSMadhavan Venkataraman 99587a18d3fSMadhavan Venkataraman if (cl->cl_waiting) { 99687a18d3fSMadhavan Venkataraman cl->cl_waiting = 0; 99787a18d3fSMadhavan Venkataraman cv_broadcast(&cl->cl_done); 9987c478bd9Sstevel@tonic-gate } 99987a18d3fSMadhavan Venkataraman } 100087a18d3fSMadhavan Venkataraman 100187a18d3fSMadhavan Venkataraman cl->cl_executor = NULL; 10027c478bd9Sstevel@tonic-gate } 10037c478bd9Sstevel@tonic-gate 10047c478bd9Sstevel@tonic-gate /* 100587a18d3fSMadhavan Venkataraman * Execute all expired callout lists for a callout table. 10067c478bd9Sstevel@tonic-gate */ 10077c478bd9Sstevel@tonic-gate static void 100887a18d3fSMadhavan Venkataraman callout_expire(callout_table_t *ct) 10097c478bd9Sstevel@tonic-gate { 101087a18d3fSMadhavan Venkataraman callout_list_t *cl, *clnext; 1011f635d46aSqiao 101287a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 10137c478bd9Sstevel@tonic-gate 101487a18d3fSMadhavan Venkataraman for (cl = ct->ct_expired.ch_head; (cl != NULL); cl = clnext) { 1015f635d46aSqiao /* 101687a18d3fSMadhavan Venkataraman * Multiple executor threads could be running at the same 101787a18d3fSMadhavan Venkataraman * time. Each callout list is processed by only one thread. 101887a18d3fSMadhavan Venkataraman * If this callout list is already being processed by another 101987a18d3fSMadhavan Venkataraman * executor, go on to the next one. 1020f635d46aSqiao */ 102187a18d3fSMadhavan Venkataraman if (cl->cl_executor != NULL) { 102287a18d3fSMadhavan Venkataraman clnext = cl->cl_next; 10237c478bd9Sstevel@tonic-gate continue; 10247c478bd9Sstevel@tonic-gate } 10257c478bd9Sstevel@tonic-gate 10267c478bd9Sstevel@tonic-gate /* 102787a18d3fSMadhavan Venkataraman * Expire all the callouts in this callout list. 102887a18d3fSMadhavan Venkataraman */ 102987a18d3fSMadhavan Venkataraman callout_list_expire(ct, cl); 103087a18d3fSMadhavan Venkataraman 103187a18d3fSMadhavan Venkataraman /* 103287a18d3fSMadhavan Venkataraman * Free the callout list. 103387a18d3fSMadhavan Venkataraman */ 103487a18d3fSMadhavan Venkataraman clnext = cl->cl_next; 103587a18d3fSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_expired, cl); 103687a18d3fSMadhavan Venkataraman cl->cl_next = ct->ct_lfree; 103787a18d3fSMadhavan Venkataraman ct->ct_lfree = cl; 103887a18d3fSMadhavan Venkataraman } 103987a18d3fSMadhavan Venkataraman } 104087a18d3fSMadhavan Venkataraman 104187a18d3fSMadhavan Venkataraman /* 104287a18d3fSMadhavan Venkataraman * The cyclic handlers below process callouts in two steps: 104387a18d3fSMadhavan Venkataraman * 104487a18d3fSMadhavan Venkataraman * 1. Find all expired callout lists and queue them in a separate 104587a18d3fSMadhavan Venkataraman * list of expired callouts. 104687a18d3fSMadhavan Venkataraman * 2. Execute the expired callout lists. 104787a18d3fSMadhavan Venkataraman * 104887a18d3fSMadhavan Venkataraman * This is done for two reasons: 104987a18d3fSMadhavan Venkataraman * 105087a18d3fSMadhavan Venkataraman * 1. We want to quickly find the next earliest expiration to program 105187a18d3fSMadhavan Venkataraman * the cyclic to and reprogram it. We can do this right at the end 105287a18d3fSMadhavan Venkataraman * of step 1. 105387a18d3fSMadhavan Venkataraman * 2. The realtime cyclic handler expires callouts in place. However, 105487a18d3fSMadhavan Venkataraman * for normal callouts, callouts are expired by a taskq thread. 105587a18d3fSMadhavan Venkataraman * So, it is simpler and more robust to have the taskq thread just 105687a18d3fSMadhavan Venkataraman * do step 2. 105787a18d3fSMadhavan Venkataraman */ 105887a18d3fSMadhavan Venkataraman 105987a18d3fSMadhavan Venkataraman /* 106087a18d3fSMadhavan Venkataraman * Realtime callout cyclic handler. 10617c478bd9Sstevel@tonic-gate */ 10627c478bd9Sstevel@tonic-gate void 106387a18d3fSMadhavan Venkataraman callout_realtime(callout_table_t *ct) 10647c478bd9Sstevel@tonic-gate { 106587a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 106687a18d3fSMadhavan Venkataraman callout_heap_delete(ct); 106787a18d3fSMadhavan Venkataraman callout_expire(ct); 106887a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 106987a18d3fSMadhavan Venkataraman } 10707c478bd9Sstevel@tonic-gate 107187a18d3fSMadhavan Venkataraman void 107287a18d3fSMadhavan Venkataraman callout_execute(callout_table_t *ct) 107387a18d3fSMadhavan Venkataraman { 107487a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 107587a18d3fSMadhavan Venkataraman callout_expire(ct); 107687a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 107787a18d3fSMadhavan Venkataraman } 10787c478bd9Sstevel@tonic-gate 107987a18d3fSMadhavan Venkataraman /* 108087a18d3fSMadhavan Venkataraman * Normal callout cyclic handler. 108187a18d3fSMadhavan Venkataraman */ 108287a18d3fSMadhavan Venkataraman void 108387a18d3fSMadhavan Venkataraman callout_normal(callout_table_t *ct) 108487a18d3fSMadhavan Venkataraman { 108587a18d3fSMadhavan Venkataraman int exec; 108687a18d3fSMadhavan Venkataraman 108787a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 108887a18d3fSMadhavan Venkataraman callout_heap_delete(ct); 108987a18d3fSMadhavan Venkataraman exec = (ct->ct_expired.ch_head != NULL); 109087a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 109187a18d3fSMadhavan Venkataraman 109287a18d3fSMadhavan Venkataraman if (exec) { 109387a18d3fSMadhavan Venkataraman ASSERT(ct->ct_taskq != NULL); 109487a18d3fSMadhavan Venkataraman (void) taskq_dispatch(ct->ct_taskq, 109587a18d3fSMadhavan Venkataraman (task_func_t *)callout_execute, ct, TQ_NOSLEEP); 109687a18d3fSMadhavan Venkataraman } 109787a18d3fSMadhavan Venkataraman } 109887a18d3fSMadhavan Venkataraman 109987a18d3fSMadhavan Venkataraman /* 110087a18d3fSMadhavan Venkataraman * Suspend callout processing. 110187a18d3fSMadhavan Venkataraman */ 110287a18d3fSMadhavan Venkataraman static void 110387a18d3fSMadhavan Venkataraman callout_suspend(void) 110487a18d3fSMadhavan Venkataraman { 110587a18d3fSMadhavan Venkataraman int t, f; 110687a18d3fSMadhavan Venkataraman callout_table_t *ct; 110787a18d3fSMadhavan Venkataraman 110887a18d3fSMadhavan Venkataraman /* 110987a18d3fSMadhavan Venkataraman * Traverse every callout table in the system and suspend callout 111087a18d3fSMadhavan Venkataraman * processing. 111187a18d3fSMadhavan Venkataraman * 111287a18d3fSMadhavan Venkataraman * We need to suspend all the tables (including the inactive ones) 111387a18d3fSMadhavan Venkataraman * so that if a table is made active while the suspend is still on, 111487a18d3fSMadhavan Venkataraman * the table remains suspended. 111587a18d3fSMadhavan Venkataraman */ 111687a18d3fSMadhavan Venkataraman for (f = 0; f < max_ncpus; f++) { 111787a18d3fSMadhavan Venkataraman for (t = 0; t < CALLOUT_NTYPES; t++) { 111887a18d3fSMadhavan Venkataraman ct = &callout_table[CALLOUT_TABLE(t, f)]; 111987a18d3fSMadhavan Venkataraman 112087a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 1121*454ab202SMadhavan Venkataraman ct->ct_suspend++; 112287a18d3fSMadhavan Venkataraman if (ct->ct_cyclic == CYCLIC_NONE) { 112387a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 112487a18d3fSMadhavan Venkataraman continue; 112587a18d3fSMadhavan Venkataraman } 1126*454ab202SMadhavan Venkataraman if (ct->ct_suspend == 1) 1127*454ab202SMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_cyclic, 1128*454ab202SMadhavan Venkataraman CY_INFINITY); 112987a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 113087a18d3fSMadhavan Venkataraman } 113187a18d3fSMadhavan Venkataraman } 113287a18d3fSMadhavan Venkataraman } 113387a18d3fSMadhavan Venkataraman 113487a18d3fSMadhavan Venkataraman static void 113587a18d3fSMadhavan Venkataraman callout_adjust(callout_table_t *ct, hrtime_t delta) 113687a18d3fSMadhavan Venkataraman { 113787a18d3fSMadhavan Venkataraman int hash, newhash; 113887a18d3fSMadhavan Venkataraman hrtime_t expiration; 113987a18d3fSMadhavan Venkataraman callout_list_t *cl; 114087a18d3fSMadhavan Venkataraman callout_hash_t list; 114187a18d3fSMadhavan Venkataraman 114287a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 114387a18d3fSMadhavan Venkataraman 114487a18d3fSMadhavan Venkataraman /* 114587a18d3fSMadhavan Venkataraman * In order to adjust the expirations, we null out the heap. Then, 114687a18d3fSMadhavan Venkataraman * we reinsert adjusted expirations in the heap. Keeps it simple. 114787a18d3fSMadhavan Venkataraman * Note that since the CALLOUT_TABLE_SUSPENDED flag is set by the 114887a18d3fSMadhavan Venkataraman * caller, the heap insert does not result in cyclic reprogramming. 114987a18d3fSMadhavan Venkataraman */ 115087a18d3fSMadhavan Venkataraman ct->ct_heap_num = 0; 115187a18d3fSMadhavan Venkataraman 115287a18d3fSMadhavan Venkataraman /* 115387a18d3fSMadhavan Venkataraman * First, remove all the callout lists from the table and string them 115487a18d3fSMadhavan Venkataraman * in a list. 115587a18d3fSMadhavan Venkataraman */ 115687a18d3fSMadhavan Venkataraman list.ch_head = list.ch_tail = NULL; 115787a18d3fSMadhavan Venkataraman for (hash = 0; hash < CALLOUT_BUCKETS; hash++) { 115887a18d3fSMadhavan Venkataraman while ((cl = ct->ct_clhash[hash].ch_head) != NULL) { 115987a18d3fSMadhavan Venkataraman CALLOUT_LIST_DELETE(ct->ct_clhash[hash], cl); 116087a18d3fSMadhavan Venkataraman CALLOUT_LIST_APPEND(list, cl); 116187a18d3fSMadhavan Venkataraman } 116287a18d3fSMadhavan Venkataraman } 116387a18d3fSMadhavan Venkataraman 116487a18d3fSMadhavan Venkataraman /* 116587a18d3fSMadhavan Venkataraman * Now, traverse the callout lists and adjust their expirations. 116687a18d3fSMadhavan Venkataraman */ 116787a18d3fSMadhavan Venkataraman while ((cl = list.ch_head) != NULL) { 116887a18d3fSMadhavan Venkataraman CALLOUT_LIST_DELETE(list, cl); 116987a18d3fSMadhavan Venkataraman /* 117087a18d3fSMadhavan Venkataraman * Set the new expiration and reinsert in the right 117187a18d3fSMadhavan Venkataraman * hash bucket. 117287a18d3fSMadhavan Venkataraman */ 117387a18d3fSMadhavan Venkataraman expiration = cl->cl_expiration; 117487a18d3fSMadhavan Venkataraman expiration += delta; 117587a18d3fSMadhavan Venkataraman cl->cl_expiration = expiration; 117687a18d3fSMadhavan Venkataraman newhash = CALLOUT_CLHASH(expiration); 117787a18d3fSMadhavan Venkataraman CALLOUT_LIST_INSERT(ct->ct_clhash[newhash], cl); 117887a18d3fSMadhavan Venkataraman callout_heap_insert(ct, expiration); 117987a18d3fSMadhavan Venkataraman } 118087a18d3fSMadhavan Venkataraman } 118187a18d3fSMadhavan Venkataraman 118287a18d3fSMadhavan Venkataraman /* 118387a18d3fSMadhavan Venkataraman * Resume callout processing. 118487a18d3fSMadhavan Venkataraman */ 118587a18d3fSMadhavan Venkataraman static void 118687a18d3fSMadhavan Venkataraman callout_resume(hrtime_t delta) 118787a18d3fSMadhavan Venkataraman { 118887a18d3fSMadhavan Venkataraman hrtime_t exp; 118987a18d3fSMadhavan Venkataraman int t, f; 119087a18d3fSMadhavan Venkataraman callout_table_t *ct; 119187a18d3fSMadhavan Venkataraman 119287a18d3fSMadhavan Venkataraman /* 119387a18d3fSMadhavan Venkataraman * Traverse every callout table in the system and resume callout 119487a18d3fSMadhavan Venkataraman * processing. For active tables, perform any hrtime adjustments 119587a18d3fSMadhavan Venkataraman * necessary. 119687a18d3fSMadhavan Venkataraman */ 119787a18d3fSMadhavan Venkataraman for (f = 0; f < max_ncpus; f++) { 119887a18d3fSMadhavan Venkataraman for (t = 0; t < CALLOUT_NTYPES; t++) { 119987a18d3fSMadhavan Venkataraman ct = &callout_table[CALLOUT_TABLE(t, f)]; 120087a18d3fSMadhavan Venkataraman 120187a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 120287a18d3fSMadhavan Venkataraman if (ct->ct_cyclic == CYCLIC_NONE) { 1203*454ab202SMadhavan Venkataraman ct->ct_suspend--; 120487a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 120587a18d3fSMadhavan Venkataraman continue; 120687a18d3fSMadhavan Venkataraman } 120787a18d3fSMadhavan Venkataraman 120887a18d3fSMadhavan Venkataraman if (delta) 120987a18d3fSMadhavan Venkataraman callout_adjust(ct, delta); 121087a18d3fSMadhavan Venkataraman 1211*454ab202SMadhavan Venkataraman ct->ct_suspend--; 1212*454ab202SMadhavan Venkataraman if (ct->ct_suspend == 0) { 121387a18d3fSMadhavan Venkataraman /* 1214*454ab202SMadhavan Venkataraman * If the expired list is non-empty, then have 1215*454ab202SMadhavan Venkataraman * the cyclic expire immediately. Else, program 1216*454ab202SMadhavan Venkataraman * the cyclic based on the heap. 121787a18d3fSMadhavan Venkataraman */ 121887a18d3fSMadhavan Venkataraman if (ct->ct_expired.ch_head != NULL) 121987a18d3fSMadhavan Venkataraman exp = gethrtime(); 122087a18d3fSMadhavan Venkataraman else if (ct->ct_heap_num > 0) 122187a18d3fSMadhavan Venkataraman exp = ct->ct_heap[0]; 122287a18d3fSMadhavan Venkataraman else 122387a18d3fSMadhavan Venkataraman exp = 0; 122487a18d3fSMadhavan Venkataraman if (exp != 0) 1225*454ab202SMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_cyclic, 1226*454ab202SMadhavan Venkataraman exp); 1227*454ab202SMadhavan Venkataraman } 122887a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 122987a18d3fSMadhavan Venkataraman } 123087a18d3fSMadhavan Venkataraman } 12317c478bd9Sstevel@tonic-gate } 12327c478bd9Sstevel@tonic-gate 12337c478bd9Sstevel@tonic-gate /* 12347c478bd9Sstevel@tonic-gate * Callback handler used by CPR to stop and resume callouts. 12357c478bd9Sstevel@tonic-gate */ 12367c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 12377c478bd9Sstevel@tonic-gate static boolean_t 12387c478bd9Sstevel@tonic-gate callout_cpr_callb(void *arg, int code) 12397c478bd9Sstevel@tonic-gate { 124087a18d3fSMadhavan Venkataraman if (code == CB_CODE_CPR_CHKPT) 124187a18d3fSMadhavan Venkataraman callout_suspend(); 124287a18d3fSMadhavan Venkataraman else 124387a18d3fSMadhavan Venkataraman callout_resume(0); 124487a18d3fSMadhavan Venkataraman 12457c478bd9Sstevel@tonic-gate return (B_TRUE); 12467c478bd9Sstevel@tonic-gate } 12477c478bd9Sstevel@tonic-gate 12487c478bd9Sstevel@tonic-gate /* 124987a18d3fSMadhavan Venkataraman * Callback handler invoked when the debugger is entered or exited. 12507c478bd9Sstevel@tonic-gate */ 125187a18d3fSMadhavan Venkataraman /*ARGSUSED*/ 125287a18d3fSMadhavan Venkataraman static boolean_t 125387a18d3fSMadhavan Venkataraman callout_debug_callb(void *arg, int code) 12547c478bd9Sstevel@tonic-gate { 125587a18d3fSMadhavan Venkataraman hrtime_t delta; 1256f635d46aSqiao 1257f635d46aSqiao /* 125887a18d3fSMadhavan Venkataraman * When the system enters the debugger. make a note of the hrtime. 125987a18d3fSMadhavan Venkataraman * When it is resumed, compute how long the system was in the 126087a18d3fSMadhavan Venkataraman * debugger. This interval should not be counted for callouts. 1261f635d46aSqiao */ 126287a18d3fSMadhavan Venkataraman if (code == 0) { 126387a18d3fSMadhavan Venkataraman callout_suspend(); 126487a18d3fSMadhavan Venkataraman callout_debug_hrtime = gethrtime(); 126587a18d3fSMadhavan Venkataraman } else { 126687a18d3fSMadhavan Venkataraman delta = gethrtime() - callout_debug_hrtime; 126787a18d3fSMadhavan Venkataraman callout_resume(delta); 126887a18d3fSMadhavan Venkataraman } 1269f635d46aSqiao 127087a18d3fSMadhavan Venkataraman return (B_TRUE); 127187a18d3fSMadhavan Venkataraman } 127287a18d3fSMadhavan Venkataraman 127387a18d3fSMadhavan Venkataraman /* 127487a18d3fSMadhavan Venkataraman * Move the hrestime callouts to the expired list. Then program the table's 127587a18d3fSMadhavan Venkataraman * cyclic to expire immediately so that the callouts can be executed 127687a18d3fSMadhavan Venkataraman * immediately. 127787a18d3fSMadhavan Venkataraman */ 127887a18d3fSMadhavan Venkataraman static void 127987a18d3fSMadhavan Venkataraman callout_hrestime_one(callout_table_t *ct) 128087a18d3fSMadhavan Venkataraman { 128187a18d3fSMadhavan Venkataraman callout_list_t *cl, *ecl; 128287a18d3fSMadhavan Venkataraman callout_t *cp; 128387a18d3fSMadhavan Venkataraman int hash; 128487a18d3fSMadhavan Venkataraman 128587a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 128687a18d3fSMadhavan Venkataraman if (ct->ct_heap_num == 0) { 128787a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 128887a18d3fSMadhavan Venkataraman return; 128987a18d3fSMadhavan Venkataraman } 129087a18d3fSMadhavan Venkataraman 129187a18d3fSMadhavan Venkataraman if (ct->ct_lfree == NULL) 129287a18d3fSMadhavan Venkataraman callout_list_alloc(ct); 129387a18d3fSMadhavan Venkataraman ecl = ct->ct_lfree; 129487a18d3fSMadhavan Venkataraman ct->ct_lfree = ecl->cl_next; 129587a18d3fSMadhavan Venkataraman 129687a18d3fSMadhavan Venkataraman for (hash = 0; hash < CALLOUT_BUCKETS; hash++) { 129787a18d3fSMadhavan Venkataraman for (cl = ct->ct_clhash[hash].ch_head; cl; cl = cl->cl_next) { 129887a18d3fSMadhavan Venkataraman for (cp = cl->cl_callouts.ch_head; cp; 129987a18d3fSMadhavan Venkataraman cp = cp->c_clnext) { 130087a18d3fSMadhavan Venkataraman if ((cp->c_xid & CALLOUT_HRESTIME) == 0) 130187a18d3fSMadhavan Venkataraman continue; 130287a18d3fSMadhavan Venkataraman CALLOUT_HASH_DELETE(cl->cl_callouts, cp, 130387a18d3fSMadhavan Venkataraman c_clnext, c_clprev); 130487a18d3fSMadhavan Venkataraman cp->c_list = ecl; 130587a18d3fSMadhavan Venkataraman CALLOUT_HASH_APPEND(ecl->cl_callouts, cp, 130687a18d3fSMadhavan Venkataraman c_clnext, c_clprev); 130787a18d3fSMadhavan Venkataraman } 130887a18d3fSMadhavan Venkataraman } 130987a18d3fSMadhavan Venkataraman } 131087a18d3fSMadhavan Venkataraman 131187a18d3fSMadhavan Venkataraman if (ecl->cl_callouts.ch_head != NULL) { 131287a18d3fSMadhavan Venkataraman CALLOUT_LIST_APPEND(ct->ct_expired, ecl); 1313*454ab202SMadhavan Venkataraman if (ct->ct_suspend == 0) 131487a18d3fSMadhavan Venkataraman (void) cyclic_reprogram(ct->ct_cyclic, gethrtime()); 131587a18d3fSMadhavan Venkataraman } else { 131687a18d3fSMadhavan Venkataraman ecl->cl_next = ct->ct_lfree; 131787a18d3fSMadhavan Venkataraman ct->ct_lfree = ecl; 131887a18d3fSMadhavan Venkataraman } 131987a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 132087a18d3fSMadhavan Venkataraman } 132187a18d3fSMadhavan Venkataraman 132287a18d3fSMadhavan Venkataraman /* 132387a18d3fSMadhavan Venkataraman * This function is called whenever system time (hrestime) is changed 132487a18d3fSMadhavan Venkataraman * explicitly. All the HRESTIME callouts must be expired at once. 132587a18d3fSMadhavan Venkataraman */ 132687a18d3fSMadhavan Venkataraman /*ARGSUSED*/ 132787a18d3fSMadhavan Venkataraman void 132887a18d3fSMadhavan Venkataraman callout_hrestime(void) 132987a18d3fSMadhavan Venkataraman { 133087a18d3fSMadhavan Venkataraman int t, f; 133187a18d3fSMadhavan Venkataraman callout_table_t *ct; 133287a18d3fSMadhavan Venkataraman 133387a18d3fSMadhavan Venkataraman /* 133487a18d3fSMadhavan Venkataraman * Traverse every callout table in the system and process the hrestime 133587a18d3fSMadhavan Venkataraman * callouts therein. 133687a18d3fSMadhavan Venkataraman * 133787a18d3fSMadhavan Venkataraman * We look at all the tables because we don't know which ones were 133887a18d3fSMadhavan Venkataraman * onlined and offlined in the past. The offlined tables may still 133987a18d3fSMadhavan Venkataraman * have active cyclics processing timers somewhere. 134087a18d3fSMadhavan Venkataraman */ 134187a18d3fSMadhavan Venkataraman for (f = 0; f < max_ncpus; f++) { 134287a18d3fSMadhavan Venkataraman for (t = 0; t < CALLOUT_NTYPES; t++) { 134387a18d3fSMadhavan Venkataraman ct = &callout_table[CALLOUT_TABLE(t, f)]; 134487a18d3fSMadhavan Venkataraman callout_hrestime_one(ct); 134587a18d3fSMadhavan Venkataraman } 134687a18d3fSMadhavan Venkataraman } 134787a18d3fSMadhavan Venkataraman } 134887a18d3fSMadhavan Venkataraman 134987a18d3fSMadhavan Venkataraman /* 135087a18d3fSMadhavan Venkataraman * Create the hash tables for this callout table. 135187a18d3fSMadhavan Venkataraman */ 135287a18d3fSMadhavan Venkataraman static void 135387a18d3fSMadhavan Venkataraman callout_hash_init(callout_table_t *ct) 135487a18d3fSMadhavan Venkataraman { 135587a18d3fSMadhavan Venkataraman size_t size; 135687a18d3fSMadhavan Venkataraman 135787a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 135887a18d3fSMadhavan Venkataraman ASSERT((ct->ct_idhash == NULL) && (ct->ct_clhash == NULL)); 135987a18d3fSMadhavan Venkataraman 136087a18d3fSMadhavan Venkataraman size = sizeof (callout_hash_t) * CALLOUT_BUCKETS; 136187a18d3fSMadhavan Venkataraman ct->ct_idhash = kmem_zalloc(size, KM_SLEEP); 136287a18d3fSMadhavan Venkataraman ct->ct_clhash = kmem_zalloc(size, KM_SLEEP); 136387a18d3fSMadhavan Venkataraman } 136487a18d3fSMadhavan Venkataraman 136587a18d3fSMadhavan Venkataraman /* 136687a18d3fSMadhavan Venkataraman * Create per-callout table kstats. 136787a18d3fSMadhavan Venkataraman */ 136887a18d3fSMadhavan Venkataraman static void 136987a18d3fSMadhavan Venkataraman callout_kstat_init(callout_table_t *ct) 137087a18d3fSMadhavan Venkataraman { 137187a18d3fSMadhavan Venkataraman callout_stat_type_t stat; 137287a18d3fSMadhavan Venkataraman kstat_t *ct_kstats; 137387a18d3fSMadhavan Venkataraman int ndx; 137487a18d3fSMadhavan Venkataraman 137587a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 137687a18d3fSMadhavan Venkataraman ASSERT(ct->ct_kstats == NULL); 137787a18d3fSMadhavan Venkataraman 137887a18d3fSMadhavan Venkataraman ndx = ct - callout_table; 137987a18d3fSMadhavan Venkataraman ct_kstats = kstat_create("unix", ndx, "callout", 138087a18d3fSMadhavan Venkataraman "misc", KSTAT_TYPE_NAMED, CALLOUT_NUM_STATS, KSTAT_FLAG_VIRTUAL); 138187a18d3fSMadhavan Venkataraman 138287a18d3fSMadhavan Venkataraman if (ct_kstats == NULL) { 138387a18d3fSMadhavan Venkataraman cmn_err(CE_WARN, "kstat_create for callout table %p failed", 138487a18d3fSMadhavan Venkataraman (void *)ct); 138587a18d3fSMadhavan Venkataraman } else { 138687a18d3fSMadhavan Venkataraman ct_kstats->ks_data = ct->ct_kstat_data; 138787a18d3fSMadhavan Venkataraman for (stat = 0; stat < CALLOUT_NUM_STATS; stat++) 138887a18d3fSMadhavan Venkataraman kstat_named_init(&ct->ct_kstat_data[stat], 138987a18d3fSMadhavan Venkataraman callout_kstat_names[stat], KSTAT_DATA_INT64); 139087a18d3fSMadhavan Venkataraman ct->ct_kstats = ct_kstats; 139187a18d3fSMadhavan Venkataraman kstat_install(ct_kstats); 139287a18d3fSMadhavan Venkataraman } 139387a18d3fSMadhavan Venkataraman } 139487a18d3fSMadhavan Venkataraman 139587a18d3fSMadhavan Venkataraman static void 139687a18d3fSMadhavan Venkataraman callout_cyclic_init(callout_table_t *ct) 139787a18d3fSMadhavan Venkataraman { 139887a18d3fSMadhavan Venkataraman cyc_handler_t hdlr; 139987a18d3fSMadhavan Venkataraman cyc_time_t when; 140087a18d3fSMadhavan Venkataraman processorid_t seqid; 140187a18d3fSMadhavan Venkataraman int t; 140287a18d3fSMadhavan Venkataraman 140387a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&ct->ct_mutex)); 140487a18d3fSMadhavan Venkataraman 140587a18d3fSMadhavan Venkataraman t = CALLOUT_TABLE_TYPE(ct); 140687a18d3fSMadhavan Venkataraman seqid = CALLOUT_TABLE_SEQID(ct); 140787a18d3fSMadhavan Venkataraman 140887a18d3fSMadhavan Venkataraman /* 140987a18d3fSMadhavan Venkataraman * Create the taskq thread if the table type is normal. 141087a18d3fSMadhavan Venkataraman * Realtime tables are handled at PIL1 by a softint 141187a18d3fSMadhavan Venkataraman * handler. 141287a18d3fSMadhavan Venkataraman */ 14137c478bd9Sstevel@tonic-gate if (t == CALLOUT_NORMAL) { 141487a18d3fSMadhavan Venkataraman ASSERT(ct->ct_taskq == NULL); 14157c478bd9Sstevel@tonic-gate /* 14167c478bd9Sstevel@tonic-gate * Each callout thread consumes exactly one 14177c478bd9Sstevel@tonic-gate * task structure while active. Therefore, 14187c478bd9Sstevel@tonic-gate * prepopulating with 2 * CALLOUT_THREADS tasks 14197c478bd9Sstevel@tonic-gate * ensures that there's at least one task per 14207c478bd9Sstevel@tonic-gate * thread that's either scheduled or on the 14217c478bd9Sstevel@tonic-gate * freelist. In turn, this guarantees that 14227c478bd9Sstevel@tonic-gate * taskq_dispatch() will always either succeed 14237c478bd9Sstevel@tonic-gate * (because there's a free task structure) or 14247c478bd9Sstevel@tonic-gate * be unnecessary (because "callout_excute(ct)" 14257c478bd9Sstevel@tonic-gate * has already scheduled). 14267c478bd9Sstevel@tonic-gate */ 14277c478bd9Sstevel@tonic-gate ct->ct_taskq = 142887a18d3fSMadhavan Venkataraman taskq_create_instance("callout_taskq", seqid, 14297c478bd9Sstevel@tonic-gate CALLOUT_THREADS, maxclsyspri, 14307c478bd9Sstevel@tonic-gate 2 * CALLOUT_THREADS, 2 * CALLOUT_THREADS, 14317c478bd9Sstevel@tonic-gate TASKQ_PREPOPULATE | TASKQ_CPR_SAFE); 14327c478bd9Sstevel@tonic-gate } 143387a18d3fSMadhavan Venkataraman 143487a18d3fSMadhavan Venkataraman /* 143587a18d3fSMadhavan Venkataraman * callouts can only be created in a table whose 143687a18d3fSMadhavan Venkataraman * cyclic has been initialized. 143787a18d3fSMadhavan Venkataraman */ 143887a18d3fSMadhavan Venkataraman ASSERT(ct->ct_heap_num == 0); 143987a18d3fSMadhavan Venkataraman 144087a18d3fSMadhavan Venkataraman /* 144187a18d3fSMadhavan Venkataraman * Create the callout table cyclics. 144287a18d3fSMadhavan Venkataraman */ 144387a18d3fSMadhavan Venkataraman ASSERT(ct->ct_cyclic == CYCLIC_NONE); 144487a18d3fSMadhavan Venkataraman 144587a18d3fSMadhavan Venkataraman hdlr.cyh_func = (cyc_func_t)CALLOUT_CYCLIC_HANDLER(t); 1446*454ab202SMadhavan Venkataraman hdlr.cyh_level = CY_LOW_LEVEL; 144787a18d3fSMadhavan Venkataraman hdlr.cyh_arg = ct; 144887a18d3fSMadhavan Venkataraman when.cyt_when = CY_INFINITY; 144987a18d3fSMadhavan Venkataraman when.cyt_interval = CY_INFINITY; 145087a18d3fSMadhavan Venkataraman 145187a18d3fSMadhavan Venkataraman ct->ct_cyclic = cyclic_add(&hdlr, &when); 145287a18d3fSMadhavan Venkataraman } 145387a18d3fSMadhavan Venkataraman 145487a18d3fSMadhavan Venkataraman void 145587a18d3fSMadhavan Venkataraman callout_cpu_online(cpu_t *cp) 145687a18d3fSMadhavan Venkataraman { 145787a18d3fSMadhavan Venkataraman lgrp_handle_t hand; 145887a18d3fSMadhavan Venkataraman callout_cache_t *cache; 145987a18d3fSMadhavan Venkataraman char s[KMEM_CACHE_NAMELEN]; 146087a18d3fSMadhavan Venkataraman callout_table_t *ct; 146187a18d3fSMadhavan Venkataraman processorid_t seqid; 146287a18d3fSMadhavan Venkataraman int t; 146387a18d3fSMadhavan Venkataraman 146487a18d3fSMadhavan Venkataraman ASSERT(MUTEX_HELD(&cpu_lock)); 146587a18d3fSMadhavan Venkataraman 146687a18d3fSMadhavan Venkataraman /* 146787a18d3fSMadhavan Venkataraman * Locate the cache corresponding to the onlined CPU's lgroup. 146887a18d3fSMadhavan Venkataraman * Note that access to callout_caches is protected by cpu_lock. 146987a18d3fSMadhavan Venkataraman */ 147087a18d3fSMadhavan Venkataraman hand = lgrp_plat_cpu_to_hand(cp->cpu_id); 147187a18d3fSMadhavan Venkataraman for (cache = callout_caches; cache != NULL; cache = cache->cc_next) { 147287a18d3fSMadhavan Venkataraman if (cache->cc_hand == hand) 147387a18d3fSMadhavan Venkataraman break; 147487a18d3fSMadhavan Venkataraman } 147587a18d3fSMadhavan Venkataraman 147687a18d3fSMadhavan Venkataraman /* 147787a18d3fSMadhavan Venkataraman * If not found, create one. The caches are never destroyed. 147887a18d3fSMadhavan Venkataraman */ 147987a18d3fSMadhavan Venkataraman if (cache == NULL) { 148087a18d3fSMadhavan Venkataraman cache = kmem_alloc(sizeof (callout_cache_t), KM_SLEEP); 148187a18d3fSMadhavan Venkataraman cache->cc_hand = hand; 148287a18d3fSMadhavan Venkataraman (void) snprintf(s, KMEM_CACHE_NAMELEN, "callout_cache%lx", 148387a18d3fSMadhavan Venkataraman (long)hand); 148487a18d3fSMadhavan Venkataraman cache->cc_cache = kmem_cache_create(s, sizeof (callout_t), 148587a18d3fSMadhavan Venkataraman CALLOUT_ALIGN, NULL, NULL, NULL, NULL, NULL, 0); 148687a18d3fSMadhavan Venkataraman (void) snprintf(s, KMEM_CACHE_NAMELEN, "callout_lcache%lx", 148787a18d3fSMadhavan Venkataraman (long)hand); 148887a18d3fSMadhavan Venkataraman cache->cc_lcache = kmem_cache_create(s, sizeof (callout_list_t), 148987a18d3fSMadhavan Venkataraman CALLOUT_ALIGN, NULL, NULL, NULL, NULL, NULL, 0); 149087a18d3fSMadhavan Venkataraman cache->cc_next = callout_caches; 149187a18d3fSMadhavan Venkataraman callout_caches = cache; 149287a18d3fSMadhavan Venkataraman } 149387a18d3fSMadhavan Venkataraman 149487a18d3fSMadhavan Venkataraman seqid = cp->cpu_seqid; 149587a18d3fSMadhavan Venkataraman 149687a18d3fSMadhavan Venkataraman for (t = 0; t < CALLOUT_NTYPES; t++) { 149787a18d3fSMadhavan Venkataraman ct = &callout_table[CALLOUT_TABLE(t, seqid)]; 149887a18d3fSMadhavan Venkataraman 149987a18d3fSMadhavan Venkataraman mutex_enter(&ct->ct_mutex); 150087a18d3fSMadhavan Venkataraman /* 150187a18d3fSMadhavan Venkataraman * Store convinience pointers to the kmem caches 150287a18d3fSMadhavan Venkataraman * in the callout table. These assignments should always be 150387a18d3fSMadhavan Venkataraman * done as callout tables can map to different physical 150487a18d3fSMadhavan Venkataraman * CPUs each time. 150587a18d3fSMadhavan Venkataraman */ 150687a18d3fSMadhavan Venkataraman ct->ct_cache = cache->cc_cache; 150787a18d3fSMadhavan Venkataraman ct->ct_lcache = cache->cc_lcache; 150887a18d3fSMadhavan Venkataraman 150987a18d3fSMadhavan Venkataraman /* 151087a18d3fSMadhavan Venkataraman * We use the heap pointer to check if stuff has been 151187a18d3fSMadhavan Venkataraman * initialized for this callout table. 151287a18d3fSMadhavan Venkataraman */ 151387a18d3fSMadhavan Venkataraman if (ct->ct_heap == NULL) { 151487a18d3fSMadhavan Venkataraman callout_heap_init(ct); 151587a18d3fSMadhavan Venkataraman callout_hash_init(ct); 151687a18d3fSMadhavan Venkataraman callout_kstat_init(ct); 151787a18d3fSMadhavan Venkataraman callout_cyclic_init(ct); 151887a18d3fSMadhavan Venkataraman } 151987a18d3fSMadhavan Venkataraman 152087a18d3fSMadhavan Venkataraman mutex_exit(&ct->ct_mutex); 152187a18d3fSMadhavan Venkataraman 152287a18d3fSMadhavan Venkataraman /* 1523*454ab202SMadhavan Venkataraman * Move the cyclic to this CPU by doing a bind. 152487a18d3fSMadhavan Venkataraman */ 152587a18d3fSMadhavan Venkataraman cyclic_bind(ct->ct_cyclic, cp, NULL); 1526*454ab202SMadhavan Venkataraman } 1527*454ab202SMadhavan Venkataraman } 1528*454ab202SMadhavan Venkataraman 1529*454ab202SMadhavan Venkataraman void 1530*454ab202SMadhavan Venkataraman callout_cpu_offline(cpu_t *cp) 1531*454ab202SMadhavan Venkataraman { 1532*454ab202SMadhavan Venkataraman callout_table_t *ct; 1533*454ab202SMadhavan Venkataraman processorid_t seqid; 1534*454ab202SMadhavan Venkataraman int t; 1535*454ab202SMadhavan Venkataraman 1536*454ab202SMadhavan Venkataraman ASSERT(MUTEX_HELD(&cpu_lock)); 1537*454ab202SMadhavan Venkataraman 1538*454ab202SMadhavan Venkataraman seqid = cp->cpu_seqid; 1539*454ab202SMadhavan Venkataraman 1540*454ab202SMadhavan Venkataraman for (t = 0; t < CALLOUT_NTYPES; t++) { 1541*454ab202SMadhavan Venkataraman ct = &callout_table[CALLOUT_TABLE(t, seqid)]; 1542*454ab202SMadhavan Venkataraman 1543*454ab202SMadhavan Venkataraman /* 1544*454ab202SMadhavan Venkataraman * Unbind the cyclic. This will allow the cyclic subsystem 1545*454ab202SMadhavan Venkataraman * to juggle the cyclic during CPU offline. 1546*454ab202SMadhavan Venkataraman */ 154787a18d3fSMadhavan Venkataraman cyclic_bind(ct->ct_cyclic, NULL, NULL); 15487c478bd9Sstevel@tonic-gate } 15497c478bd9Sstevel@tonic-gate } 155087a18d3fSMadhavan Venkataraman 155187a18d3fSMadhavan Venkataraman /* 155287a18d3fSMadhavan Venkataraman * This is called to perform per-CPU initialization for slave CPUs at 155387a18d3fSMadhavan Venkataraman * boot time. 155487a18d3fSMadhavan Venkataraman */ 155587a18d3fSMadhavan Venkataraman void 155687a18d3fSMadhavan Venkataraman callout_mp_init(void) 155787a18d3fSMadhavan Venkataraman { 155887a18d3fSMadhavan Venkataraman cpu_t *cp; 155987a18d3fSMadhavan Venkataraman 156087a18d3fSMadhavan Venkataraman mutex_enter(&cpu_lock); 156187a18d3fSMadhavan Venkataraman 156287a18d3fSMadhavan Venkataraman cp = cpu_active; 156387a18d3fSMadhavan Venkataraman do { 156487a18d3fSMadhavan Venkataraman callout_cpu_online(cp); 156587a18d3fSMadhavan Venkataraman } while ((cp = cp->cpu_next_onln) != cpu_active); 156687a18d3fSMadhavan Venkataraman 156787a18d3fSMadhavan Venkataraman mutex_exit(&cpu_lock); 156887a18d3fSMadhavan Venkataraman } 156987a18d3fSMadhavan Venkataraman 157087a18d3fSMadhavan Venkataraman /* 157187a18d3fSMadhavan Venkataraman * Initialize all callout tables. Called at boot time just before clkstart(). 157287a18d3fSMadhavan Venkataraman */ 157387a18d3fSMadhavan Venkataraman void 157487a18d3fSMadhavan Venkataraman callout_init(void) 157587a18d3fSMadhavan Venkataraman { 157687a18d3fSMadhavan Venkataraman int f, t; 157787a18d3fSMadhavan Venkataraman size_t size; 157887a18d3fSMadhavan Venkataraman int table_id; 157987a18d3fSMadhavan Venkataraman callout_table_t *ct; 158087a18d3fSMadhavan Venkataraman long bits, fanout; 158187a18d3fSMadhavan Venkataraman uintptr_t buf; 158287a18d3fSMadhavan Venkataraman 158387a18d3fSMadhavan Venkataraman /* 158487a18d3fSMadhavan Venkataraman * Initialize callout globals. 158587a18d3fSMadhavan Venkataraman */ 158687a18d3fSMadhavan Venkataraman bits = 0; 158787a18d3fSMadhavan Venkataraman for (fanout = 1; (fanout < max_ncpus); fanout <<= 1) 158887a18d3fSMadhavan Venkataraman bits++; 158987a18d3fSMadhavan Venkataraman callout_table_bits = CALLOUT_TYPE_BITS + bits; 159087a18d3fSMadhavan Venkataraman callout_table_mask = (1 << callout_table_bits) - 1; 159187a18d3fSMadhavan Venkataraman callout_counter_low = 1 << CALLOUT_COUNTER_SHIFT; 159287a18d3fSMadhavan Venkataraman callout_longterm = TICK_TO_NSEC(CALLOUT_LONGTERM_TICKS); 1593*454ab202SMadhavan Venkataraman callout_max_ticks = CALLOUT_MAX_TICKS; 159487a18d3fSMadhavan Venkataraman 159587a18d3fSMadhavan Venkataraman /* 159687a18d3fSMadhavan Venkataraman * Because of the variability in timing behavior across systems with 159787a18d3fSMadhavan Venkataraman * different architectures, we cannot allow arbitrarily low 159887a18d3fSMadhavan Venkataraman * resolutions. The minimum resolution has to be determined in a 159987a18d3fSMadhavan Venkataraman * platform-specific way. Until then, we define a blanket minimum 160087a18d3fSMadhavan Venkataraman * resolution for callouts of CALLOUT_MIN_RESOLUTION. 160187a18d3fSMadhavan Venkataraman * 160287a18d3fSMadhavan Venkataraman * If, in the future, someone requires lower resolution timers, they 160387a18d3fSMadhavan Venkataraman * can do one of two things: 160487a18d3fSMadhavan Venkataraman * 160587a18d3fSMadhavan Venkataraman * - Define a lower value for callout_min_resolution. This would 160687a18d3fSMadhavan Venkataraman * affect all clients of the callout subsystem. If this done 160787a18d3fSMadhavan Venkataraman * via /etc/system, then no code changes are required and it 160887a18d3fSMadhavan Venkataraman * would affect only that customer. 160987a18d3fSMadhavan Venkataraman * 161087a18d3fSMadhavan Venkataraman * - Define a flag to be passed to timeout creation that allows 161187a18d3fSMadhavan Venkataraman * the lower resolution. This involves code changes. But it 161287a18d3fSMadhavan Venkataraman * would affect only the calling module. It is the developer's 161387a18d3fSMadhavan Venkataraman * responsibility to test on all systems and make sure that 161487a18d3fSMadhavan Venkataraman * everything works. 161587a18d3fSMadhavan Venkataraman */ 161687a18d3fSMadhavan Venkataraman if (callout_min_resolution <= 0) 161787a18d3fSMadhavan Venkataraman callout_min_resolution = CALLOUT_MIN_RESOLUTION; 161887a18d3fSMadhavan Venkataraman 161987a18d3fSMadhavan Venkataraman /* 162087a18d3fSMadhavan Venkataraman * Allocate all the callout tables based on max_ncpus. We have chosen 162187a18d3fSMadhavan Venkataraman * to do boot-time allocation instead of dynamic allocation because: 162287a18d3fSMadhavan Venkataraman * 162387a18d3fSMadhavan Venkataraman * - the size of the callout tables is not too large. 162487a18d3fSMadhavan Venkataraman * - there are race conditions involved in making this dynamic. 162587a18d3fSMadhavan Venkataraman * - the hash tables that go with the callout tables consume 162687a18d3fSMadhavan Venkataraman * most of the memory and they are only allocated in 162787a18d3fSMadhavan Venkataraman * callout_cpu_online(). 162887a18d3fSMadhavan Venkataraman * 162987a18d3fSMadhavan Venkataraman * Each CPU has two tables that are consecutive in the array. The first 163087a18d3fSMadhavan Venkataraman * one is for realtime callouts and the second one is for normal ones. 163187a18d3fSMadhavan Venkataraman * 163287a18d3fSMadhavan Venkataraman * We do this alignment dance to make sure that callout table 163387a18d3fSMadhavan Venkataraman * structures will always be on a cache line boundary. 163487a18d3fSMadhavan Venkataraman */ 163587a18d3fSMadhavan Venkataraman size = sizeof (callout_table_t) * CALLOUT_NTYPES * max_ncpus; 163687a18d3fSMadhavan Venkataraman size += CALLOUT_ALIGN; 163787a18d3fSMadhavan Venkataraman buf = (uintptr_t)kmem_zalloc(size, KM_SLEEP); 163887a18d3fSMadhavan Venkataraman callout_table = (callout_table_t *)P2ROUNDUP(buf, CALLOUT_ALIGN); 163987a18d3fSMadhavan Venkataraman 164087a18d3fSMadhavan Venkataraman size = sizeof (kstat_named_t) * CALLOUT_NUM_STATS; 164187a18d3fSMadhavan Venkataraman /* 164287a18d3fSMadhavan Venkataraman * Now, initialize the tables for all the CPUs. 164387a18d3fSMadhavan Venkataraman */ 164487a18d3fSMadhavan Venkataraman for (f = 0; f < max_ncpus; f++) { 164587a18d3fSMadhavan Venkataraman for (t = 0; t < CALLOUT_NTYPES; t++) { 164687a18d3fSMadhavan Venkataraman table_id = CALLOUT_TABLE(t, f); 164787a18d3fSMadhavan Venkataraman ct = &callout_table[table_id]; 1648*454ab202SMadhavan Venkataraman ct->ct_type = t; 164987a18d3fSMadhavan Venkataraman mutex_init(&ct->ct_mutex, NULL, MUTEX_DEFAULT, NULL); 165087a18d3fSMadhavan Venkataraman /* 165187a18d3fSMadhavan Venkataraman * Precompute the base IDs for long and short-term 165287a18d3fSMadhavan Venkataraman * legacy IDs. This makes ID generation during 165387a18d3fSMadhavan Venkataraman * timeout() fast. 165487a18d3fSMadhavan Venkataraman */ 165587a18d3fSMadhavan Venkataraman ct->ct_short_id = CALLOUT_SHORT_ID(table_id); 165687a18d3fSMadhavan Venkataraman ct->ct_long_id = CALLOUT_LONG_ID(table_id); 165787a18d3fSMadhavan Venkataraman /* 165887a18d3fSMadhavan Venkataraman * Precompute the base ID for generation-based IDs. 165987a18d3fSMadhavan Venkataraman * Note that when the first ID gets allocated, the 166087a18d3fSMadhavan Venkataraman * ID will wrap. This will cause the generation 166187a18d3fSMadhavan Venkataraman * number to be incremented to 1. 166287a18d3fSMadhavan Venkataraman */ 166387a18d3fSMadhavan Venkataraman ct->ct_gen_id = CALLOUT_SHORT_ID(table_id); 166487a18d3fSMadhavan Venkataraman /* 166587a18d3fSMadhavan Venkataraman * Initialize the cyclic as NONE. This will get set 166687a18d3fSMadhavan Venkataraman * during CPU online. This is so that partially 166787a18d3fSMadhavan Venkataraman * populated systems will only have the required 166887a18d3fSMadhavan Venkataraman * number of cyclics, not more. 166987a18d3fSMadhavan Venkataraman */ 167087a18d3fSMadhavan Venkataraman ct->ct_cyclic = CYCLIC_NONE; 167187a18d3fSMadhavan Venkataraman ct->ct_kstat_data = kmem_zalloc(size, KM_SLEEP); 167287a18d3fSMadhavan Venkataraman } 167387a18d3fSMadhavan Venkataraman } 167487a18d3fSMadhavan Venkataraman 167587a18d3fSMadhavan Venkataraman /* 167687a18d3fSMadhavan Venkataraman * Add the callback for CPR. This is called during checkpoint 167787a18d3fSMadhavan Venkataraman * resume to suspend and resume callouts. 167887a18d3fSMadhavan Venkataraman */ 167987a18d3fSMadhavan Venkataraman (void) callb_add(callout_cpr_callb, 0, CB_CL_CPR_CALLOUT, 168087a18d3fSMadhavan Venkataraman "callout_cpr"); 168187a18d3fSMadhavan Venkataraman (void) callb_add(callout_debug_callb, 0, CB_CL_ENTER_DEBUGGER, 168287a18d3fSMadhavan Venkataraman "callout_debug"); 168387a18d3fSMadhavan Venkataraman 168487a18d3fSMadhavan Venkataraman /* 168587a18d3fSMadhavan Venkataraman * Call the per-CPU initialization function for the boot CPU. This 168687a18d3fSMadhavan Venkataraman * is done here because the function is not called automatically for 168787a18d3fSMadhavan Venkataraman * the boot CPU from the CPU online/offline hooks. Note that the 168887a18d3fSMadhavan Venkataraman * CPU lock is taken here because of convention. 168987a18d3fSMadhavan Venkataraman */ 169087a18d3fSMadhavan Venkataraman mutex_enter(&cpu_lock); 169187a18d3fSMadhavan Venkataraman callout_boot_ct = &callout_table[CALLOUT_TABLE(0, CPU->cpu_seqid)]; 169287a18d3fSMadhavan Venkataraman callout_cpu_online(CPU); 169387a18d3fSMadhavan Venkataraman mutex_exit(&cpu_lock); 16947c478bd9Sstevel@tonic-gate } 1695