1 /* $NetBSD: cyclic.c,v 1.5 2016/04/09 14:50:08 riastradh Exp $ */
2
3 /*
4 * CDDL HEADER START
5 *
6 * The contents of this file are subject to the terms of the
7 * Common Development and Distribution License, Version 1.0 only
8 * (the "License"). You may not use this file except in compliance
9 * with the License.
10 *
11 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
12 * or http://www.opensolaris.org/os/licensing.
13 * See the License for the specific language governing permissions
14 * and limitations under the License.
15 *
16 * When distributing Covered Code, include this CDDL HEADER in each
17 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
18 * If applicable, add the following below this CDDL HEADER, with the
19 * fields enclosed by brackets "[]" replaced with your own identifying
20 * information: Portions Copyright [yyyy] [name of copyright owner]
21 *
22 * CDDL HEADER END
23 *
24 * Portions Copyright 2008 John Birrell <jb@freebsd.org>
25 *
26 * $FreeBSD$
27 *
28 * This is a simplified version of the cyclic timer subsystem from
29 * OpenSolaris. In the FreeBSD version, we don't use interrupt levels.
30 */
31
32 /*
33 * Copyright 2004 Sun Microsystems, Inc. All rights reserved.
34 * Use is subject to license terms.
35 */
36
37 /*
38 * The Cyclic Subsystem
39 * --------------------
40 *
41 * Prehistory
42 *
43 * Historically, most computer architectures have specified interval-based
44 * timer parts (e.g. SPARCstation's counter/timer; Intel's i8254). While
45 * these parts deal in relative (i.e. not absolute) time values, they are
46 * typically used by the operating system to implement the abstraction of
47 * absolute time. As a result, these parts cannot typically be reprogrammed
48 * without introducing error in the system's notion of time.
49 *
50 * Starting in about 1994, chip architectures began specifying high resolution
51 * timestamp registers. As of this writing (1999), all major chip families
52 * (UltraSPARC, PentiumPro, MIPS, PowerPC, Alpha) have high resolution
53 * timestamp registers, and two (UltraSPARC and MIPS) have added the capacity
54 * to interrupt based on timestamp values. These timestamp-compare registers
55 * present a time-based interrupt source which can be reprogrammed arbitrarily
56 * often without introducing error. Given the low cost of implementing such a
57 * timestamp-compare register (and the tangible benefit of eliminating
58 * discrete timer parts), it is reasonable to expect that future chip
59 * architectures will adopt this feature.
60 *
61 * The cyclic subsystem has been designed to take advantage of chip
62 * architectures with the capacity to interrupt based on absolute, high
63 * resolution values of time.
64 *
65 * Subsystem Overview
66 *
67 * The cyclic subsystem is a low-level kernel subsystem designed to provide
68 * arbitrarily high resolution, per-CPU interval timers (to avoid colliding
69 * with existing terms, we dub such an interval timer a "cyclic").
70 * Alternatively, a cyclic may be specified to be "omnipresent", denoting
71 * firing on all online CPUs.
72 *
73 * Cyclic Subsystem Interface Overview
74 * -----------------------------------
75 *
76 * The cyclic subsystem has interfaces with the kernel at-large, with other
77 * kernel subsystems (e.g. the processor management subsystem, the checkpoint
78 * resume subsystem) and with the platform (the cyclic backend). Each
79 * of these interfaces is given a brief synopsis here, and is described
80 * in full above the interface's implementation.
81 *
82 * The following diagram displays the cyclic subsystem's interfaces to
83 * other kernel components. The arrows denote a "calls" relationship, with
84 * the large arrow indicating the cyclic subsystem's consumer interface.
85 * Each arrow is labeled with the section in which the corresponding
86 * interface is described.
87 *
88 * Kernel at-large consumers
89 * -----------++------------
90 * ||
91 * ||
92 * _||_
93 * \ /
94 * \/
95 * +---------------------+
96 * | |
97 * | Cyclic subsystem |<----------- Other kernel subsystems
98 * | |
99 * +---------------------+
100 * ^ |
101 * | |
102 * | |
103 * | v
104 * +---------------------+
105 * | |
106 * | Cyclic backend |
107 * | (platform specific) |
108 * | |
109 * +---------------------+
110 *
111 *
112 * Kernel At-Large Interfaces
113 *
114 * cyclic_add() <-- Creates a cyclic
115 * cyclic_add_omni() <-- Creates an omnipresent cyclic
116 * cyclic_remove() <-- Removes a cyclic
117 *
118 * Backend Interfaces
119 *
120 * cyclic_init() <-- Initializes the cyclic subsystem
121 * cyclic_fire() <-- Interrupt entry point
122 *
123 * The backend-supplied interfaces (through the cyc_backend structure) are
124 * documented in detail in <sys/cyclic_impl.h>
125 *
126 *
127 * Cyclic Subsystem Implementation Overview
128 * ----------------------------------------
129 *
130 * The cyclic subsystem is designed to minimize interference between cyclics
131 * on different CPUs. Thus, all of the cyclic subsystem's data structures
132 * hang off of a per-CPU structure, cyc_cpu.
133 *
134 * Each cyc_cpu has a power-of-two sized array of cyclic structures (the
135 * cyp_cyclics member of the cyc_cpu structure). If cyclic_add() is called
136 * and there does not exist a free slot in the cyp_cyclics array, the size of
137 * the array will be doubled. The array will never shrink. Cyclics are
138 * referred to by their index in the cyp_cyclics array, which is of type
139 * cyc_index_t.
140 *
141 * The cyclics are kept sorted by expiration time in the cyc_cpu's heap. The
142 * heap is keyed by cyclic expiration time, with parents expiring earlier
143 * than their children.
144 *
145 * Heap Management
146 *
147 * The heap is managed primarily by cyclic_fire(). Upon entry, cyclic_fire()
148 * compares the root cyclic's expiration time to the current time. If the
149 * expiration time is in the past, cyclic_expire() is called on the root
150 * cyclic. Upon return from cyclic_expire(), the cyclic's new expiration time
151 * is derived by adding its interval to its old expiration time, and a
152 * downheap operation is performed. After the downheap, cyclic_fire()
153 * examines the (potentially changed) root cyclic, repeating the
154 * cyclic_expire()/add interval/cyclic_downheap() sequence until the root
155 * cyclic has an expiration time in the future. This expiration time
156 * (guaranteed to be the earliest in the heap) is then communicated to the
157 * backend via cyb_reprogram. Optimal backends will next call cyclic_fire()
158 * shortly after the root cyclic's expiration time.
159 *
160 * To allow efficient, deterministic downheap operations, we implement the
161 * heap as an array (the cyp_heap member of the cyc_cpu structure), with each
162 * element containing an index into the CPU's cyp_cyclics array.
163 *
164 * The heap is laid out in the array according to the following:
165 *
166 * 1. The root of the heap is always in the 0th element of the heap array
167 * 2. The left and right children of the nth element are element
168 * (((n + 1) << 1) - 1) and element ((n + 1) << 1), respectively.
169 *
170 * This layout is standard (see, e.g., Cormen's "Algorithms"); the proof
171 * that these constraints correctly lay out a heap (or indeed, any binary
172 * tree) is trivial and left to the reader.
173 *
174 * To see the heap by example, assume our cyclics array has the following
175 * members (at time t):
176 *
177 * cy_handler cy_expire
178 * ---------------------------------------------
179 * [ 0] clock() t+10000000
180 * [ 1] deadman() t+1000000000
181 * [ 2] clock_highres_fire() t+100
182 * [ 3] clock_highres_fire() t+1000
183 * [ 4] clock_highres_fire() t+500
184 * [ 5] (free) --
185 * [ 6] (free) --
186 * [ 7] (free) --
187 *
188 * The heap array could be:
189 *
190 * [0] [1] [2] [3] [4] [5] [6] [7]
191 * +-----+-----+-----+-----+-----+-----+-----+-----+
192 * | | | | | | | | |
193 * | 2 | 3 | 4 | 0 | 1 | x | x | x |
194 * | | | | | | | | |
195 * +-----+-----+-----+-----+-----+-----+-----+-----+
196 *
197 * Graphically, this array corresponds to the following (excuse the ASCII art):
198 *
199 * 2
200 * |
201 * +------------------+------------------+
202 * 3 4
203 * |
204 * +---------+--------+
205 * 0 1
206 *
207 * Note that the heap is laid out by layer: all nodes at a given depth are
208 * stored in consecutive elements of the array. Moreover, layers of
209 * consecutive depths are in adjacent element ranges. This property
210 * guarantees high locality of reference during downheap operations.
211 * Specifically, we are guaranteed that we can downheap to a depth of
212 *
213 * lg (cache_line_size / sizeof (cyc_index_t))
214 *
215 * nodes with at most one cache miss. On UltraSPARC (64 byte e-cache line
216 * size), this corresponds to a depth of four nodes. Thus, if there are
217 * fewer than sixteen cyclics in the heap, downheaps on UltraSPARC miss at
218 * most once in the e-cache.
219 *
220 * Downheaps are required to compare siblings as they proceed down the
221 * heap. For downheaps proceeding beyond the one-cache-miss depth, every
222 * access to a left child could potentially miss in the cache. However,
223 * if we assume
224 *
225 * (cache_line_size / sizeof (cyc_index_t)) > 2,
226 *
227 * then all siblings are guaranteed to be on the same cache line. Thus, the
228 * miss on the left child will guarantee a hit on the right child; downheaps
229 * will incur at most one cache miss per layer beyond the one-cache-miss
230 * depth. The total number of cache misses for heap management during a
231 * downheap operation is thus bounded by
232 *
233 * lg (n) - lg (cache_line_size / sizeof (cyc_index_t))
234 *
235 * Traditional pointer-based heaps are implemented without regard to
236 * locality. Downheaps can thus incur two cache misses per layer (one for
237 * each child), but at most one cache miss at the root. This yields a bound
238 * of
239 *
240 * 2 * lg (n) - 1
241 *
242 * on the total cache misses.
243 *
244 * This difference may seem theoretically trivial (the difference is, after
245 * all, constant), but can become substantial in practice -- especially for
246 * caches with very large cache lines and high miss penalties (e.g. TLBs).
247 *
248 * Heaps must always be full, balanced trees. Heap management must therefore
249 * track the next point-of-insertion into the heap. In pointer-based heaps,
250 * recomputing this point takes O(lg (n)). Given the layout of the
251 * array-based implementation, however, the next point-of-insertion is
252 * always:
253 *
254 * heap[number_of_elements]
255 *
256 * We exploit this property by implementing the free-list in the usused
257 * heap elements. Heap insertion, therefore, consists only of filling in
258 * the cyclic at cyp_cyclics[cyp_heap[number_of_elements]], incrementing
259 * the number of elements, and performing an upheap. Heap deletion consists
260 * of decrementing the number of elements, swapping the to-be-deleted element
261 * with the element at cyp_heap[number_of_elements], and downheaping.
262 *
263 * Filling in more details in our earlier example:
264 *
265 * +--- free list head
266 * |
267 * V
268 *
269 * [0] [1] [2] [3] [4] [5] [6] [7]
270 * +-----+-----+-----+-----+-----+-----+-----+-----+
271 * | | | | | | | | |
272 * | 2 | 3 | 4 | 0 | 1 | 5 | 6 | 7 |
273 * | | | | | | | | |
274 * +-----+-----+-----+-----+-----+-----+-----+-----+
275 *
276 * To insert into this heap, we would just need to fill in the cyclic at
277 * cyp_cyclics[5], bump the number of elements (from 5 to 6) and perform
278 * an upheap.
279 *
280 * If we wanted to remove, say, cyp_cyclics[3], we would first scan for it
281 * in the cyp_heap, and discover it at cyp_heap[1]. We would then decrement
282 * the number of elements (from 5 to 4), swap cyp_heap[1] with cyp_heap[4],
283 * and perform a downheap from cyp_heap[1]. The linear scan is required
284 * because the cyclic does not keep a backpointer into the heap. This makes
285 * heap manipulation (e.g. downheaps) faster at the expense of removal
286 * operations.
287 *
288 * Expiry processing
289 *
290 * As alluded to above, cyclic_expire() is called by cyclic_fire() to expire
291 * a cyclic. Cyclic subsystem consumers are guaranteed that for an arbitrary
292 * time t in the future, their cyclic handler will have been called
293 * (t - cyt_when) / cyt_interval times. cyclic_expire() simply needs to call
294 * the handler.
295 *
296 * Resizing
297 *
298 * All of the discussion thus far has assumed a static number of cyclics.
299 * Obviously, static limitations are not practical; we need the capacity
300 * to resize our data structures dynamically.
301 *
302 * We resize our data structures lazily, and only on a per-CPU basis.
303 * The size of the data structures always doubles and never shrinks. We
304 * serialize adds (and thus resizes) on cpu_lock; we never need to deal
305 * with concurrent resizes. Resizes should be rare; they may induce jitter
306 * on the CPU being resized, but should not affect cyclic operation on other
307 * CPUs.
308 *
309 * Three key cyc_cpu data structures need to be resized: the cyclics array,
310 * nad the heap array. Resizing is relatively straightforward:
311 *
312 * 1. The new, larger arrays are allocated in cyclic_expand() (called
313 * from cyclic_add()).
314 * 2. The contents of the old arrays are copied into the new arrays.
315 * 3. The old cyclics array is bzero()'d
316 * 4. The pointers are updated.
317 *
318 * Removals
319 *
320 * Cyclic removals should be rare. To simplify the implementation (and to
321 * allow optimization for the cyclic_fire()/cyclic_expire()
322 * path), we force removals and adds to serialize on cpu_lock.
323 *
324 */
325 #include <sys/cdefs.h>
326 #include <sys/param.h>
327 #include <sys/conf.h>
328 #include <sys/kernel.h>
329 #ifdef __FreeBSD___
330 #include <sys/lock.h>
331 #include <sys/sx.h>
332 #endif
333 #include <sys/cyclic_impl.h>
334 #include <sys/module.h>
335 #include <sys/systm.h>
336 #include <sys/atomic.h>
337 #include <sys/kmem.h>
338 #include <sys/cmn_err.h>
339 #include <sys/dtrace_bsd.h>
340 #ifdef __FreeBSD__
341 #include <machine/cpu.h>
342 #endif
343
344 #ifdef __NetBSD__
345 #include <sys/cpu.h>
346 #include <sys/malloc.h>
347 #include <sys/xcall.h>
348
349 #undef mutex_init
350 #define mtx_init(m, d, p, f) mutex_init(m, MUTEX_DEFAULT, IPL_CLOCK)
351 #define mtx_lock_spin(x) mutex_spin_enter(x)
352 #define mtx_unlock_spin(x) mutex_spin_exit(x)
353 #define mtx_destroy(x) mutex_destroy(x)
354
355 #define ASSERT(x) KASSERT(x)
356 #define SYSINIT(a1, a2, a3, a4, a5)
357 #define SYSUNINIT(a1, a2, a3, a4, a5)
358 #define CPU_FOREACH(var) \
359 CPU_INFO_ITERATOR cii; \
360 struct cpu_info *ci; \
361 for (CPU_INFO_FOREACH(cii, ci))
362 #define MAXCPU MAXCPUS
363 #define TRAPF_USERMODE(x) CLKF_USERMODE(x)
364 #define TRAPF_PC(x) CLKF_PC(x)
365 #endif
366
367 static kmem_cache_t *cyclic_id_cache;
368 static cyc_id_t *cyclic_id_head;
369 static cyc_backend_t cyclic_backend;
370
371 MALLOC_DEFINE(M_CYCLIC, "cyclic", "Cyclic timer subsystem");
372
373 /*
374 * Returns 1 if the upheap propagated to the root, 0 if it did not. This
375 * allows the caller to reprogram the backend only when the root has been
376 * modified.
377 */
378 static int
cyclic_upheap(cyc_cpu_t * cpu,cyc_index_t ndx)379 cyclic_upheap(cyc_cpu_t *cpu, cyc_index_t ndx)
380 {
381 cyclic_t *cyclics;
382 cyc_index_t *heap;
383 cyc_index_t heap_parent, heap_current = ndx;
384 cyc_index_t parent, current;
385
386 if (heap_current == 0)
387 return (1);
388
389 heap = cpu->cyp_heap;
390 cyclics = cpu->cyp_cyclics;
391 heap_parent = CYC_HEAP_PARENT(heap_current);
392
393 for (;;) {
394 current = heap[heap_current];
395 parent = heap[heap_parent];
396
397 /*
398 * We have an expiration time later than our parent; we're
399 * done.
400 */
401 if (cyclics[current].cy_expire >= cyclics[parent].cy_expire)
402 return (0);
403
404 /*
405 * We need to swap with our parent, and continue up the heap.
406 */
407 heap[heap_parent] = current;
408 heap[heap_current] = parent;
409
410 /*
411 * If we just reached the root, we're done.
412 */
413 if (heap_parent == 0)
414 return (1);
415
416 heap_current = heap_parent;
417 heap_parent = CYC_HEAP_PARENT(heap_current);
418 }
419 }
420
421 static void
cyclic_downheap(cyc_cpu_t * cpu,cyc_index_t ndx)422 cyclic_downheap(cyc_cpu_t *cpu, cyc_index_t ndx)
423 {
424 cyclic_t *cyclics = cpu->cyp_cyclics;
425 cyc_index_t *heap = cpu->cyp_heap;
426
427 cyc_index_t heap_left, heap_right, heap_me = ndx;
428 cyc_index_t left, right, me;
429 cyc_index_t nelems = cpu->cyp_nelems;
430
431 for (;;) {
432 /*
433 * If we don't have a left child (i.e., we're a leaf), we're
434 * done.
435 */
436 if ((heap_left = CYC_HEAP_LEFT(heap_me)) >= nelems)
437 return;
438
439 left = heap[heap_left];
440 me = heap[heap_me];
441
442 heap_right = CYC_HEAP_RIGHT(heap_me);
443
444 /*
445 * Even if we don't have a right child, we still need to compare
446 * our expiration time against that of our left child.
447 */
448 if (heap_right >= nelems)
449 goto comp_left;
450
451 right = heap[heap_right];
452
453 /*
454 * We have both a left and a right child. We need to compare
455 * the expiration times of the children to determine which
456 * expires earlier.
457 */
458 if (cyclics[right].cy_expire < cyclics[left].cy_expire) {
459 /*
460 * Our right child is the earlier of our children.
461 * We'll now compare our expiration time to its; if
462 * ours is the earlier, we're done.
463 */
464 if (cyclics[me].cy_expire <= cyclics[right].cy_expire)
465 return;
466
467 /*
468 * Our right child expires earlier than we do; swap
469 * with our right child, and descend right.
470 */
471 heap[heap_right] = me;
472 heap[heap_me] = right;
473 heap_me = heap_right;
474 continue;
475 }
476
477 comp_left:
478 /*
479 * Our left child is the earlier of our children (or we have
480 * no right child). We'll now compare our expiration time
481 * to its; if ours is the earlier, we're done.
482 */
483 if (cyclics[me].cy_expire <= cyclics[left].cy_expire)
484 return;
485
486 /*
487 * Our left child expires earlier than we do; swap with our
488 * left child, and descend left.
489 */
490 heap[heap_left] = me;
491 heap[heap_me] = left;
492 heap_me = heap_left;
493 }
494 }
495
496 static void
cyclic_expire(cyc_cpu_t * cpu,cyc_index_t ndx,cyclic_t * cyclic)497 cyclic_expire(cyc_cpu_t *cpu, cyc_index_t ndx, cyclic_t *cyclic)
498 {
499 cyc_func_t handler = cyclic->cy_handler;
500 void *arg = cyclic->cy_arg;
501
502 (*handler)(arg);
503 }
504
505 /*
506 * cyclic_fire(cpu_t *)
507 *
508 * Overview
509 *
510 * cyclic_fire() is the cyclic subsystem's interrupt handler.
511 * Called by the cyclic backend.
512 *
513 * Arguments and notes
514 *
515 * The only argument is the CPU on which the interrupt is executing;
516 * backends must call into cyclic_fire() on the specified CPU.
517 *
518 * cyclic_fire() may be called spuriously without ill effect. Optimal
519 * backends will call into cyclic_fire() at or shortly after the time
520 * requested via cyb_reprogram(). However, calling cyclic_fire()
521 * arbitrarily late will only manifest latency bubbles; the correctness
522 * of the cyclic subsystem does not rely on the timeliness of the backend.
523 *
524 * cyclic_fire() is wait-free; it will not block or spin.
525 *
526 * Return values
527 *
528 * None.
529 *
530 */
531 static void
cyclic_fire(cpu_t * c)532 cyclic_fire(cpu_t *c)
533 {
534 cyc_cpu_t *cpu = c->cpu_cyclic;
535 cyc_backend_t *be = cpu->cyp_backend;
536 cyc_index_t *heap = cpu->cyp_heap;
537 cyclic_t *cyclic, *cyclics = cpu->cyp_cyclics;
538 void *arg = be->cyb_arg;
539 hrtime_t now = gethrtime();
540 hrtime_t exp;
541
542 if (cpu->cyp_nelems == 0) {
543 /* This is a spurious fire. */
544 return;
545 }
546
547 for (;;) {
548 cyc_index_t ndx = heap[0];
549
550 cyclic = &cyclics[ndx];
551
552 ASSERT(!(cyclic->cy_flags & CYF_FREE));
553
554 if ((exp = cyclic->cy_expire) > now)
555 break;
556
557 cyclic_expire(cpu, ndx, cyclic);
558
559 /*
560 * If this cyclic will be set to next expire in the distant
561 * past, we have one of two situations:
562 *
563 * a) This is the first firing of a cyclic which had
564 * cy_expire set to 0.
565 *
566 * b) We are tragically late for a cyclic -- most likely
567 * due to being in the debugger.
568 *
569 * In either case, we set the new expiration time to be the
570 * the next interval boundary. This assures that the
571 * expiration time modulo the interval is invariant.
572 *
573 * We arbitrarily define "distant" to be one second (one second
574 * is chosen because it's shorter than any foray to the
575 * debugger while still being longer than any legitimate
576 * stretch).
577 */
578 exp += cyclic->cy_interval;
579
580 if (now - exp > NANOSEC) {
581 hrtime_t interval = cyclic->cy_interval;
582
583 exp += ((now - exp) / interval + 1) * interval;
584 }
585
586 cyclic->cy_expire = exp;
587 cyclic_downheap(cpu, 0);
588 }
589
590 /*
591 * Now we have a cyclic in the root slot which isn't in the past;
592 * reprogram the interrupt source.
593 */
594 be->cyb_reprogram(arg, exp);
595 }
596
597 static void
cyclic_expand_xcall(cyc_xcallarg_t * arg)598 cyclic_expand_xcall(cyc_xcallarg_t *arg)
599 {
600 cyc_cpu_t *cpu = arg->cyx_cpu;
601 cyc_index_t new_size = arg->cyx_size, size = cpu->cyp_size, i;
602 cyc_index_t *new_heap = arg->cyx_heap;
603 cyclic_t *cyclics = cpu->cyp_cyclics, *new_cyclics = arg->cyx_cyclics;
604
605 /* Disable preemption and interrupts. */
606 mtx_lock_spin(&cpu->cyp_mtx);
607
608 /*
609 * Assert that the new size is a power of 2.
610 */
611 ASSERT((new_size & (new_size - 1)) == 0);
612 ASSERT(new_size == (size << 1));
613 ASSERT(cpu->cyp_heap != NULL && cpu->cyp_cyclics != NULL);
614
615 bcopy(cpu->cyp_heap, new_heap, sizeof (cyc_index_t) * size);
616 bcopy(cyclics, new_cyclics, sizeof (cyclic_t) * size);
617
618 /*
619 * Set up the free list, and set all of the new cyclics to be CYF_FREE.
620 */
621 for (i = size; i < new_size; i++) {
622 new_heap[i] = i;
623 new_cyclics[i].cy_flags = CYF_FREE;
624 }
625
626 /*
627 * We can go ahead and plow the value of cyp_heap and cyp_cyclics;
628 * cyclic_expand() has kept a copy.
629 */
630 cpu->cyp_heap = new_heap;
631 cpu->cyp_cyclics = new_cyclics;
632 cpu->cyp_size = new_size;
633 mtx_unlock_spin(&cpu->cyp_mtx);
634 }
635
636 /*
637 * cyclic_expand() will cross call onto the CPU to perform the actual
638 * expand operation.
639 */
640 static void
cyclic_expand(cyc_cpu_t * cpu)641 cyclic_expand(cyc_cpu_t *cpu)
642 {
643 cyc_index_t new_size, old_size;
644 cyc_index_t *new_heap, *old_heap;
645 cyclic_t *new_cyclics, *old_cyclics;
646 cyc_xcallarg_t arg;
647 cyc_backend_t *be = cpu->cyp_backend;
648
649 ASSERT(MUTEX_HELD(&cpu_lock));
650
651 old_heap = cpu->cyp_heap;
652 old_cyclics = cpu->cyp_cyclics;
653
654 if ((new_size = ((old_size = cpu->cyp_size) << 1)) == 0) {
655 new_size = CY_DEFAULT_PERCPU;
656 ASSERT(old_heap == NULL && old_cyclics == NULL);
657 }
658
659 /*
660 * Check that the new_size is a power of 2.
661 */
662 ASSERT(((new_size - 1) & new_size) == 0);
663
664 new_heap = malloc(sizeof(cyc_index_t) * new_size, M_CYCLIC, M_WAITOK);
665 new_cyclics = malloc(sizeof(cyclic_t) * new_size, M_CYCLIC, M_ZERO | M_WAITOK);
666
667 arg.cyx_cpu = cpu;
668 arg.cyx_heap = new_heap;
669 arg.cyx_cyclics = new_cyclics;
670 arg.cyx_size = new_size;
671
672 be->cyb_xcall(be->cyb_arg, cpu->cyp_cpu,
673 (cyc_func_t)cyclic_expand_xcall, &arg);
674
675 if (old_cyclics != NULL) {
676 ASSERT(old_heap != NULL);
677 ASSERT(old_size != 0);
678 free(old_cyclics, M_CYCLIC);
679 free(old_heap, M_CYCLIC);
680 }
681 }
682
683 static void
cyclic_add_xcall(cyc_xcallarg_t * arg)684 cyclic_add_xcall(cyc_xcallarg_t *arg)
685 {
686 cyc_cpu_t *cpu = arg->cyx_cpu;
687 cyc_handler_t *hdlr = arg->cyx_hdlr;
688 cyc_time_t *when = arg->cyx_when;
689 cyc_backend_t *be = cpu->cyp_backend;
690 cyc_index_t ndx, nelems;
691 cyb_arg_t bar = be->cyb_arg;
692 cyclic_t *cyclic;
693
694 ASSERT(cpu->cyp_nelems < cpu->cyp_size);
695
696 /* Disable preemption and interrupts. */
697 mtx_lock_spin(&cpu->cyp_mtx);
698 nelems = cpu->cyp_nelems++;
699
700 if (nelems == 0) {
701 /*
702 * If this is the first element, we need to enable the
703 * backend on this CPU.
704 */
705 be->cyb_enable(bar);
706 }
707
708 ndx = cpu->cyp_heap[nelems];
709 cyclic = &cpu->cyp_cyclics[ndx];
710
711 ASSERT(cyclic->cy_flags == CYF_FREE);
712 cyclic->cy_interval = when->cyt_interval;
713
714 if (when->cyt_when == 0) {
715 /*
716 * If a start time hasn't been explicitly specified, we'll
717 * start on the next interval boundary.
718 */
719 cyclic->cy_expire = (gethrtime() / cyclic->cy_interval + 1) *
720 cyclic->cy_interval;
721 } else {
722 cyclic->cy_expire = when->cyt_when;
723 }
724
725 cyclic->cy_handler = hdlr->cyh_func;
726 cyclic->cy_arg = hdlr->cyh_arg;
727 cyclic->cy_flags = arg->cyx_flags;
728
729 if (cyclic_upheap(cpu, nelems)) {
730 hrtime_t exp = cyclic->cy_expire;
731
732 /*
733 * If our upheap propagated to the root, we need to
734 * reprogram the interrupt source.
735 */
736 be->cyb_reprogram(bar, exp);
737 }
738 mtx_unlock_spin(&cpu->cyp_mtx);
739
740 arg->cyx_ndx = ndx;
741 }
742
743 static cyc_index_t
cyclic_add_here(cyc_cpu_t * cpu,cyc_handler_t * hdlr,cyc_time_t * when,uint16_t flags)744 cyclic_add_here(cyc_cpu_t *cpu, cyc_handler_t *hdlr,
745 cyc_time_t *when, uint16_t flags)
746 {
747 cyc_backend_t *be = cpu->cyp_backend;
748 cyb_arg_t bar = be->cyb_arg;
749 cyc_xcallarg_t arg;
750
751 ASSERT(MUTEX_HELD(&cpu_lock));
752 ASSERT(!(cpu->cyp_cpu->cpu_flags & CPU_OFFLINE));
753 ASSERT(when->cyt_when >= 0 && when->cyt_interval > 0);
754
755 if (cpu->cyp_nelems == cpu->cyp_size) {
756 /*
757 * This is expensive; it will cross call onto the other
758 * CPU to perform the expansion.
759 */
760 cyclic_expand(cpu);
761 ASSERT(cpu->cyp_nelems < cpu->cyp_size);
762 }
763
764 /*
765 * By now, we know that we're going to be able to successfully
766 * perform the add. Now cross call over to the CPU of interest to
767 * actually add our cyclic.
768 */
769 arg.cyx_cpu = cpu;
770 arg.cyx_hdlr = hdlr;
771 arg.cyx_when = when;
772 arg.cyx_flags = flags;
773
774 be->cyb_xcall(bar, cpu->cyp_cpu, (cyc_func_t)cyclic_add_xcall, &arg);
775
776 return (arg.cyx_ndx);
777 }
778
779 static void
cyclic_remove_xcall(cyc_xcallarg_t * arg)780 cyclic_remove_xcall(cyc_xcallarg_t *arg)
781 {
782 cyc_cpu_t *cpu = arg->cyx_cpu;
783 cyc_backend_t *be = cpu->cyp_backend;
784 cyb_arg_t bar = be->cyb_arg;
785 cyc_index_t ndx = arg->cyx_ndx, nelems = cpu->cyp_nelems, i;
786 cyc_index_t *heap = cpu->cyp_heap, last;
787 cyclic_t *cyclic;
788
789 ASSERT(nelems > 0);
790
791 /* Disable preemption and interrupts. */
792 mtx_lock_spin(&cpu->cyp_mtx);
793 cyclic = &cpu->cyp_cyclics[ndx];
794
795 /*
796 * Grab the current expiration time. If this cyclic is being
797 * removed as part of a juggling operation, the expiration time
798 * will be used when the cyclic is added to the new CPU.
799 */
800 if (arg->cyx_when != NULL) {
801 arg->cyx_when->cyt_when = cyclic->cy_expire;
802 arg->cyx_when->cyt_interval = cyclic->cy_interval;
803 }
804
805 /*
806 * Now set the flags to CYF_FREE. We don't need a membar_enter()
807 * between zeroing pend and setting the flags because we're at
808 * CY_HIGH_LEVEL (that is, the zeroing of pend and the setting
809 * of cy_flags appear atomic to softints).
810 */
811 cyclic->cy_flags = CYF_FREE;
812
813 for (i = 0; i < nelems; i++) {
814 if (heap[i] == ndx)
815 break;
816 }
817
818 if (i == nelems)
819 panic("attempt to remove non-existent cyclic");
820
821 cpu->cyp_nelems = --nelems;
822
823 if (nelems == 0) {
824 /*
825 * If we just removed the last element, then we need to
826 * disable the backend on this CPU.
827 */
828 be->cyb_disable(bar);
829 }
830
831 if (i == nelems) {
832 /*
833 * If we just removed the last element of the heap, then
834 * we don't have to downheap.
835 */
836 goto out;
837 }
838
839 /*
840 * Swap the last element of the heap with the one we want to
841 * remove, and downheap (this has the implicit effect of putting
842 * the newly freed element on the free list).
843 */
844 heap[i] = (last = heap[nelems]);
845 heap[nelems] = ndx;
846
847 if (i == 0) {
848 cyclic_downheap(cpu, 0);
849 } else {
850 if (cyclic_upheap(cpu, i) == 0) {
851 /*
852 * The upheap didn't propagate to the root; if it
853 * didn't propagate at all, we need to downheap.
854 */
855 if (heap[i] == last) {
856 cyclic_downheap(cpu, i);
857 }
858 goto out;
859 }
860 }
861
862 /*
863 * We're here because we changed the root; we need to reprogram
864 * the clock source.
865 */
866 cyclic = &cpu->cyp_cyclics[heap[0]];
867
868 ASSERT(nelems != 0);
869 be->cyb_reprogram(bar, cyclic->cy_expire);
870 out:
871 mtx_unlock_spin(&cpu->cyp_mtx);
872 }
873
874 static int
cyclic_remove_here(cyc_cpu_t * cpu,cyc_index_t ndx,cyc_time_t * when,int wait)875 cyclic_remove_here(cyc_cpu_t *cpu, cyc_index_t ndx, cyc_time_t *when, int wait)
876 {
877 cyc_backend_t *be = cpu->cyp_backend;
878 cyc_xcallarg_t arg;
879
880 ASSERT(MUTEX_HELD(&cpu_lock));
881 ASSERT(wait == CY_WAIT || wait == CY_NOWAIT);
882
883 arg.cyx_ndx = ndx;
884 arg.cyx_cpu = cpu;
885 arg.cyx_when = when;
886 arg.cyx_wait = wait;
887
888 be->cyb_xcall(be->cyb_arg, cpu->cyp_cpu,
889 (cyc_func_t)cyclic_remove_xcall, &arg);
890
891 return (1);
892 }
893
894 static void
cyclic_configure(cpu_t * c)895 cyclic_configure(cpu_t *c)
896 {
897 cyc_cpu_t *cpu = malloc(sizeof(cyc_cpu_t), M_CYCLIC, M_ZERO | M_WAITOK);
898 cyc_backend_t *nbe = malloc(sizeof(cyc_backend_t), M_CYCLIC, M_ZERO | M_WAITOK);
899
900 ASSERT(MUTEX_HELD(&cpu_lock));
901
902 if (cyclic_id_cache == NULL)
903 cyclic_id_cache = kmem_cache_create(__UNCONST("cyclic_id_cache"),
904 sizeof (cyc_id_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
905
906 cpu->cyp_cpu = c;
907
908 cpu->cyp_size = 1;
909 cpu->cyp_heap = malloc(sizeof(cyc_index_t), M_CYCLIC, M_ZERO | M_WAITOK);
910 cpu->cyp_cyclics = malloc(sizeof(cyclic_t), M_CYCLIC, M_ZERO | M_WAITOK);
911 cpu->cyp_cyclics->cy_flags = CYF_FREE;
912
913 mtx_init(&cpu->cyp_mtx, "cyclic cpu", NULL, MTX_SPIN);
914
915 /*
916 * Setup the backend for this CPU.
917 */
918 bcopy(&cyclic_backend, nbe, sizeof (cyc_backend_t));
919 if (nbe->cyb_configure != NULL)
920 nbe->cyb_arg = nbe->cyb_configure(c);
921 cpu->cyp_backend = nbe;
922
923 /*
924 * On platforms where stray interrupts may be taken during startup,
925 * the CPU's cpu_cyclic pointer serves as an indicator that the
926 * cyclic subsystem for this CPU is prepared to field interrupts.
927 */
928 membar_producer();
929
930 c->cpu_cyclic = cpu;
931 }
932
933 static void
cyclic_unconfigure(cpu_t * c)934 cyclic_unconfigure(cpu_t *c)
935 {
936 cyc_cpu_t *cpu = c->cpu_cyclic;
937 cyc_backend_t *be = cpu->cyp_backend;
938 cyb_arg_t bar = be->cyb_arg;
939
940 ASSERT(MUTEX_HELD(&cpu_lock));
941
942 c->cpu_cyclic = NULL;
943
944 /*
945 * Let the backend know that the CPU is being yanked, and free up
946 * the backend structure.
947 */
948 if (be->cyb_unconfigure != NULL)
949 be->cyb_unconfigure(bar);
950 free(be, M_CYCLIC);
951 cpu->cyp_backend = NULL;
952
953 mtx_destroy(&cpu->cyp_mtx);
954
955 /* Finally, clean up our remaining dynamic structures. */
956 free(cpu->cyp_cyclics, M_CYCLIC);
957 free(cpu->cyp_heap, M_CYCLIC);
958 free(cpu, M_CYCLIC);
959 }
960
961 static void
cyclic_omni_start(cyc_id_t * idp,cyc_cpu_t * cpu)962 cyclic_omni_start(cyc_id_t *idp, cyc_cpu_t *cpu)
963 {
964 cyc_omni_handler_t *omni = &idp->cyi_omni_hdlr;
965 cyc_omni_cpu_t *ocpu = malloc(sizeof(cyc_omni_cpu_t), M_CYCLIC , M_WAITOK);
966 cyc_handler_t hdlr;
967 cyc_time_t when;
968
969 ASSERT(MUTEX_HELD(&cpu_lock));
970 ASSERT(idp->cyi_cpu == NULL);
971
972 hdlr.cyh_func = NULL;
973 hdlr.cyh_arg = NULL;
974
975 when.cyt_when = 0;
976 when.cyt_interval = 0;
977
978 omni->cyo_online(omni->cyo_arg, cpu->cyp_cpu, &hdlr, &when);
979
980 ASSERT(hdlr.cyh_func != NULL);
981 ASSERT(when.cyt_when >= 0 && when.cyt_interval > 0);
982
983 ocpu->cyo_cpu = cpu;
984 ocpu->cyo_arg = hdlr.cyh_arg;
985 ocpu->cyo_ndx = cyclic_add_here(cpu, &hdlr, &when, 0);
986 ocpu->cyo_next = idp->cyi_omni_list;
987 idp->cyi_omni_list = ocpu;
988 }
989
990 static void
cyclic_omni_stop(cyc_id_t * idp,cyc_cpu_t * cpu)991 cyclic_omni_stop(cyc_id_t *idp, cyc_cpu_t *cpu)
992 {
993 cyc_omni_handler_t *omni = &idp->cyi_omni_hdlr;
994 cyc_omni_cpu_t *ocpu = idp->cyi_omni_list, *prev = NULL;
995
996 ASSERT(MUTEX_HELD(&cpu_lock));
997 ASSERT(idp->cyi_cpu == NULL);
998 ASSERT(ocpu != NULL);
999
1000 while (ocpu != NULL && ocpu->cyo_cpu != cpu) {
1001 prev = ocpu;
1002 ocpu = ocpu->cyo_next;
1003 }
1004
1005 /*
1006 * We _must_ have found an cyc_omni_cpu which corresponds to this
1007 * CPU -- the definition of an omnipresent cyclic is that it runs
1008 * on all online CPUs.
1009 */
1010 ASSERT(ocpu != NULL);
1011
1012 if (prev == NULL) {
1013 idp->cyi_omni_list = ocpu->cyo_next;
1014 } else {
1015 prev->cyo_next = ocpu->cyo_next;
1016 }
1017
1018 (void) cyclic_remove_here(ocpu->cyo_cpu, ocpu->cyo_ndx, NULL, CY_WAIT);
1019
1020 /*
1021 * The cyclic has been removed from this CPU; time to call the
1022 * omnipresent offline handler.
1023 */
1024 if (omni->cyo_offline != NULL)
1025 omni->cyo_offline(omni->cyo_arg, cpu->cyp_cpu, ocpu->cyo_arg);
1026
1027 free(ocpu, M_CYCLIC);
1028 }
1029
1030 static cyc_id_t *
cyclic_new_id(void)1031 cyclic_new_id(void)
1032 {
1033 cyc_id_t *idp;
1034
1035 ASSERT(MUTEX_HELD(&cpu_lock));
1036
1037 idp = kmem_cache_alloc(cyclic_id_cache, KM_SLEEP);
1038
1039 /*
1040 * The cyi_cpu field of the cyc_id_t structure tracks the CPU
1041 * associated with the cyclic. If and only if this field is NULL, the
1042 * cyc_id_t is an omnipresent cyclic. Note that cyi_omni_list may be
1043 * NULL for an omnipresent cyclic while the cyclic is being created
1044 * or destroyed.
1045 */
1046 idp->cyi_cpu = NULL;
1047 idp->cyi_ndx = 0;
1048
1049 idp->cyi_next = cyclic_id_head;
1050 idp->cyi_prev = NULL;
1051 idp->cyi_omni_list = NULL;
1052
1053 if (cyclic_id_head != NULL) {
1054 ASSERT(cyclic_id_head->cyi_prev == NULL);
1055 cyclic_id_head->cyi_prev = idp;
1056 }
1057
1058 cyclic_id_head = idp;
1059
1060 return (idp);
1061 }
1062
1063 /*
1064 * cyclic_id_t cyclic_add(cyc_handler_t *, cyc_time_t *)
1065 *
1066 * Overview
1067 *
1068 * cyclic_add() will create an unbound cyclic with the specified handler and
1069 * interval. The cyclic will run on a CPU which both has interrupts enabled
1070 * and is in the system CPU partition.
1071 *
1072 * Arguments and notes
1073 *
1074 * As its first argument, cyclic_add() takes a cyc_handler, which has the
1075 * following members:
1076 *
1077 * cyc_func_t cyh_func <-- Cyclic handler
1078 * void *cyh_arg <-- Argument to cyclic handler
1079 *
1080 * In addition to a cyc_handler, cyclic_add() takes a cyc_time, which
1081 * has the following members:
1082 *
1083 * hrtime_t cyt_when <-- Absolute time, in nanoseconds since boot, at
1084 * which to start firing
1085 * hrtime_t cyt_interval <-- Length of interval, in nanoseconds
1086 *
1087 * gethrtime() is the time source for nanoseconds since boot. If cyt_when
1088 * is set to 0, the cyclic will start to fire when cyt_interval next
1089 * divides the number of nanoseconds since boot.
1090 *
1091 * The cyt_interval field _must_ be filled in by the caller; one-shots are
1092 * _not_ explicitly supported by the cyclic subsystem (cyclic_add() will
1093 * assert that cyt_interval is non-zero). The maximum value for either
1094 * field is INT64_MAX; the caller is responsible for assuring that
1095 * cyt_when + cyt_interval <= INT64_MAX. Neither field may be negative.
1096 *
1097 * For an arbitrary time t in the future, the cyclic handler is guaranteed
1098 * to have been called (t - cyt_when) / cyt_interval times. This will
1099 * be true even if interrupts have been disabled for periods greater than
1100 * cyt_interval nanoseconds. In order to compensate for such periods,
1101 * the cyclic handler may be called a finite number of times with an
1102 * arbitrarily small interval.
1103 *
1104 * The cyclic subsystem will not enforce any lower bound on the interval;
1105 * if the interval is less than the time required to process an interrupt,
1106 * the CPU will wedge. It's the responsibility of the caller to assure that
1107 * either the value of the interval is sane, or that its caller has
1108 * sufficient privilege to deny service (i.e. its caller is root).
1109 *
1110 * Return value
1111 *
1112 * cyclic_add() returns a cyclic_id_t, which is guaranteed to be a value
1113 * other than CYCLIC_NONE. cyclic_add() cannot fail.
1114 *
1115 * Caller's context
1116 *
1117 * cpu_lock must be held by the caller, and the caller must not be in
1118 * interrupt context. cyclic_add() will perform a KM_SLEEP kernel
1119 * memory allocation, so the usual rules (e.g. p_lock cannot be held)
1120 * apply. A cyclic may be added even in the presence of CPUs that have
1121 * not been configured with respect to the cyclic subsystem, but only
1122 * configured CPUs will be eligible to run the new cyclic.
1123 *
1124 * Cyclic handler's context
1125 *
1126 * Cyclic handlers will be executed in the interrupt context corresponding
1127 * to the specified level (i.e. either high, lock or low level). The
1128 * usual context rules apply.
1129 *
1130 * A cyclic handler may not grab ANY locks held by the caller of any of
1131 * cyclic_add() or cyclic_remove(); the implementation of these functions
1132 * may require blocking on cyclic handler completion.
1133 * Moreover, cyclic handlers may not make any call back into the cyclic
1134 * subsystem.
1135 */
1136 cyclic_id_t
cyclic_add(cyc_handler_t * hdlr,cyc_time_t * when)1137 cyclic_add(cyc_handler_t *hdlr, cyc_time_t *when)
1138 {
1139 cyc_id_t *idp = cyclic_new_id();
1140 solaris_cpu_t *c = &solaris_cpu[cpu_number()];
1141
1142 ASSERT(MUTEX_HELD(&cpu_lock));
1143 ASSERT(when->cyt_when >= 0 && when->cyt_interval > 0);
1144
1145 idp->cyi_cpu = c->cpu_cyclic;
1146 idp->cyi_ndx = cyclic_add_here(idp->cyi_cpu, hdlr, when, 0);
1147
1148 return ((uintptr_t)idp);
1149 }
1150
1151 /*
1152 * cyclic_id_t cyclic_add_omni(cyc_omni_handler_t *)
1153 *
1154 * Overview
1155 *
1156 * cyclic_add_omni() will create an omnipresent cyclic with the specified
1157 * online and offline handlers. Omnipresent cyclics run on all online
1158 * CPUs, including CPUs which have unbound interrupts disabled.
1159 *
1160 * Arguments
1161 *
1162 * As its only argument, cyclic_add_omni() takes a cyc_omni_handler, which
1163 * has the following members:
1164 *
1165 * void (*cyo_online)() <-- Online handler
1166 * void (*cyo_offline)() <-- Offline handler
1167 * void *cyo_arg <-- Argument to be passed to on/offline handlers
1168 *
1169 * Online handler
1170 *
1171 * The cyo_online member is a pointer to a function which has the following
1172 * four arguments:
1173 *
1174 * void * <-- Argument (cyo_arg)
1175 * cpu_t * <-- Pointer to CPU about to be onlined
1176 * cyc_handler_t * <-- Pointer to cyc_handler_t; must be filled in
1177 * by omni online handler
1178 * cyc_time_t * <-- Pointer to cyc_time_t; must be filled in by
1179 * omni online handler
1180 *
1181 * The omni cyclic online handler is always called _before_ the omni
1182 * cyclic begins to fire on the specified CPU. As the above argument
1183 * description implies, the online handler must fill in the two structures
1184 * passed to it: the cyc_handler_t and the cyc_time_t. These are the
1185 * same two structures passed to cyclic_add(), outlined above. This
1186 * allows the omni cyclic to have maximum flexibility; different CPUs may
1187 * optionally
1188 *
1189 * (a) have different intervals
1190 * (b) be explicitly in or out of phase with one another
1191 * (c) have different handlers
1192 * (d) have different handler arguments
1193 * (e) fire at different levels
1194 *
1195 * Of these, (e) seems somewhat dubious, but is nonetheless allowed.
1196 *
1197 * The omni online handler is called in the same context as cyclic_add(),
1198 * and has the same liberties: omni online handlers may perform KM_SLEEP
1199 * kernel memory allocations, and may grab locks which are also acquired
1200 * by cyclic handlers. However, omni cyclic online handlers may _not_
1201 * call back into the cyclic subsystem, and should be generally careful
1202 * about calling into arbitrary kernel subsystems.
1203 *
1204 * Offline handler
1205 *
1206 * The cyo_offline member is a pointer to a function which has the following
1207 * three arguments:
1208 *
1209 * void * <-- Argument (cyo_arg)
1210 * cpu_t * <-- Pointer to CPU about to be offlined
1211 * void * <-- CPU's cyclic argument (that is, value
1212 * to which cyh_arg member of the cyc_handler_t
1213 * was set in the omni online handler)
1214 *
1215 * The omni cyclic offline handler is always called _after_ the omni
1216 * cyclic has ceased firing on the specified CPU. Its purpose is to
1217 * allow cleanup of any resources dynamically allocated in the omni cyclic
1218 * online handler. The context of the offline handler is identical to
1219 * that of the online handler; the same constraints and liberties apply.
1220 *
1221 * The offline handler is optional; it may be NULL.
1222 *
1223 * Return value
1224 *
1225 * cyclic_add_omni() returns a cyclic_id_t, which is guaranteed to be a
1226 * value other than CYCLIC_NONE. cyclic_add_omni() cannot fail.
1227 *
1228 * Caller's context
1229 *
1230 * The caller's context is identical to that of cyclic_add(), specified
1231 * above.
1232 */
1233 cyclic_id_t
cyclic_add_omni(cyc_omni_handler_t * omni)1234 cyclic_add_omni(cyc_omni_handler_t *omni)
1235 {
1236 cyc_id_t *idp = cyclic_new_id();
1237 cyc_cpu_t *cpu;
1238 cpu_t *c;
1239 int i;
1240
1241 ASSERT(MUTEX_HELD(&cpu_lock));
1242 ASSERT(omni != NULL && omni->cyo_online != NULL);
1243
1244 idp->cyi_omni_hdlr = *omni;
1245
1246 CPU_FOREACH(i) {
1247 i = cpu_index(ci);
1248 c = &solaris_cpu[i];
1249 if ((cpu = c->cpu_cyclic) == NULL)
1250 continue;
1251 cyclic_omni_start(idp, cpu);
1252 }
1253
1254 /*
1255 * We must have found at least one online CPU on which to run
1256 * this cyclic.
1257 */
1258 ASSERT(idp->cyi_omni_list != NULL);
1259 ASSERT(idp->cyi_cpu == NULL);
1260
1261 return ((uintptr_t)idp);
1262 }
1263
1264 /*
1265 * void cyclic_remove(cyclic_id_t)
1266 *
1267 * Overview
1268 *
1269 * cyclic_remove() will remove the specified cyclic from the system.
1270 *
1271 * Arguments and notes
1272 *
1273 * The only argument is a cyclic_id returned from either cyclic_add() or
1274 * cyclic_add_omni().
1275 *
1276 * By the time cyclic_remove() returns, the caller is guaranteed that the
1277 * removed cyclic handler has completed execution (this is the same
1278 * semantic that untimeout() provides). As a result, cyclic_remove() may
1279 * need to block, waiting for the removed cyclic to complete execution.
1280 * This leads to an important constraint on the caller: no lock may be
1281 * held across cyclic_remove() that also may be acquired by a cyclic
1282 * handler.
1283 *
1284 * Return value
1285 *
1286 * None; cyclic_remove() always succeeds.
1287 *
1288 * Caller's context
1289 *
1290 * cpu_lock must be held by the caller, and the caller must not be in
1291 * interrupt context. The caller may not hold any locks which are also
1292 * grabbed by any cyclic handler. See "Arguments and notes", above.
1293 */
1294 void
cyclic_remove(cyclic_id_t id)1295 cyclic_remove(cyclic_id_t id)
1296 {
1297 cyc_id_t *idp = (cyc_id_t *)id;
1298 cyc_id_t *prev = idp->cyi_prev, *next = idp->cyi_next;
1299 cyc_cpu_t *cpu = idp->cyi_cpu;
1300
1301 ASSERT(MUTEX_HELD(&cpu_lock));
1302
1303 if (cpu != NULL) {
1304 (void) cyclic_remove_here(cpu, idp->cyi_ndx, NULL, CY_WAIT);
1305 } else {
1306 ASSERT(idp->cyi_omni_list != NULL);
1307 while (idp->cyi_omni_list != NULL)
1308 cyclic_omni_stop(idp, idp->cyi_omni_list->cyo_cpu);
1309 }
1310
1311 if (prev != NULL) {
1312 ASSERT(cyclic_id_head != idp);
1313 prev->cyi_next = next;
1314 } else {
1315 ASSERT(cyclic_id_head == idp);
1316 cyclic_id_head = next;
1317 }
1318
1319 if (next != NULL)
1320 next->cyi_prev = prev;
1321
1322 kmem_cache_free(cyclic_id_cache, idp);
1323 }
1324
1325 static void
cyclic_init(cyc_backend_t * be)1326 cyclic_init(cyc_backend_t *be)
1327 {
1328 ASSERT(MUTEX_HELD(&cpu_lock));
1329
1330 /*
1331 * Copy the passed cyc_backend into the backend template. This must
1332 * be done before the CPU can be configured.
1333 */
1334 bcopy(be, &cyclic_backend, sizeof (cyc_backend_t));
1335
1336 cyclic_configure(&solaris_cpu[cpu_number()]);
1337 }
1338
1339 /*
1340 * It is assumed that cyclic_mp_init() is called some time after cyclic
1341 * init (and therefore, after cpu0 has been initialized). We grab cpu_lock,
1342 * find the already initialized CPU, and initialize every other CPU with the
1343 * same backend.
1344 */
1345 static void
cyclic_mp_init(void)1346 cyclic_mp_init(void)
1347 {
1348 cpu_t *c;
1349 int i;
1350
1351 #ifndef __NetBSD__
1352 mutex_enter(&cpu_lock);
1353 #endif
1354
1355 CPU_FOREACH(i) {
1356 i = cpu_index(ci);
1357 c = &solaris_cpu[i];
1358 if (c->cpu_cyclic == NULL)
1359 cyclic_configure(c);
1360 }
1361
1362 #ifndef __NetBSD__
1363 mutex_exit(&cpu_lock);
1364 #endif
1365 }
1366
1367 static void
cyclic_uninit(void)1368 cyclic_uninit(void)
1369 {
1370 cpu_t *c;
1371 int id;
1372
1373 CPU_FOREACH(id) {
1374 id = cpu_index(ci);
1375 c = &solaris_cpu[id];
1376 if (c->cpu_cyclic == NULL)
1377 continue;
1378 cyclic_unconfigure(c);
1379 }
1380
1381 if (cyclic_id_cache != NULL)
1382 kmem_cache_destroy(cyclic_id_cache);
1383 }
1384
1385 #include "cyclic_machdep.c"
1386
1387 /*
1388 * Cyclic subsystem initialisation.
1389 */
1390 static void
cyclic_load(void * dummy)1391 cyclic_load(void *dummy)
1392 {
1393 mutex_enter(&cpu_lock);
1394
1395 /* Initialise the machine-dependent backend. */
1396 cyclic_machdep_init();
1397
1398 mutex_exit(&cpu_lock);
1399 }
1400
1401 SYSINIT(cyclic_register, SI_SUB_CYCLIC, SI_ORDER_SECOND, cyclic_load, NULL);
1402
1403 static void
cyclic_unload(void)1404 cyclic_unload(void)
1405 {
1406 mutex_enter(&cpu_lock);
1407
1408 /* Uninitialise the machine-dependent backend. */
1409 cyclic_machdep_uninit();
1410
1411 mutex_exit(&cpu_lock);
1412 }
1413
1414 SYSUNINIT(cyclic_unregister, SI_SUB_CYCLIC, SI_ORDER_SECOND, cyclic_unload, NULL);
1415
1416 #ifdef __FreeBSD__
1417 /* ARGSUSED */
1418 static int
cyclic_modevent(module_t mod __unused,int type,void * data __unused)1419 cyclic_modevent(module_t mod __unused, int type, void *data __unused)
1420 {
1421 int error = 0;
1422
1423 switch (type) {
1424 case MOD_LOAD:
1425 break;
1426
1427 case MOD_UNLOAD:
1428 break;
1429
1430 case MOD_SHUTDOWN:
1431 break;
1432
1433 default:
1434 error = EOPNOTSUPP;
1435 break;
1436
1437 }
1438 return (error);
1439 }
1440
1441 DEV_MODULE(cyclic, cyclic_modevent, NULL);
1442 MODULE_VERSION(cyclic, 1);
1443 MODULE_DEPEND(cyclic, opensolaris, 1, 1, 1);
1444 #endif
1445
1446 #ifdef __NetBSD__
1447 static int
cyclic_modcmd(modcmd_t cmd,void * data)1448 cyclic_modcmd(modcmd_t cmd, void *data)
1449 {
1450 switch (cmd) {
1451 case MODULE_CMD_INIT:
1452 cyclic_load(NULL);
1453 return 0;
1454
1455 case MODULE_CMD_FINI:
1456 cyclic_unload();
1457 return 0;
1458
1459 case MODULE_CMD_AUTOUNLOAD:
1460 if (cyclic_id_head != NULL)
1461 return EBUSY;
1462 return 0;
1463
1464 default:
1465 return ENOTTY;
1466 }
1467 }
1468
1469 MODULE(MODULE_CLASS_MISC, cyclic, "dtrace");
1470 #endif
1471