xref: /illumos-gate/usr/src/uts/common/sys/cyclic_impl.h (revision 87a18d3f)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef _SYS_CYCLIC_IMPL_H
#define	_SYS_CYCLIC_IMPL_H

#ifdef	__cplusplus
extern "C" {
#endif

#include <sys/cyclic.h>
#include <sys/rwlock.h>

/*
 *  Cyclic Subsystem Backend-supplied Interfaces
 *  --------------------------------------------
 *
 *  0  Background
 *
 *    The design, implementation and interfaces of the cyclic subsystem are
 *    covered in detail in block comments in the implementation.  This
 *    comment covers the interface from the cyclic subsystem into the cyclic
 *    backend.  The backend is specified by a structure of function pointers
 *    defined below.
 *
 *  1  Overview
 *
 *      cyb_configure()      <-- Configures the backend on the specified CPU
 *      cyb_unconfigure()    <-- Unconfigures the backend
 *      cyb_enable()         <-- Enables the CY_HIGH_LEVEL interrupt source
 *      cyb_disable()        <-- Disables the CY_HIGH_LEVEL interrupt source
 *      cyb_reprogram()      <-- Reprograms the CY_HIGH_LEVEL interrupt source
 *      cyb_softint()        <-- Generates a soft interrupt
 *      cyb_set_level()      <-- Sets the programmable interrupt level
 *      cyb_restore_level()  <-- Restores the programmable interrupt level
 *      cyb_xcall()          <-- Cross calls to the specified CPU
 *      cyb_suspend()        <-- Suspends the backend
 *      cyb_resume()         <-- Resumes the backend
 *
 *  2  cyb_arg_t cyb_configure(cpu_t *)
 *
 *  2.1  Overview
 *
 *    cyb_configure() should configure the specified CPU for cyclic operation.
 *
 *  2.2  Arguments and notes
 *
 *    cyb_configure() should initialize any backend-specific per-CPU
 *    structures for the specified CPU.  cyb_configure() will be called for
 *    each CPU (including the boot CPU) during boot.  If the platform
 *    supports dynamic reconfiguration, cyb_configure() will be called for
 *    new CPUs as they are configured into the system.
 *
 *  2.3  Return value
 *
 *    cyb_configure() is expected to return a cookie (a cyb_arg_t, which is
 *    of type void *) which will be used as the first argument for all future
 *    cyclic calls into the backend on the specified CPU.
 *
 *  2.4  Caller's context
 *
 *    cpu_lock will be held.  The caller's CPU is unspecified, and may or
 *    may not be the CPU specified to cyb_configure().
 *
 *  3  void cyb_unconfigure(cyb_arg_t arg)
 *
 *  3.1  Overview
 *
 *    cyb_unconfigure() should unconfigure the specified backend.
 *
 *  3.2  Arguments and notes
 *
 *    The only argument to cyb_unconfigure() is a cookie as returned from
 *    cyb_configure().
 *
 *    cyb_unconfigure() should free any backend-specific per-CPU structures
 *    for the specified backend.  cyb_unconfigure() will _only_ be called on
 *    platforms which support dynamic reconfiguration.  If the platform does
 *    not support dynamic reconfiguration, cyb_unconfigure() may panic.
 *
 *    After cyb_unconfigure() returns, the backend must not call cyclic_fire()
 *    on the corresponding CPU; doing so will result in a bad trap.
 *
 *  3.3  Return value
 *
 *    None.
 *
 *  3.4  Caller's context
 *
 *    cpu_lock will be held.  The caller's CPU is unspecified, and may or
 *    may not be the CPU specified to cyb_unconfigure().  The specified
 *    CPU is guaranteed to exist at the time cyb_unconfigure() is called.
 *    The cyclic subsystem is guaranteed to be suspended when cyb_unconfigure()
 *    is called, and interrupts are guaranteed to be disabled.
 *
 *  4  void cyb_enable(cyb_arg_t arg)
 *
 *  4.1  Overview
 *
 *    cyb_enable() should enable the CY_HIGH_LEVEL interrupt source on
 *    the specified backend.
 *
 *  4.2  Arguments and notes
 *
 *    The only argument to cyb_enable() is a backend cookie as returned from
 *    cyb_configure().
 *
 *    cyb_enable() will only be called if a) the specified backend has never
 *    been enabled or b) the specified backend has been explicitly disabled with
 *    cyb_disable().  In either case, cyb_enable() will only be called if
 *    the cyclic subsystem wishes to add a cyclic to the CPU corresponding
 *    to the specified backend.  cyb_enable() will be called before
 *    cyb_reprogram() for a given backend.
 *
 *    cyclic_fire() should not be called on a CPU which has not had its backend
 *    explicitly cyb_enable()'d, but to do so does not constitute fatal error.
 *
 *  4.3  Return value
 *
 *    None.
 *
 *  4.4  Caller's context
 *
 *    cyb_enable() will only be called from CY_HIGH_LEVEL context on the CPU
 *    corresponding to the specified backend.
 *
 *  5  void cyb_disable(cyb_arg_t arg)
 *
 *  5.1  Overview
 *
 *    cyb_disable() should disable the CY_HIGH_LEVEL interrupt source on
 *    the specified backend.
 *
 *  5.2  Arguments and notes
 *
 *    The only argument to cyb_disable() is a backend cookie as returned from
 *    cyb_configure().
 *
 *    cyb_disable() will only be called on backends which have been previously
 *    been cyb_enable()'d.  cyb_disable() will be called when all cyclics have
 *    been juggled away or removed from a cyb_enable()'d CPU.
 *
 *    cyclic_fire() should not be called on a CPU which has had its backend
 *    explicitly cyb_disable()'d, but to do so does not constitute fatal
 *    error.  cyb_disable() is thus not required to check for a pending
 *    CY_HIGH_LEVEL interrupt.
 *
 *  5.3  Return value
 *
 *    None.
 *
 *  5.4  Caller's context
 *
 *    cyb_disable() will only be called from CY_HIGH_LEVEL context on the CPU
 *    corresponding to the specified backend.
 *
 *  6  void cyb_reprogram(cyb_arg_t arg, hrtime_t time)
 *
 *  6.1  Overview
 *
 *    cyb_reprogram() should reprogram the CY_HIGH_LEVEL interrupt source
 *    to fire at the absolute time specified.
 *
 *  6.2  Arguments and notes
 *
 *    The first argument to cyb_reprogram() is a backend cookie as returned from
 *    cyb_configure().
 *
 *    The second argument is an absolute time at which the CY_HIGH_LEVEL
 *    interrupt should fire.  The specified time _may_ be in the past (albeit
 *    the very recent past).  If this is the case, the backend should generate
 *    a CY_HIGH_LEVEL interrupt as soon as possible.
 *
 *    The platform should not assume that cyb_reprogram() will be called with
 *    monotonically increasing values.
 *
 *    If the platform does not allow for interrupts at arbitrary times in the
 *    future, cyb_reprogram() may do nothing -- as long as cyclic_fire() is
 *    called periodically at CY_HIGH_LEVEL.  While this is clearly suboptimal
 *    (cyclic granularity will be bounded by the length of the period between
 *    cyclic_fire()'s), it allows the cyclic subsystem to be implemented on
 *    inferior hardware.
 *
 *  6.3  Return value
 *
 *     None.
 *
 *  6.4  Caller's context
 *
 *    cyb_reprogram() will only be called from CY_HIGH_LEVEL context on the CPU
 *    corresponding to the specified backend.
 *
 *  7  void cyb_softint(cyb_arg_t arg, cyc_level_t level)
 *
 *  7.1  Overview
 *
 *    cyb_softint() should generate a software interrupt on the specified
 *    backend at the specified level.
 *
 *  7.2  Arguments and notes
 *
 *    The first argument to cyb_softint() is a backend cookie as returned from
 *    cyb_configure().  The second argument is the interrupt level at which
 *    the software interrupt should be generated; it will be either
 *    CY_LOCK_LEVEL or CY_LOW_LEVEL.
 *
 *    The software interrupt _must_ be generated on the CPU corresponding
 *    to the specified backend; platforms are _required_ to have a per-CPU
 *    notion of a software interrupt.
 *
 *    Unless a software interrupt is already pending at the specified level,
 *    the software interrupt _must_ be generated.  Once cyclic_softint()
 *    has been called at a given level, the software interrupt at that level
 *    should no longer be considered pending; an intervening CY_HIGH_LEVEL
 *    interrupt and subsequent cyb_softint() must generate another software
 *    interrupt.
 *
 *  7.3  Return value
 *
 *    None.
 *
 *  7.4  Caller's context
 *
 *    cyb_softint() will only be called at a level higher than the one
 *    specified:  if CY_LOCK_LEVEL is specified, the caller will be at
 *    CY_HIGH_LEVEL; if CY_LOW_LEVEL is specified, the caller will be at
 *    either CY_HIGH_LEVEL or CY_LOCK_LEVEL.  cyb_softint() will only be
 *    called on the CPU corresponding to the specified backend.
 *
 *  8  cyb_set_level(cyb_arg_t arg, cyc_level_t level)
 *
 *  8.1  Overview
 *
 *    cyb_set_level() should set the programmable interrupt level to the
 *    level specified.
 *
 *  8.2  Arguments and notes
 *
 *    The first argument to cyb_set_level() is a backend cookie as returned
 *    from cyb_configure().  The second argument is the level to which
 *    the programmable interrupt level should be set; it will be one of
 *    CY_HIGH_LEVEL, CY_LOCK_LEVEL or CY_LOW_LEVEL.
 *
 *    After cyb_set_level() returns, the CPU associated with the specified
 *    backend should accept no interrupt at a level greater than or equal to
 *    the specified level.  This will generally be a wrapper around splx().
 *
 *    The cyclic subsystem will never call cyb_set_level() twice consecutively
 *    on the same backend; there will always be an intervening
 *    cyb_restore_level();
 *
 *  8.3  Return value
 *
 *    cyb_set_level() should return a cookie to be passed back to
 *    cyb_restore_level().  On most implementations, this cookie will be
 *    the spl at the time of cyb_set_level().
 *
 *  8.4  Caller's context
 *
 *    cyb_set_level() is unique in that it is the only backend-provided
 *    interface which may be called in cross call context (see cyb_xcall(),
 *    below).  cyb_set_level() may also be called from any of the cyclic
 *
 *  9  cyb_restore_level(cyb_arg_t arg, cyc_cookie_t cookie)
 *
 *  9.1  Overview
 *
 *    cyb_restore_level() should restore the programmable interrupt level
 *    based upon the specified cookie.
 *
 *  9.2  Arguments and notes
 *
 *    The first argument to cyb_restore_level() is a backend cookie as returned
 *    from cyb_configure().  The second argument is a cookie as returned from
 *    cyb_set_level().
 *
 *    cyb_restore_level() should restore the programmable interrupt level
 *    to its value when cyb_set_level() was called; the cookie is used
 *    to provide a hint to the backend.  cyb_restore_level() will not be
 *    called without a proceeding call to cyb_set_level(), and
 *    cyb_restore_level() will never be called twice consecutively on the
 *    same backend.
 *
 *  9.3  Return value
 *
 *    None.
 *
 *  9.4  Caller's context
 *
 *    The constraints outlined in 5.9.2 imply that cyb_restore_level() can
 *    only be called from CY_HIGH_LEVEL, CY_LOCK_LEVEL or CY_LOW_LEVEL context.
 *    cyb_restore_level() is always called on the CPU associated with the
 *    specified backend.
 *
 *  10  cyb_xcall(cyb_arg_t arg, cpu_t *, void(*func)(void *), void *farg)
 *
 *  10.1  Overview
 *
 *    cyb_xcall() should execute the specified function on the specified CPU.
 *
 *  10.2  Arguments and notes
 *
 *    The first argument to cyb_restore_level() is a backend cookie as returned
 *    from cyb_configure().  The second argument is a CPU on which the third
 *    argument, a function pointer, should be executed.  The fourth argument,
 *    a void *, should be passed as the argument to the specified function.
 *
 *    cyb_xcall() must provide exactly-once semantics.  If the specified
 *    function is called more than once, or not at all, the cyclic subsystem
 *    will become internally inconsistent.  The specified function must be
 *    be executed on the specified CPU, but may be executed in any context
 *    (any interrupt context or kernel context).
 *
 *    cyb_xcall() cannot block.  Any resources which cyb_xcall() needs to
 *    acquire must thus be protected by synchronization primitives which
 *    never require the caller to block.
 *
 *  10.3  Return value
 *
 *    None.
 *
 *  10.4  Caller's context
 *
 *    cpu_lock will be held and kernel preemption may be disabled.  The caller
 *    may be unable to block, giving rise to the constraint outlined in
 *    10.2, above.
 *
 *  11  cyb_suspend(cyb_arg_t arg)
 *
 *  11.1  Overview
 *
 *    cyb_suspend() should suspend the specified backend.
 *
 *  11.2  Arguments and notes
 *
 *    The only argument to cyb_suspend() is a backend cookie as returned from
 *    cyb_configure().
 *
 *    cyb_suspend() will never be called on enabled backends.  The backend
 *    should assume that the machine may be subsequently powered off; any
 *    volatile hardware state should be preserved and restored in cyb_resume().
 *    However, the backend should not _assume_ that the machine will be
 *    powered off; cyb_suspend() may also be called as part of dynamic
 *    reconfiguration.
 *
 *    cyb_suspend() will be called on the corresponding backend of each
 *    CPU in the system in succession, regardless of CPU state (P_ONLINE,
 *    P_OFFLINE, P_NOINTR).  The cyclic subsystem will not suspend only a
 *    fraction of the CPUs.
 *
 *  11.3  Return value
 *
 *    None.
 *
 *  11.4  Caller's context
 *
 *    cyb_suspend() will be called in cross call context on the CPU associated
 *    with the specified backend.
 *
 *  12  cyb_resume(cyb_arg_t arg)
 *
 *  12.1  Overview
 *
 *    cyb_resume() should resume the specified backend.
 *
 *  12.2  Arguments and notes
 *
 *    The only argument to cyb_resume() is a backend cookie as returned from
 *    cyb_resume().
 *
 *    Calls to cyb_resume() will always have been proceeded by corresponding
 *    calls to cyb_suspend().  The machine may have been powered off between
 *    cyb_suspend() and the call to cyb_resume().  cyb_resume() may decide
 *    to restore hardware to its state at the time cyb_suspend() was called.
 *
 *    The cyclic subsystem will make no calls into the backend between
 *    cyb_suspend() and cyb_resume().
 *
 *  12.3  Return value
 *
 *    None.
 *
 *  12.4  Caller's context
 *
 *    cyb_resume() will be called in cross call context on the CPU associated
 *    with the specified backend.
 */
typedef struct cyc_backend {
	cyb_arg_t (*cyb_configure)(cpu_t *);
	void (*cyb_unconfigure)(cyb_arg_t);
	void (*cyb_enable)(cyb_arg_t);
	void (*cyb_disable)(cyb_arg_t);
	void (*cyb_reprogram)(cyb_arg_t, hrtime_t);
	void (*cyb_softint)(cyb_arg_t, cyc_level_t);
	cyc_cookie_t (*cyb_set_level)(cyb_arg_t, cyc_level_t);
	void (*cyb_restore_level)(cyb_arg_t, cyc_cookie_t);
	void (*cyb_xcall)(cyb_arg_t, cpu_t *, cyc_func_t, void *);
	void (*cyb_suspend)(cyb_arg_t);
	void (*cyb_resume)(cyb_arg_t);
	cyb_arg_t cyb_arg;
} cyc_backend_t;

extern void cyclic_init(cyc_backend_t *be, hrtime_t resolution);
extern void cyclic_mp_init();

#ifdef DEBUG
#define	CYCLIC_TRACE
#endif

typedef enum {
	CYS_ONLINE,
	CYS_OFFLINE,
	CYS_EXPANDING,
	CYS_REMOVING,
	CYS_SUSPENDED
} cyc_state_t;

#define	CYF_FREE		0x0001
#define	CYF_CPU_BOUND		0x0002
#define	CYF_PART_BOUND		0x0004

typedef struct cyclic {
	hrtime_t cy_expire;
	hrtime_t cy_interval;
	void (*cy_handler)(void *);
	void *cy_arg;
	uint32_t cy_pend;
	uint16_t cy_flags;
	cyc_level_t cy_level;
} cyclic_t;

typedef struct cyc_pcbuffer {
	cyc_index_t *cypc_buf;
	int cypc_prodndx;
	int cypc_consndx;
	int cypc_sizemask;
} cyc_pcbuffer_t;

typedef struct cyc_softbuf {
	uchar_t cys_hard;		/* Can only be zero or one */
	uchar_t cys_soft;		/* Can only be zero or one */
	cyc_pcbuffer_t cys_buf[2];
} cyc_softbuf_t;

#define	CY_NTRACEREC		512

typedef struct cyc_tracerec {
	hrtime_t cyt_tstamp;
	char *cyt_why;
	uint64_t cyt_arg0;
	uint64_t cyt_arg1;
} cyc_tracerec_t;

typedef struct cyc_tracebuf {
	int cyt_ndx;
	cyc_tracerec_t cyt_buf[CY_NTRACEREC];
} cyc_tracebuf_t;

#define	CY_NCOVERAGE	127

typedef struct cyc_coverage {
	char *cyv_why;
	int cyv_passive_count;
	int cyv_count[CY_LEVELS];
	uint64_t cyv_arg0;
	uint64_t cyv_arg1;
} cyc_coverage_t;

typedef struct cyc_cpu {
	cpu_t *cyp_cpu;
	cyc_index_t *cyp_heap;
	cyclic_t *cyp_cyclics;
	cyc_index_t cyp_nelems;
	cyc_index_t cyp_size;
	cyc_state_t cyp_state;
	cyc_softbuf_t cyp_softbuf[CY_SOFT_LEVELS];
	cyc_backend_t *cyp_backend;
	ksema_t cyp_modify_wait;
	uint32_t cyp_modify_levels;
	uint32_t cyp_rpend;
#ifdef CYCLIC_TRACE
	cyc_tracebuf_t cyp_trace[CY_LEVELS];
#endif
} cyc_cpu_t;

typedef struct cyc_omni_cpu {
	cyc_cpu_t *cyo_cpu;
	cyc_index_t cyo_ndx;
	void *cyo_arg;
	struct cyc_omni_cpu *cyo_next;
} cyc_omni_cpu_t;

typedef struct cyc_id {
	krwlock_t cyi_lock;
	cyc_cpu_t *cyi_cpu;
	cyc_index_t cyi_ndx;
	struct cyc_id *cyi_prev;
	struct cyc_id *cyi_next;
	cyc_omni_handler_t cyi_omni_hdlr;
	cyc_omni_cpu_t *cyi_omni_list;
} cyc_id_t;

typedef struct cyc_xcallarg {
	cyc_cpu_t *cyx_cpu;
	cyc_handler_t *cyx_hdlr;
	cyc_time_t *cyx_when;
	cyc_index_t cyx_ndx;
	cyc_index_t *cyx_heap;
	cyclic_t *cyx_cyclics;
	cyc_index_t cyx_size;
	uint16_t cyx_flags;
	int cyx_wait;
} cyc_xcallarg_t;

#define	CY_DEFAULT_PERCPU	1
#define	CY_PASSIVE_LEVEL	-1

#define	CY_WAIT			0
#define	CY_NOWAIT		1

#define	CYC_HEAP_PARENT(ndx)		(((ndx) - 1) >> 1)
#define	CYC_HEAP_RIGHT(ndx)		(((ndx) + 1) << 1)
#define	CYC_HEAP_LEFT(ndx)		((((ndx) + 1) << 1) - 1)

#ifdef	__cplusplus
}
#endif

#endif /* _SYS_CYCLIC_IMPL_H */