xref: /illumos-gate/usr/src/uts/common/sys/poll_impl.h (revision f23ed011)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2003 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright 2017 Joyent, Inc.
 * Copyright 2022 Oxide Computer Company
 */

#ifndef _SYS_POLL_IMPL_H
#define	_SYS_POLL_IMPL_H

/*
 * Caching Poll Subsystem:
 *
 * Each kernel thread (1), if engaged in poll system call, has a reference to
 * a pollstate_t (2), which contains relevant flags and locks.  The pollstate_t
 * contains a pointer to a pollcache_t (3), which caches the state of previous
 * calls to poll.  A bitmap (4) is stored inside the poll cache, where each
 * bit represents a file descriptor.  The bits are set if the corresponding
 * device has a polled event pending.  Only fds with their bit set will be
 * examined on the next poll invocation.  The pollstate_t also contains a list
 * of fd sets (5), which are represented by the pollcacheset_t type.  These
 * structures keep track of the pollfd_t arrays (6) passed in from userland.
 * Each polled file descriptor has a corresponding polldat_t which can be
 * chained onto a device's pollhead, and these are kept in a hash table (7)
 * inside the pollcache_t.  The hash table allows efficient conversion of a
 * given fd to its corresponding polldat_t.
 *
 * (1)              (2)
 * +-----------+    +-------------+
 * | kthread_t |--->| pollstate_t |-->+-------------+  (6)
 * +-----------+    +-------------+(5)| pcacheset_t |->[_][_][_][_] pollfd_t
 *                          |         +-------------+
 *                          |         | pcacheset_t |->[_][_][_][_] pollfd_t
 * (1a)                     |         +-------------+
 * +---------------+	    |
 * | /dev/poll tbl |	    |
 * +-v-------------+	    |
 *   |			    |
 *   +------------------+   |
 * (7)              (3) V   v
 * polldat hash     +-------------+    (4) bitmap representing fd space
 * [_][_][_][_]<----|             |--->000010010010001010101010101010110
 *  |  |  |  |      | pollcache_t |
 *  .  v  .  .      |             |
 *    [polldat_t]   +-------------+
 *     |
 *    [polldat_t]
 *     |
 *     v
 *     NULL
 *
 *
 * Both poll system call and /dev/poll use the pollcache_t structure
 * definition and the routines managing the structure. But poll(2) and
 * /dev/poll have their own copy of the structures. The /dev/poll driver
 * table (1a) contains an array of pointers, each pointing at a pollcache_t
 * struct (3). A device minor number is used as an device table index.
 *
 */
#include <sys/poll.h>

#if defined(_KERNEL) || defined(_KMEMUSER)

#include <sys/thread.h>
#include <sys/file.h>
#include <sys/port_kernel.h>

#ifdef	__cplusplus
extern "C" {
#endif

/*
 * Typedefs
 */
struct pollcache;
struct pollstate;
struct pcachelink;
struct polldat;

typedef struct pollcache pollcache_t;
typedef struct pollstate pollstate_t;
typedef struct pcachelink pcachelink_t;
typedef struct polldat polldat_t;

/*
 * description of pollcacheset structure
 */
typedef struct pollcacheset {
	uintptr_t	pcs_usradr;	/* usr pollfd array address */
	pollfd_t	*pcs_pollfd;	/* cached poll lists */
	size_t		pcs_nfds;	/* number of poll fd in cached list */
	ulong_t		pcs_count;	/* for LU replacement policy */
} pollcacheset_t;

#define	POLLFDSETS	2

/*
 * Maximum depth for recusive poll operations.
 */
#define	POLLMAXDEPTH	5

/*
 * State information kept by each polling thread
 */
struct pollstate {
	pollfd_t	*ps_pollfd;	/* hold the current poll list */
	size_t		ps_nfds;	/* size of ps_pollfd */
	kmutex_t	ps_lock;	/* mutex for sleep/wakeup */
	pollcache_t	*ps_pcache;	/* cached poll fd set */
	pollcacheset_t	*ps_pcacheset;	/* cached poll lists */
	int		ps_nsets;	/* no. of cached poll sets */
	pollfd_t	*ps_dpbuf;	/* return pollfd buf used by devpoll */
	size_t		ps_dpbufsize;	/* size of ps_dpbuf */
	int		ps_depth;	/* epoll recursion depth */
	pollcache_t	*ps_pc_stack[POLLMAXDEPTH]; /* epoll recursion state */
	pollcache_t	*ps_contend_pc;		/* pollcache waited on */
	pollstate_t	*ps_contend_nextp;	/* next in contender list */
	pollstate_t	**ps_contend_pnextp;	/* pointer-to-previous-next */
	int		ps_flags;	/* state flags */
};

/* pollstate flags */
#define	POLLSTATE_STALEMATE	0x1
#define	POLLSTATE_ULFAIL	0x2

/* pollstate_enter results */
#define	PSE_SUCCESS		0
#define	PSE_FAIL_DEPTH		1
#define	PSE_FAIL_LOOP		2
#define	PSE_FAIL_DEADLOCK	3
#define	PSE_FAIL_POLLSTATE	4

/*
 * poll cache size defines
 */
#define	POLLCHUNKSHIFT		8	/* hash table increment size is 256 */
#define	POLLHASHCHUNKSZ		(1 << POLLCHUNKSHIFT)
#define	POLLHASHINC		2	/* poll hash table growth factor */
#define	POLLHASHTHRESHOLD	2	/* poll hash list length threshold */
#define	POLLHASH(x, y)	((y) % (x))	/* poll hash function */

/*
 * poll.c assumes the POLLMAPCHUNK is power of 2
 */
#define	POLLMAPCHUNK	2048	/* bitmap inc -- each for 2K of polled fd's */

/*
 * used to refrence from watched fd back to the fd position in cached
 * poll list for quick revents update.
 */
typedef struct xref {
	ssize_t	xf_position;    /* xref fd position in poll fd list */
	short	xf_refcnt;	/* ref cnt of same fd in poll list */
} xref_t;

#define	POLLPOSINVAL	(-1L)	/* xf_position is invalid */
#define	POLLPOSTRANS	(-2L)	/* xf_position is transient state */


typedef enum pclstate {
	PCL_INIT = 0,	/* just allocated/zeroed, prior */
	PCL_VALID,	/* linked with both parent and child pollcaches */
	PCL_STALE,	/* still linked but marked stale, pending refresh */
	PCL_INVALID,	/* dissociated from one pollcache, awaiting cleanup */
	PCL_FREE	/* only meant to indicate use-after-free */
} pclstate_t;

/*
 * The pcachelink struct creates an association between parent and child
 * pollcaches in a recursive /dev/poll operation.  Fields are protected by
 * pcl_lock although manipulation of pcl_child_next or pcl_parent_next also
 * requires holding pc_lock in the respective pcl_parent_pc or pcl_child_pc
 * pollcache.
 */
struct pcachelink {
	kmutex_t	pcl_lock;		/* protects contents */
	pclstate_t	pcl_state;		/* status of link entry */
	int		pcl_refcnt;		/* ref cnt of linked pcaches */
	pollcache_t	*pcl_child_pc;		/* child pollcache */
	pollcache_t	*pcl_parent_pc;		/* parent pollcache */
	pcachelink_t	*pcl_child_next;	/* next in child list */
	pcachelink_t	*pcl_parent_next;	/* next in parents list */
};


/*
 * polldat is an entry for a cached poll fd. A polldat struct can be in
 * poll cache table as well as on pollhead ph_list, which is used by
 * pollwakeup to wake up a sleeping poller. There should be one polldat
 * per polled fd hanging off pollstate struct.
 */
struct polldat {
	int		pd_fd;		/* cached poll fd */
	int		pd_events;	/* union of all polled events */
	file_t		*pd_fp;		/* used to detect fd reuse */
	pollhead_t	*pd_php;	/* used to undo poll registration */
	kthread_t	*pd_thread;	/* used for waking up a sleep thrd */
	pollcache_t	*pd_pcache;	/* a ptr to the pollcache of this fd */
	polldat_t	*pd_next;	/* next on pollhead's ph_list */
	polldat_t	*pd_hashnext;	/* next on pollhead's ph_list */
	int		pd_count;	/* total count from all ref'ed sets */
	int		pd_nsets;	/* num of xref sets, used by poll(2) */
	xref_t		*pd_ref;	/* ptr to xref info, 1 for each set */
	port_kevent_t	*pd_portev;	/* associated port event struct */
	uf_entry_gen_t	pd_gen;		/* fd generation at cache time */
	uint64_t	pd_epolldata;	/* epoll data, if any */
};

/*
 * One cache for each thread that polls. Points to a bitmap (used by pollwakeup)
 * and a hash table of polldats.
 *
 * Because of the handling required in pollrelock(), portfs abuses the notion of
 * an active pollcache (t_pollcache), providing its own struct port_fdcache_t.
 * It has matching pc_lock and pc_flag members at the correct offsets, but none
 * of its other fields can be accessed (through t_pollcache) safetly.
 */
struct pollcache {
	kmutex_t	pc_lock;	/* lock to protect pollcache */
	ulong_t		*pc_bitmap;	/* point to poll fd bitmap */
	polldat_t	**pc_hash;	/* points to a hash table of ptrs */
	int		pc_mapend;	/* the largest fd encountered so far */
	int		pc_mapsize;	/* the size of current map */
	int		pc_hashsize;	/* the size of current hash table */
	int		pc_fdcount;	/* track how many fd's are hashed */
	int		pc_flag;	/* see pc_flag define below */
	int		pc_busy;	/* can only exit when its 0 */
	kmutex_t	pc_no_exit;	/* protects pc_busy*, can't be nested */
	kcondvar_t	pc_busy_cv;	/* cv to wait on if ps_busy != 0 */
	kcondvar_t	pc_cv;		/* cv to wait on if needed */
	pid_t		pc_pid;		/* for check acc rights, devpoll only */
	int		pc_mapstart;	/* where search start, devpoll only */
	pcachelink_t	*pc_parents;	/* linked list of epoll parents */
	pcachelink_t	*pc_children;	/* linked list of epoll children */
};

/* pc_flag */
#define	PC_POLLWAKE	0x02	/* pollwakeup() occurred */
#define	PC_EPOLL	0x04	/* pollcache is epoll-enabled */
/*
 * PC_PORTFS is not a flag for "real" pollcaches, but rather an indicator for
 * when portfs sets t_pollcache to a port_fdcache_t pointer.  If, while
 * debugging a system, one sees PC_PORTFS in pc_flag, they will know to
 * disregard the other fields, as it is not a pollcache.
 */
#define	PC_PORTFS	0x08

#if defined(_KERNEL)
/*
 * Internal routines.
 */
extern void pollnotify(pollcache_t *, int);

/*
 * public poll head interfaces (see poll.h):
 *
 *  pollhead_clean      clean up all polldats on a pollhead list
 */
extern void pollhead_clean(pollhead_t *);

/*
 * private poll head interfaces:
 *
 *  polldat_associate		adds a polldat to a pollhead list
 *  polldat_disassociate	remove polldat from its assoc'd pollhead list
 */
extern void polldat_associate(polldat_t *, pollhead_t *);
extern void polldat_disassociate(polldat_t *);

/*
 * poll state interfaces:
 *
 *  pollstate_create	initializes per-thread pollstate
 *  pollstate_destroy	cleans up per-thread pollstate
 *  pollstate_enter	safely lock pollcache for pollstate
 *  pollstate_exit	unlock pollcache from pollstate
 */
extern pollstate_t *pollstate_create(void);
extern void pollstate_destroy(pollstate_t *);
extern int pollstate_enter(pollcache_t *);
extern void pollstate_exit(pollcache_t *);

/*
 * public pcache interfaces:
 *
 *  pcache_alloc	allocate a poll cache skeleton
 *  pcache_create       creates all poll cache supporting data struct
 *  pcache_insert	cache a poll fd, calls pcache_insert_fd
 *  pcache_lookup       given an fd list, returns a cookie
 *  pcache_poll         polls the cache for fd's having events on them
 *  pcache_clean        clean up all the pollhead and fpollinfo reference
 *  pcache_destroy      destroys the pcache
 */
extern pollcache_t *pcache_alloc();
extern void pcache_create(pollcache_t *, nfds_t);
extern int pcache_insert(pollstate_t *, file_t *, pollfd_t *, int *, ssize_t,
    int);
extern int pcache_poll(pollfd_t *, pollstate_t *, nfds_t, int *, int);
extern void pcache_clean(pollcache_t *);
extern void pcache_destroy(pollcache_t *);

/*
 * private pcache interfaces:
 *
 *  pcache_lookup_fd	lookup an fd, returns a polldat
 *  pcache_alloc_fd	allocates and returns a polldat
 *  pcache_insert_fd	insert an fd into pcache (called by pcache_insert)
 *  pcache_delete_fd	insert an fd into pcache (called by pcacheset_delete_fd)
 *  pcache_grow_hashtbl	grows the pollcache hash table and rehash
 *  pcache_grow_map	grows the pollcache bitmap
 *  pcache_update_xref	update cross ref (from polldat back to cacheset) info
 *  pcache_clean_entry	cleanup an entry in pcache and more...
 *  pcache_wake_parents	wake linked parent pollcaches
 */
extern polldat_t *pcache_lookup_fd(pollcache_t *, int);
extern polldat_t *pcache_alloc_fd(int);
extern void pcache_insert_fd(pollcache_t *, polldat_t *, nfds_t);
extern int pcache_delete_fd(pollstate_t *, int, size_t, int, uint_t);
extern void pcache_grow_hashtbl(pollcache_t *, nfds_t);
extern void pcache_grow_map(pollcache_t *, int);
extern void pcache_update_xref(pollcache_t *, int, ssize_t, int);
extern void pcache_clean_entry(pollstate_t *, int);
extern void pcache_wake_parents(pollcache_t *);

/*
 * pcacheset interfaces:
 *
 * pcacheset_create     creates new pcachesets (easier for dynamic pcachesets)
 * pcacheset_destroy    destroys a pcacheset
 * pcacheset_cache_list caches and polls a new poll list
 * pcacheset_remove_list removes (usually a partial) cached poll list
 * pcacheset_resolve    resolves extant pcacheset and fd list
 * pcacheset_cmp        compares a pcacheset with an fd list
 * pcacheset_invalidate invalidate entries in pcachesets
 * pcacheset_reset_count resets the usage counter of pcachesets
 * pcacheset_replace	selects a poll cacheset for replacement
 */
extern pollcacheset_t *pcacheset_create(int);
extern void pcacheset_destroy(pollcacheset_t *, int);
extern int pcacheset_cache_list(pollstate_t *, pollfd_t *, int *, int);
extern void pcacheset_remove_list(pollstate_t *, pollfd_t *, int, int, int,
    int);
extern int pcacheset_resolve(pollstate_t *, nfds_t, int *, int);
extern int pcacheset_cmp(pollfd_t *, pollfd_t *, pollfd_t *, int);
extern void pcacheset_invalidate(pollstate_t *, polldat_t *);
extern void pcacheset_reset_count(pollstate_t *, int);
extern int pcacheset_replace(pollstate_t *);

#endif /* defined(_KERNEL) */

#ifdef	__cplusplus
}
#endif

#endif /* defined(_KERNEL) || defined(_KMEMUSER) */

#endif	/* _SYS_POLL_IMPL_H */