xref: /dragonfly/sys/kern/kern_sysref.c (revision dd0e3cd7)

/*
 * Copyright (c) 2007 The DragonFly Project.  All rights reserved.
 *
 * This code is derived from software contributed to The DragonFly Project
 * by Matthew Dillon <dillon@backplane.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name of The DragonFly Project nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific, prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
/*
 * System resource control module for all cluster-addressable system resource
 * structures.
 *
 * This module implements the core ref counting, sysid registration, and
 * objcache-backed allocation mechanism for all major system resource
 * structures.
 *
 * sysid registrations operate via the objcache ctor/dtor mechanism and
 * sysids will be reused if the resource is not explicitly accessed via
 * its sysid.  This removes all RB tree handling overhead from the critical
 * path for locally used resources.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/tree.h>
#include <sys/spinlock.h>
#include <machine/atomic.h>
#include <machine/cpufunc.h>

#include <sys/spinlock2.h>
#include <sys/sysref2.h>

static boolean_t sysref_ctor(void *data, void *privdata, int ocflags);
static void sysref_dtor(void *data, void *privdata);

/*
 * Red-Black tree support
 */
static int rb_sysref_compare(struct sysref *sr1, struct sysref *sr2);
RB_GENERATE2(sysref_rb_tree, sysref, rbnode, rb_sysref_compare, sysid_t, sysid);

static struct srpercpu {
	struct sysref_rb_tree rbtree;
	struct spinlock spin;
} sysref_array[MAXCPU];

static void
sysrefbootinit(void *dummy __unused)
{
	struct srpercpu *sa;
	int i;

	for (i = 0; i < ncpus; ++i) {
		sa = &sysref_array[i];
		spin_init(&sa->spin, "sysrefbootinit");
		RB_INIT(&sa->rbtree);
	}
}

SYSINIT(sysref, SI_BOOT2_MACHDEP, SI_ORDER_ANY, sysrefbootinit, NULL);

static
int
rb_sysref_compare(struct sysref *sr1, struct sysref *sr2)
{
	if (sr1->sysid < sr2->sysid)
		return(-1);
	if (sr1->sysid > sr2->sysid)
		return(1);
	return(0);
}

/*
 * Manual initialization of a resource structure's sysref, only used during
 * booting to set up certain statically declared resources which cannot
 * be deallocated.
 */
void
sysref_init(struct sysref *sr, struct sysref_class *srclass)
{
	struct srpercpu *sa;
	globaldata_t gd;

	gd = mycpu;
	crit_enter_gd(gd);
	gd->gd_sysid_alloc += ncpus_fit; /* next unique sysid */
	sr->sysid = gd->gd_sysid_alloc;
	KKASSERT(((int)sr->sysid & ncpus_fit_mask) == gd->gd_cpuid);
	sr->refcnt = -0x40000000;
	sr->flags = 0;
	sr->srclass = srclass;

	sa = &sysref_array[gd->gd_cpuid];
	spin_lock(&sa->spin);
	sysref_rb_tree_RB_INSERT(&sa->rbtree, sr);
	spin_unlock(&sa->spin);
	crit_exit_gd(gd);
}

/*
 * Allocate a resource structure of the specified class, initialize a
 * sysid and add the resource to the RB tree.  The caller must complete
 * initialization of the resource and call sysref_activate() to activate it.
 */
void *
sysref_alloc(struct sysref_class *srclass)
{
	struct sysref *sr;
	char *data;
	int n;

	/*
	 * Create the object cache backing store.
	 */
	if (srclass->oc == NULL) {
		KKASSERT(srclass->mtype != NULL);
		srclass->oc = objcache_create_mbacked(
				srclass->mtype, srclass->objsize,
				0, srclass->nom_cache,
				sysref_ctor, sysref_dtor, srclass);
	}

	/*
	 * Allocate the resource.
	 */
	data = objcache_get(srclass->oc, M_WAITOK);
	sr = (struct sysref *)(data + srclass->offset);
	KKASSERT(sr->flags & SRF_PUTAWAY);
	sr->flags &= ~SRF_PUTAWAY;

	/*
	 * Refcnt isn't touched while it is zero.  The objcache ctor
	 * function has already allocated a sysid and emplaced the
	 * structure in the RB tree.
	 */
	KKASSERT(sr->refcnt == 0);
	sr->refcnt = -0x40000000;

	/*
	 * Clean out the structure unless the caller wants to deal with
	 * it (e.g. like the vmspace code).
	 */
	if ((srclass->flags & SRC_MANAGEDINIT) == 0) {
		if (srclass->offset != 0)
			bzero(data, srclass->offset);
		n = srclass->offset + sizeof(struct sysref);
		KKASSERT(n <= srclass->objsize);
		if (n != srclass->objsize)
			bzero(data + n, srclass->objsize - n);
	}
	return(data);
}

/*
 * Object cache backing store ctor function.
 *
 * This allocates the sysid and associates the structure with the
 * red-black tree, allowing it to be looked up.  The actual resource
 * structure has NOT yet been allocated so it is marked free.
 *
 * If the sysid is not used to access the resource, we will just
 * allow the sysid to be reused when the resource structure is reused,
 * allowing the RB tree operation to be 'cached'.  This results in
 * virtually no performance penalty for using the sysref facility.
 */
static
boolean_t
sysref_ctor(void *data, void *privdata, int ocflags)
{
	globaldata_t gd;
	struct srpercpu *sa;
	struct sysref_class *srclass = privdata;
	struct sysref *sr = (void *)((char *)data + srclass->offset);

	/*
	 * Resource structures need to be cleared when allocating from
	 * malloc backing store.  This is different from the zeroing
	 * that we do in sysref_alloc().
	 */
	bzero(data, srclass->objsize);

	/*
	 * Resources managed by our objcache do the sysid and RB tree
	 * handling in the objcache ctor/dtor, so we can reuse the
	 * structure without re-treeing it over and over again.
	 */
	gd = mycpu;
	crit_enter_gd(gd);
	gd->gd_sysid_alloc += ncpus_fit; /* next unique sysid */
	sr->sysid = gd->gd_sysid_alloc;
	KKASSERT(((int)sr->sysid & ncpus_fit_mask) == gd->gd_cpuid);
	/* sr->refcnt= 0; already zero */
	sr->flags = SRF_ALLOCATED | SRF_PUTAWAY;
	sr->srclass = srclass;

	sa = &sysref_array[gd->gd_cpuid];
	spin_lock(&sa->spin);
	sysref_rb_tree_RB_INSERT(&sa->rbtree, sr);
	spin_unlock(&sa->spin);
	crit_exit_gd(gd);

	/*
	 * Execute the class's ctor function, if any.  NOTE: The class
	 * should not try to zero out the structure, we've already handled
	 * that and preinitialized the sysref.
	 *
	 * XXX ignores return value for now
	 */
	if (srclass->ctor)
		srclass->ctor(data, privdata, ocflags);
	return TRUE;
}

/*
 * Object cache destructor, allowing the structure to be returned
 * to the system memory pool.  The resource structure must be
 * removed from the RB tree.  All other references have already
 * been destroyed and the RB tree will not create any new references
 * to the structure in its current state.
 */
static
void
sysref_dtor(void *data, void *privdata)
{
	struct srpercpu *sa;
	struct sysref_class *srclass = privdata;
	struct sysref *sr = (void *)((char *)data + srclass->offset);

	KKASSERT(sr->refcnt == 0);
	sa = &sysref_array[(int)sr->sysid & ncpus_fit_mask];
	spin_lock(&sa->spin);
	sysref_rb_tree_RB_REMOVE(&sa->rbtree, sr);
	spin_unlock(&sa->spin);
	if (srclass->dtor)
		srclass->dtor(data, privdata);
}

/*
 * Activate or reactivate a resource. 0x40000001 is added to the ref count
 * so -0x40000000 (during initialization) will translate to a ref count of 1.
 * Any references made during initialization will translate to additional
 * positive ref counts.
 *
 * MPSAFE
 */
void
sysref_activate(struct sysref *sr)
{
	int count;

	for (;;) {
		count = sr->refcnt;
		KASSERT(count < 0 && count + 0x40000001 > 0,
			("sysref_activate: bad count %08x", count));
		if (atomic_cmpset_int(&sr->refcnt, count, count + 0x40000001))
			break;
		cpu_pause();
	}
}

/*
 * Release a reference under special circumstances.  This call is made
 * from the sysref_put() inline from sys/sysref2.h for any 1->0 transitions,
 * negative->negative 'termination in progress' transitions, and when the
 * cmpset instruction fails during a normal transition.
 *
 * This function is called from the sysref_put() inline in sys/sysref2.h,
 * but handles all cases regardless.
 */
void
_sysref_put(struct sysref *sr)
{
	int count;
	void *data;

	KKASSERT((sr->flags & SRF_PUTAWAY) == 0);

	for (;;) {
		count = sr->refcnt;
		if (count > 1) {
			/*
			 * release 1 count, nominal case, active resource
			 * structure, no other action required.
			 */
			if (atomic_cmpset_int(&sr->refcnt, count, count - 1))
				break;
		} else if (count == 1) {
			/*
			 * 1->0 transitions transition to -0x40000000 instead,
			 * placing the resource structure into a termination-
			 * in-progress state.  The termination function is
			 * then called.
			 */
			data = (char *)sr - sr->srclass->offset;
			sr->srclass->ops.lock(data);
			if (atomic_cmpset_int(&sr->refcnt, count, -0x40000000)) {
				sr->srclass->ops.terminate(data);
				/* callback unlocks */
				break;
			}
			sr->srclass->ops.unlock(data);
		} else if (count > -0x40000000) {
			/*
			 * release 1 count, nominal case, resource undergoing
			 * termination.  The Resource can be ref'd and
			 * deref'd while undergoing termination.
			 */
			if (atomic_cmpset_int(&sr->refcnt, count, count - 1))
				break;
		} else {
			/*
			 * Final release, set refcnt to 0.
			 * Resource must have been allocated.
			 *
			 * If SRF_SYSIDUSED is not set just objcache_put() the
			 * resource, otherwise objcache_dtor() the resource.
			 */
			KKASSERT(count == -0x40000000);
			if (atomic_cmpset_int(&sr->refcnt, count, 0)) {
				KKASSERT(sr->flags & SRF_ALLOCATED);
				sr->flags |= SRF_PUTAWAY;
				data = (char *)sr - sr->srclass->offset;
				if (sr->flags & SRF_SYSIDUSED)
					objcache_dtor(sr->srclass->oc, data);
				else
					objcache_put(sr->srclass->oc, data);
				break;
			}
		}
		/* loop until the cmpset succeeds */
		cpu_pause();
	}
}

sysid_t
allocsysid(void)
{
	globaldata_t gd = mycpu;
	sysid_t sysid;

	crit_enter_gd(gd);
	gd->gd_sysid_alloc += ncpus_fit;
	sysid = gd->gd_sysid_alloc;
	crit_exit_gd(gd);
	return(sysid);
}