18a268428SJeffrey Hsu /*
28a268428SJeffrey Hsu * Copyright (c) 2005 Jeffrey M. Hsu. All rights reserved.
38a268428SJeffrey Hsu *
48a268428SJeffrey Hsu * This code is derived from software contributed to The DragonFly Project
58a268428SJeffrey Hsu * by Jeffrey M. Hsu.
68a268428SJeffrey Hsu *
78a268428SJeffrey Hsu * Redistribution and use in source and binary forms, with or without
88a268428SJeffrey Hsu * modification, are permitted provided that the following conditions
98a268428SJeffrey Hsu * are met:
108a268428SJeffrey Hsu * 1. Redistributions of source code must retain the above copyright
118a268428SJeffrey Hsu * notice, this list of conditions and the following disclaimer.
128a268428SJeffrey Hsu * 2. Redistributions in binary form must reproduce the above copyright
138a268428SJeffrey Hsu * notice, this list of conditions and the following disclaimer in the
148a268428SJeffrey Hsu * documentation and/or other materials provided with the distribution.
158a268428SJeffrey Hsu * 3. Neither the name of The DragonFly Project nor the names of its
168a268428SJeffrey Hsu * contributors may be used to endorse or promote products derived
178a268428SJeffrey Hsu * from this software without specific, prior written permission.
188a268428SJeffrey Hsu *
198a268428SJeffrey Hsu * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
208a268428SJeffrey Hsu * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
218a268428SJeffrey Hsu * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
228a268428SJeffrey Hsu * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
238a268428SJeffrey Hsu * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
248a268428SJeffrey Hsu * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
258a268428SJeffrey Hsu * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
268a268428SJeffrey Hsu * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
278a268428SJeffrey Hsu * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
288a268428SJeffrey Hsu * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
298a268428SJeffrey Hsu * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
308a268428SJeffrey Hsu * SUCH DAMAGE.
318a268428SJeffrey Hsu */
328a268428SJeffrey Hsu
338a268428SJeffrey Hsu #include <sys/param.h>
348a268428SJeffrey Hsu #include <sys/kernel.h>
358a268428SJeffrey Hsu #include <sys/systm.h>
368a268428SJeffrey Hsu #include <sys/callout.h>
378a268428SJeffrey Hsu #include <sys/globaldata.h>
388a268428SJeffrey Hsu #include <sys/malloc.h>
398a268428SJeffrey Hsu #include <sys/queue.h>
408a268428SJeffrey Hsu #include <sys/objcache.h>
4154d017fdSMatthew Dillon #include <sys/spinlock.h>
428a268428SJeffrey Hsu #include <sys/thread.h>
438a268428SJeffrey Hsu #include <sys/thread2.h>
4454d017fdSMatthew Dillon #include <sys/spinlock2.h>
45c6bb9a90SSepherosa Ziehau #include <sys/sysctl.h>
468a268428SJeffrey Hsu
478a268428SJeffrey Hsu static MALLOC_DEFINE(M_OBJCACHE, "objcache", "Object Cache");
48c6cd37a0SMatthew Dillon static MALLOC_DEFINE(M_OBJMAG, "objcache mag", "Object Cache Magazine");
498a268428SJeffrey Hsu
50a1f6a1feSMatthew Dillon #define INITIAL_MAG_CAPACITY 64
518a268428SJeffrey Hsu
528a268428SJeffrey Hsu struct magazine {
538a268428SJeffrey Hsu int rounds;
548a268428SJeffrey Hsu int capacity;
558a268428SJeffrey Hsu SLIST_ENTRY(magazine) nextmagazine;
568a268428SJeffrey Hsu void *objects[];
578a268428SJeffrey Hsu };
588a268428SJeffrey Hsu
598a268428SJeffrey Hsu SLIST_HEAD(magazinelist, magazine);
608a268428SJeffrey Hsu
611ad7b4a7SSepherosa Ziehau #define MAGAZINE_HDRSIZE __offsetof(struct magazine, objects[0])
62791a85dcSMatthew Dillon #define MAGAZINE_CAPACITY_MAX 4096
631ad7b4a7SSepherosa Ziehau #define MAGAZINE_CAPACITY_MIN 4
641ad7b4a7SSepherosa Ziehau
658a268428SJeffrey Hsu /*
668a268428SJeffrey Hsu * per-cluster cache of magazines
6754d017fdSMatthew Dillon *
6854d017fdSMatthew Dillon * All fields in this structure are protected by the spinlock.
698a268428SJeffrey Hsu */
708a268428SJeffrey Hsu struct magazinedepot {
718a268428SJeffrey Hsu /*
728a268428SJeffrey Hsu * The per-cpu object caches only exchanges completely full or
738a268428SJeffrey Hsu * completely empty magazines with the depot layer, so only have
748a268428SJeffrey Hsu * to cache these two types of magazines.
758a268428SJeffrey Hsu */
768a268428SJeffrey Hsu struct magazinelist fullmagazines;
778a268428SJeffrey Hsu struct magazinelist emptymagazines;
788a268428SJeffrey Hsu int magcapacity;
798a268428SJeffrey Hsu
8077e294a1SMatthew Dillon /* protect this structure */
8154d017fdSMatthew Dillon struct spinlock spin;
828a268428SJeffrey Hsu
8377e294a1SMatthew Dillon /* magazines not yet allocated towards limit */
8477e294a1SMatthew Dillon int unallocated_objects;
85e98a16b6SMatthew Dillon int cluster_limit; /* ref for adjustments */
868a268428SJeffrey Hsu
878a268428SJeffrey Hsu /* infrequently used fields */
888a268428SJeffrey Hsu int waiting; /* waiting for another cpu to
898a268428SJeffrey Hsu * return a full magazine to
908a268428SJeffrey Hsu * the depot */
918a268428SJeffrey Hsu int contested; /* depot contention count */
92a1882035SSepherosa Ziehau } __cachealign;
938a268428SJeffrey Hsu
948a268428SJeffrey Hsu /*
958a268428SJeffrey Hsu * per-cpu object cache
968a268428SJeffrey Hsu * All fields in this structure are protected by crit_enter().
978a268428SJeffrey Hsu */
988a268428SJeffrey Hsu struct percpu_objcache {
998a268428SJeffrey Hsu struct magazine *loaded_magazine; /* active magazine */
1008a268428SJeffrey Hsu struct magazine *previous_magazine; /* backup magazine */
1018a268428SJeffrey Hsu
1028a268428SJeffrey Hsu /* statistics */
103b5d55e77SSepherosa Ziehau u_long gets_cumulative; /* total calls to get */
104b5d55e77SSepherosa Ziehau u_long gets_null; /* objcache_get returned NULL */
105b5d55e77SSepherosa Ziehau u_long allocs_cumulative; /* total calls to alloc */
106b5d55e77SSepherosa Ziehau u_long puts_cumulative; /* total calls to put */
107da22de6fSSepherosa Ziehau u_long gets_exhausted; /* # of gets hit exhaustion */
108b5d55e77SSepherosa Ziehau #ifdef notyet
109b5d55e77SSepherosa Ziehau u_long puts_othercluster; /* returned to other cluster */
110b5d55e77SSepherosa Ziehau #endif
1118a268428SJeffrey Hsu
1128a268428SJeffrey Hsu /* infrequently used fields */
113114b500dSSepherosa Ziehau int waiting; /* waiting for a thread on this
114114b500dSSepherosa Ziehau * cpu to return an obj to the
115114b500dSSepherosa Ziehau * per-cpu cache */
116a1882035SSepherosa Ziehau } __cachealign;
1178a268428SJeffrey Hsu
1188a268428SJeffrey Hsu /* only until we have NUMA cluster topology information XXX */
1198a268428SJeffrey Hsu #define MAXCLUSTERS 1
1208a268428SJeffrey Hsu #define myclusterid 0
1218a268428SJeffrey Hsu #define CLUSTER_OF(obj) 0
1228a268428SJeffrey Hsu
1238a268428SJeffrey Hsu /*
1245b694eafSSepherosa Ziehau * Rarely accessed but useful bits of objcache.
1255b694eafSSepherosa Ziehau */
1265b694eafSSepherosa Ziehau struct objcache_desc {
1275b694eafSSepherosa Ziehau LIST_ENTRY(objcache_desc) next;
1285b694eafSSepherosa Ziehau struct objcache *objcache;
1295b694eafSSepherosa Ziehau int total_objects;
130c6bb9a90SSepherosa Ziehau int reserved;
1315b694eafSSepherosa Ziehau char name[OBJCACHE_NAMELEN];
1325b694eafSSepherosa Ziehau };
1335b694eafSSepherosa Ziehau
1345b694eafSSepherosa Ziehau /*
1358a268428SJeffrey Hsu * Two-level object cache consisting of NUMA cluster-level depots of
1368a268428SJeffrey Hsu * fully loaded or completely empty magazines and cpu-level caches of
1378a268428SJeffrey Hsu * individual objects.
1388a268428SJeffrey Hsu */
1398a268428SJeffrey Hsu struct objcache {
1408a268428SJeffrey Hsu /* object constructor and destructor from blank storage */
1418a268428SJeffrey Hsu objcache_ctor_fn *ctor;
1428a268428SJeffrey Hsu objcache_dtor_fn *dtor;
143698331b0SMatthew Dillon void *privdata;
1448a268428SJeffrey Hsu
1458a268428SJeffrey Hsu /* interface to underlying allocator */
1468a268428SJeffrey Hsu objcache_alloc_fn *alloc;
1478a268428SJeffrey Hsu objcache_free_fn *free;
1488a268428SJeffrey Hsu void *allocator_args;
1498a268428SJeffrey Hsu
1505b694eafSSepherosa Ziehau struct objcache_desc *desc;
1518a268428SJeffrey Hsu
1528a268428SJeffrey Hsu /* NUMA-cluster level caches */
1538a268428SJeffrey Hsu struct magazinedepot depot[MAXCLUSTERS];
1548a268428SJeffrey Hsu
1558a268428SJeffrey Hsu struct percpu_objcache cache_percpu[]; /* per-cpu caches */
1568a268428SJeffrey Hsu };
1578a268428SJeffrey Hsu
158c6bb9a90SSepherosa Ziehau SYSCTL_NODE(_kern, OID_AUTO, objcache, CTLFLAG_RW, 0, "objcache");
159c6bb9a90SSepherosa Ziehau
16054d017fdSMatthew Dillon static struct spinlock objcachelist_spin;
1615b694eafSSepherosa Ziehau static LIST_HEAD(objcachelist, objcache_desc) allobjcaches;
1621ad7b4a7SSepherosa Ziehau static int magazine_capmin;
1631ad7b4a7SSepherosa Ziehau static int magazine_capmax;
1648a268428SJeffrey Hsu
1658a268428SJeffrey Hsu static struct magazine *
mag_alloc(int capacity)1668a268428SJeffrey Hsu mag_alloc(int capacity)
1678a268428SJeffrey Hsu {
1688a268428SJeffrey Hsu struct magazine *mag;
1691ad7b4a7SSepherosa Ziehau int size;
1708a268428SJeffrey Hsu
1711ad7b4a7SSepherosa Ziehau size = __offsetof(struct magazine, objects[capacity]);
1721ad7b4a7SSepherosa Ziehau KASSERT(size > 0 && (size & __VM_CACHELINE_MASK) == 0,
1731ad7b4a7SSepherosa Ziehau ("magazine size is not multiple cache line size"));
1741ad7b4a7SSepherosa Ziehau
17562938642SMatthew Dillon mag = kmalloc(size, M_OBJMAG, M_INTWAIT | M_ZERO | M_CACHEALIGN);
1768a268428SJeffrey Hsu mag->capacity = capacity;
1778a268428SJeffrey Hsu mag->rounds = 0;
1788a268428SJeffrey Hsu return (mag);
1798a268428SJeffrey Hsu }
1808a268428SJeffrey Hsu
1811ad7b4a7SSepherosa Ziehau static int
mag_capacity_align(int mag_capacity)1821ad7b4a7SSepherosa Ziehau mag_capacity_align(int mag_capacity)
1831ad7b4a7SSepherosa Ziehau {
1841ad7b4a7SSepherosa Ziehau int mag_size;
1851ad7b4a7SSepherosa Ziehau
1861ad7b4a7SSepherosa Ziehau mag_size = __VM_CACHELINE_ALIGN(
1871ad7b4a7SSepherosa Ziehau __offsetof(struct magazine, objects[mag_capacity]));
1881ad7b4a7SSepherosa Ziehau mag_capacity = (mag_size - MAGAZINE_HDRSIZE) / sizeof(void *);
1891ad7b4a7SSepherosa Ziehau
1901ad7b4a7SSepherosa Ziehau return mag_capacity;
1911ad7b4a7SSepherosa Ziehau }
1921ad7b4a7SSepherosa Ziehau
1938a268428SJeffrey Hsu /*
1945b7da64aSMatthew Dillon * Utility routine for objects that don't require any de-construction.
1955b7da64aSMatthew Dillon */
1965b7da64aSMatthew Dillon
1975b7da64aSMatthew Dillon static void
null_dtor(void * obj,void * privdata)198698331b0SMatthew Dillon null_dtor(void *obj, void *privdata)
1995b7da64aSMatthew Dillon {
2005b7da64aSMatthew Dillon /* do nothing */
2015b7da64aSMatthew Dillon }
2025b7da64aSMatthew Dillon
2035b7da64aSMatthew Dillon static boolean_t
null_ctor(void * obj,void * privdata,int ocflags)204698331b0SMatthew Dillon null_ctor(void *obj, void *privdata, int ocflags)
2055b7da64aSMatthew Dillon {
2065b7da64aSMatthew Dillon return TRUE;
2075b7da64aSMatthew Dillon }
2085b7da64aSMatthew Dillon
2095b7da64aSMatthew Dillon /*
2108a268428SJeffrey Hsu * Create an object cache.
2118a268428SJeffrey Hsu */
2128a268428SJeffrey Hsu struct objcache *
objcache_create(const char * name,int cluster_limit,int nom_cache,objcache_ctor_fn * ctor,objcache_dtor_fn * dtor,void * privdata,objcache_alloc_fn * alloc,objcache_free_fn * free,void * allocator_args)2132fce2579SSepherosa Ziehau objcache_create(const char *name, int cluster_limit, int nom_cache,
214698331b0SMatthew Dillon objcache_ctor_fn *ctor, objcache_dtor_fn *dtor, void *privdata,
2158a268428SJeffrey Hsu objcache_alloc_fn *alloc, objcache_free_fn *free,
2168a268428SJeffrey Hsu void *allocator_args)
2178a268428SJeffrey Hsu {
2185b694eafSSepherosa Ziehau struct objcache_desc *desc;
2198a268428SJeffrey Hsu struct objcache *oc;
2208a268428SJeffrey Hsu struct magazinedepot *depot;
2218a268428SJeffrey Hsu int cpuid;
2221a21fef5SNicolas Thery int nmagdepot;
223765b1ae0SMatthew Dillon int mag_capacity;
2241a21fef5SNicolas Thery int i;
2258a268428SJeffrey Hsu
226765b1ae0SMatthew Dillon /*
2275b694eafSSepherosa Ziehau * Allocate objcache descriptor.
2285b694eafSSepherosa Ziehau */
2295b694eafSSepherosa Ziehau desc = kmalloc(sizeof(*desc), M_OBJCACHE, M_WAITOK | M_ZERO);
2305b694eafSSepherosa Ziehau
2315b694eafSSepherosa Ziehau /*
232765b1ae0SMatthew Dillon * Allocate object cache structure
233765b1ae0SMatthew Dillon */
23462938642SMatthew Dillon oc = kmalloc(__offsetof(struct objcache, cache_percpu[ncpus]),
23562938642SMatthew Dillon M_OBJCACHE,
23662938642SMatthew Dillon M_WAITOK | M_ZERO | M_CACHEALIGN);
2375b7da64aSMatthew Dillon oc->ctor = ctor ? ctor : null_ctor;
2385b7da64aSMatthew Dillon oc->dtor = dtor ? dtor : null_dtor;
239698331b0SMatthew Dillon oc->privdata = privdata;
240765b1ae0SMatthew Dillon oc->alloc = alloc;
2418a268428SJeffrey Hsu oc->free = free;
2428a268428SJeffrey Hsu oc->allocator_args = allocator_args;
2438a268428SJeffrey Hsu
244765b1ae0SMatthew Dillon /*
2455b694eafSSepherosa Ziehau * Link objcache and its descriptor.
2465b694eafSSepherosa Ziehau */
2475b694eafSSepherosa Ziehau oc->desc = desc;
2485b694eafSSepherosa Ziehau desc->objcache = oc;
2495b694eafSSepherosa Ziehau strlcpy(desc->name, name, sizeof(desc->name));
2505b694eafSSepherosa Ziehau
2515b694eafSSepherosa Ziehau /*
252765b1ae0SMatthew Dillon * Initialize depot list(s).
253765b1ae0SMatthew Dillon */
2548a268428SJeffrey Hsu depot = &oc->depot[0];
25577e294a1SMatthew Dillon
256ba87a4abSSascha Wildner spin_init(&depot->spin, "objcachedepot");
2578a268428SJeffrey Hsu SLIST_INIT(&depot->fullmagazines);
2588a268428SJeffrey Hsu SLIST_INIT(&depot->emptymagazines);
25977e294a1SMatthew Dillon
260765b1ae0SMatthew Dillon /*
261765b1ae0SMatthew Dillon * Figure out the nominal number of free objects to cache and
262765b1ae0SMatthew Dillon * the magazine capacity. By default we want to cache up to
263765b1ae0SMatthew Dillon * half the cluster_limit. If there is no cluster_limit then
264765b1ae0SMatthew Dillon * we want to cache up to 128 objects.
265765b1ae0SMatthew Dillon */
266765b1ae0SMatthew Dillon if (nom_cache == 0)
267765b1ae0SMatthew Dillon nom_cache = cluster_limit / 2;
268765b1ae0SMatthew Dillon if (cluster_limit && nom_cache > cluster_limit)
269765b1ae0SMatthew Dillon nom_cache = cluster_limit;
270765b1ae0SMatthew Dillon if (nom_cache == 0)
271765b1ae0SMatthew Dillon nom_cache = INITIAL_MAG_CAPACITY * 2;
27277e294a1SMatthew Dillon
27377e294a1SMatthew Dillon /*
274765b1ae0SMatthew Dillon * Magazine capacity for 2 active magazines per cpu plus 2
2751ad7b4a7SSepherosa Ziehau * magazines in the depot.
276765b1ae0SMatthew Dillon */
2771ad7b4a7SSepherosa Ziehau mag_capacity = mag_capacity_align(nom_cache / (ncpus + 1) / 2 + 1);
2781ad7b4a7SSepherosa Ziehau if (mag_capacity > magazine_capmax)
2791ad7b4a7SSepherosa Ziehau mag_capacity = magazine_capmax;
2801ad7b4a7SSepherosa Ziehau else if (mag_capacity < magazine_capmin)
2811ad7b4a7SSepherosa Ziehau mag_capacity = magazine_capmin;
282765b1ae0SMatthew Dillon depot->magcapacity = mag_capacity;
283765b1ae0SMatthew Dillon
284765b1ae0SMatthew Dillon /*
285765b1ae0SMatthew Dillon * The cluster_limit must be sufficient to have two magazines per
286765b1ae0SMatthew Dillon * cpu plus at least two magazines in the depot. However, because
287765b1ae0SMatthew Dillon * partial magazines can stay on the cpus what we really need here
288765b1ae0SMatthew Dillon * is to specify the number of extra magazines we allocate for the
289765b1ae0SMatthew Dillon * depot.
290e98a16b6SMatthew Dillon *
291e98a16b6SMatthew Dillon * Use ~1B objects to mean 'unlimited'. A negative unallocated
292e98a16b6SMatthew Dillon * object count is possible due to dynamic adjustments so we can't
293e98a16b6SMatthew Dillon * use a negative number to mean 'unlimited'. We need some overflow
294e98a16b6SMatthew Dillon * capacity too due to the preallocated mags.
29577e294a1SMatthew Dillon */
29677e294a1SMatthew Dillon if (cluster_limit == 0) {
297c6bb9a90SSepherosa Ziehau depot->unallocated_objects = OBJCACHE_UNLIMITED;
29877e294a1SMatthew Dillon } else {
299765b1ae0SMatthew Dillon depot->unallocated_objects = ncpus * mag_capacity * 2 +
300765b1ae0SMatthew Dillon cluster_limit;
301a1f6a1feSMatthew Dillon }
3028a268428SJeffrey Hsu
3035b694eafSSepherosa Ziehau /* Save # of total objects. */
3045b694eafSSepherosa Ziehau desc->total_objects = depot->unallocated_objects;
3055b694eafSSepherosa Ziehau
306765b1ae0SMatthew Dillon /*
307e98a16b6SMatthew Dillon * This is a dynamic adjustment aid initialized to the callers
308e98a16b6SMatthew Dillon * expectations of the current limit.
309e98a16b6SMatthew Dillon */
310e98a16b6SMatthew Dillon depot->cluster_limit = cluster_limit;
311e98a16b6SMatthew Dillon
312e98a16b6SMatthew Dillon /*
313765b1ae0SMatthew Dillon * Initialize per-cpu caches
314765b1ae0SMatthew Dillon */
3158a268428SJeffrey Hsu for (cpuid = 0; cpuid < ncpus; cpuid++) {
3168a268428SJeffrey Hsu struct percpu_objcache *cache_percpu = &oc->cache_percpu[cpuid];
3178a268428SJeffrey Hsu
3188a268428SJeffrey Hsu cache_percpu->loaded_magazine = mag_alloc(mag_capacity);
3198a268428SJeffrey Hsu cache_percpu->previous_magazine = mag_alloc(mag_capacity);
3208a268428SJeffrey Hsu }
3211a21fef5SNicolas Thery
322765b1ae0SMatthew Dillon /*
323765b1ae0SMatthew Dillon * Compute how many empty magazines to place in the depot. This
324765b1ae0SMatthew Dillon * determines the retained cache size and is based on nom_cache.
325765b1ae0SMatthew Dillon *
326765b1ae0SMatthew Dillon * The actual cache size is larger because there are two magazines
327765b1ae0SMatthew Dillon * for each cpu as well but those can be in any fill state so we
328765b1ae0SMatthew Dillon * just can't count them.
329765b1ae0SMatthew Dillon *
330765b1ae0SMatthew Dillon * There is a minimum of two magazines in the depot.
331765b1ae0SMatthew Dillon */
332765b1ae0SMatthew Dillon nmagdepot = nom_cache / mag_capacity + 1;
3331a21fef5SNicolas Thery if (nmagdepot < 2)
3341a21fef5SNicolas Thery nmagdepot = 2;
3351a21fef5SNicolas Thery
336765b1ae0SMatthew Dillon /*
337765b1ae0SMatthew Dillon * Put empty magazines in depot
338765b1ae0SMatthew Dillon */
3391a21fef5SNicolas Thery for (i = 0; i < nmagdepot; i++) {
3401a21fef5SNicolas Thery struct magazine *mag = mag_alloc(mag_capacity);
3411a21fef5SNicolas Thery SLIST_INSERT_HEAD(&depot->emptymagazines, mag, nextmagazine);
3421a21fef5SNicolas Thery }
3431a21fef5SNicolas Thery
344287a8577SAlex Hornung spin_lock(&objcachelist_spin);
3455b694eafSSepherosa Ziehau LIST_INSERT_HEAD(&allobjcaches, desc, next);
346287a8577SAlex Hornung spin_unlock(&objcachelist_spin);
3478a268428SJeffrey Hsu
3488a268428SJeffrey Hsu return (oc);
3498a268428SJeffrey Hsu }
3508a268428SJeffrey Hsu
351e98a16b6SMatthew Dillon /*
352e98a16b6SMatthew Dillon * Adjust the cluster limit. This is allowed to cause unallocated_objects
353e98a16b6SMatthew Dillon * to go negative. Note that due to the magazine hysteresis there is a
354e98a16b6SMatthew Dillon * limit to how much of the objcache can be reclaimed using this API to
355e98a16b6SMatthew Dillon * reduce its size.
356e98a16b6SMatthew Dillon */
357e98a16b6SMatthew Dillon void
objcache_set_cluster_limit(struct objcache * oc,int cluster_limit)358e98a16b6SMatthew Dillon objcache_set_cluster_limit(struct objcache *oc, int cluster_limit)
359e98a16b6SMatthew Dillon {
360e98a16b6SMatthew Dillon struct magazinedepot *depot;
361e98a16b6SMatthew Dillon
362e98a16b6SMatthew Dillon depot = &oc->depot[myclusterid];
363e98a16b6SMatthew Dillon if (depot->cluster_limit != cluster_limit) {
3645b694eafSSepherosa Ziehau int delta;
3655b694eafSSepherosa Ziehau
366e98a16b6SMatthew Dillon spin_lock(&depot->spin);
367e98a16b6SMatthew Dillon delta = cluster_limit - depot->cluster_limit;
368e98a16b6SMatthew Dillon depot->unallocated_objects += delta;
369e98a16b6SMatthew Dillon depot->cluster_limit = cluster_limit;
370e98a16b6SMatthew Dillon spin_unlock(&depot->spin);
371e98a16b6SMatthew Dillon wakeup(depot);
3725b694eafSSepherosa Ziehau
3735b694eafSSepherosa Ziehau oc->desc->total_objects += delta;
374e98a16b6SMatthew Dillon }
375e98a16b6SMatthew Dillon }
376e98a16b6SMatthew Dillon
37770aac194SMatthew Dillon struct objcache *
objcache_create_simple(malloc_type_t mtype,size_t objsize)37870aac194SMatthew Dillon objcache_create_simple(malloc_type_t mtype, size_t objsize)
37970aac194SMatthew Dillon {
38070aac194SMatthew Dillon struct objcache_malloc_args *margs;
38170aac194SMatthew Dillon struct objcache *oc;
38270aac194SMatthew Dillon
383efda3bd0SMatthew Dillon margs = kmalloc(sizeof(*margs), M_OBJCACHE, M_WAITOK|M_ZERO);
38470aac194SMatthew Dillon margs->objsize = objsize;
38570aac194SMatthew Dillon margs->mtype = mtype;
3862fce2579SSepherosa Ziehau oc = objcache_create(mtype->ks_shortdesc, 0, 0,
3875b7da64aSMatthew Dillon NULL, NULL, NULL,
38870aac194SMatthew Dillon objcache_malloc_alloc, objcache_malloc_free,
38970aac194SMatthew Dillon margs);
39070aac194SMatthew Dillon return (oc);
39170aac194SMatthew Dillon }
39270aac194SMatthew Dillon
393b1641984SMatthew Dillon struct objcache *
objcache_create_mbacked(malloc_type_t mtype,size_t objsize,int cluster_limit,int nom_cache,objcache_ctor_fn * ctor,objcache_dtor_fn * dtor,void * privdata)394b1641984SMatthew Dillon objcache_create_mbacked(malloc_type_t mtype, size_t objsize,
3952fce2579SSepherosa Ziehau int cluster_limit, int nom_cache,
396b1641984SMatthew Dillon objcache_ctor_fn *ctor, objcache_dtor_fn *dtor,
397698331b0SMatthew Dillon void *privdata)
398b1641984SMatthew Dillon {
399b1641984SMatthew Dillon struct objcache_malloc_args *margs;
400b1641984SMatthew Dillon struct objcache *oc;
401b1641984SMatthew Dillon
402b1641984SMatthew Dillon margs = kmalloc(sizeof(*margs), M_OBJCACHE, M_WAITOK|M_ZERO);
403b1641984SMatthew Dillon margs->objsize = objsize;
404b1641984SMatthew Dillon margs->mtype = mtype;
405b1641984SMatthew Dillon oc = objcache_create(mtype->ks_shortdesc,
406765b1ae0SMatthew Dillon cluster_limit, nom_cache,
407698331b0SMatthew Dillon ctor, dtor, privdata,
408b1641984SMatthew Dillon objcache_malloc_alloc, objcache_malloc_free,
409b1641984SMatthew Dillon margs);
410b1641984SMatthew Dillon return(oc);
411b1641984SMatthew Dillon }
412b1641984SMatthew Dillon
413b1641984SMatthew Dillon
4148a268428SJeffrey Hsu #define MAGAZINE_EMPTY(mag) (mag->rounds == 0)
41577e294a1SMatthew Dillon #define MAGAZINE_NOTEMPTY(mag) (mag->rounds != 0)
4168a268428SJeffrey Hsu #define MAGAZINE_FULL(mag) (mag->rounds == mag->capacity)
4178a268428SJeffrey Hsu
4188a268428SJeffrey Hsu #define swap(x, y) ({ struct magazine *t = x; x = y; y = t; })
4198a268428SJeffrey Hsu
4208a268428SJeffrey Hsu /*
4218a268428SJeffrey Hsu * Get an object from the object cache.
4229bfc4d6dSMatthew Dillon *
4239bfc4d6dSMatthew Dillon * WARNING! ocflags are only used when we have to go to the underlying
4249bfc4d6dSMatthew Dillon * allocator, so we cannot depend on flags such as M_ZERO.
4258a268428SJeffrey Hsu */
4268a268428SJeffrey Hsu void *
objcache_get(struct objcache * oc,int ocflags)4278a268428SJeffrey Hsu objcache_get(struct objcache *oc, int ocflags)
4288a268428SJeffrey Hsu {
4298a268428SJeffrey Hsu struct percpu_objcache *cpucache = &oc->cache_percpu[mycpuid];
4308a268428SJeffrey Hsu struct magazine *loadedmag;
43177e294a1SMatthew Dillon struct magazine *emptymag;
4328a268428SJeffrey Hsu void *obj;
4338a268428SJeffrey Hsu struct magazinedepot *depot;
4348a268428SJeffrey Hsu
4355b7da64aSMatthew Dillon KKASSERT((ocflags & M_ZERO) == 0);
4368a268428SJeffrey Hsu crit_enter();
4378a268428SJeffrey Hsu ++cpucache->gets_cumulative;
4388a268428SJeffrey Hsu
4398a268428SJeffrey Hsu retry:
4408a268428SJeffrey Hsu /*
4418a268428SJeffrey Hsu * Loaded magazine has an object. This is the hot path.
4428a268428SJeffrey Hsu * It is lock-free and uses a critical section to block
4438a268428SJeffrey Hsu * out interrupt handlers on the same processor.
4448a268428SJeffrey Hsu */
4458a268428SJeffrey Hsu loadedmag = cpucache->loaded_magazine;
44677e294a1SMatthew Dillon if (MAGAZINE_NOTEMPTY(loadedmag)) {
44777e294a1SMatthew Dillon obj = loadedmag->objects[--loadedmag->rounds];
4488a268428SJeffrey Hsu crit_exit();
4498a268428SJeffrey Hsu return (obj);
4508a268428SJeffrey Hsu }
4518a268428SJeffrey Hsu
4528a268428SJeffrey Hsu /* Previous magazine has an object. */
45377e294a1SMatthew Dillon if (MAGAZINE_NOTEMPTY(cpucache->previous_magazine)) {
4548a268428SJeffrey Hsu swap(cpucache->loaded_magazine, cpucache->previous_magazine);
4558a268428SJeffrey Hsu loadedmag = cpucache->loaded_magazine;
45677e294a1SMatthew Dillon obj = loadedmag->objects[--loadedmag->rounds];
4575b7da64aSMatthew Dillon crit_exit();
4585b7da64aSMatthew Dillon return (obj);
4598a268428SJeffrey Hsu }
4608a268428SJeffrey Hsu
4618a268428SJeffrey Hsu /*
46277e294a1SMatthew Dillon * Both magazines empty. Get a full magazine from the depot and
463208c0e5bSMatthew Dillon * move one of the empty ones to the depot.
46477e294a1SMatthew Dillon *
46554d017fdSMatthew Dillon * Obtain the depot spinlock.
466b2560af1SMatthew Dillon *
4675b7da64aSMatthew Dillon * NOTE: Beyond this point, M_* flags are handled via oc->alloc()
4688a268428SJeffrey Hsu */
4698a268428SJeffrey Hsu depot = &oc->depot[myclusterid];
470287a8577SAlex Hornung spin_lock(&depot->spin);
471208c0e5bSMatthew Dillon
472208c0e5bSMatthew Dillon /*
47354d017fdSMatthew Dillon * Recheck the cpucache after obtaining the depot spinlock. This
47454d017fdSMatthew Dillon * shouldn't be necessary now but don't take any chances.
475208c0e5bSMatthew Dillon */
476208c0e5bSMatthew Dillon if (MAGAZINE_NOTEMPTY(cpucache->loaded_magazine) ||
477208c0e5bSMatthew Dillon MAGAZINE_NOTEMPTY(cpucache->previous_magazine)
478208c0e5bSMatthew Dillon ) {
479287a8577SAlex Hornung spin_unlock(&depot->spin);
480208c0e5bSMatthew Dillon goto retry;
4818a268428SJeffrey Hsu }
4828a268428SJeffrey Hsu
4838a268428SJeffrey Hsu /* Check if depot has a full magazine. */
4848a268428SJeffrey Hsu if (!SLIST_EMPTY(&depot->fullmagazines)) {
48577e294a1SMatthew Dillon emptymag = cpucache->previous_magazine;
4868a268428SJeffrey Hsu cpucache->previous_magazine = cpucache->loaded_magazine;
4878a268428SJeffrey Hsu cpucache->loaded_magazine = SLIST_FIRST(&depot->fullmagazines);
4888a268428SJeffrey Hsu SLIST_REMOVE_HEAD(&depot->fullmagazines, nextmagazine);
48977e294a1SMatthew Dillon
49077e294a1SMatthew Dillon /*
491208c0e5bSMatthew Dillon * Return emptymag to the depot.
49277e294a1SMatthew Dillon */
493208c0e5bSMatthew Dillon KKASSERT(MAGAZINE_EMPTY(emptymag));
49477e294a1SMatthew Dillon SLIST_INSERT_HEAD(&depot->emptymagazines,
49577e294a1SMatthew Dillon emptymag, nextmagazine);
496287a8577SAlex Hornung spin_unlock(&depot->spin);
49777e294a1SMatthew Dillon goto retry;
4988a268428SJeffrey Hsu }
4998a268428SJeffrey Hsu
5008a268428SJeffrey Hsu /*
50177e294a1SMatthew Dillon * The depot does not have any non-empty magazines. If we have
50277e294a1SMatthew Dillon * not hit our object limit we can allocate a new object using
50377e294a1SMatthew Dillon * the back-end allocator.
50477e294a1SMatthew Dillon *
505e98a16b6SMatthew Dillon * NOTE: unallocated_objects can wind up being negative due to
506e98a16b6SMatthew Dillon * objcache_set_cluster_limit() calls.
5078a268428SJeffrey Hsu */
508c5ec3350SSepherosa Ziehau if (__predict_true(depot->unallocated_objects > 0)) {
50977e294a1SMatthew Dillon --depot->unallocated_objects;
510287a8577SAlex Hornung spin_unlock(&depot->spin);
511b5d55e77SSepherosa Ziehau ++cpucache->allocs_cumulative;
51277e294a1SMatthew Dillon crit_exit();
5138a268428SJeffrey Hsu
51477e294a1SMatthew Dillon obj = oc->alloc(oc->allocator_args, ocflags);
51577e294a1SMatthew Dillon if (obj) {
516698331b0SMatthew Dillon if (oc->ctor(obj, oc->privdata, ocflags))
51777e294a1SMatthew Dillon return (obj);
51877e294a1SMatthew Dillon oc->free(obj, oc->allocator_args);
5198d968f1dSSepherosa Ziehau obj = NULL;
5208d968f1dSSepherosa Ziehau }
5218d968f1dSSepherosa Ziehau if (obj == NULL) {
522287a8577SAlex Hornung spin_lock(&depot->spin);
52377e294a1SMatthew Dillon ++depot->unallocated_objects;
524287a8577SAlex Hornung spin_unlock(&depot->spin);
52577e294a1SMatthew Dillon if (depot->waiting)
52677e294a1SMatthew Dillon wakeup(depot);
5278d968f1dSSepherosa Ziehau
52877e294a1SMatthew Dillon crit_enter();
52969982f91SMatthew Dillon /*
53069982f91SMatthew Dillon * makes debugging easier when gets_cumulative does
53169982f91SMatthew Dillon * not include gets_null.
53269982f91SMatthew Dillon */
53377e294a1SMatthew Dillon ++cpucache->gets_null;
53469982f91SMatthew Dillon --cpucache->gets_cumulative;
53577e294a1SMatthew Dillon crit_exit();
53677e294a1SMatthew Dillon }
53777e294a1SMatthew Dillon return(obj);
53877e294a1SMatthew Dillon }
539da22de6fSSepherosa Ziehau if (__predict_false(cpucache->gets_exhausted++ == 0)) {
540da22de6fSSepherosa Ziehau kprintf("Warning: objcache(%s) exhausted on cpu%d!\n",
541da22de6fSSepherosa Ziehau oc->desc->name, mycpuid);
542a1f6a1feSMatthew Dillon }
54377e294a1SMatthew Dillon
54477e294a1SMatthew Dillon /*
54577e294a1SMatthew Dillon * Otherwise block if allowed to.
54677e294a1SMatthew Dillon */
54777e294a1SMatthew Dillon if ((ocflags & (M_WAITOK|M_NULLOK)) == M_WAITOK) {
5488a268428SJeffrey Hsu ++cpucache->waiting;
5498a268428SJeffrey Hsu ++depot->waiting;
550e590ee86SMatthew Dillon ssleep(depot, &depot->spin, 0, "objcache_get", 0);
5518a268428SJeffrey Hsu --cpucache->waiting;
5528a268428SJeffrey Hsu --depot->waiting;
553287a8577SAlex Hornung spin_unlock(&depot->spin);
5548a268428SJeffrey Hsu goto retry;
5558a268428SJeffrey Hsu }
55669982f91SMatthew Dillon
55769982f91SMatthew Dillon /*
55869982f91SMatthew Dillon * Otherwise fail
55969982f91SMatthew Dillon */
5608a268428SJeffrey Hsu ++cpucache->gets_null;
56169982f91SMatthew Dillon --cpucache->gets_cumulative;
5628a268428SJeffrey Hsu crit_exit();
563287a8577SAlex Hornung spin_unlock(&depot->spin);
5648a268428SJeffrey Hsu return (NULL);
5658a268428SJeffrey Hsu }
5668a268428SJeffrey Hsu
5678a268428SJeffrey Hsu /*
5688a268428SJeffrey Hsu * Wrapper for malloc allocation routines.
5698a268428SJeffrey Hsu */
5708a268428SJeffrey Hsu void *
objcache_malloc_alloc(void * allocator_args,int ocflags)5718a268428SJeffrey Hsu objcache_malloc_alloc(void *allocator_args, int ocflags)
5728a268428SJeffrey Hsu {
5738a268428SJeffrey Hsu struct objcache_malloc_args *alloc_args = allocator_args;
5748a268428SJeffrey Hsu
575efda3bd0SMatthew Dillon return (kmalloc(alloc_args->objsize, alloc_args->mtype,
5768a268428SJeffrey Hsu ocflags & OC_MFLAGS));
5778a268428SJeffrey Hsu }
5788a268428SJeffrey Hsu
5797d4ac97cSMatthew Dillon /*
5807d4ac97cSMatthew Dillon * Wrapper for malloc allocation routines, with initial zeroing
5817d4ac97cSMatthew Dillon * (but objects are not zerod on reuse from cache).
5827d4ac97cSMatthew Dillon */
5837d4ac97cSMatthew Dillon void *
objcache_malloc_alloc_zero(void * allocator_args,int ocflags)5847d4ac97cSMatthew Dillon objcache_malloc_alloc_zero(void *allocator_args, int ocflags)
5857d4ac97cSMatthew Dillon {
5867d4ac97cSMatthew Dillon struct objcache_malloc_args *alloc_args = allocator_args;
5877d4ac97cSMatthew Dillon
5887d4ac97cSMatthew Dillon return (kmalloc(alloc_args->objsize, alloc_args->mtype,
5897d4ac97cSMatthew Dillon (ocflags & OC_MFLAGS) | M_ZERO));
5907d4ac97cSMatthew Dillon }
5917d4ac97cSMatthew Dillon
5927d4ac97cSMatthew Dillon
5938a268428SJeffrey Hsu void
objcache_malloc_free(void * obj,void * allocator_args)5948a268428SJeffrey Hsu objcache_malloc_free(void *obj, void *allocator_args)
5958a268428SJeffrey Hsu {
5968a268428SJeffrey Hsu struct objcache_malloc_args *alloc_args = allocator_args;
5978a268428SJeffrey Hsu
598efda3bd0SMatthew Dillon kfree(obj, alloc_args->mtype);
5998a268428SJeffrey Hsu }
6008a268428SJeffrey Hsu
6018a268428SJeffrey Hsu /*
6028a268428SJeffrey Hsu * Wrapper for allocation policies that pre-allocate at initialization time
6038a268428SJeffrey Hsu * and don't do run-time allocation.
6048a268428SJeffrey Hsu */
6058a268428SJeffrey Hsu void *
objcache_nop_alloc(void * allocator_args,int ocflags)6068a268428SJeffrey Hsu objcache_nop_alloc(void *allocator_args, int ocflags)
6078a268428SJeffrey Hsu {
6088a268428SJeffrey Hsu return (NULL);
6098a268428SJeffrey Hsu }
6108a268428SJeffrey Hsu
6118a268428SJeffrey Hsu void
objcache_nop_free(void * obj,void * allocator_args)6128a268428SJeffrey Hsu objcache_nop_free(void *obj, void *allocator_args)
6138a268428SJeffrey Hsu {
6148a268428SJeffrey Hsu }
6158a268428SJeffrey Hsu
6168a268428SJeffrey Hsu /*
6178a268428SJeffrey Hsu * Return an object to the object cache.
6188a268428SJeffrey Hsu */
6198a268428SJeffrey Hsu void
objcache_put(struct objcache * oc,void * obj)6208a268428SJeffrey Hsu objcache_put(struct objcache *oc, void *obj)
6218a268428SJeffrey Hsu {
6228a268428SJeffrey Hsu struct percpu_objcache *cpucache = &oc->cache_percpu[mycpuid];
6238a268428SJeffrey Hsu struct magazine *loadedmag;
6248a268428SJeffrey Hsu struct magazinedepot *depot;
6258a268428SJeffrey Hsu
6268a268428SJeffrey Hsu crit_enter();
6278a268428SJeffrey Hsu ++cpucache->puts_cumulative;
6288a268428SJeffrey Hsu
6298a268428SJeffrey Hsu if (CLUSTER_OF(obj) != myclusterid) {
6308a268428SJeffrey Hsu #ifdef notyet
6318a268428SJeffrey Hsu /* use lazy IPI to send object to owning cluster XXX todo */
6328a268428SJeffrey Hsu ++cpucache->puts_othercluster;
63369982f91SMatthew Dillon crit_exit();
6348a268428SJeffrey Hsu return;
6358a268428SJeffrey Hsu #endif
6368a268428SJeffrey Hsu }
6378a268428SJeffrey Hsu
6388a268428SJeffrey Hsu retry:
6398a268428SJeffrey Hsu /*
6408a268428SJeffrey Hsu * Free slot available in loaded magazine. This is the hot path.
6418a268428SJeffrey Hsu * It is lock-free and uses a critical section to block out interrupt
6428a268428SJeffrey Hsu * handlers on the same processor.
6438a268428SJeffrey Hsu */
6448a268428SJeffrey Hsu loadedmag = cpucache->loaded_magazine;
6458a268428SJeffrey Hsu if (!MAGAZINE_FULL(loadedmag)) {
64677e294a1SMatthew Dillon loadedmag->objects[loadedmag->rounds++] = obj;
6478a268428SJeffrey Hsu if (cpucache->waiting)
6486f266dc1SMatthew Dillon wakeup_mycpu(&oc->depot[myclusterid]);
6498a268428SJeffrey Hsu crit_exit();
6508a268428SJeffrey Hsu return;
6518a268428SJeffrey Hsu }
6528a268428SJeffrey Hsu
65377e294a1SMatthew Dillon /*
65477e294a1SMatthew Dillon * Current magazine full, but previous magazine has room. XXX
65577e294a1SMatthew Dillon */
6568a268428SJeffrey Hsu if (!MAGAZINE_FULL(cpucache->previous_magazine)) {
6578a268428SJeffrey Hsu swap(cpucache->loaded_magazine, cpucache->previous_magazine);
6588a268428SJeffrey Hsu loadedmag = cpucache->loaded_magazine;
65977e294a1SMatthew Dillon loadedmag->objects[loadedmag->rounds++] = obj;
66077e294a1SMatthew Dillon if (cpucache->waiting)
6616f266dc1SMatthew Dillon wakeup_mycpu(&oc->depot[myclusterid]);
66277e294a1SMatthew Dillon crit_exit();
66377e294a1SMatthew Dillon return;
6648a268428SJeffrey Hsu }
6658a268428SJeffrey Hsu
6668a268428SJeffrey Hsu /*
66777e294a1SMatthew Dillon * Both magazines full. Get an empty magazine from the depot and
66877e294a1SMatthew Dillon * move a full loaded magazine to the depot. Even though the
66977e294a1SMatthew Dillon * magazine may wind up with space available after we block on
67054d017fdSMatthew Dillon * the spinlock, we still cycle it through to avoid the non-optimal
67177e294a1SMatthew Dillon * corner-case.
67277e294a1SMatthew Dillon *
67354d017fdSMatthew Dillon * Obtain the depot spinlock.
6748a268428SJeffrey Hsu */
6758a268428SJeffrey Hsu depot = &oc->depot[myclusterid];
676287a8577SAlex Hornung spin_lock(&depot->spin);
67777e294a1SMatthew Dillon
67877e294a1SMatthew Dillon /*
67977e294a1SMatthew Dillon * If an empty magazine is available in the depot, cycle it
68077e294a1SMatthew Dillon * through and retry.
68177e294a1SMatthew Dillon */
68277e294a1SMatthew Dillon if (!SLIST_EMPTY(&depot->emptymagazines)) {
68377e294a1SMatthew Dillon loadedmag = cpucache->previous_magazine;
68477e294a1SMatthew Dillon cpucache->previous_magazine = cpucache->loaded_magazine;
68577e294a1SMatthew Dillon cpucache->loaded_magazine = SLIST_FIRST(&depot->emptymagazines);
68677e294a1SMatthew Dillon SLIST_REMOVE_HEAD(&depot->emptymagazines, nextmagazine);
68777e294a1SMatthew Dillon
68877e294a1SMatthew Dillon /*
68977e294a1SMatthew Dillon * Return loadedmag to the depot. Due to blocking it may
69077e294a1SMatthew Dillon * not be entirely full and could even be empty.
69177e294a1SMatthew Dillon */
69277e294a1SMatthew Dillon if (MAGAZINE_EMPTY(loadedmag)) {
69377e294a1SMatthew Dillon SLIST_INSERT_HEAD(&depot->emptymagazines,
69477e294a1SMatthew Dillon loadedmag, nextmagazine);
695287a8577SAlex Hornung spin_unlock(&depot->spin);
69677e294a1SMatthew Dillon } else {
69777e294a1SMatthew Dillon SLIST_INSERT_HEAD(&depot->fullmagazines,
69877e294a1SMatthew Dillon loadedmag, nextmagazine);
699287a8577SAlex Hornung spin_unlock(&depot->spin);
70077e294a1SMatthew Dillon if (depot->waiting)
70177e294a1SMatthew Dillon wakeup(depot);
70277e294a1SMatthew Dillon }
7038a268428SJeffrey Hsu goto retry;
7048a268428SJeffrey Hsu }
7058a268428SJeffrey Hsu
70677e294a1SMatthew Dillon /*
70777e294a1SMatthew Dillon * An empty mag is not available. This is a corner case which can
70877e294a1SMatthew Dillon * occur due to cpus holding partially full magazines. Do not try
70977e294a1SMatthew Dillon * to allocate a mag, just free the object.
71077e294a1SMatthew Dillon */
71177e294a1SMatthew Dillon ++depot->unallocated_objects;
712287a8577SAlex Hornung spin_unlock(&depot->spin);
7138a268428SJeffrey Hsu if (depot->waiting)
7148a268428SJeffrey Hsu wakeup(depot);
7158a268428SJeffrey Hsu crit_exit();
716698331b0SMatthew Dillon oc->dtor(obj, oc->privdata);
7178a268428SJeffrey Hsu oc->free(obj, oc->allocator_args);
71877e294a1SMatthew Dillon }
71977e294a1SMatthew Dillon
72077e294a1SMatthew Dillon /*
72177e294a1SMatthew Dillon * The object is being put back into the cache, but the caller has
72277e294a1SMatthew Dillon * indicated that the object is not in any shape to be reused and should
72377e294a1SMatthew Dillon * be dtor'd immediately.
72477e294a1SMatthew Dillon */
72577e294a1SMatthew Dillon void
objcache_dtor(struct objcache * oc,void * obj)72677e294a1SMatthew Dillon objcache_dtor(struct objcache *oc, void *obj)
72777e294a1SMatthew Dillon {
72877e294a1SMatthew Dillon struct magazinedepot *depot;
72977e294a1SMatthew Dillon
73077e294a1SMatthew Dillon depot = &oc->depot[myclusterid];
731287a8577SAlex Hornung spin_lock(&depot->spin);
73277e294a1SMatthew Dillon ++depot->unallocated_objects;
733287a8577SAlex Hornung spin_unlock(&depot->spin);
73477e294a1SMatthew Dillon if (depot->waiting)
73577e294a1SMatthew Dillon wakeup(depot);
736698331b0SMatthew Dillon oc->dtor(obj, oc->privdata);
73777e294a1SMatthew Dillon oc->free(obj, oc->allocator_args);
7388a268428SJeffrey Hsu }
7398a268428SJeffrey Hsu
7408a268428SJeffrey Hsu /*
74154d017fdSMatthew Dillon * Deallocate all objects in a magazine and free the magazine if requested.
742aa1f2da3SMatthew Dillon * When freeit is TRUE the magazine must already be disassociated from the
743aa1f2da3SMatthew Dillon * depot.
74454d017fdSMatthew Dillon *
74554d017fdSMatthew Dillon * Must be called with a critical section held when called with a per-cpu
74654d017fdSMatthew Dillon * magazine. The magazine may be indirectly modified during the loop.
74754d017fdSMatthew Dillon *
748aa1f2da3SMatthew Dillon * If the magazine moves during a dtor the operation is aborted. This is
749aa1f2da3SMatthew Dillon * only allowed when freeit is FALSE.
750aa1f2da3SMatthew Dillon *
75154d017fdSMatthew Dillon * The number of objects freed is returned.
7528a268428SJeffrey Hsu */
7538a268428SJeffrey Hsu static int
mag_purge(struct objcache * oc,struct magazine ** magp,int freeit)754aa1f2da3SMatthew Dillon mag_purge(struct objcache *oc, struct magazine **magp, int freeit)
7558a268428SJeffrey Hsu {
756aa1f2da3SMatthew Dillon struct magazine *mag = *magp;
75754d017fdSMatthew Dillon int count;
7588a268428SJeffrey Hsu void *obj;
7598a268428SJeffrey Hsu
76054d017fdSMatthew Dillon count = 0;
76177e294a1SMatthew Dillon while (mag->rounds) {
76277e294a1SMatthew Dillon obj = mag->objects[--mag->rounds];
763698331b0SMatthew Dillon oc->dtor(obj, oc->privdata); /* MAY BLOCK */
76454d017fdSMatthew Dillon oc->free(obj, oc->allocator_args); /* MAY BLOCK */
76554d017fdSMatthew Dillon ++count;
76654d017fdSMatthew Dillon
76754d017fdSMatthew Dillon /*
768aa1f2da3SMatthew Dillon * Cycle for interrupts.
76954d017fdSMatthew Dillon */
77054d017fdSMatthew Dillon if ((count & 15) == 0) {
77177e294a1SMatthew Dillon crit_exit();
77277e294a1SMatthew Dillon crit_enter();
7738a268428SJeffrey Hsu }
774aa1f2da3SMatthew Dillon
775aa1f2da3SMatthew Dillon /*
776aa1f2da3SMatthew Dillon * mag may have become invalid either due to dtor/free
777aa1f2da3SMatthew Dillon * blocking or interrupt cycling, do not derefernce it
778aa1f2da3SMatthew Dillon * until we check.
779aa1f2da3SMatthew Dillon */
780aa1f2da3SMatthew Dillon if (*magp != mag) {
781aa1f2da3SMatthew Dillon kprintf("mag_purge: mag ripped out\n");
782aa1f2da3SMatthew Dillon break;
78354d017fdSMatthew Dillon }
784aa1f2da3SMatthew Dillon }
785aa1f2da3SMatthew Dillon if (freeit) {
786aa1f2da3SMatthew Dillon KKASSERT(*magp == mag);
787aa1f2da3SMatthew Dillon *magp = NULL;
78854d017fdSMatthew Dillon kfree(mag, M_OBJMAG);
789aa1f2da3SMatthew Dillon }
79054d017fdSMatthew Dillon return(count);
7918a268428SJeffrey Hsu }
7928a268428SJeffrey Hsu
7938a268428SJeffrey Hsu /*
79454d017fdSMatthew Dillon * Disassociate zero or more magazines from a magazine list associated with
79554d017fdSMatthew Dillon * the depot, update the depot, and move the magazines to a temporary
79654d017fdSMatthew Dillon * list.
79754d017fdSMatthew Dillon *
79854d017fdSMatthew Dillon * The caller must check the depot for waiters and wake it up, typically
79954d017fdSMatthew Dillon * after disposing of the magazines this function loads onto the temporary
80054d017fdSMatthew Dillon * list.
8018a268428SJeffrey Hsu */
80254d017fdSMatthew Dillon static void
maglist_disassociate(struct magazinedepot * depot,struct magazinelist * maglist,struct magazinelist * tmplist,boolean_t purgeall)80354d017fdSMatthew Dillon maglist_disassociate(struct magazinedepot *depot, struct magazinelist *maglist,
80454d017fdSMatthew Dillon struct magazinelist *tmplist, boolean_t purgeall)
8058a268428SJeffrey Hsu {
8068a268428SJeffrey Hsu struct magazine *mag;
8078a268428SJeffrey Hsu
80854d017fdSMatthew Dillon while ((mag = SLIST_FIRST(maglist)) != NULL) {
8098a268428SJeffrey Hsu SLIST_REMOVE_HEAD(maglist, nextmagazine);
81054d017fdSMatthew Dillon SLIST_INSERT_HEAD(tmplist, mag, nextmagazine);
81154d017fdSMatthew Dillon depot->unallocated_objects += mag->rounds;
8128a268428SJeffrey Hsu }
81354d017fdSMatthew Dillon }
81454d017fdSMatthew Dillon
81554d017fdSMatthew Dillon /*
81654d017fdSMatthew Dillon * Deallocate all magazines and their contents from the passed temporary
81754d017fdSMatthew Dillon * list. The magazines have already been accounted for by their depots.
81854d017fdSMatthew Dillon *
81954d017fdSMatthew Dillon * The total number of rounds freed is returned. This number is typically
82054d017fdSMatthew Dillon * only used to determine whether a wakeup on the depot is needed or not.
82154d017fdSMatthew Dillon */
82254d017fdSMatthew Dillon static int
maglist_purge(struct objcache * oc,struct magazinelist * maglist)82354d017fdSMatthew Dillon maglist_purge(struct objcache *oc, struct magazinelist *maglist)
82454d017fdSMatthew Dillon {
82554d017fdSMatthew Dillon struct magazine *mag;
82654d017fdSMatthew Dillon int count = 0;
82754d017fdSMatthew Dillon
82854d017fdSMatthew Dillon /*
82954d017fdSMatthew Dillon * can't use SLIST_FOREACH because blocking releases the depot
83054d017fdSMatthew Dillon * spinlock
83154d017fdSMatthew Dillon */
832dd00f6f3SMatthew Dillon crit_enter();
83354d017fdSMatthew Dillon while ((mag = SLIST_FIRST(maglist)) != NULL) {
83454d017fdSMatthew Dillon SLIST_REMOVE_HEAD(maglist, nextmagazine);
835aa1f2da3SMatthew Dillon count += mag_purge(oc, &mag, TRUE);
83654d017fdSMatthew Dillon }
837dd00f6f3SMatthew Dillon crit_exit();
83854d017fdSMatthew Dillon return(count);
8398a268428SJeffrey Hsu }
8408a268428SJeffrey Hsu
8418a268428SJeffrey Hsu /*
8428a268428SJeffrey Hsu * De-allocates all magazines on the full and empty magazine lists.
84354d017fdSMatthew Dillon *
84454d017fdSMatthew Dillon * Because this routine is called with a spinlock held, the magazines
84554d017fdSMatthew Dillon * can only be disassociated and moved to a temporary list, not freed.
84654d017fdSMatthew Dillon *
84754d017fdSMatthew Dillon * The caller is responsible for freeing the magazines.
8488a268428SJeffrey Hsu */
8498a268428SJeffrey Hsu static void
depot_disassociate(struct magazinedepot * depot,struct magazinelist * tmplist)85054d017fdSMatthew Dillon depot_disassociate(struct magazinedepot *depot, struct magazinelist *tmplist)
8518a268428SJeffrey Hsu {
85254d017fdSMatthew Dillon maglist_disassociate(depot, &depot->fullmagazines, tmplist, TRUE);
85354d017fdSMatthew Dillon maglist_disassociate(depot, &depot->emptymagazines, tmplist, TRUE);
8548a268428SJeffrey Hsu }
8558a268428SJeffrey Hsu
8568a268428SJeffrey Hsu /*
85754d017fdSMatthew Dillon * Try to free up some memory. Return as soon as some free memory is found.
8588a268428SJeffrey Hsu * For each object cache on the reclaim list, first try the current per-cpu
8598a268428SJeffrey Hsu * cache, then the full magazine depot.
8608a268428SJeffrey Hsu */
8618a268428SJeffrey Hsu boolean_t
objcache_reclaimlist(struct objcache * oclist[],int nlist)862*8a7a7510SAaron LI objcache_reclaimlist(struct objcache *oclist[], int nlist)
8638a268428SJeffrey Hsu {
8648a268428SJeffrey Hsu struct objcache *oc;
8658a268428SJeffrey Hsu struct percpu_objcache *cpucache;
8668a268428SJeffrey Hsu struct magazinedepot *depot;
86754d017fdSMatthew Dillon struct magazinelist tmplist;
86854d017fdSMatthew Dillon int i, count;
86954d017fdSMatthew Dillon
87054d017fdSMatthew Dillon SLIST_INIT(&tmplist);
8718a268428SJeffrey Hsu
8728a268428SJeffrey Hsu for (i = 0; i < nlist; i++) {
8738a268428SJeffrey Hsu oc = oclist[i];
8748a268428SJeffrey Hsu cpucache = &oc->cache_percpu[mycpuid];
8758a268428SJeffrey Hsu depot = &oc->depot[myclusterid];
8768a268428SJeffrey Hsu
8778a268428SJeffrey Hsu crit_enter();
878aa1f2da3SMatthew Dillon count = mag_purge(oc, &cpucache->loaded_magazine, FALSE);
87954d017fdSMatthew Dillon if (count == 0)
880aa1f2da3SMatthew Dillon count += mag_purge(oc, &cpucache->previous_magazine, FALSE);
8818a268428SJeffrey Hsu crit_exit();
88254d017fdSMatthew Dillon if (count > 0) {
883287a8577SAlex Hornung spin_lock(&depot->spin);
88454d017fdSMatthew Dillon depot->unallocated_objects += count;
885287a8577SAlex Hornung spin_unlock(&depot->spin);
88654d017fdSMatthew Dillon if (depot->waiting)
88777e294a1SMatthew Dillon wakeup(depot);
8888a268428SJeffrey Hsu return (TRUE);
8898a268428SJeffrey Hsu }
890287a8577SAlex Hornung spin_lock(&depot->spin);
89154d017fdSMatthew Dillon maglist_disassociate(depot, &depot->fullmagazines,
89254d017fdSMatthew Dillon &tmplist, FALSE);
893287a8577SAlex Hornung spin_unlock(&depot->spin);
89454d017fdSMatthew Dillon count = maglist_purge(oc, &tmplist);
89554d017fdSMatthew Dillon if (count > 0) {
89654d017fdSMatthew Dillon if (depot->waiting)
89777e294a1SMatthew Dillon wakeup(depot);
8988a268428SJeffrey Hsu return (TRUE);
8998a268428SJeffrey Hsu }
9008a268428SJeffrey Hsu }
9018a268428SJeffrey Hsu return (FALSE);
9028a268428SJeffrey Hsu }
9038a268428SJeffrey Hsu
9048a268428SJeffrey Hsu /*
9058a268428SJeffrey Hsu * Destroy an object cache. Must have no existing references.
9068a268428SJeffrey Hsu */
9078a268428SJeffrey Hsu void
objcache_destroy(struct objcache * oc)9088a268428SJeffrey Hsu objcache_destroy(struct objcache *oc)
9098a268428SJeffrey Hsu {
9105b694eafSSepherosa Ziehau struct objcache_desc *desc = oc->desc;
9118a268428SJeffrey Hsu struct percpu_objcache *cache_percpu;
91254d017fdSMatthew Dillon struct magazinedepot *depot;
9138a268428SJeffrey Hsu int clusterid, cpuid;
91454d017fdSMatthew Dillon struct magazinelist tmplist;
9158a268428SJeffrey Hsu
916287a8577SAlex Hornung spin_lock(&objcachelist_spin);
9175b694eafSSepherosa Ziehau LIST_REMOVE(desc, next);
918287a8577SAlex Hornung spin_unlock(&objcachelist_spin);
91900de24deSNicolas Thery
92054d017fdSMatthew Dillon SLIST_INIT(&tmplist);
92154d017fdSMatthew Dillon for (clusterid = 0; clusterid < MAXCLUSTERS; clusterid++) {
92254d017fdSMatthew Dillon depot = &oc->depot[clusterid];
923287a8577SAlex Hornung spin_lock(&depot->spin);
92454d017fdSMatthew Dillon depot_disassociate(depot, &tmplist);
925287a8577SAlex Hornung spin_unlock(&depot->spin);
92654d017fdSMatthew Dillon }
92754d017fdSMatthew Dillon maglist_purge(oc, &tmplist);
9288a268428SJeffrey Hsu
9298a268428SJeffrey Hsu for (cpuid = 0; cpuid < ncpus; cpuid++) {
9308a268428SJeffrey Hsu cache_percpu = &oc->cache_percpu[cpuid];
9318a268428SJeffrey Hsu
932dd00f6f3SMatthew Dillon crit_enter();
933aa1f2da3SMatthew Dillon mag_purge(oc, &cache_percpu->loaded_magazine, TRUE);
934aa1f2da3SMatthew Dillon mag_purge(oc, &cache_percpu->previous_magazine, TRUE);
935dd00f6f3SMatthew Dillon crit_exit();
93654d017fdSMatthew Dillon cache_percpu->loaded_magazine = NULL;
93754d017fdSMatthew Dillon cache_percpu->previous_magazine = NULL;
93854d017fdSMatthew Dillon /* don't bother adjusting depot->unallocated_objects */
9398a268428SJeffrey Hsu }
9408a268428SJeffrey Hsu
9415b694eafSSepherosa Ziehau kfree(desc, M_OBJCACHE);
942efda3bd0SMatthew Dillon kfree(oc, M_OBJCACHE);
9438a268428SJeffrey Hsu }
9448a268428SJeffrey Hsu
945c6bb9a90SSepherosa Ziehau static int
sysctl_ocstats(SYSCTL_HANDLER_ARGS)946c6bb9a90SSepherosa Ziehau sysctl_ocstats(SYSCTL_HANDLER_ARGS)
947c6bb9a90SSepherosa Ziehau {
948c6bb9a90SSepherosa Ziehau struct objcache_stats stat;
949c6bb9a90SSepherosa Ziehau struct objcache_desc marker, *desc;
950c6bb9a90SSepherosa Ziehau int error;
951c6bb9a90SSepherosa Ziehau
952c6bb9a90SSepherosa Ziehau memset(&marker, 0, sizeof(marker));
953c6bb9a90SSepherosa Ziehau
954c6bb9a90SSepherosa Ziehau spin_lock(&objcachelist_spin);
955c6bb9a90SSepherosa Ziehau
956c6bb9a90SSepherosa Ziehau LIST_INSERT_HEAD(&allobjcaches, &marker, next);
957c6bb9a90SSepherosa Ziehau while ((desc = LIST_NEXT(&marker, next)) != NULL) {
958c6bb9a90SSepherosa Ziehau u_long puts, unalloc;
959c6bb9a90SSepherosa Ziehau int cpu;
960c6bb9a90SSepherosa Ziehau
961c6bb9a90SSepherosa Ziehau LIST_REMOVE(&marker, next);
962c6bb9a90SSepherosa Ziehau LIST_INSERT_AFTER(desc, &marker, next);
963c6bb9a90SSepherosa Ziehau
964c6bb9a90SSepherosa Ziehau if (desc->total_objects == 0) {
965c6bb9a90SSepherosa Ziehau /* Marker inserted by another thread. */
966c6bb9a90SSepherosa Ziehau continue;
967c6bb9a90SSepherosa Ziehau }
968c6bb9a90SSepherosa Ziehau
969c6bb9a90SSepherosa Ziehau memset(&stat, 0, sizeof(stat));
970c6bb9a90SSepherosa Ziehau strlcpy(stat.oc_name, desc->name, sizeof(stat.oc_name));
971c6bb9a90SSepherosa Ziehau stat.oc_limit = desc->total_objects;
972c6bb9a90SSepherosa Ziehau /* XXX domain aware */
973c6bb9a90SSepherosa Ziehau unalloc = desc->objcache->depot[0].unallocated_objects;
974c6bb9a90SSepherosa Ziehau
975c6bb9a90SSepherosa Ziehau puts = 0;
976c6bb9a90SSepherosa Ziehau for (cpu = 0; cpu < ncpus; ++cpu) {
977c6bb9a90SSepherosa Ziehau const struct percpu_objcache *cache;
978c6bb9a90SSepherosa Ziehau
979c6bb9a90SSepherosa Ziehau cache = &desc->objcache->cache_percpu[cpu];
980c6bb9a90SSepherosa Ziehau puts += cache->puts_cumulative;
981c6bb9a90SSepherosa Ziehau
982c6bb9a90SSepherosa Ziehau stat.oc_requested += cache->gets_cumulative;
983c6bb9a90SSepherosa Ziehau stat.oc_exhausted += cache->gets_exhausted;
984c6bb9a90SSepherosa Ziehau stat.oc_failed += cache->gets_null;
985c6bb9a90SSepherosa Ziehau stat.oc_allocated += cache->allocs_cumulative;
986c6bb9a90SSepherosa Ziehau }
987c6bb9a90SSepherosa Ziehau spin_unlock(&objcachelist_spin);
988c6bb9a90SSepherosa Ziehau
989c6bb9a90SSepherosa Ziehau /*
990c6bb9a90SSepherosa Ziehau * Apply fixup.
991c6bb9a90SSepherosa Ziehau */
992c6bb9a90SSepherosa Ziehau if (stat.oc_requested > puts)
993c6bb9a90SSepherosa Ziehau stat.oc_used = stat.oc_requested - puts;
994c6bb9a90SSepherosa Ziehau if (stat.oc_limit > unalloc + stat.oc_used) {
995c6bb9a90SSepherosa Ziehau stat.oc_cached = stat.oc_limit -
996c6bb9a90SSepherosa Ziehau (unalloc + stat.oc_used);
997c6bb9a90SSepherosa Ziehau }
998c6bb9a90SSepherosa Ziehau stat.oc_requested += stat.oc_failed;
999c6bb9a90SSepherosa Ziehau
1000c6bb9a90SSepherosa Ziehau /* Send out. */
1001c6bb9a90SSepherosa Ziehau error = SYSCTL_OUT(req, &stat, sizeof(stat));
1002c6bb9a90SSepherosa Ziehau
1003c6bb9a90SSepherosa Ziehau /* Hold the lock before we return. */
1004c6bb9a90SSepherosa Ziehau spin_lock(&objcachelist_spin);
1005c6bb9a90SSepherosa Ziehau
1006c6bb9a90SSepherosa Ziehau if (error)
1007c6bb9a90SSepherosa Ziehau break;
1008c6bb9a90SSepherosa Ziehau }
1009c6bb9a90SSepherosa Ziehau LIST_REMOVE(&marker, next);
1010c6bb9a90SSepherosa Ziehau
1011c6bb9a90SSepherosa Ziehau spin_unlock(&objcachelist_spin);
1012c6bb9a90SSepherosa Ziehau
1013c6bb9a90SSepherosa Ziehau return error;
1014c6bb9a90SSepherosa Ziehau }
1015c6bb9a90SSepherosa Ziehau SYSCTL_PROC(_kern_objcache, OID_AUTO, stats, (CTLTYPE_OPAQUE | CTLFLAG_RD),
1016c6bb9a90SSepherosa Ziehau 0, 0, sysctl_ocstats, "S,objcache_stats", "objcache statistics");
1017c6bb9a90SSepherosa Ziehau
10188a268428SJeffrey Hsu static void
objcache_init(void)10198a268428SJeffrey Hsu objcache_init(void)
10208a268428SJeffrey Hsu {
1021ba87a4abSSascha Wildner spin_init(&objcachelist_spin, "objcachelist");
10221ad7b4a7SSepherosa Ziehau
10231ad7b4a7SSepherosa Ziehau magazine_capmin = mag_capacity_align(MAGAZINE_CAPACITY_MIN);
10241ad7b4a7SSepherosa Ziehau magazine_capmax = mag_capacity_align(MAGAZINE_CAPACITY_MAX);
10251ad7b4a7SSepherosa Ziehau if (bootverbose) {
10261ad7b4a7SSepherosa Ziehau kprintf("objcache: magazine cap [%d, %d]\n",
10271ad7b4a7SSepherosa Ziehau magazine_capmin, magazine_capmax);
10281ad7b4a7SSepherosa Ziehau }
102977e294a1SMatthew Dillon #if 0
1030bf0ecf68SMatthew Dillon callout_init_mp(&objcache_callout);
103177e294a1SMatthew Dillon objcache_rebalance_period = 60 * hz;
10328a268428SJeffrey Hsu callout_reset(&objcache_callout, objcache_rebalance_period,
10338a268428SJeffrey Hsu objcache_timer, NULL);
103477e294a1SMatthew Dillon #endif
10358a268428SJeffrey Hsu }
1036ba39e2e0SMatthew Dillon SYSINIT(objcache, SI_BOOT2_OBJCACHE, SI_ORDER_FIRST, objcache_init, 0);
1037