1*cf1d77f7Schristos /*	$NetBSD: sl_malloc.c,v 1.3 2021/08/14 16:14:58 christos Exp $	*/
24e6df137Slukem 
32de962bdSlukem /* sl_malloc.c - malloc routines using a per-thread slab */
433197c6aStron /* $OpenLDAP$ */
52de962bdSlukem /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
62de962bdSlukem  *
7*cf1d77f7Schristos  * Copyright 2003-2021 The OpenLDAP Foundation.
82de962bdSlukem  * All rights reserved.
92de962bdSlukem  *
102de962bdSlukem  * Redistribution and use in source and binary forms, with or without
112de962bdSlukem  * modification, are permitted only as authorized by the OpenLDAP
122de962bdSlukem  * Public License.
132de962bdSlukem  *
142de962bdSlukem  * A copy of this license is available in the file LICENSE in the
152de962bdSlukem  * top-level directory of the distribution or, alternatively, at
162de962bdSlukem  * <http://www.OpenLDAP.org/license.html>.
172de962bdSlukem  */
182de962bdSlukem 
198bd9f7cdSchristos #include <sys/cdefs.h>
20*cf1d77f7Schristos __RCSID("$NetBSD: sl_malloc.c,v 1.3 2021/08/14 16:14:58 christos Exp $");
218bd9f7cdSchristos 
222de962bdSlukem #include "portable.h"
232de962bdSlukem 
242de962bdSlukem #include <stdio.h>
252de962bdSlukem #include <ac/string.h>
262de962bdSlukem 
272de962bdSlukem #include "slap.h"
282de962bdSlukem 
2933197c6aStron #ifdef USE_VALGRIND
3033197c6aStron /* Get debugging help from Valgrind */
3133197c6aStron #include <valgrind/memcheck.h>
3233197c6aStron #define	VGMEMP_MARK(m,s)	VALGRIND_MAKE_MEM_NOACCESS(m,s)
3333197c6aStron #define VGMEMP_CREATE(h,r,z)	VALGRIND_CREATE_MEMPOOL(h,r,z)
3433197c6aStron #define VGMEMP_TRIM(h,a,s)	VALGRIND_MEMPOOL_TRIM(h,a,s)
3533197c6aStron #define VGMEMP_ALLOC(h,a,s)	VALGRIND_MEMPOOL_ALLOC(h,a,s)
3633197c6aStron #define VGMEMP_CHANGE(h,a,b,s)	VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
3733197c6aStron #else
3833197c6aStron #define	VGMEMP_MARK(m,s)
3933197c6aStron #define VGMEMP_CREATE(h,r,z)
4033197c6aStron #define VGMEMP_TRIM(h,a,s)
4133197c6aStron #define VGMEMP_ALLOC(h,a,s)
4233197c6aStron #define VGMEMP_CHANGE(h,a,b,s)
4333197c6aStron #endif
4433197c6aStron 
4533197c6aStron /*
4633197c6aStron  * This allocator returns temporary memory from a slab in a given memory
4733197c6aStron  * context, aligned on a 2-int boundary.  It cannot be used for data
4833197c6aStron  * which will outlive the task allocating it.
4933197c6aStron  *
5033197c6aStron  * A new memory context attaches to the creator's thread context, if any.
5133197c6aStron  * Threads cannot use other threads' memory contexts; there are no locks.
5233197c6aStron  *
5333197c6aStron  * The caller of slap_sl_malloc, usually a thread pool task, must
5433197c6aStron  * slap_sl_free the memory before finishing: New tasks reuse the context
5533197c6aStron  * and normally reset it, reclaiming memory left over from last task.
5633197c6aStron  *
5733197c6aStron  * The allocator helps memory fragmentation, speed and memory leaks.
5833197c6aStron  * It is not (yet) reliable as a garbage collector:
5933197c6aStron  *
6033197c6aStron  * It falls back to context NULL - plain ber_memalloc() - when the
6133197c6aStron  * context's slab is full.  A reset does not reclaim such memory.
6233197c6aStron  * Conversely, free/realloc of data not from the given context assumes
6333197c6aStron  * context NULL.  The data must not belong to another memory context.
6433197c6aStron  *
6533197c6aStron  * Code which has lost track of the current memory context can try
6633197c6aStron  * slap_sl_context() or ch_malloc.c:ch_free/ch_realloc().
6733197c6aStron  *
6833197c6aStron  * Allocations cannot yet return failure.  Like ch_malloc, they succeed
6933197c6aStron  * or abort slapd.  This will change, do fix code which assumes success.
7033197c6aStron  */
7133197c6aStron 
7233197c6aStron /*
7333197c6aStron  * The stack-based allocator stores (ber_len_t)sizeof(head+block) at
7433197c6aStron  * allocated blocks' head - and in freed blocks also at the tail, marked
7533197c6aStron  * by ORing *next* block's head with 1.  Freed blocks are only reclaimed
7633197c6aStron  * from the last block forward.  This is fast, but when a block is never
7733197c6aStron  * freed, older blocks will not be reclaimed until the slab is reset...
7833197c6aStron  */
7933197c6aStron 
8033197c6aStron #ifdef SLAP_NO_SL_MALLOC /* Useful with memory debuggers like Valgrind */
8133197c6aStron enum { No_sl_malloc = 1 };
8233197c6aStron #else
8333197c6aStron enum { No_sl_malloc = 0 };
8433197c6aStron #endif
8533197c6aStron 
8633197c6aStron #define SLAP_SLAB_SOBLOCK 64
8733197c6aStron 
8833197c6aStron struct slab_object {
8933197c6aStron     void *so_ptr;
9033197c6aStron 	int so_blockhead;
9133197c6aStron     LDAP_LIST_ENTRY(slab_object) so_link;
9233197c6aStron };
9333197c6aStron 
9433197c6aStron struct slab_heap {
9533197c6aStron     void *sh_base;
9633197c6aStron     void *sh_last;
9733197c6aStron     void *sh_end;
9833197c6aStron 	int sh_stack;
9933197c6aStron 	int sh_maxorder;
10033197c6aStron     unsigned char **sh_map;
10133197c6aStron     LDAP_LIST_HEAD(sh_freelist, slab_object) *sh_free;
10233197c6aStron 	LDAP_LIST_HEAD(sh_so, slab_object) sh_sopool;
10333197c6aStron };
10433197c6aStron 
10533197c6aStron enum {
10633197c6aStron 	Align = sizeof(ber_len_t) > 2*sizeof(int)
10733197c6aStron 		? sizeof(ber_len_t) : 2*sizeof(int),
10833197c6aStron 	Align_log2 = 1 + (Align>2) + (Align>4) + (Align>8) + (Align>16),
10933197c6aStron 	order_start = Align_log2 - 1,
11033197c6aStron 	pad = Align - 1
11133197c6aStron };
11233197c6aStron 
1132de962bdSlukem static struct slab_object * slap_replenish_sopool(struct slab_heap* sh);
1142de962bdSlukem #ifdef SLAPD_UNUSED
1152de962bdSlukem static void print_slheap(int level, void *ctx);
1162de962bdSlukem #endif
1172de962bdSlukem 
118*cf1d77f7Schristos /* Keep memory context in a thread-local var */
11933197c6aStron # define memctx_key ((void *) slap_sl_mem_init)
12033197c6aStron # define SET_MEMCTX(thrctx, memctx, kfree) \
12133197c6aStron 	ldap_pvt_thread_pool_setkey(thrctx,memctx_key, memctx,kfree, NULL,NULL)
12233197c6aStron # define GET_MEMCTX(thrctx, memctxp) \
12333197c6aStron 	((void) (*(memctxp) = NULL), \
12433197c6aStron 	 (void) ldap_pvt_thread_pool_getkey(thrctx,memctx_key, memctxp,NULL), \
12533197c6aStron 	 *(memctxp))
12633197c6aStron 
12733197c6aStron /* Destroy the context, or if key==NULL clean it up for reuse. */
1282de962bdSlukem void
slap_sl_mem_destroy(void * key,void * data)1292de962bdSlukem slap_sl_mem_destroy(
1302de962bdSlukem 	void *key,
1312de962bdSlukem 	void *data
1322de962bdSlukem )
1332de962bdSlukem {
1342de962bdSlukem 	struct slab_heap *sh = data;
1352de962bdSlukem 	struct slab_object *so;
13633197c6aStron 	int i;
1372de962bdSlukem 
138*cf1d77f7Schristos 	if (!sh)
139*cf1d77f7Schristos 		return;
140*cf1d77f7Schristos 
14133197c6aStron 	if (!sh->sh_stack) {
1422de962bdSlukem 		for (i = 0; i <= sh->sh_maxorder - order_start; i++) {
1432de962bdSlukem 			so = LDAP_LIST_FIRST(&sh->sh_free[i]);
1442de962bdSlukem 			while (so) {
1452de962bdSlukem 				struct slab_object *so_tmp = so;
1462de962bdSlukem 				so = LDAP_LIST_NEXT(so, so_link);
1472de962bdSlukem 				LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_tmp, so_link);
1482de962bdSlukem 			}
1492de962bdSlukem 			ch_free(sh->sh_map[i]);
1502de962bdSlukem 		}
1512de962bdSlukem 		ch_free(sh->sh_free);
1522de962bdSlukem 		ch_free(sh->sh_map);
1532de962bdSlukem 
1542de962bdSlukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
1552de962bdSlukem 		while (so) {
1562de962bdSlukem 			struct slab_object *so_tmp = so;
1572de962bdSlukem 			so = LDAP_LIST_NEXT(so, so_link);
1582de962bdSlukem 			if (!so_tmp->so_blockhead) {
1592de962bdSlukem 				LDAP_LIST_REMOVE(so_tmp, so_link);
1602de962bdSlukem 			}
1612de962bdSlukem 		}
1622de962bdSlukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
1632de962bdSlukem 		while (so) {
1642de962bdSlukem 			struct slab_object *so_tmp = so;
1652de962bdSlukem 			so = LDAP_LIST_NEXT(so, so_link);
1662de962bdSlukem 			ch_free(so_tmp);
1672de962bdSlukem 		}
16833197c6aStron 	}
16933197c6aStron 
17033197c6aStron 	if (key != NULL) {
1712de962bdSlukem 		ber_memfree_x(sh->sh_base, NULL);
1722de962bdSlukem 		ber_memfree_x(sh, NULL);
1732de962bdSlukem 	}
1742de962bdSlukem }
1752de962bdSlukem 
1762de962bdSlukem BerMemoryFunctions slap_sl_mfuncs =
1772de962bdSlukem 	{ slap_sl_malloc, slap_sl_calloc, slap_sl_realloc, slap_sl_free };
1782de962bdSlukem 
1792de962bdSlukem void
slap_sl_mem_init()1802de962bdSlukem slap_sl_mem_init()
1812de962bdSlukem {
18233197c6aStron 	assert( Align == 1 << Align_log2 );
18333197c6aStron 
1842de962bdSlukem 	ber_set_option( NULL, LBER_OPT_MEMORY_FNS, &slap_sl_mfuncs );
1852de962bdSlukem }
1862de962bdSlukem 
18733197c6aStron /* Create, reset or just return the memory context of the current thread. */
1882de962bdSlukem void *
slap_sl_mem_create(ber_len_t size,int stack,void * thrctx,int new)1892de962bdSlukem slap_sl_mem_create(
1902de962bdSlukem 	ber_len_t size,
1912de962bdSlukem 	int stack,
19233197c6aStron 	void *thrctx,
1932de962bdSlukem 	int new
1942de962bdSlukem )
1952de962bdSlukem {
19633197c6aStron 	void *memctx;
1972de962bdSlukem 	struct slab_heap *sh;
1982de962bdSlukem 	ber_len_t size_shift;
1992de962bdSlukem 	struct slab_object *so;
20033197c6aStron 	char *base, *newptr;
20133197c6aStron 	enum { Base_offset = (unsigned) -sizeof(ber_len_t) % Align };
2022de962bdSlukem 
20333197c6aStron 	sh = GET_MEMCTX(thrctx, &memctx);
2044e6df137Slukem 	if ( sh && !new )
2052de962bdSlukem 		return sh;
2062de962bdSlukem 
20733197c6aStron 	/* Round up to doubleword boundary, then make room for initial
20833197c6aStron 	 * padding, preserving expected available size for pool version */
20933197c6aStron 	size = ((size + Align-1) & -Align) + Base_offset;
2102de962bdSlukem 
2112de962bdSlukem 	if (!sh) {
2122de962bdSlukem 		sh = ch_malloc(sizeof(struct slab_heap));
21333197c6aStron 		base = ch_malloc(size);
21433197c6aStron 		SET_MEMCTX(thrctx, sh, slap_sl_mem_destroy);
21533197c6aStron 		VGMEMP_MARK(base, size);
21633197c6aStron 		VGMEMP_CREATE(sh, 0, 0);
2172de962bdSlukem 	} else {
21833197c6aStron 		slap_sl_mem_destroy(NULL, sh);
21933197c6aStron 		base = sh->sh_base;
22033197c6aStron 		if (size > (ber_len_t) ((char *) sh->sh_end - base)) {
22133197c6aStron 			newptr = ch_realloc(base, size);
22233197c6aStron 			if ( newptr == NULL ) return NULL;
22333197c6aStron 			VGMEMP_CHANGE(sh, base, newptr, size);
22433197c6aStron 			base = newptr;
22533197c6aStron 		}
22633197c6aStron 		VGMEMP_TRIM(sh, base, 0);
22733197c6aStron 	}
22833197c6aStron 	sh->sh_base = base;
22933197c6aStron 	sh->sh_end = base + size;
23033197c6aStron 
23133197c6aStron 	/* Align (base + head of first block) == first returned block */
23233197c6aStron 	base += Base_offset;
23333197c6aStron 	size -= Base_offset;
23433197c6aStron 
23533197c6aStron 	sh->sh_stack = stack;
23633197c6aStron 	if (stack) {
23733197c6aStron 		sh->sh_last = base;
23833197c6aStron 
23933197c6aStron 	} else {
24033197c6aStron 		int i, order = -1, order_end = -1;
24133197c6aStron 
2422de962bdSlukem 		size_shift = size - 1;
2432de962bdSlukem 		do {
2442de962bdSlukem 			order_end++;
2452de962bdSlukem 		} while (size_shift >>= 1);
2462de962bdSlukem 		order = order_end - order_start + 1;
2472de962bdSlukem 		sh->sh_maxorder = order_end;
2482de962bdSlukem 
2492de962bdSlukem 		sh->sh_free = (struct sh_freelist *)
2502de962bdSlukem 						ch_malloc(order * sizeof(struct sh_freelist));
2512de962bdSlukem 		for (i = 0; i < order; i++) {
2522de962bdSlukem 			LDAP_LIST_INIT(&sh->sh_free[i]);
2532de962bdSlukem 		}
2542de962bdSlukem 
2552de962bdSlukem 		LDAP_LIST_INIT(&sh->sh_sopool);
2562de962bdSlukem 
2572de962bdSlukem 		if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
2582de962bdSlukem 			slap_replenish_sopool(sh);
2592de962bdSlukem 		}
2602de962bdSlukem 		so = LDAP_LIST_FIRST(&sh->sh_sopool);
2612de962bdSlukem 		LDAP_LIST_REMOVE(so, so_link);
26233197c6aStron 		so->so_ptr = base;
2632de962bdSlukem 
2642de962bdSlukem 		LDAP_LIST_INSERT_HEAD(&sh->sh_free[order-1], so, so_link);
2652de962bdSlukem 
2662de962bdSlukem 		sh->sh_map = (unsigned char **)
2672de962bdSlukem 					ch_malloc(order * sizeof(unsigned char *));
2682de962bdSlukem 		for (i = 0; i < order; i++) {
2692de962bdSlukem 			int shiftamt = order_start + 1 + i;
2702de962bdSlukem 			int nummaps = size >> shiftamt;
2712de962bdSlukem 			assert(nummaps);
2722de962bdSlukem 			nummaps >>= 3;
2732de962bdSlukem 			if (!nummaps) nummaps = 1;
2742de962bdSlukem 			sh->sh_map[i] = (unsigned char *) ch_malloc(nummaps);
2752de962bdSlukem 			memset(sh->sh_map[i], 0, nummaps);
2762de962bdSlukem 		}
2772de962bdSlukem 	}
2782de962bdSlukem 
27933197c6aStron 	return sh;
28033197c6aStron }
28133197c6aStron 
28233197c6aStron /*
2838bd9f7cdSchristos  * Assign memory context to thread context. Use NULL to detach
2848bd9f7cdSchristos  * current memory context from thread. Future users must
28533197c6aStron  * know the context, since ch_free/slap_sl_context() cannot find it.
28633197c6aStron  */
2872de962bdSlukem void
slap_sl_mem_setctx(void * thrctx,void * memctx)2888bd9f7cdSchristos slap_sl_mem_setctx(
28933197c6aStron 	void *thrctx,
2902de962bdSlukem 	void *memctx
2912de962bdSlukem )
2922de962bdSlukem {
2938bd9f7cdSchristos 	SET_MEMCTX(thrctx, memctx, slap_sl_mem_destroy);
2942de962bdSlukem }
2952de962bdSlukem 
2962de962bdSlukem void *
slap_sl_malloc(ber_len_t size,void * ctx)2972de962bdSlukem slap_sl_malloc(
2982de962bdSlukem     ber_len_t	size,
2992de962bdSlukem     void *ctx
3002de962bdSlukem )
3012de962bdSlukem {
3022de962bdSlukem 	struct slab_heap *sh = ctx;
3032de962bdSlukem 	ber_len_t *ptr, *newptr;
3042de962bdSlukem 
3052de962bdSlukem 	/* ber_set_option calls us like this */
30633197c6aStron 	if (No_sl_malloc || !ctx) {
3074e6df137Slukem 		newptr = ber_memalloc_x( size, NULL );
3084e6df137Slukem 		if ( newptr ) return newptr;
30933197c6aStron 		Debug(LDAP_DEBUG_ANY, "slap_sl_malloc of %lu bytes failed\n",
310*cf1d77f7Schristos 			(unsigned long) size );
3114e6df137Slukem 		assert( 0 );
3124e6df137Slukem 		exit( EXIT_FAILURE );
3134e6df137Slukem 	}
3142de962bdSlukem 
31533197c6aStron 	/* Add room for head, ensure room for tail when freed, and
31633197c6aStron 	 * round up to doubleword boundary. */
31733197c6aStron 	size = (size + sizeof(ber_len_t) + Align-1 + !size) & -Align;
3182de962bdSlukem 
3192de962bdSlukem 	if (sh->sh_stack) {
32033197c6aStron 		if (size < (ber_len_t) ((char *) sh->sh_end - (char *) sh->sh_last)) {
3212de962bdSlukem 			newptr = sh->sh_last;
3222de962bdSlukem 			sh->sh_last = (char *) sh->sh_last + size;
32333197c6aStron 			VGMEMP_ALLOC(sh, newptr, size);
3244e6df137Slukem 			*newptr++ = size;
3252de962bdSlukem 			return( (void *)newptr );
32633197c6aStron 		}
32733197c6aStron 
32833197c6aStron 		size -= sizeof(ber_len_t);
32933197c6aStron 
3302de962bdSlukem 	} else {
3314e6df137Slukem 		struct slab_object *so_new, *so_left, *so_right;
3324e6df137Slukem 		ber_len_t size_shift;
3334e6df137Slukem 		unsigned long diff;
33433197c6aStron 		int i, j, order = -1;
3354e6df137Slukem 
3362de962bdSlukem 		size_shift = size - 1;
3372de962bdSlukem 		do {
3382de962bdSlukem 			order++;
3392de962bdSlukem 		} while (size_shift >>= 1);
3402de962bdSlukem 
34133197c6aStron 		size -= sizeof(ber_len_t);
3422de962bdSlukem 
3432de962bdSlukem 		for (i = order; i <= sh->sh_maxorder &&
3442de962bdSlukem 				LDAP_LIST_EMPTY(&sh->sh_free[i-order_start]); i++);
3452de962bdSlukem 
3462de962bdSlukem 		if (i == order) {
3472de962bdSlukem 			so_new = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
3482de962bdSlukem 			LDAP_LIST_REMOVE(so_new, so_link);
3492de962bdSlukem 			ptr = so_new->so_ptr;
3502de962bdSlukem 			diff = (unsigned long)((char*)ptr -
3512de962bdSlukem 					(char*)sh->sh_base) >> (order + 1);
3522de962bdSlukem 			sh->sh_map[order-order_start][diff>>3] |= (1 << (diff & 0x7));
35333197c6aStron 			*ptr++ = size;
3542de962bdSlukem 			LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_new, so_link);
3552de962bdSlukem 			return((void*)ptr);
3562de962bdSlukem 		} else if (i <= sh->sh_maxorder) {
3572de962bdSlukem 			for (j = i; j > order; j--) {
3582de962bdSlukem 				so_left = LDAP_LIST_FIRST(&sh->sh_free[j-order_start]);
3592de962bdSlukem 				LDAP_LIST_REMOVE(so_left, so_link);
3602de962bdSlukem 				if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
3612de962bdSlukem 					slap_replenish_sopool(sh);
3622de962bdSlukem 				}
3632de962bdSlukem 				so_right = LDAP_LIST_FIRST(&sh->sh_sopool);
3642de962bdSlukem 				LDAP_LIST_REMOVE(so_right, so_link);
3652de962bdSlukem 				so_right->so_ptr = (void *)((char *)so_left->so_ptr + (1 << j));
3662de962bdSlukem 				if (j == order + 1) {
3672de962bdSlukem 					ptr = so_left->so_ptr;
3682de962bdSlukem 					diff = (unsigned long)((char*)ptr -
3692de962bdSlukem 							(char*)sh->sh_base) >> (order+1);
3702de962bdSlukem 					sh->sh_map[order-order_start][diff>>3] |=
3712de962bdSlukem 							(1 << (diff & 0x7));
37233197c6aStron 					*ptr++ = size;
3732de962bdSlukem 					LDAP_LIST_INSERT_HEAD(
3742de962bdSlukem 							&sh->sh_free[j-1-order_start], so_right, so_link);
3752de962bdSlukem 					LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, so_left, so_link);
3762de962bdSlukem 					return((void*)ptr);
3772de962bdSlukem 				} else {
3782de962bdSlukem 					LDAP_LIST_INSERT_HEAD(
3792de962bdSlukem 							&sh->sh_free[j-1-order_start], so_right, so_link);
3802de962bdSlukem 					LDAP_LIST_INSERT_HEAD(
3812de962bdSlukem 							&sh->sh_free[j-1-order_start], so_left, so_link);
3822de962bdSlukem 				}
3832de962bdSlukem 			}
3842de962bdSlukem 		}
38533197c6aStron 		/* FIXME: missing return; guessing we failed... */
3862de962bdSlukem 	}
3872de962bdSlukem 
38833197c6aStron 	Debug(LDAP_DEBUG_TRACE,
38933197c6aStron 		"sl_malloc %lu: ch_malloc\n",
390*cf1d77f7Schristos 		(unsigned long) size );
39133197c6aStron 	return ch_malloc(size);
3922de962bdSlukem }
3932de962bdSlukem 
39433197c6aStron #define LIM_SQRT(t) /* some value < sqrt(max value of unsigned type t) */ \
39533197c6aStron 	((0UL|(t)-1) >>31>>31 > 1 ? ((t)1 <<32) - 1 : \
39633197c6aStron 	 (0UL|(t)-1) >>31 ? 65535U : (0UL|(t)-1) >>15 ? 255U : 15U)
39733197c6aStron 
3982de962bdSlukem void *
slap_sl_calloc(ber_len_t n,ber_len_t size,void * ctx)3992de962bdSlukem slap_sl_calloc( ber_len_t n, ber_len_t size, void *ctx )
4002de962bdSlukem {
4012de962bdSlukem 	void *newptr;
40233197c6aStron 	ber_len_t total = n * size;
4032de962bdSlukem 
40433197c6aStron 	/* The sqrt test is a slight optimization: often avoids the division */
40533197c6aStron 	if ((n | size) <= LIM_SQRT(ber_len_t) || n == 0 || total/n == size) {
40633197c6aStron 		newptr = slap_sl_malloc( total, ctx );
4072de962bdSlukem 		memset( newptr, 0, n*size );
40833197c6aStron 	} else {
40933197c6aStron 		Debug(LDAP_DEBUG_ANY, "slap_sl_calloc(%lu,%lu) out of range\n",
410*cf1d77f7Schristos 			(unsigned long) n, (unsigned long) size );
41133197c6aStron 		assert(0);
41233197c6aStron 		exit(EXIT_FAILURE);
4132de962bdSlukem 	}
4142de962bdSlukem 	return newptr;
4152de962bdSlukem }
4162de962bdSlukem 
4172de962bdSlukem void *
slap_sl_realloc(void * ptr,ber_len_t size,void * ctx)4182de962bdSlukem slap_sl_realloc(void *ptr, ber_len_t size, void *ctx)
4192de962bdSlukem {
4202de962bdSlukem 	struct slab_heap *sh = ctx;
42133197c6aStron 	ber_len_t oldsize, *p = (ber_len_t *) ptr, *nextp;
42233197c6aStron 	void *newptr;
4232de962bdSlukem 
4242de962bdSlukem 	if (ptr == NULL)
4252de962bdSlukem 		return slap_sl_malloc(size, ctx);
4262de962bdSlukem 
4272de962bdSlukem 	/* Not our memory? */
42833197c6aStron 	if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
42933197c6aStron 		/* Like ch_realloc(), except not trying a new context */
4302de962bdSlukem 		newptr = ber_memrealloc_x(ptr, size, NULL);
4312de962bdSlukem 		if (newptr) {
4322de962bdSlukem 			return newptr;
4332de962bdSlukem 		}
43433197c6aStron 		Debug(LDAP_DEBUG_ANY, "slap_sl_realloc of %lu bytes failed\n",
435*cf1d77f7Schristos 			(unsigned long) size );
4362de962bdSlukem 		assert(0);
4372de962bdSlukem 		exit( EXIT_FAILURE );
4382de962bdSlukem 	}
4392de962bdSlukem 
4402de962bdSlukem 	if (size == 0) {
4412de962bdSlukem 		slap_sl_free(ptr, ctx);
4422de962bdSlukem 		return NULL;
4432de962bdSlukem 	}
4442de962bdSlukem 
44533197c6aStron 	oldsize = p[-1];
44633197c6aStron 
4472de962bdSlukem 	if (sh->sh_stack) {
44833197c6aStron 		/* Add room for head, round up to doubleword boundary */
44933197c6aStron 		size = (size + sizeof(ber_len_t) + Align-1) & -Align;
4502de962bdSlukem 
4514e6df137Slukem 		p--;
4524e6df137Slukem 
4532de962bdSlukem 		/* Never shrink blocks */
45433197c6aStron 		if (size <= oldsize) {
45533197c6aStron 			return ptr;
45633197c6aStron 		}
4572de962bdSlukem 
45833197c6aStron 		oldsize &= -2;
45933197c6aStron 		nextp = (ber_len_t *) ((char *) p + oldsize);
46033197c6aStron 
46133197c6aStron 		/* If reallocing the last block, try to grow it */
46233197c6aStron 		if (nextp == sh->sh_last) {
46333197c6aStron 			if (size < (ber_len_t) ((char *) sh->sh_end - (char *) p)) {
46433197c6aStron 				sh->sh_last = (char *) p + size;
46533197c6aStron 				p[0] = (p[0] & 1) | size;
46633197c6aStron 				return ptr;
46733197c6aStron 			}
4682de962bdSlukem 
4692de962bdSlukem 		/* Nowhere to grow, need to alloc and copy */
4702de962bdSlukem 		} else {
47133197c6aStron 			/* Slight optimization of the final realloc variant */
4724e6df137Slukem 			newptr = slap_sl_malloc(size-sizeof(ber_len_t), ctx);
47333197c6aStron 			AC_MEMCPY(newptr, ptr, oldsize-sizeof(ber_len_t));
47433197c6aStron 			/* Not last block, can just mark old region as free */
47533197c6aStron 			nextp[-1] = oldsize;
47633197c6aStron 			nextp[0] |= 1;
4772de962bdSlukem 			return newptr;
47833197c6aStron 		}
4792de962bdSlukem 
48033197c6aStron 		size -= sizeof(ber_len_t);
48133197c6aStron 		oldsize -= sizeof(ber_len_t);
48233197c6aStron 
48333197c6aStron 	} else if (oldsize > size) {
48433197c6aStron 		oldsize = size;
4852de962bdSlukem 	}
48633197c6aStron 
48733197c6aStron 	newptr = slap_sl_malloc(size, ctx);
48833197c6aStron 	AC_MEMCPY(newptr, ptr, oldsize);
4892de962bdSlukem 	slap_sl_free(ptr, ctx);
49033197c6aStron 	return newptr;
4912de962bdSlukem }
4922de962bdSlukem 
4932de962bdSlukem void
slap_sl_free(void * ptr,void * ctx)4942de962bdSlukem slap_sl_free(void *ptr, void *ctx)
4952de962bdSlukem {
4962de962bdSlukem 	struct slab_heap *sh = ctx;
4974e6df137Slukem 	ber_len_t size;
49833197c6aStron 	ber_len_t *p = ptr, *nextp, *tmpp;
4992de962bdSlukem 
5002de962bdSlukem 	if (!ptr)
5012de962bdSlukem 		return;
5022de962bdSlukem 
50333197c6aStron 	if (No_sl_malloc || !sh || ptr < sh->sh_base || ptr >= sh->sh_end) {
5042de962bdSlukem 		ber_memfree_x(ptr, NULL);
5052de962bdSlukem 		return;
5064e6df137Slukem 	}
5074e6df137Slukem 
5082de962bdSlukem 	size = *(--p);
50933197c6aStron 
51033197c6aStron 	if (sh->sh_stack) {
51133197c6aStron 		size &= -2;
51233197c6aStron 		nextp = (ber_len_t *) ((char *) p + size);
51333197c6aStron 		if (sh->sh_last != nextp) {
51433197c6aStron 			/* Mark it free: tail = size, head of next block |= 1 */
51533197c6aStron 			nextp[-1] = size;
51633197c6aStron 			nextp[0] |= 1;
51733197c6aStron 			/* We can't tell Valgrind about it yet, because we
51833197c6aStron 			 * still need read/write access to this block for
51933197c6aStron 			 * when we eventually get to reclaim it.
52033197c6aStron 			 */
52133197c6aStron 		} else {
52233197c6aStron 			/* Reclaim freed block(s) off tail */
52333197c6aStron 			while (*p & 1) {
52433197c6aStron 				p = (ber_len_t *) ((char *) p - p[-1]);
52533197c6aStron 			}
52633197c6aStron 			sh->sh_last = p;
52733197c6aStron 			VGMEMP_TRIM(sh, sh->sh_base,
52833197c6aStron 				(char *) sh->sh_last - (char *) sh->sh_base);
52933197c6aStron 		}
53033197c6aStron 
53133197c6aStron 	} else {
53233197c6aStron 		int size_shift, order_size;
53333197c6aStron 		struct slab_object *so;
53433197c6aStron 		unsigned long diff;
53533197c6aStron 		int i, inserted = 0, order = -1;
53633197c6aStron 
5372de962bdSlukem 		size_shift = size + sizeof(ber_len_t) - 1;
5382de962bdSlukem 		do {
5392de962bdSlukem 			order++;
5402de962bdSlukem 		} while (size_shift >>= 1);
5412de962bdSlukem 
5422de962bdSlukem 		for (i = order, tmpp = p; i <= sh->sh_maxorder; i++) {
5432de962bdSlukem 			order_size = 1 << (i+1);
5442de962bdSlukem 			diff = (unsigned long)((char*)tmpp - (char*)sh->sh_base) >> (i+1);
5452de962bdSlukem 			sh->sh_map[i-order_start][diff>>3] &= (~(1 << (diff & 0x7)));
5462de962bdSlukem 			if (diff == ((diff>>1)<<1)) {
5472de962bdSlukem 				if (!(sh->sh_map[i-order_start][(diff+1)>>3] &
5482de962bdSlukem 						(1<<((diff+1)&0x7)))) {
5492de962bdSlukem 					so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
5502de962bdSlukem 					while (so) {
5512de962bdSlukem 						if ((char*)so->so_ptr == (char*)tmpp) {
5522de962bdSlukem 							LDAP_LIST_REMOVE( so, so_link );
5532de962bdSlukem 						} else if ((char*)so->so_ptr ==
5542de962bdSlukem 								(char*)tmpp + order_size) {
5552de962bdSlukem 							LDAP_LIST_REMOVE(so, so_link);
5562de962bdSlukem 							break;
5572de962bdSlukem 						}
5582de962bdSlukem 						so = LDAP_LIST_NEXT(so, so_link);
5592de962bdSlukem 					}
5602de962bdSlukem 					if (so) {
5612de962bdSlukem 						if (i < sh->sh_maxorder) {
5622de962bdSlukem 							inserted = 1;
5632de962bdSlukem 							so->so_ptr = tmpp;
5642de962bdSlukem 							LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],
5652de962bdSlukem 									so, so_link);
5662de962bdSlukem 						}
5672de962bdSlukem 						continue;
5682de962bdSlukem 					} else {
5692de962bdSlukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
5702de962bdSlukem 							slap_replenish_sopool(sh);
5712de962bdSlukem 						}
5722de962bdSlukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
5732de962bdSlukem 						LDAP_LIST_REMOVE(so, so_link);
5742de962bdSlukem 						so->so_ptr = tmpp;
5752de962bdSlukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
5762de962bdSlukem 								so, so_link);
5772de962bdSlukem 						break;
5782de962bdSlukem 
5792de962bdSlukem 						Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
580*cf1d77f7Schristos 							"free object not found while bit is clear.\n" );
5812de962bdSlukem 						assert(so != NULL);
5822de962bdSlukem 
5832de962bdSlukem 					}
5842de962bdSlukem 				} else {
5852de962bdSlukem 					if (!inserted) {
5862de962bdSlukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
5872de962bdSlukem 							slap_replenish_sopool(sh);
5882de962bdSlukem 						}
5892de962bdSlukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
5902de962bdSlukem 						LDAP_LIST_REMOVE(so, so_link);
5912de962bdSlukem 						so->so_ptr = tmpp;
5922de962bdSlukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
5932de962bdSlukem 								so, so_link);
5942de962bdSlukem 					}
5952de962bdSlukem 					break;
5962de962bdSlukem 				}
5972de962bdSlukem 			} else {
5982de962bdSlukem 				if (!(sh->sh_map[i-order_start][(diff-1)>>3] &
5992de962bdSlukem 						(1<<((diff-1)&0x7)))) {
6002de962bdSlukem 					so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
6012de962bdSlukem 					while (so) {
6022de962bdSlukem 						if ((char*)so->so_ptr == (char*)tmpp) {
6032de962bdSlukem 							LDAP_LIST_REMOVE(so, so_link);
6042de962bdSlukem 						} else if ((char*)tmpp == (char *)so->so_ptr + order_size) {
6052de962bdSlukem 							LDAP_LIST_REMOVE(so, so_link);
6062de962bdSlukem 							tmpp = so->so_ptr;
6072de962bdSlukem 							break;
6082de962bdSlukem 						}
6092de962bdSlukem 						so = LDAP_LIST_NEXT(so, so_link);
6102de962bdSlukem 					}
6112de962bdSlukem 					if (so) {
6122de962bdSlukem 						if (i < sh->sh_maxorder) {
6132de962bdSlukem 							inserted = 1;
6142de962bdSlukem 							LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start+1],									so, so_link);
6152de962bdSlukem 							continue;
6162de962bdSlukem 						}
6172de962bdSlukem 					} else {
6182de962bdSlukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
6192de962bdSlukem 							slap_replenish_sopool(sh);
6202de962bdSlukem 						}
6212de962bdSlukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
6222de962bdSlukem 						LDAP_LIST_REMOVE(so, so_link);
6232de962bdSlukem 						so->so_ptr = tmpp;
6242de962bdSlukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
6252de962bdSlukem 								so, so_link);
6262de962bdSlukem 						break;
6272de962bdSlukem 
6282de962bdSlukem 						Debug(LDAP_DEBUG_TRACE, "slap_sl_free: "
629*cf1d77f7Schristos 							"free object not found while bit is clear.\n" );
6302de962bdSlukem 						assert(so != NULL);
6312de962bdSlukem 
6322de962bdSlukem 					}
6332de962bdSlukem 				} else {
6342de962bdSlukem 					if ( !inserted ) {
6352de962bdSlukem 						if (LDAP_LIST_EMPTY(&sh->sh_sopool)) {
6362de962bdSlukem 							slap_replenish_sopool(sh);
6372de962bdSlukem 						}
6382de962bdSlukem 						so = LDAP_LIST_FIRST(&sh->sh_sopool);
6392de962bdSlukem 						LDAP_LIST_REMOVE(so, so_link);
6402de962bdSlukem 						so->so_ptr = tmpp;
6412de962bdSlukem 						LDAP_LIST_INSERT_HEAD(&sh->sh_free[i-order_start],
6422de962bdSlukem 								so, so_link);
6432de962bdSlukem 					}
6442de962bdSlukem 					break;
6452de962bdSlukem 				}
6462de962bdSlukem 			}
6472de962bdSlukem 		}
6482de962bdSlukem 	}
6492de962bdSlukem }
6502de962bdSlukem 
651*cf1d77f7Schristos void
slap_sl_release(void * ptr,void * ctx)652*cf1d77f7Schristos slap_sl_release( void *ptr, void *ctx )
653*cf1d77f7Schristos {
654*cf1d77f7Schristos 	struct slab_heap *sh = ctx;
655*cf1d77f7Schristos 	if ( sh && ptr >= sh->sh_base && ptr <= sh->sh_end )
656*cf1d77f7Schristos 		sh->sh_last = ptr;
657*cf1d77f7Schristos }
658*cf1d77f7Schristos 
659*cf1d77f7Schristos void *
slap_sl_mark(void * ctx)660*cf1d77f7Schristos slap_sl_mark( void *ctx )
661*cf1d77f7Schristos {
662*cf1d77f7Schristos 	struct slab_heap *sh = ctx;
663*cf1d77f7Schristos 	return sh->sh_last;
664*cf1d77f7Schristos }
665*cf1d77f7Schristos 
66633197c6aStron /*
66733197c6aStron  * Return the memory context of the current thread if the given block of
66833197c6aStron  * memory belongs to it, otherwise return NULL.
66933197c6aStron  */
6702de962bdSlukem void *
slap_sl_context(void * ptr)6712de962bdSlukem slap_sl_context( void *ptr )
6722de962bdSlukem {
67333197c6aStron 	void *memctx;
6742de962bdSlukem 	struct slab_heap *sh;
6752de962bdSlukem 
6762de962bdSlukem 	if ( slapMode & SLAP_TOOL_MODE ) return NULL;
6772de962bdSlukem 
67833197c6aStron 	sh = GET_MEMCTX(ldap_pvt_thread_pool_context(), &memctx);
6792de962bdSlukem 	if (sh && ptr >= sh->sh_base && ptr <= sh->sh_end) {
6802de962bdSlukem 		return sh;
6812de962bdSlukem 	}
6822de962bdSlukem 	return NULL;
6832de962bdSlukem }
6842de962bdSlukem 
6852de962bdSlukem static struct slab_object *
slap_replenish_sopool(struct slab_heap * sh)6862de962bdSlukem slap_replenish_sopool(
6872de962bdSlukem     struct slab_heap* sh
6882de962bdSlukem )
6892de962bdSlukem {
6902de962bdSlukem     struct slab_object *so_block;
6912de962bdSlukem     int i;
6922de962bdSlukem 
6932de962bdSlukem     so_block = (struct slab_object *)ch_malloc(
6942de962bdSlukem                     SLAP_SLAB_SOBLOCK * sizeof(struct slab_object));
6952de962bdSlukem 
6962de962bdSlukem     if ( so_block == NULL ) {
6972de962bdSlukem         return NULL;
6982de962bdSlukem     }
6992de962bdSlukem 
7002de962bdSlukem     so_block[0].so_blockhead = 1;
7012de962bdSlukem     LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[0], so_link);
7022de962bdSlukem     for (i = 1; i < SLAP_SLAB_SOBLOCK; i++) {
7032de962bdSlukem         so_block[i].so_blockhead = 0;
7042de962bdSlukem         LDAP_LIST_INSERT_HEAD(&sh->sh_sopool, &so_block[i], so_link );
7052de962bdSlukem     }
7062de962bdSlukem 
7072de962bdSlukem     return so_block;
7082de962bdSlukem }
7092de962bdSlukem 
7102de962bdSlukem #ifdef SLAPD_UNUSED
7112de962bdSlukem static void
print_slheap(int level,void * ctx)7122de962bdSlukem print_slheap(int level, void *ctx)
7132de962bdSlukem {
7142de962bdSlukem 	struct slab_heap *sh = ctx;
7152de962bdSlukem 	struct slab_object *so;
7162de962bdSlukem 	int i, j, once = 0;
7172de962bdSlukem 
7182de962bdSlukem 	if (!ctx) {
719*cf1d77f7Schristos 		Debug(level, "NULL memctx\n" );
7202de962bdSlukem 		return;
7212de962bdSlukem 	}
7222de962bdSlukem 
723*cf1d77f7Schristos 	Debug(level, "sh->sh_maxorder=%d\n", sh->sh_maxorder );
7242de962bdSlukem 
7252de962bdSlukem 	for (i = order_start; i <= sh->sh_maxorder; i++) {
7262de962bdSlukem 		once = 0;
727*cf1d77f7Schristos 		Debug(level, "order=%d\n", i );
7282de962bdSlukem 		for (j = 0; j < (1<<(sh->sh_maxorder-i))/8; j++) {
729*cf1d77f7Schristos 			Debug(level, "%02x ", sh->sh_map[i-order_start][j] );
7302de962bdSlukem 			once = 1;
7312de962bdSlukem 		}
7322de962bdSlukem 		if (!once) {
733*cf1d77f7Schristos 			Debug(level, "%02x ", sh->sh_map[i-order_start][0] );
7342de962bdSlukem 		}
735*cf1d77f7Schristos 		Debug(level, "\n" );
736*cf1d77f7Schristos 		Debug(level, "free list:\n" );
7372de962bdSlukem 		so = LDAP_LIST_FIRST(&sh->sh_free[i-order_start]);
7382de962bdSlukem 		while (so) {
739*cf1d77f7Schristos 			Debug(level, "%p\n", so->so_ptr );
7402de962bdSlukem 			so = LDAP_LIST_NEXT(so, so_link);
7412de962bdSlukem 		}
7422de962bdSlukem 	}
7432de962bdSlukem }
7442de962bdSlukem #endif
745