1 /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3 * Slabs memory allocation, based on powers-of-N. Slabs are up to 1MB in size
4 * and are divided into chunks. The chunk sizes start off at the size of the
5 * "item" structure plus space for a small key and value. They increase by
6 * a multiplier factor from there, up to half the maximum slab size. The last
7 * slab size is always 1MB, since that's the maximum item size allowed by the
8 * memcached protocol.
9 */
10 #include "config.h"
11
12 #include <fcntl.h>
13 #include <errno.h>
14 #include <stdlib.h>
15 #include <stdio.h>
16 #include <string.h>
17 #include <assert.h>
18 #include <pthread.h>
19 #include <inttypes.h>
20 #include <stdarg.h>
21
22 #include "default_engine.h"
23
24 /*
25 * Forward Declarations
26 */
27 static int do_slabs_newslab(struct default_engine *engine, const unsigned int id);
28 static void *memory_allocate(struct default_engine *engine, size_t size);
29
30 #ifndef DONT_PREALLOC_SLABS
31 /* Preallocate as many slab pages as possible (called from slabs_init)
32 on start-up, so users don't get confused out-of-memory errors when
33 they do have free (in-slab) space, but no space to make new slabs.
34 if maxslabs is 18 (POWER_LARGEST - POWER_SMALLEST + 1), then all
35 slab types can be made. if max memory is less than 18 MB, only the
36 smaller ones will be made. */
37 static void slabs_preallocate (const unsigned int maxslabs);
38 #endif
39
40 /*
41 * Figures out which slab class (chunk size) is required to store an item of
42 * a given size.
43 *
44 * Given object size, return id to use when allocating/freeing memory for object
45 * 0 means error: can't store such a large object
46 */
47
slabs_clsid(struct default_engine * engine,const size_t size)48 unsigned int slabs_clsid(struct default_engine *engine, const size_t size) {
49 int res = POWER_SMALLEST;
50
51 if (size == 0)
52 return 0;
53 while (size > engine->slabs.slabclass[res].size)
54 if (res++ == engine->slabs.power_largest) /* won't fit in the biggest slab */
55 return 0;
56 return res;
57 }
58
59 /**
60 * Determines the chunk sizes and initializes the slab class descriptors
61 * accordingly.
62 */
slabs_init(struct default_engine * engine,const size_t limit,const double factor,const bool prealloc)63 ENGINE_ERROR_CODE slabs_init(struct default_engine *engine,
64 const size_t limit,
65 const double factor,
66 const bool prealloc) {
67 int i = POWER_SMALLEST - 1;
68 unsigned int size = sizeof(hash_item) + engine->config.chunk_size;
69
70 engine->slabs.mem_limit = limit;
71
72 EXTENSION_LOGGER_DESCRIPTOR *logger;
73 logger = (void*)engine->server.extension->get_extension(EXTENSION_LOGGER);
74
75 if (prealloc) {
76 /* Allocate everything in a big chunk with malloc */
77 engine->slabs.mem_base = malloc(engine->slabs.mem_limit);
78 if (engine->slabs.mem_base != NULL) {
79 engine->slabs.mem_current = engine->slabs.mem_base;
80 engine->slabs.mem_avail = engine->slabs.mem_limit;
81 } else {
82 logger->log(EXTENSION_LOG_WARNING, NULL,
83 "default_engine: Failed attempt to preallocate %zu bytes.",
84 engine->slabs.mem_limit);
85 return ENGINE_ENOMEM;
86 }
87 }
88
89 memset(engine->slabs.slabclass, 0, sizeof(engine->slabs.slabclass));
90
91 while (++i < POWER_LARGEST && size <= engine->config.item_size_max / factor) {
92 /* Make sure items are always n-byte aligned */
93 if (size % CHUNK_ALIGN_BYTES)
94 size += CHUNK_ALIGN_BYTES - (size % CHUNK_ALIGN_BYTES);
95
96 engine->slabs.slabclass[i].size = size;
97 engine->slabs.slabclass[i].perslab = engine->config.item_size_max / engine->slabs.slabclass[i].size;
98 size *= factor;
99 if (engine->config.verbose > 1) {
100 logger->log(EXTENSION_LOG_INFO, NULL,
101 "slab class %3d: chunk size %9u perslab %7u\n",
102 i, engine->slabs.slabclass[i].size,
103 engine->slabs.slabclass[i].perslab);
104 }
105 }
106
107 engine->slabs.power_largest = i;
108 engine->slabs.slabclass[engine->slabs.power_largest].size = engine->config.item_size_max;
109 engine->slabs.slabclass[engine->slabs.power_largest].perslab = 1;
110 if (engine->config.verbose > 1) {
111 logger->log(EXTENSION_LOG_INFO, NULL,
112 "slab class %3d: chunk size %9u perslab %7u\n",
113 i, engine->slabs.slabclass[i].size,
114 engine->slabs.slabclass[i].perslab);
115 }
116
117 /* for the test suite: faking of how much we've already malloc'd */
118 {
119 char *t_initial_malloc = getenv("T_MEMD_INITIAL_MALLOC");
120 if (t_initial_malloc) {
121 engine->slabs.mem_malloced = (size_t)atol(t_initial_malloc);
122 }
123
124 }
125
126 #ifndef DONT_PREALLOC_SLABS
127 {
128 char *pre_alloc = getenv("T_MEMD_SLABS_ALLOC");
129
130 if (pre_alloc == NULL || atoi(pre_alloc) != 0) {
131 slabs_preallocate(power_largest);
132 }
133 }
134 #endif
135
136 return ENGINE_SUCCESS;
137 }
138
139 #ifndef DONT_PREALLOC_SLABS
slabs_preallocate(const unsigned int maxslabs)140 static void slabs_preallocate (const unsigned int maxslabs) {
141 int i;
142 unsigned int prealloc = 0;
143
144 /* pre-allocate a 1MB slab in every size class so people don't get
145 confused by non-intuitive "SERVER_ERROR out of memory"
146 messages. this is the most common question on the mailing
147 list. if you really don't want this, you can rebuild without
148 these three lines. */
149
150 for (i = POWER_SMALLEST; i <= POWER_LARGEST; i++) {
151 if (++prealloc > maxslabs)
152 return;
153 do_slabs_newslab(i);
154 }
155
156 }
157 #endif
158
grow_slab_list(struct default_engine * engine,const unsigned int id)159 static int grow_slab_list (struct default_engine *engine, const unsigned int id) {
160 slabclass_t *p = &engine->slabs.slabclass[id];
161 if (p->slabs == p->list_size) {
162 size_t new_size = (p->list_size != 0) ? p->list_size * 2 : 16;
163 void *new_list = realloc(p->slab_list, new_size * sizeof(void *));
164 if (new_list == 0) return 0;
165 p->list_size = new_size;
166 p->slab_list = new_list;
167 }
168 return 1;
169 }
170
do_slabs_newslab(struct default_engine * engine,const unsigned int id)171 static int do_slabs_newslab(struct default_engine *engine, const unsigned int id) {
172 slabclass_t *p = &engine->slabs.slabclass[id];
173 int len = p->size * p->perslab;
174 char *ptr;
175
176 if ((engine->slabs.mem_limit && engine->slabs.mem_malloced + len > engine->slabs.mem_limit && p->slabs > 0) ||
177 (grow_slab_list(engine, id) == 0) ||
178 ((ptr = memory_allocate(engine, (size_t)len)) == 0)) {
179
180 MEMCACHED_SLABS_SLABCLASS_ALLOCATE_FAILED(id);
181 return 0;
182 }
183
184 memset(ptr, 0, (size_t)len);
185 p->end_page_ptr = ptr;
186 p->end_page_free = p->perslab;
187
188 p->slab_list[p->slabs++] = ptr;
189 engine->slabs.mem_malloced += len;
190 MEMCACHED_SLABS_SLABCLASS_ALLOCATE(id);
191
192 return 1;
193 }
194
195 /*@null@*/
do_slabs_alloc(struct default_engine * engine,const size_t size,unsigned int id)196 static void *do_slabs_alloc(struct default_engine *engine, const size_t size, unsigned int id) {
197 slabclass_t *p;
198 void *ret = NULL;
199
200 if (id < POWER_SMALLEST || id > engine->slabs.power_largest) {
201 MEMCACHED_SLABS_ALLOCATE_FAILED(size, 0);
202 return NULL;
203 }
204
205 p = &engine->slabs.slabclass[id];
206
207 #ifdef USE_SYSTEM_MALLOC
208 if (engine->slabs.mem_limit && engine->slabs.mem_malloced + size > engine->slabs.mem_limit) {
209 MEMCACHED_SLABS_ALLOCATE_FAILED(size, id);
210 return 0;
211 }
212 engine->slabs.mem_malloced += size;
213 ret = malloc(size);
214 MEMCACHED_SLABS_ALLOCATE(size, id, 0, ret);
215 return ret;
216 #endif
217
218 /* fail unless we have space at the end of a recently allocated page,
219 we have something on our freelist, or we could allocate a new page */
220 if (! (p->end_page_ptr != 0 || p->sl_curr != 0 ||
221 do_slabs_newslab(engine, id) != 0)) {
222 /* We don't have more memory available */
223 ret = NULL;
224 } else if (p->sl_curr != 0) {
225 /* return off our freelist */
226 ret = p->slots[--p->sl_curr];
227 } else {
228 /* if we recently allocated a whole page, return from that */
229 assert(p->end_page_ptr != NULL);
230 ret = p->end_page_ptr;
231 if (--p->end_page_free != 0) {
232 p->end_page_ptr = ((caddr_t)p->end_page_ptr) + p->size;
233 } else {
234 p->end_page_ptr = 0;
235 }
236 }
237
238 if (ret) {
239 p->requested += size;
240 MEMCACHED_SLABS_ALLOCATE(size, id, p->size, ret);
241 } else {
242 MEMCACHED_SLABS_ALLOCATE_FAILED(size, id);
243 }
244
245 return ret;
246 }
247
do_slabs_free(struct default_engine * engine,void * ptr,const size_t size,unsigned int id)248 static void do_slabs_free(struct default_engine *engine, void *ptr, const size_t size, unsigned int id) {
249 slabclass_t *p;
250
251 if (id < POWER_SMALLEST || id > engine->slabs.power_largest)
252 return;
253
254 MEMCACHED_SLABS_FREE(size, id, ptr);
255 p = &engine->slabs.slabclass[id];
256
257 #ifdef USE_SYSTEM_MALLOC
258 engine->slabs.mem_malloced -= size;
259 free(ptr);
260 return;
261 #endif
262
263 if (p->sl_curr == p->sl_total) { /* need more space on the free list */
264 int new_size = (p->sl_total != 0) ? p->sl_total * 2 : 16; /* 16 is arbitrary */
265 void **new_slots = realloc(p->slots, new_size * sizeof(void *));
266 if (new_slots == 0)
267 return;
268 p->slots = new_slots;
269 p->sl_total = new_size;
270 }
271 p->slots[p->sl_curr++] = ptr;
272 p->requested -= size;
273 return;
274 }
275
add_statistics(const void * cookie,ADD_STAT add_stats,const char * prefix,int num,const char * key,const char * fmt,...)276 void add_statistics(const void *cookie, ADD_STAT add_stats,
277 const char* prefix, int num, const char *key,
278 const char *fmt, ...) {
279 char name[80], val[80];
280 int klen = 0, vlen;
281 va_list ap;
282
283 assert(cookie);
284 assert(add_stats);
285 assert(key);
286
287 va_start(ap, fmt);
288 vlen = vsnprintf(val, sizeof(val) - 1, fmt, ap);
289 va_end(ap);
290
291 if (prefix != NULL) {
292 klen = snprintf(name, sizeof(name), "%s:", prefix);
293 }
294
295 if (num != -1) {
296 klen += snprintf(name + klen, sizeof(name) - klen, "%d:", num);
297 }
298
299 klen += snprintf(name + klen, sizeof(name) - klen, "%s", key);
300
301 add_stats(name, klen, val, vlen, cookie);
302 }
303
304 /*@null@*/
do_slabs_stats(struct default_engine * engine,ADD_STAT add_stats,const void * cookie)305 static void do_slabs_stats(struct default_engine *engine, ADD_STAT add_stats, const void *cookie) {
306 int i, total;
307 /* Get the per-thread stats which contain some interesting aggregates */
308 #ifdef FUTURE
309 struct conn *conn = (struct conn*)cookie;
310 struct thread_stats thread_stats;
311 threadlocal_stats_aggregate(c, &thread_stats);
312 #endif
313
314 total = 0;
315 for(i = POWER_SMALLEST; i <= engine->slabs.power_largest; i++) {
316 slabclass_t *p = &engine->slabs.slabclass[i];
317 if (p->slabs != 0) {
318 uint32_t perslab, slabs;
319 slabs = p->slabs;
320 perslab = p->perslab;
321
322 add_statistics(cookie, add_stats, NULL, i, "chunk_size", "%u",
323 p->size);
324 add_statistics(cookie, add_stats, NULL, i, "chunks_per_page", "%u",
325 perslab);
326 add_statistics(cookie, add_stats, NULL, i, "total_pages", "%u",
327 slabs);
328 add_statistics(cookie, add_stats, NULL, i, "total_chunks", "%u",
329 slabs * perslab);
330 add_statistics(cookie, add_stats, NULL, i, "used_chunks", "%u",
331 slabs*perslab - p->sl_curr - p->end_page_free);
332 add_statistics(cookie, add_stats, NULL, i, "free_chunks", "%u",
333 p->sl_curr);
334 add_statistics(cookie, add_stats, NULL, i, "free_chunks_end", "%u",
335 p->end_page_free);
336 add_statistics(cookie, add_stats, NULL, i, "mem_requested", "%zu",
337 p->requested);
338 #ifdef FUTURE
339 add_statistics(cookie, add_stats, NULL, i, "get_hits", "%"PRIu64,
340 thread_stats.slab_stats[i].get_hits);
341 add_statistics(cookie, add_stats, NULL, i, "cmd_set", "%"PRIu64,
342 thread_stats.slab_stats[i].set_cmds);
343 add_statistics(cookie, add_stats, NULL, i, "delete_hits", "%"PRIu64,
344 thread_stats.slab_stats[i].delete_hits);
345 add_statistics(cookie, add_stats, NULL, i, "cas_hits", "%"PRIu64,
346 thread_stats.slab_stats[i].cas_hits);
347 add_statistics(cookie, add_stats, NULL, i, "cas_badval", "%"PRIu64,
348 thread_stats.slab_stats[i].cas_badval);
349 #endif
350 total++;
351 }
352 }
353
354 /* add overall slab stats and append terminator */
355
356 add_statistics(cookie, add_stats, NULL, -1, "active_slabs", "%d", total);
357 add_statistics(cookie, add_stats, NULL, -1, "total_malloced", "%zu",
358 engine->slabs.mem_malloced);
359 }
360
memory_allocate(struct default_engine * engine,size_t size)361 static void *memory_allocate(struct default_engine *engine, size_t size) {
362 void *ret;
363
364 if (engine->slabs.mem_base == NULL) {
365 /* We are not using a preallocated large memory chunk */
366 ret = malloc(size);
367 } else {
368 ret = engine->slabs.mem_current;
369
370 if (size > engine->slabs.mem_avail) {
371 return NULL;
372 }
373
374 /* mem_current pointer _must_ be aligned!!! */
375 if (size % CHUNK_ALIGN_BYTES) {
376 size += CHUNK_ALIGN_BYTES - (size % CHUNK_ALIGN_BYTES);
377 }
378
379 engine->slabs.mem_current = ((char*)engine->slabs.mem_current) + size;
380 if (size < engine->slabs.mem_avail) {
381 engine->slabs.mem_avail -= size;
382 } else {
383 engine->slabs.mem_avail = 0;
384 }
385 }
386
387 return ret;
388 }
389
slabs_alloc(struct default_engine * engine,size_t size,unsigned int id)390 void *slabs_alloc(struct default_engine *engine, size_t size, unsigned int id) {
391 void *ret;
392
393 pthread_mutex_lock(&engine->slabs.lock);
394 ret = do_slabs_alloc(engine, size, id);
395 pthread_mutex_unlock(&engine->slabs.lock);
396 return ret;
397 }
398
slabs_free(struct default_engine * engine,void * ptr,size_t size,unsigned int id)399 void slabs_free(struct default_engine *engine, void *ptr, size_t size, unsigned int id) {
400 pthread_mutex_lock(&engine->slabs.lock);
401 do_slabs_free(engine, ptr, size, id);
402 pthread_mutex_unlock(&engine->slabs.lock);
403 }
404
slabs_stats(struct default_engine * engine,ADD_STAT add_stats,const void * c)405 void slabs_stats(struct default_engine *engine, ADD_STAT add_stats, const void *c) {
406 pthread_mutex_lock(&engine->slabs.lock);
407 do_slabs_stats(engine, add_stats, c);
408 pthread_mutex_unlock(&engine->slabs.lock);
409 }
410
slabs_adjust_mem_requested(struct default_engine * engine,unsigned int id,size_t old,size_t ntotal)411 void slabs_adjust_mem_requested(struct default_engine *engine, unsigned int id, size_t old, size_t ntotal)
412 {
413 pthread_mutex_lock(&engine->slabs.lock);
414 slabclass_t *p;
415 if (id < POWER_SMALLEST || id > engine->slabs.power_largest) {
416 EXTENSION_LOGGER_DESCRIPTOR *logger;
417 logger = (void*)engine->server.extension->get_extension(EXTENSION_LOGGER);
418 logger->log(EXTENSION_LOG_WARNING, NULL,
419 "Internal error! Invalid slab class\n");
420 abort();
421 }
422
423 p = &engine->slabs.slabclass[id];
424 p->requested = p->requested - old + ntotal;
425 pthread_mutex_unlock(&engine->slabs.lock);
426 }
427