1 /*
2 * Memory management functions.
3 *
4 * Copyright 2000-2007 Willy Tarreau <w@1wt.eu>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12 #include <errno.h>
13
14 #include <haproxy/activity-t.h>
15 #include <haproxy/api.h>
16 #include <haproxy/applet-t.h>
17 #include <haproxy/cfgparse.h>
18 #include <haproxy/channel.h>
19 #include <haproxy/cli.h>
20 #include <haproxy/errors.h>
21 #include <haproxy/global.h>
22 #include <haproxy/list.h>
23 #include <haproxy/pool.h>
24 #include <haproxy/stats-t.h>
25 #include <haproxy/stream_interface.h>
26 #include <haproxy/thread.h>
27 #include <haproxy/tools.h>
28
29
30 #ifdef CONFIG_HAP_POOLS
31 /* These ones are initialized per-thread on startup by init_pools() */
32 THREAD_LOCAL size_t pool_cache_bytes = 0; /* total cache size */
33 THREAD_LOCAL size_t pool_cache_count = 0; /* #cache objects */
34 #endif
35
36 static struct list pools = LIST_HEAD_INIT(pools);
37 int mem_poison_byte = -1;
38
39 #ifdef DEBUG_FAIL_ALLOC
40 static int mem_fail_rate = 0;
41 #endif
42
43 #if defined(HA_HAVE_MALLOC_TRIM)
44 static int using_libc_allocator = 0;
45
46 /* ask the allocator to trim memory pools.
47 * This must run under thread isolation so that competing threads trying to
48 * allocate or release memory do not prevent the allocator from completing
49 * its job. We just have to be careful as callers might already be isolated
50 * themselves.
51 */
trim_all_pools(void)52 static void trim_all_pools(void)
53 {
54 int isolated = thread_isolated();
55
56 if (!isolated)
57 thread_isolate();
58
59 if (using_libc_allocator)
60 malloc_trim(0);
61
62 if (!isolated)
63 thread_release();
64 }
65
66 /* check if we're using the same allocator as the one that provides
67 * malloc_trim() and mallinfo(). The principle is that on glibc, both
68 * malloc_trim() and mallinfo() are provided, and using mallinfo() we
69 * can check if malloc() is performed through glibc or any other one
70 * the executable was linked against (e.g. jemalloc).
71 */
detect_allocator(void)72 static void detect_allocator(void)
73 {
74 #ifdef HA_HAVE_MALLINFO2
75 struct mallinfo2 mi1, mi2;
76 #else
77 struct mallinfo mi1, mi2;
78 #endif
79 void *ptr;
80
81 #ifdef HA_HAVE_MALLINFO2
82 mi1 = mallinfo2();
83 #else
84 mi1 = mallinfo();
85 #endif
86 ptr = DISGUISE(malloc(1));
87 #ifdef HA_HAVE_MALLINFO2
88 mi2 = mallinfo2();
89 #else
90 mi2 = mallinfo();
91 #endif
92 free(DISGUISE(ptr));
93
94 using_libc_allocator = !!memcmp(&mi1, &mi2, sizeof(mi1));
95 }
96 #else
97
trim_all_pools(void)98 static void trim_all_pools(void)
99 {
100 }
101
detect_allocator(void)102 static void detect_allocator(void)
103 {
104 }
105 #endif
106
107 /* Try to find an existing shared pool with the same characteristics and
108 * returns it, otherwise creates this one. NULL is returned if no memory
109 * is available for a new creation. Two flags are supported :
110 * - MEM_F_SHARED to indicate that the pool may be shared with other users
111 * - MEM_F_EXACT to indicate that the size must not be rounded up
112 */
create_pool(char * name,unsigned int size,unsigned int flags)113 struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags)
114 {
115 struct pool_head *pool;
116 struct pool_head *entry;
117 struct list *start;
118 unsigned int align;
119 int thr __maybe_unused;
120
121 /* We need to store a (void *) at the end of the chunks. Since we know
122 * that the malloc() function will never return such a small size,
123 * let's round the size up to something slightly bigger, in order to
124 * ease merging of entries. Note that the rounding is a power of two.
125 * This extra (void *) is not accounted for in the size computation
126 * so that the visible parts outside are not affected.
127 *
128 * Note: for the LRU cache, we need to store 2 doubly-linked lists.
129 */
130
131 if (!(flags & MEM_F_EXACT)) {
132 align = 4 * sizeof(void *); // 2 lists = 4 pointers min
133 size = ((size + POOL_EXTRA + align - 1) & -align) - POOL_EXTRA;
134 }
135
136 /* TODO: thread: we do not lock pool list for now because all pools are
137 * created during HAProxy startup (so before threads creation) */
138 start = &pools;
139 pool = NULL;
140
141 list_for_each_entry(entry, &pools, list) {
142 if (entry->size == size) {
143 /* either we can share this place and we take it, or
144 * we look for a shareable one or for the next position
145 * before which we will insert a new one.
146 */
147 if ((flags & entry->flags & MEM_F_SHARED)
148 #ifdef DEBUG_DONT_SHARE_POOLS
149 && strcmp(name, entry->name) == 0
150 #endif
151 ) {
152 /* we can share this one */
153 pool = entry;
154 DPRINTF(stderr, "Sharing %s with %s\n", name, pool->name);
155 break;
156 }
157 }
158 else if (entry->size > size) {
159 /* insert before this one */
160 start = &entry->list;
161 break;
162 }
163 }
164
165 if (!pool) {
166 if (!pool)
167 pool = calloc(1, sizeof(*pool));
168
169 if (!pool)
170 return NULL;
171 if (name)
172 strlcpy2(pool->name, name, sizeof(pool->name));
173 pool->size = size;
174 pool->flags = flags;
175 LIST_APPEND(start, &pool->list);
176
177 #ifdef CONFIG_HAP_POOLS
178 /* update per-thread pool cache if necessary */
179 for (thr = 0; thr < MAX_THREADS; thr++) {
180 LIST_INIT(&pool->cache[thr].list);
181 }
182 #endif
183 HA_SPIN_INIT(&pool->lock);
184 }
185 pool->users++;
186 return pool;
187 }
188
189 /* Tries to allocate an object for the pool <pool> using the system's allocator
190 * and directly returns it. The pool's allocated counter is checked and updated,
191 * but no other checks are performed. The pool's lock is not used and is not a
192 * problem either.
193 */
pool_get_from_os(struct pool_head * pool)194 void *pool_get_from_os(struct pool_head *pool)
195 {
196 if (!pool->limit || pool->allocated < pool->limit) {
197 void *ptr = pool_alloc_area(pool->size + POOL_EXTRA);
198 if (ptr) {
199 _HA_ATOMIC_INC(&pool->allocated);
200 return ptr;
201 }
202 _HA_ATOMIC_INC(&pool->failed);
203 }
204 activity[tid].pool_fail++;
205 return NULL;
206
207 }
208
209 /* Releases a pool item back to the operating system and atomically updates
210 * the allocation counter.
211 */
pool_put_to_os(struct pool_head * pool,void * ptr)212 void pool_put_to_os(struct pool_head *pool, void *ptr)
213 {
214 #ifdef DEBUG_UAF
215 /* This object will be released for real in order to detect a use after
216 * free. We also force a write to the area to ensure we crash on double
217 * free or free of a const area.
218 */
219 *(uint32_t *)ptr = 0xDEADADD4;
220 #endif /* DEBUG_UAF */
221
222 pool_free_area(ptr, pool->size + POOL_EXTRA);
223 _HA_ATOMIC_DEC(&pool->allocated);
224 }
225
226 /* Tries to allocate an object for the pool <pool> using the system's allocator
227 * and directly returns it. The pool's counters are updated but the object is
228 * never cached, so this is usable with and without local or shared caches.
229 * This may be called with or without the pool lock held, so it must not use
230 * the pool's lock.
231 */
pool_alloc_nocache(struct pool_head * pool)232 void *pool_alloc_nocache(struct pool_head *pool)
233 {
234 void *ptr = NULL;
235
236 ptr = pool_get_from_os(pool);
237 if (!ptr)
238 return NULL;
239
240 swrate_add_scaled(&pool->needed_avg, POOL_AVG_SAMPLES, pool->used, POOL_AVG_SAMPLES/4);
241 _HA_ATOMIC_INC(&pool->used);
242
243 #ifdef DEBUG_MEMORY_POOLS
244 /* keep track of where the element was allocated from */
245 *POOL_LINK(pool, ptr) = (void *)pool;
246 #endif
247 return ptr;
248 }
249
250 /* Release a pool item back to the OS and keeps the pool's counters up to date.
251 * This is always defined even when pools are not enabled (their usage stats
252 * are maintained).
253 */
pool_free_nocache(struct pool_head * pool,void * ptr)254 void pool_free_nocache(struct pool_head *pool, void *ptr)
255 {
256 _HA_ATOMIC_DEC(&pool->used);
257 swrate_add(&pool->needed_avg, POOL_AVG_SAMPLES, pool->used);
258 pool_put_to_os(pool, ptr);
259 }
260
261
262 #ifdef CONFIG_HAP_POOLS
263
264 /* Evicts some of the oldest objects from one local cache, until its number of
265 * objects is no more than 16+1/8 of the total number of locally cached objects
266 * or the total size of the local cache is no more than 75% of its maximum (i.e.
267 * we don't want a single cache to use all the cache for itself). For this, the
268 * list is scanned in reverse.
269 */
pool_evict_from_local_cache(struct pool_head * pool)270 void pool_evict_from_local_cache(struct pool_head *pool)
271 {
272 struct pool_cache_head *ph = &pool->cache[tid];
273 struct pool_cache_item *item;
274
275 while (ph->count >= 16 + pool_cache_count / 8 &&
276 pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 3 / 4) {
277 item = LIST_NEXT(&ph->list, typeof(item), by_pool);
278 ph->count--;
279 pool_cache_bytes -= pool->size;
280 pool_cache_count--;
281 LIST_DELETE(&item->by_pool);
282 LIST_DELETE(&item->by_lru);
283 pool_put_to_shared_cache(pool, item);
284 }
285 }
286
287 /* Evicts some of the oldest objects from the local cache, pushing them to the
288 * global pool.
289 */
pool_evict_from_local_caches()290 void pool_evict_from_local_caches()
291 {
292 struct pool_cache_item *item;
293 struct pool_cache_head *ph;
294 struct pool_head *pool;
295
296 do {
297 item = LIST_PREV(&ti->pool_lru_head, struct pool_cache_item *, by_lru);
298 /* note: by definition we remove oldest objects so they also are the
299 * oldest in their own pools, thus their next is the pool's head.
300 */
301 ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list);
302 pool = container_of(ph - tid, struct pool_head, cache);
303 LIST_DELETE(&item->by_pool);
304 LIST_DELETE(&item->by_lru);
305 ph->count--;
306 pool_cache_count--;
307 pool_cache_bytes -= pool->size;
308 pool_put_to_shared_cache(pool, item);
309 } while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8);
310 }
311
312 /* Frees an object to the local cache, possibly pushing oldest objects to the
313 * shared cache, which itself may decide to release some of them to the OS.
314 * While it is unspecified what the object becomes past this point, it is
315 * guaranteed to be released from the users' perpective.
316 */
pool_put_to_cache(struct pool_head * pool,void * ptr)317 void pool_put_to_cache(struct pool_head *pool, void *ptr)
318 {
319 struct pool_cache_item *item = (struct pool_cache_item *)ptr;
320 struct pool_cache_head *ph = &pool->cache[tid];
321
322 LIST_INSERT(&ph->list, &item->by_pool);
323 LIST_INSERT(&ti->pool_lru_head, &item->by_lru);
324 ph->count++;
325 pool_cache_count++;
326 pool_cache_bytes += pool->size;
327
328 if (unlikely(pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 3 / 4)) {
329 if (ph->count >= 16 + pool_cache_count / 8)
330 pool_evict_from_local_cache(pool);
331 if (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE)
332 pool_evict_from_local_caches();
333 }
334 }
335
336 #if defined(CONFIG_HAP_NO_GLOBAL_POOLS)
337
338 /* legacy stuff */
pool_flush(struct pool_head * pool)339 void pool_flush(struct pool_head *pool)
340 {
341 }
342
343 /* This function might ask the malloc library to trim its buffers. */
pool_gc(struct pool_head * pool_ctx)344 void pool_gc(struct pool_head *pool_ctx)
345 {
346 trim_all_pools();
347 }
348
349 #else /* CONFIG_HAP_NO_GLOBAL_POOLS */
350
351 #if defined(CONFIG_HAP_LOCKLESS_POOLS)
352
353 /*
354 * This function frees whatever can be freed in pool <pool>.
355 */
pool_flush(struct pool_head * pool)356 void pool_flush(struct pool_head *pool)
357 {
358 void *next, *temp;
359
360 if (!pool)
361 return;
362
363 /* The loop below atomically detaches the head of the free list and
364 * replaces it with a NULL. Then the list can be released.
365 */
366 next = pool->free_list;
367 do {
368 while (unlikely(next == POOL_BUSY)) {
369 __ha_cpu_relax();
370 next = _HA_ATOMIC_LOAD(&pool->free_list);
371 }
372 if (next == NULL)
373 return;
374 } while (unlikely((next = _HA_ATOMIC_XCHG(&pool->free_list, POOL_BUSY)) == POOL_BUSY));
375 _HA_ATOMIC_STORE(&pool->free_list, NULL);
376 __ha_barrier_atomic_store();
377
378 while (next) {
379 temp = next;
380 next = *POOL_LINK(pool, temp);
381 pool_put_to_os(pool, temp);
382 }
383 /* here, we should have pool->allocated == pool->used */
384 }
385
386 #else /* CONFIG_HAP_LOCKLESS_POOLS */
387
388 /*
389 * This function frees whatever can be freed in pool <pool>.
390 */
pool_flush(struct pool_head * pool)391 void pool_flush(struct pool_head *pool)
392 {
393 void *temp, **next;
394
395 if (!pool)
396 return;
397
398 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
399 next = pool->free_list;
400 pool->free_list = NULL;
401 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
402
403 while (next) {
404 temp = next;
405 next = *POOL_LINK(pool, temp);
406 pool_put_to_os(pool, temp);
407 }
408 /* here, we should have pool->allocated == pool->used */
409 }
410
411 #endif /* CONFIG_HAP_LOCKLESS_POOLS */
412
413 /*
414 * This function frees whatever can be freed in all pools, but respecting
415 * the minimum thresholds imposed by owners. It makes sure to be alone to
416 * run by using thread_isolate(). <pool_ctx> is unused.
417 */
pool_gc(struct pool_head * pool_ctx)418 void pool_gc(struct pool_head *pool_ctx)
419 {
420 struct pool_head *entry;
421 int isolated = thread_isolated();
422
423 if (!isolated)
424 thread_isolate();
425
426 list_for_each_entry(entry, &pools, list) {
427 void *temp;
428 //qfprintf(stderr, "Flushing pool %s\n", entry->name);
429 while (entry->free_list &&
430 (int)(entry->allocated - entry->used) > (int)entry->minavail) {
431 temp = entry->free_list;
432 entry->free_list = *POOL_LINK(entry, temp);
433 pool_put_to_os(entry, temp);
434 }
435 }
436
437 trim_all_pools();
438
439 if (!isolated)
440 thread_release();
441 }
442 #endif /* CONFIG_HAP_NO_GLOBAL_POOLS */
443
444 #else /* CONFIG_HAP_POOLS */
445
446 /* legacy stuff */
pool_flush(struct pool_head * pool)447 void pool_flush(struct pool_head *pool)
448 {
449 }
450
451 /* This function might ask the malloc library to trim its buffers. */
pool_gc(struct pool_head * pool_ctx)452 void pool_gc(struct pool_head *pool_ctx)
453 {
454 trim_all_pools();
455 }
456
457 #endif /* CONFIG_HAP_POOLS */
458
459 /*
460 * This function destroys a pool by freeing it completely, unless it's still
461 * in use. This should be called only under extreme circumstances. It always
462 * returns NULL if the resulting pool is empty, easing the clearing of the old
463 * pointer, otherwise it returns the pool.
464 * .
465 */
pool_destroy(struct pool_head * pool)466 void *pool_destroy(struct pool_head *pool)
467 {
468 if (pool) {
469 pool_flush(pool);
470 if (pool->used)
471 return pool;
472 pool->users--;
473 if (!pool->users) {
474 LIST_DELETE(&pool->list);
475 #ifndef CONFIG_HAP_LOCKLESS_POOLS
476 HA_SPIN_DESTROY(&pool->lock);
477 #endif
478 /* note that if used == 0, the cache is empty */
479 free(pool);
480 }
481 }
482 return NULL;
483 }
484
485 /* This destroys all pools on exit. It is *not* thread safe. */
pool_destroy_all()486 void pool_destroy_all()
487 {
488 struct pool_head *entry, *back;
489
490 list_for_each_entry_safe(entry, back, &pools, list)
491 pool_destroy(entry);
492 }
493
494 /* This function dumps memory usage information into the trash buffer. */
dump_pools_to_trash()495 void dump_pools_to_trash()
496 {
497 struct pool_head *entry;
498 unsigned long allocated, used;
499 int nbpools;
500
501 allocated = used = nbpools = 0;
502 chunk_printf(&trash, "Dumping pools usage. Use SIGQUIT to flush them.\n");
503 list_for_each_entry(entry, &pools, list) {
504 #ifndef CONFIG_HAP_LOCKLESS_POOLS
505 HA_SPIN_LOCK(POOL_LOCK, &entry->lock);
506 #endif
507 chunk_appendf(&trash, " - Pool %s (%u bytes) : %u allocated (%u bytes), %u used, needed_avg %u, %u failures, %u users, @%p%s\n",
508 entry->name, entry->size, entry->allocated,
509 entry->size * entry->allocated, entry->used,
510 swrate_avg(entry->needed_avg, POOL_AVG_SAMPLES), entry->failed,
511 entry->users, entry,
512 (entry->flags & MEM_F_SHARED) ? " [SHARED]" : "");
513
514 allocated += entry->allocated * entry->size;
515 used += entry->used * entry->size;
516 nbpools++;
517 #ifndef CONFIG_HAP_LOCKLESS_POOLS
518 HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock);
519 #endif
520 }
521 chunk_appendf(&trash, "Total: %d pools, %lu bytes allocated, %lu used.\n",
522 nbpools, allocated, used);
523 }
524
525 /* Dump statistics on pools usage. */
dump_pools(void)526 void dump_pools(void)
527 {
528 dump_pools_to_trash();
529 qfprintf(stderr, "%s", trash.area);
530 }
531
532 /* This function returns the total number of failed pool allocations */
pool_total_failures()533 int pool_total_failures()
534 {
535 struct pool_head *entry;
536 int failed = 0;
537
538 list_for_each_entry(entry, &pools, list)
539 failed += entry->failed;
540 return failed;
541 }
542
543 /* This function returns the total amount of memory allocated in pools (in bytes) */
pool_total_allocated()544 unsigned long pool_total_allocated()
545 {
546 struct pool_head *entry;
547 unsigned long allocated = 0;
548
549 list_for_each_entry(entry, &pools, list)
550 allocated += entry->allocated * entry->size;
551 return allocated;
552 }
553
554 /* This function returns the total amount of memory used in pools (in bytes) */
pool_total_used()555 unsigned long pool_total_used()
556 {
557 struct pool_head *entry;
558 unsigned long used = 0;
559
560 list_for_each_entry(entry, &pools, list)
561 used += entry->used * entry->size;
562 return used;
563 }
564
565 /* This function dumps memory usage information onto the stream interface's
566 * read buffer. It returns 0 as long as it does not complete, non-zero upon
567 * completion. No state is used.
568 */
cli_io_handler_dump_pools(struct appctx * appctx)569 static int cli_io_handler_dump_pools(struct appctx *appctx)
570 {
571 struct stream_interface *si = appctx->owner;
572
573 dump_pools_to_trash();
574 if (ci_putchk(si_ic(si), &trash) == -1) {
575 si_rx_room_blk(si);
576 return 0;
577 }
578 return 1;
579 }
580
581 /* callback used to create early pool <name> of size <size> and store the
582 * resulting pointer into <ptr>. If the allocation fails, it quits with after
583 * emitting an error message.
584 */
create_pool_callback(struct pool_head ** ptr,char * name,unsigned int size)585 void create_pool_callback(struct pool_head **ptr, char *name, unsigned int size)
586 {
587 *ptr = create_pool(name, size, MEM_F_SHARED);
588 if (!*ptr) {
589 ha_alert("Failed to allocate pool '%s' of size %u : %s. Aborting.\n",
590 name, size, strerror(errno));
591 exit(1);
592 }
593 }
594
595 /* Initializes all per-thread arrays on startup */
init_pools()596 static void init_pools()
597 {
598 #ifdef CONFIG_HAP_POOLS
599 int thr;
600
601 for (thr = 0; thr < MAX_THREADS; thr++) {
602 LIST_INIT(&ha_thread_info[thr].pool_lru_head);
603 }
604 #endif
605 detect_allocator();
606 }
607
608 INITCALL0(STG_PREPARE, init_pools);
609
610 /* register cli keywords */
611 static struct cli_kw_list cli_kws = {{ },{
612 { { "show", "pools", NULL }, "show pools : report information about the memory pools usage", NULL, cli_io_handler_dump_pools },
613 {{},}
614 }};
615
616 INITCALL1(STG_REGISTER, cli_register_kw, &cli_kws);
617
618 #ifdef DEBUG_FAIL_ALLOC
619
mem_should_fail(const struct pool_head * pool)620 int mem_should_fail(const struct pool_head *pool)
621 {
622 int ret = 0;
623
624 if (mem_fail_rate > 0 && !(global.mode & MODE_STARTING)) {
625 if (mem_fail_rate > statistical_prng_range(100))
626 ret = 1;
627 else
628 ret = 0;
629 }
630 return ret;
631
632 }
633
634 /* config parser for global "tune.fail-alloc" */
mem_parse_global_fail_alloc(char ** args,int section_type,struct proxy * curpx,const struct proxy * defpx,const char * file,int line,char ** err)635 static int mem_parse_global_fail_alloc(char **args, int section_type, struct proxy *curpx,
636 const struct proxy *defpx, const char *file, int line,
637 char **err)
638 {
639 if (too_many_args(1, args, err, NULL))
640 return -1;
641 mem_fail_rate = atoi(args[1]);
642 if (mem_fail_rate < 0 || mem_fail_rate > 100) {
643 memprintf(err, "'%s' expects a numeric value between 0 and 100.", args[0]);
644 return -1;
645 }
646 return 0;
647 }
648 #endif
649
650 /* register global config keywords */
651 static struct cfg_kw_list mem_cfg_kws = {ILH, {
652 #ifdef DEBUG_FAIL_ALLOC
653 { CFG_GLOBAL, "tune.fail-alloc", mem_parse_global_fail_alloc },
654 #endif
655 { 0, NULL, NULL }
656 }};
657
658 INITCALL1(STG_REGISTER, cfg_register_keywords, &mem_cfg_kws);
659
660 /*
661 * Local variables:
662 * c-indent-level: 8
663 * c-basic-offset: 8
664 * End:
665 */
666