1/**
2* Copyright (C) Mellanox Technologies Ltd. 2001-2015.  ALL RIGHTS RESERVED.
3*
4* See file LICENSE for terms.
5*/
6
7#ifndef UCS_MPOOL_INL_
8#define UCS_MPOOL_INL_
9
10#include "mpool.h"
11
12#include <ucs/config/global_opts.h>
13#include <ucs/sys/checker.h>
14#include <ucs/sys/sys.h>
15
16
17static inline void *ucs_mpool_get_inline(ucs_mpool_t *mp)
18{
19    ucs_mpool_elem_t *elem;
20    void *obj;
21
22    if (ucs_unlikely(mp->freelist == NULL)) {
23        return ucs_mpool_get_grow(mp);
24    }
25
26    /* Disconnect an element from the pool */
27    elem = mp->freelist;
28    VALGRIND_MAKE_MEM_DEFINED(elem, sizeof *elem);
29    mp->freelist = elem->next;
30    elem->mpool = mp;
31    VALGRIND_MAKE_MEM_NOACCESS(elem, sizeof *elem);
32
33    obj = elem + 1;
34    VALGRIND_MEMPOOL_ALLOC(mp, obj, mp->data->elem_size - sizeof(ucs_mpool_elem_t));
35    return obj;
36}
37
38static inline void ucs_mpool_add_to_freelist(ucs_mpool_t *mp, ucs_mpool_elem_t *elem,
39                                             int add_to_tail)
40{
41    ucs_mpool_elem_t *tail;
42
43    if (add_to_tail) {
44        elem->next = NULL;
45        if (mp->freelist == NULL) {
46            mp->freelist = elem;
47        } else {
48            tail = mp->data->tail;
49            VALGRIND_MAKE_MEM_DEFINED(tail, sizeof *tail);
50            tail->next = elem;
51            VALGRIND_MAKE_MEM_NOACCESS(tail, sizeof *tail);
52        }
53        mp->data->tail = elem;
54    } else {
55        elem->next = mp->freelist;
56        mp->freelist = elem;
57    }
58}
59
60static inline ucs_mpool_elem_t *ucs_mpool_obj_to_elem(void *obj)
61{
62    ucs_mpool_elem_t *elem = (ucs_mpool_elem_t*)obj - 1;
63    VALGRIND_MAKE_MEM_DEFINED(elem, sizeof *elem);
64    return elem;
65}
66
67static inline ucs_mpool_t *ucs_mpool_obj_owner(void *obj)
68{
69    return ucs_mpool_obj_to_elem(obj)->mpool;
70}
71
72static inline void ucs_mpool_put_inline(void *obj)
73{
74    ucs_mpool_elem_t *elem;
75    ucs_mpool_t *mp;
76
77    elem = ucs_mpool_obj_to_elem(obj);
78    mp   = elem->mpool;
79    ucs_mpool_add_to_freelist(mp, elem,
80                              ENABLE_DEBUG_DATA && ucs_global_opts.mpool_fifo);
81    VALGRIND_MAKE_MEM_NOACCESS(elem, sizeof *elem);
82    VALGRIND_MEMPOOL_FREE(mp, obj);
83}
84
85#endif
86