1/** 2* Copyright (C) Mellanox Technologies Ltd. 2001-2015. ALL RIGHTS RESERVED. 3* 4* See file LICENSE for terms. 5*/ 6 7#ifndef UCS_MPOOL_INL_ 8#define UCS_MPOOL_INL_ 9 10#include "mpool.h" 11 12#include <ucs/config/global_opts.h> 13#include <ucs/sys/checker.h> 14#include <ucs/sys/sys.h> 15 16 17static inline void *ucs_mpool_get_inline(ucs_mpool_t *mp) 18{ 19 ucs_mpool_elem_t *elem; 20 void *obj; 21 22 if (ucs_unlikely(mp->freelist == NULL)) { 23 return ucs_mpool_get_grow(mp); 24 } 25 26 /* Disconnect an element from the pool */ 27 elem = mp->freelist; 28 VALGRIND_MAKE_MEM_DEFINED(elem, sizeof *elem); 29 mp->freelist = elem->next; 30 elem->mpool = mp; 31 VALGRIND_MAKE_MEM_NOACCESS(elem, sizeof *elem); 32 33 obj = elem + 1; 34 VALGRIND_MEMPOOL_ALLOC(mp, obj, mp->data->elem_size - sizeof(ucs_mpool_elem_t)); 35 return obj; 36} 37 38static inline void ucs_mpool_add_to_freelist(ucs_mpool_t *mp, ucs_mpool_elem_t *elem, 39 int add_to_tail) 40{ 41 ucs_mpool_elem_t *tail; 42 43 if (add_to_tail) { 44 elem->next = NULL; 45 if (mp->freelist == NULL) { 46 mp->freelist = elem; 47 } else { 48 tail = mp->data->tail; 49 VALGRIND_MAKE_MEM_DEFINED(tail, sizeof *tail); 50 tail->next = elem; 51 VALGRIND_MAKE_MEM_NOACCESS(tail, sizeof *tail); 52 } 53 mp->data->tail = elem; 54 } else { 55 elem->next = mp->freelist; 56 mp->freelist = elem; 57 } 58} 59 60static inline ucs_mpool_elem_t *ucs_mpool_obj_to_elem(void *obj) 61{ 62 ucs_mpool_elem_t *elem = (ucs_mpool_elem_t*)obj - 1; 63 VALGRIND_MAKE_MEM_DEFINED(elem, sizeof *elem); 64 return elem; 65} 66 67static inline ucs_mpool_t *ucs_mpool_obj_owner(void *obj) 68{ 69 return ucs_mpool_obj_to_elem(obj)->mpool; 70} 71 72static inline void ucs_mpool_put_inline(void *obj) 73{ 74 ucs_mpool_elem_t *elem; 75 ucs_mpool_t *mp; 76 77 elem = ucs_mpool_obj_to_elem(obj); 78 mp = elem->mpool; 79 ucs_mpool_add_to_freelist(mp, elem, 80 ENABLE_DEBUG_DATA && ucs_global_opts.mpool_fifo); 81 VALGRIND_MAKE_MEM_NOACCESS(elem, sizeof *elem); 82 VALGRIND_MEMPOOL_FREE(mp, obj); 83} 84 85#endif 86