1 /** 2 * Copyright (C) Mellanox Technologies Ltd. 2001-2018. ALL RIGHTS RESERVED. 3 * 4 * See file LICENSE for terms. 5 */ 6 7 #ifndef UCS_REG_CACHE_H_ 8 #define UCS_REG_CACHE_H_ 9 10 /* 11 * Memory registration cache - holds registered memory regions, takes care of 12 * memory invalidation (if it's unmapped), merging of regions, protection flags. 13 * This data structure is thread safe. 14 */ 15 #include <ucs/datastruct/pgtable.h> 16 #include <ucs/datastruct/list.h> 17 #include <ucs/datastruct/queue_types.h> 18 #include <ucs/datastruct/mpool.h> 19 #include <ucs/stats/stats_fwd.h> 20 #include <sys/mman.h> 21 22 23 #define UCS_RCACHE_PROT_FMT "%c%c" 24 #define UCS_RCACHE_PROT_ARG(_prot) \ 25 ((_prot) & PROT_READ) ? 'r' : '-', \ 26 ((_prot) & PROT_WRITE) ? 'w' : '-' 27 28 29 typedef struct ucs_rcache ucs_rcache_t; 30 typedef struct ucs_rcache_ops ucs_rcache_ops_t; 31 typedef struct ucs_rcache_params ucs_rcache_params_t; 32 typedef struct ucs_rcache_region ucs_rcache_region_t; 33 34 /* 35 * Memory region flags. 36 */ 37 enum { 38 UCS_RCACHE_REGION_FLAG_REGISTERED = UCS_BIT(0), /**< Memory registered */ 39 UCS_RCACHE_REGION_FLAG_PGTABLE = UCS_BIT(1) /**< In the page table */ 40 }; 41 42 /* 43 * Memory registration flags. 44 */ 45 enum { 46 UCS_RCACHE_MEM_REG_HIDE_ERRORS = UCS_BIT(0) /**< Hide errors on memory registration */ 47 }; 48 49 /* 50 * Rcache flags. 51 */ 52 enum { 53 UCS_RCACHE_FLAG_NO_PFN_CHECK = UCS_BIT(0), /**< PFN check not supported for this rcache */ 54 }; 55 56 /* 57 * Registration cache operations. 58 */ 59 struct ucs_rcache_ops { 60 /** 61 * Register a memory region. 62 * 63 * @param [in] context User context, as passed to @ref ucs_rcache_create 64 * @param [in] rcache Pointer to the registration cache. 65 * @param [in] arg Custom argument passed to @ref ucs_rcache_get(). 66 * @param [in] region Memory region to register. This may point to a larger 67 * user-defined structure, as specified by the field 68 * `region_struct_size' in @ref ucs_rcache_params. 69 * This function may store relevant information (such 70 * as memory keys) inside the larger structure. 71 * @param [in] flags Memory registration flags. 72 * 73 * @return UCS_OK if registration is successful, error otherwise. 74 * 75 * @note This function should be able to handle inaccessible memory addresses 76 * and return error status in this case, without any destructive consequences 77 * such as error messages or fatal failure. 78 */ 79 ucs_status_t (*mem_reg)(void *context, ucs_rcache_t *rcache, 80 void *arg, ucs_rcache_region_t *region, 81 uint16_t flags); 82 /** 83 * Deregister a memory region. 84 * 85 * @param [in] context User context, as passed to @ref ucs_rcache_create 86 * @param [in] rcache Pointer to the registration cache. 87 * @param [in] region Memory region to deregister. 88 */ 89 void (*mem_dereg)(void *context, ucs_rcache_t *rcache, 90 ucs_rcache_region_t *region); 91 92 /** 93 * Dump memory region information to a string buffer. 94 * (Only the user-defined part of the memory regoin should be dumped) 95 * 96 * @param [in] context User context, as passed to @ref ucs_rcache_create 97 * @param [in] rcache Pointer to the registration cache. 98 * @param [in] region Memory region to dump. 99 * @param [in] buf String buffer to dump to. 100 * @param [in] max Maximal length of the string buffer. 101 */ 102 void (*dump_region)(void *context, ucs_rcache_t *rcache, 103 ucs_rcache_region_t *region, 104 char *buf, size_t max); 105 }; 106 107 108 struct ucs_rcache_params { 109 size_t region_struct_size; /**< Size of memory region structure, 110 must be at least the size 111 of @ref ucs_rcache_region_t */ 112 size_t alignment; /**< Force-align regions to this size. 113 Must be smaller or equal to 114 system page size. */ 115 size_t max_alignment; /**< Maximum alignment */ 116 int ucm_events; /**< UCM events to register. Currently 117 UCM_EVENT_VM_UNMAPPED and 118 UCM_EVENT_MEM_TYPE_FREE are supported */ 119 int ucm_event_priority; /**< Priority of memory events */ 120 const ucs_rcache_ops_t *ops; /**< Memory operations functions */ 121 void *context; /**< User-defined context that will 122 be passed to mem_reg/mem_dereg */ 123 int flags; /**< Flags */ 124 }; 125 126 127 struct ucs_rcache_region { 128 ucs_pgt_region_t super; /**< Base class - page table region */ 129 ucs_list_link_t list; /**< List element */ 130 volatile uint32_t refcount; /**< Reference count, including +1 if it's 131 in the page table */ 132 ucs_status_t status; /**< Current status code */ 133 uint8_t prot; /**< Protection bits */ 134 uint16_t flags; /**< Status flags. Protected by page table lock. */ 135 union { 136 uint64_t priv; /**< Used internally */ 137 unsigned long *pfn; /**< Pointer to PFN array. In case if requested 138 evaluation more than 1 page - PFN array is 139 allocated, if 1 page requested - used 140 in-place priv value. */ 141 }; 142 }; 143 144 145 /** 146 * Create a memory registration cache. 147 * 148 * @param [in] params Registration cache parameters. 149 * @param [in] name Registration cache name, for debugging. 150 * @param [in] stats_parent Pointer to statistics parent node. 151 * @param [out] rcache_p Filled with a pointer to the registration cache. 152 */ 153 ucs_status_t ucs_rcache_create(const ucs_rcache_params_t *params, const char *name, 154 ucs_stats_node_t *stats_parent, ucs_rcache_t **rcache_p); 155 156 157 /** 158 * Destroy a memory registration cache. 159 * 160 * @param [in] rcache Registration cache to destroy. 161 */ 162 void ucs_rcache_destroy(ucs_rcache_t *rcache); 163 164 165 /** 166 * Resolve buffer in the registration cache, or register it if not found. 167 * TODO register after N usages. 168 * 169 * @param [in] rcache Memory registration cache. 170 * @param [in] address Address to register or resolve. 171 * @param [in] length Length of buffer to register or resolve. 172 * @param [in] prot Requested access flags, PROT_xx (same as passed to mmap). 173 * @param [in] arg Custom argument passed down to memory registration 174 * callback, if a memory registration happens during 175 * this call. 176 * @param [out] region_p On success, filled with a pointer to the memory 177 * region. The user could put more data in the region 178 * structure in mem_reg() function. 179 * 180 * On success succeeds, the memory region reference count is incremented by 1. 181 * 182 * @return Error code. 183 */ 184 ucs_status_t ucs_rcache_get(ucs_rcache_t *rcache, void *address, size_t length, 185 int prot, void *arg, ucs_rcache_region_t **region_p); 186 187 188 /** 189 * Increment memory region reference count. 190 * 191 * @param [in] rcache Memory registration cache. 192 * @param [in] region Memory region whose reference count to increment. 193 */ 194 void ucs_rcache_region_hold(ucs_rcache_t *rcache, ucs_rcache_region_t *region); 195 196 197 /** 198 * Decrement memory region reference count and possibly destroy it. 199 * 200 * @param [in] rcache Memory registration cache. 201 * @param [in] region Memory region to release. 202 */ 203 void ucs_rcache_region_put(ucs_rcache_t *rcache, ucs_rcache_region_t *region); 204 205 206 #endif 207