1 /******************************************************************************/ 2 #ifdef JEMALLOC_H_TYPES 3 4 #ifdef _WIN32 5 # ifdef _WIN64 6 # define FMT64_PREFIX "ll" 7 # define FMTPTR_PREFIX "ll" 8 # else 9 # define FMT64_PREFIX "ll" 10 # define FMTPTR_PREFIX "" 11 # endif 12 # define FMTd32 "d" 13 # define FMTu32 "u" 14 # define FMTx32 "x" 15 # define FMTd64 FMT64_PREFIX "d" 16 # define FMTu64 FMT64_PREFIX "u" 17 # define FMTx64 FMT64_PREFIX "x" 18 # define FMTdPTR FMTPTR_PREFIX "d" 19 # define FMTuPTR FMTPTR_PREFIX "u" 20 # define FMTxPTR FMTPTR_PREFIX "x" 21 #else 22 # include <inttypes.h> 23 # define FMTd32 PRId32 24 # define FMTu32 PRIu32 25 # define FMTx32 PRIx32 26 # define FMTd64 PRId64 27 # define FMTu64 PRIu64 28 # define FMTx64 PRIx64 29 # define FMTdPTR PRIdPTR 30 # define FMTuPTR PRIuPTR 31 # define FMTxPTR PRIxPTR 32 #endif 33 34 /* Size of stack-allocated buffer passed to buferror(). */ 35 #define BUFERROR_BUF 64 36 37 /* 38 * Size of stack-allocated buffer used by malloc_{,v,vc}printf(). This must be 39 * large enough for all possible uses within jemalloc. 40 */ 41 #define MALLOC_PRINTF_BUFSIZE 4096 42 43 /* 44 * Wrap a cpp argument that contains commas such that it isn't broken up into 45 * multiple arguments. 46 */ 47 #define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__ 48 49 /* 50 * Silence compiler warnings due to uninitialized values. This is used 51 * wherever the compiler fails to recognize that the variable is never used 52 * uninitialized. 53 */ 54 #ifdef JEMALLOC_CC_SILENCE 55 # define JEMALLOC_CC_SILENCE_INIT(v) = v 56 #else 57 # define JEMALLOC_CC_SILENCE_INIT(v) 58 #endif 59 60 #define JEMALLOC_GNUC_PREREQ(major, minor) \ 61 (!defined(__clang__) && \ 62 (__GNUC__ > (major) || (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))) 63 #ifndef __has_builtin 64 # define __has_builtin(builtin) (0) 65 #endif 66 #define JEMALLOC_CLANG_HAS_BUILTIN(builtin) \ 67 (defined(__clang__) && __has_builtin(builtin)) 68 69 #ifdef __GNUC__ 70 # define likely(x) __builtin_expect(!!(x), 1) 71 # define unlikely(x) __builtin_expect(!!(x), 0) 72 # if JEMALLOC_GNUC_PREREQ(4, 6) || \ 73 JEMALLOC_CLANG_HAS_BUILTIN(__builtin_unreachable) 74 # define unreachable() __builtin_unreachable() 75 # else 76 # define unreachable() 77 # endif 78 #else 79 # define likely(x) !!(x) 80 # define unlikely(x) !!(x) 81 # define unreachable() 82 #endif 83 84 #include "jemalloc/internal/assert.h" 85 86 /* Use to assert a particular configuration, e.g., cassert(config_debug). */ 87 #define cassert(c) do { \ 88 if (unlikely(!(c))) \ 89 not_reached(); \ 90 } while (0) 91 92 #endif /* JEMALLOC_H_TYPES */ 93 /******************************************************************************/ 94 #ifdef JEMALLOC_H_STRUCTS 95 96 #endif /* JEMALLOC_H_STRUCTS */ 97 /******************************************************************************/ 98 #ifdef JEMALLOC_H_EXTERNS 99 100 int buferror(int err, char *buf, size_t buflen); 101 uintmax_t malloc_strtoumax(const char *restrict nptr, 102 char **restrict endptr, int base); 103 void malloc_write(const char *s); 104 105 /* 106 * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating 107 * point math. 108 */ 109 int malloc_vsnprintf(char *str, size_t size, const char *format, 110 va_list ap); 111 int malloc_snprintf(char *str, size_t size, const char *format, ...) 112 JEMALLOC_FORMAT_PRINTF(3, 4); 113 void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, 114 const char *format, va_list ap); 115 void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque, 116 const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4); 117 void malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2); 118 119 #endif /* JEMALLOC_H_EXTERNS */ 120 /******************************************************************************/ 121 #ifdef JEMALLOC_H_INLINES 122 123 #ifndef JEMALLOC_ENABLE_INLINE 124 unsigned ffs_llu(unsigned long long bitmap); 125 unsigned ffs_lu(unsigned long bitmap); 126 unsigned ffs_u(unsigned bitmap); 127 unsigned ffs_zu(size_t bitmap); 128 unsigned ffs_u64(uint64_t bitmap); 129 unsigned ffs_u32(uint32_t bitmap); 130 uint64_t pow2_ceil_u64(uint64_t x); 131 uint32_t pow2_ceil_u32(uint32_t x); 132 size_t pow2_ceil_zu(size_t x); 133 unsigned lg_floor(size_t x); 134 void set_errno(int errnum); 135 int get_errno(void); 136 #endif 137 138 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_)) 139 140 /* Sanity check. */ 141 #if !defined(JEMALLOC_INTERNAL_FFSLL) || !defined(JEMALLOC_INTERNAL_FFSL) \ 142 || !defined(JEMALLOC_INTERNAL_FFS) 143 # error JEMALLOC_INTERNAL_FFS{,L,LL} should have been defined by configure 144 #endif 145 146 JEMALLOC_ALWAYS_INLINE unsigned 147 ffs_llu(unsigned long long bitmap) 148 { 149 150 return (JEMALLOC_INTERNAL_FFSLL(bitmap)); 151 } 152 153 JEMALLOC_ALWAYS_INLINE unsigned 154 ffs_lu(unsigned long bitmap) 155 { 156 157 return (JEMALLOC_INTERNAL_FFSL(bitmap)); 158 } 159 160 JEMALLOC_ALWAYS_INLINE unsigned 161 ffs_u(unsigned bitmap) 162 { 163 164 return (JEMALLOC_INTERNAL_FFS(bitmap)); 165 } 166 167 JEMALLOC_ALWAYS_INLINE unsigned 168 ffs_zu(size_t bitmap) 169 { 170 171 #if LG_SIZEOF_PTR == LG_SIZEOF_INT 172 return (ffs_u(bitmap)); 173 #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG 174 return (ffs_lu(bitmap)); 175 #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG 176 return (ffs_llu(bitmap)); 177 #else 178 #error No implementation for size_t ffs() 179 #endif 180 } 181 182 JEMALLOC_ALWAYS_INLINE unsigned 183 ffs_u64(uint64_t bitmap) 184 { 185 186 #if LG_SIZEOF_LONG == 3 187 return (ffs_lu(bitmap)); 188 #elif LG_SIZEOF_LONG_LONG == 3 189 return (ffs_llu(bitmap)); 190 #else 191 #error No implementation for 64-bit ffs() 192 #endif 193 } 194 195 JEMALLOC_ALWAYS_INLINE unsigned 196 ffs_u32(uint32_t bitmap) 197 { 198 199 #if LG_SIZEOF_INT == 2 200 return (ffs_u(bitmap)); 201 #else 202 #error No implementation for 32-bit ffs() 203 #endif 204 return (ffs_u(bitmap)); 205 } 206 207 JEMALLOC_INLINE uint64_t 208 pow2_ceil_u64(uint64_t x) 209 { 210 211 x--; 212 x |= x >> 1; 213 x |= x >> 2; 214 x |= x >> 4; 215 x |= x >> 8; 216 x |= x >> 16; 217 x |= x >> 32; 218 x++; 219 return (x); 220 } 221 222 JEMALLOC_INLINE uint32_t 223 pow2_ceil_u32(uint32_t x) 224 { 225 226 x--; 227 x |= x >> 1; 228 x |= x >> 2; 229 x |= x >> 4; 230 x |= x >> 8; 231 x |= x >> 16; 232 x++; 233 return (x); 234 } 235 236 /* Compute the smallest power of 2 that is >= x. */ 237 JEMALLOC_INLINE size_t 238 pow2_ceil_zu(size_t x) 239 { 240 241 #if (LG_SIZEOF_PTR == 3) 242 return (pow2_ceil_u64(x)); 243 #else 244 return (pow2_ceil_u32(x)); 245 #endif 246 } 247 248 #if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) 249 JEMALLOC_INLINE unsigned 250 lg_floor(size_t x) 251 { 252 size_t ret; 253 254 assert(x != 0); 255 256 asm ("bsr %1, %0" 257 : "=r"(ret) // Outputs. 258 : "r"(x) // Inputs. 259 ); 260 assert(ret < UINT_MAX); 261 return ((unsigned)ret); 262 } 263 #elif (defined(_MSC_VER)) 264 JEMALLOC_INLINE unsigned 265 lg_floor(size_t x) 266 { 267 unsigned long ret; 268 269 assert(x != 0); 270 271 #if (LG_SIZEOF_PTR == 3) 272 _BitScanReverse64(&ret, x); 273 #elif (LG_SIZEOF_PTR == 2) 274 _BitScanReverse(&ret, x); 275 #else 276 # error "Unsupported type size for lg_floor()" 277 #endif 278 assert(ret < UINT_MAX); 279 return ((unsigned)ret); 280 } 281 #elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) 282 JEMALLOC_INLINE unsigned 283 lg_floor(size_t x) 284 { 285 286 assert(x != 0); 287 288 #if (LG_SIZEOF_PTR == LG_SIZEOF_INT) 289 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x)); 290 #elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG) 291 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clzl(x)); 292 #else 293 # error "Unsupported type size for lg_floor()" 294 #endif 295 } 296 #else 297 JEMALLOC_INLINE unsigned 298 lg_floor(size_t x) 299 { 300 301 assert(x != 0); 302 303 x |= (x >> 1); 304 x |= (x >> 2); 305 x |= (x >> 4); 306 x |= (x >> 8); 307 x |= (x >> 16); 308 #if (LG_SIZEOF_PTR == 3) 309 x |= (x >> 32); 310 #endif 311 if (x == SIZE_T_MAX) 312 return ((8 << LG_SIZEOF_PTR) - 1); 313 x++; 314 return (ffs_zu(x) - 2); 315 } 316 #endif 317 318 /* Set error code. */ 319 JEMALLOC_INLINE void 320 set_errno(int errnum) 321 { 322 323 #ifdef _WIN32 324 SetLastError(errnum); 325 #else 326 errno = errnum; 327 #endif 328 } 329 330 /* Get last error code. */ 331 JEMALLOC_INLINE int 332 get_errno(void) 333 { 334 335 #ifdef _WIN32 336 return (GetLastError()); 337 #else 338 return (errno); 339 #endif 340 } 341 #endif 342 343 #endif /* JEMALLOC_H_INLINES */ 344 /******************************************************************************/ 345