1 /******************************************************************************/ 2 #ifdef JEMALLOC_H_TYPES 3 4 #ifdef _WIN32 5 # ifdef _WIN64 6 # define FMT64_PREFIX "ll" 7 # define FMTPTR_PREFIX "ll" 8 # else 9 # define FMT64_PREFIX "ll" 10 # define FMTPTR_PREFIX "" 11 # endif 12 # define FMTd32 "d" 13 # define FMTu32 "u" 14 # define FMTx32 "x" 15 # define FMTd64 FMT64_PREFIX "d" 16 # define FMTu64 FMT64_PREFIX "u" 17 # define FMTx64 FMT64_PREFIX "x" 18 # define FMTdPTR FMTPTR_PREFIX "d" 19 # define FMTuPTR FMTPTR_PREFIX "u" 20 # define FMTxPTR FMTPTR_PREFIX "x" 21 #else 22 # include <inttypes.h> 23 # define FMTd32 PRId32 24 # define FMTu32 PRIu32 25 # define FMTx32 PRIx32 26 # define FMTd64 PRId64 27 # define FMTu64 PRIu64 28 # define FMTx64 PRIx64 29 # define FMTdPTR PRIdPTR 30 # define FMTuPTR PRIuPTR 31 # define FMTxPTR PRIxPTR 32 #endif 33 34 /* Size of stack-allocated buffer passed to buferror(). */ 35 #define BUFERROR_BUF 64 36 37 /* 38 * Size of stack-allocated buffer used by malloc_{,v,vc}printf(). This must be 39 * large enough for all possible uses within jemalloc. 40 */ 41 #define MALLOC_PRINTF_BUFSIZE 4096 42 43 /* Junk fill patterns. */ 44 #ifndef JEMALLOC_ALLOC_JUNK 45 # define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5) 46 #endif 47 #ifndef JEMALLOC_FREE_JUNK 48 # define JEMALLOC_FREE_JUNK ((uint8_t)0x5a) 49 #endif 50 51 /* 52 * Wrap a cpp argument that contains commas such that it isn't broken up into 53 * multiple arguments. 54 */ 55 #define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__ 56 57 /* 58 * Silence compiler warnings due to uninitialized values. This is used 59 * wherever the compiler fails to recognize that the variable is never used 60 * uninitialized. 61 */ 62 #ifdef JEMALLOC_CC_SILENCE 63 # define JEMALLOC_CC_SILENCE_INIT(v) = v 64 #else 65 # define JEMALLOC_CC_SILENCE_INIT(v) 66 #endif 67 68 #ifdef __GNUC__ 69 # define likely(x) __builtin_expect(!!(x), 1) 70 # define unlikely(x) __builtin_expect(!!(x), 0) 71 #else 72 # define likely(x) !!(x) 73 # define unlikely(x) !!(x) 74 #endif 75 76 #if !defined(JEMALLOC_INTERNAL_UNREACHABLE) 77 # error JEMALLOC_INTERNAL_UNREACHABLE should have been defined by configure 78 #endif 79 80 #define unreachable() JEMALLOC_INTERNAL_UNREACHABLE() 81 82 #include "jemalloc/internal/assert.h" 83 84 /* Use to assert a particular configuration, e.g., cassert(config_debug). */ 85 #define cassert(c) do { \ 86 if (unlikely(!(c))) \ 87 not_reached(); \ 88 } while (0) 89 90 #endif /* JEMALLOC_H_TYPES */ 91 /******************************************************************************/ 92 #ifdef JEMALLOC_H_STRUCTS 93 94 #endif /* JEMALLOC_H_STRUCTS */ 95 /******************************************************************************/ 96 #ifdef JEMALLOC_H_EXTERNS 97 98 int buferror(int err, char *buf, size_t buflen); 99 uintmax_t malloc_strtoumax(const char *restrict nptr, 100 char **restrict endptr, int base); 101 void malloc_write(const char *s); 102 103 /* 104 * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating 105 * point math. 106 */ 107 size_t malloc_vsnprintf(char *str, size_t size, const char *format, 108 va_list ap); 109 size_t malloc_snprintf(char *str, size_t size, const char *format, ...) 110 JEMALLOC_FORMAT_PRINTF(3, 4); 111 void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, 112 const char *format, va_list ap); 113 void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque, 114 const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4); 115 void malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2); 116 117 #endif /* JEMALLOC_H_EXTERNS */ 118 /******************************************************************************/ 119 #ifdef JEMALLOC_H_INLINES 120 121 #ifndef JEMALLOC_ENABLE_INLINE 122 unsigned ffs_llu(unsigned long long bitmap); 123 unsigned ffs_lu(unsigned long bitmap); 124 unsigned ffs_u(unsigned bitmap); 125 unsigned ffs_zu(size_t bitmap); 126 unsigned ffs_u64(uint64_t bitmap); 127 unsigned ffs_u32(uint32_t bitmap); 128 uint64_t pow2_ceil_u64(uint64_t x); 129 uint32_t pow2_ceil_u32(uint32_t x); 130 size_t pow2_ceil_zu(size_t x); 131 unsigned lg_floor(size_t x); 132 void set_errno(int errnum); 133 int get_errno(void); 134 #endif 135 136 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_)) 137 138 /* Sanity check. */ 139 #if !defined(JEMALLOC_INTERNAL_FFSLL) || !defined(JEMALLOC_INTERNAL_FFSL) \ 140 || !defined(JEMALLOC_INTERNAL_FFS) 141 # error JEMALLOC_INTERNAL_FFS{,L,LL} should have been defined by configure 142 #endif 143 144 JEMALLOC_ALWAYS_INLINE unsigned 145 ffs_llu(unsigned long long bitmap) 146 { 147 148 return (JEMALLOC_INTERNAL_FFSLL(bitmap)); 149 } 150 151 JEMALLOC_ALWAYS_INLINE unsigned 152 ffs_lu(unsigned long bitmap) 153 { 154 155 return (JEMALLOC_INTERNAL_FFSL(bitmap)); 156 } 157 158 JEMALLOC_ALWAYS_INLINE unsigned 159 ffs_u(unsigned bitmap) 160 { 161 162 return (JEMALLOC_INTERNAL_FFS(bitmap)); 163 } 164 165 JEMALLOC_ALWAYS_INLINE unsigned 166 ffs_zu(size_t bitmap) 167 { 168 169 #if LG_SIZEOF_PTR == LG_SIZEOF_INT 170 return (ffs_u(bitmap)); 171 #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG 172 return (ffs_lu(bitmap)); 173 #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG 174 return (ffs_llu(bitmap)); 175 #else 176 #error No implementation for size_t ffs() 177 #endif 178 } 179 180 JEMALLOC_ALWAYS_INLINE unsigned 181 ffs_u64(uint64_t bitmap) 182 { 183 184 #if LG_SIZEOF_LONG == 3 185 return (ffs_lu(bitmap)); 186 #elif LG_SIZEOF_LONG_LONG == 3 187 return (ffs_llu(bitmap)); 188 #else 189 #error No implementation for 64-bit ffs() 190 #endif 191 } 192 193 JEMALLOC_ALWAYS_INLINE unsigned 194 ffs_u32(uint32_t bitmap) 195 { 196 197 #if LG_SIZEOF_INT == 2 198 return (ffs_u(bitmap)); 199 #else 200 #error No implementation for 32-bit ffs() 201 #endif 202 return (ffs_u(bitmap)); 203 } 204 205 JEMALLOC_INLINE uint64_t 206 pow2_ceil_u64(uint64_t x) 207 { 208 209 x--; 210 x |= x >> 1; 211 x |= x >> 2; 212 x |= x >> 4; 213 x |= x >> 8; 214 x |= x >> 16; 215 x |= x >> 32; 216 x++; 217 return (x); 218 } 219 220 JEMALLOC_INLINE uint32_t 221 pow2_ceil_u32(uint32_t x) 222 { 223 224 x--; 225 x |= x >> 1; 226 x |= x >> 2; 227 x |= x >> 4; 228 x |= x >> 8; 229 x |= x >> 16; 230 x++; 231 return (x); 232 } 233 234 /* Compute the smallest power of 2 that is >= x. */ 235 JEMALLOC_INLINE size_t 236 pow2_ceil_zu(size_t x) 237 { 238 239 #if (LG_SIZEOF_PTR == 3) 240 return (pow2_ceil_u64(x)); 241 #else 242 return (pow2_ceil_u32(x)); 243 #endif 244 } 245 246 #if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) 247 JEMALLOC_INLINE unsigned 248 lg_floor(size_t x) 249 { 250 size_t ret; 251 252 assert(x != 0); 253 254 asm ("bsr %1, %0" 255 : "=r"(ret) // Outputs. 256 : "r"(x) // Inputs. 257 ); 258 assert(ret < UINT_MAX); 259 return ((unsigned)ret); 260 } 261 #elif (defined(_MSC_VER)) 262 JEMALLOC_INLINE unsigned 263 lg_floor(size_t x) 264 { 265 unsigned long ret; 266 267 assert(x != 0); 268 269 #if (LG_SIZEOF_PTR == 3) 270 _BitScanReverse64(&ret, x); 271 #elif (LG_SIZEOF_PTR == 2) 272 _BitScanReverse(&ret, x); 273 #else 274 # error "Unsupported type size for lg_floor()" 275 #endif 276 assert(ret < UINT_MAX); 277 return ((unsigned)ret); 278 } 279 #elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) 280 JEMALLOC_INLINE unsigned 281 lg_floor(size_t x) 282 { 283 284 assert(x != 0); 285 286 #if (LG_SIZEOF_PTR == LG_SIZEOF_INT) 287 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x)); 288 #elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG) 289 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clzl(x)); 290 #else 291 # error "Unsupported type size for lg_floor()" 292 #endif 293 } 294 #else 295 JEMALLOC_INLINE unsigned 296 lg_floor(size_t x) 297 { 298 299 assert(x != 0); 300 301 x |= (x >> 1); 302 x |= (x >> 2); 303 x |= (x >> 4); 304 x |= (x >> 8); 305 x |= (x >> 16); 306 #if (LG_SIZEOF_PTR == 3) 307 x |= (x >> 32); 308 #endif 309 if (x == SIZE_T_MAX) 310 return ((8 << LG_SIZEOF_PTR) - 1); 311 x++; 312 return (ffs_zu(x) - 2); 313 } 314 #endif 315 316 /* Set error code. */ 317 JEMALLOC_INLINE void 318 set_errno(int errnum) 319 { 320 321 #ifdef _WIN32 322 SetLastError(errnum); 323 #else 324 errno = errnum; 325 #endif 326 } 327 328 /* Get last error code. */ 329 JEMALLOC_INLINE int 330 get_errno(void) 331 { 332 333 #ifdef _WIN32 334 return (GetLastError()); 335 #else 336 return (errno); 337 #endif 338 } 339 #endif 340 341 #endif /* JEMALLOC_H_INLINES */ 342 /******************************************************************************/ 343