1 /* 2 Copyright(c) 2002-2017 Anatoliy Kuznetsov(anatoliy_kuznetsov at yahoo.com) 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 16 For more information please visit: http://bitmagic.io 17 */ 18 19 /*! \file bmdef.h 20 \brief Definitions(internal) 21 */ 22 23 #include <climits> 24 #include <stdint.h> 25 26 // Incorporate appropriate tuneups when the NCBI C++ Toolkit's core 27 // headers have been included. 28 // 29 #ifdef NCBI_ASSERT 30 # define BM_ASSERT _ASSERT 31 32 # ifdef HAVE_RESTRICT_CXX 33 # define BM_HASRESTRICT 34 # define BMRESTRICT NCBI_RESTRICT 35 # endif 36 37 # if defined(NCBI_FORCEINLINE) && \ 38 ( !defined(NCBI_COMPILER_GCC) || NCBI_COMPILER_VERSION >= 400 || \ 39 defined(__OPTIMIZE__)) 40 # define BM_HASFORCEINLINE 41 # define BMFORCEINLINE NCBI_FORCEINLINE 42 # endif 43 44 # ifdef NCBI_SSE 45 # if NCBI_SSE >= 20 46 # define BMSSE2OPT 1 47 # endif 48 # if NCBI_SSE >= 40 49 # define BMSSE2OPT 1 50 # endif 51 # if NCBI_SSE >= 42 52 # define BMSSE42OPT 1 53 # endif 54 # endif 55 #endif 56 57 58 // macro to define/undefine unaligned memory access (x86, PowerPC) 59 // 60 #if defined(__i386) || defined(__x86_64) || defined(__ppc__) || \ 61 defined(__ppc64__) || defined(_M_IX86) || defined(_M_AMD64) || \ 62 defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64) || \ 63 defined(_M_ARM) || defined(_M_ARM64) || \ 64 defined(__arm__) || defined(__aarch64__) || \ 65 (defined(_M_MPPC) && !defined(BM_FORBID_UNALIGNED_ACCESS)) 66 #define BM_UNALIGNED_ACCESS_OK 1 67 #endif 68 69 #if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64) || \ 70 defined(__i386) || defined(__x86_64) || defined(_M_AMD64) || \ 71 defined(BMSSE2OPT) || defined(BMSSE42OPT) 72 #define BM_x86 73 #endif 74 75 // cxx11 features 76 // 77 #if defined(BM_NO_CXX11) || (defined(_MSC_VER) && _MSC_VER < 1900) 78 # define BMNOEXCEPT 79 # define BMNOEXCEPT2 80 #else 81 # ifndef BMNOEXCEPT 82 # define BMNOEXCEPT noexcept 83 #if defined(__EMSCRIPTEN__) 84 #else 85 # define BMNOEXCEPT2 86 #endif 87 # endif 88 #endif 89 90 // WebASM compilation settings 91 // 92 #if defined(__EMSCRIPTEN__) 93 94 // EMSCRIPTEN specific tweaks 95 // WebAssemply compiles into 32-bit memory system but offers 64-bit wordsize 96 // WebASM also benefits from use GCC extensions (buildins like popcnt, lzcnt) 97 // 98 // BMNOEXCEPT2 is to declare "noexcept" for WebAsm only where needed 99 // and silence GCC warnings 100 // 101 # define BM64OPT 102 # define BM_USE_GCC_BUILD 103 # define BMNOEXCEPT2 noexcept 104 105 #else 106 # define BMNOEXCEPT2 107 #endif 108 109 110 // Enable MSVC 8.0 (2005) specific optimization options 111 // 112 #if(_MSC_VER >= 1400) 113 # define BM_HASFORCEINLINE 114 # ifndef BMRESTRICT 115 # define BMRESTRICT __restrict 116 # endif 117 #endif 118 119 #ifdef __GNUG__ 120 # ifndef BMRESTRICT 121 # define BMRESTRICT __restrict__ 122 # endif 123 124 # ifdef __OPTIMIZE__ 125 # define BM_NOASSERT 126 # endif 127 #endif 128 129 # ifdef NDEBUG 130 # define BM_NOASSERT 131 # endif 132 133 134 #ifndef BM_ASSERT 135 # ifndef BM_NOASSERT 136 # include <cassert> 137 # define BM_ASSERT assert 138 # else 139 # ifndef BM_ASSERT 140 # define BM_ASSERT(x) 141 # endif 142 # endif 143 #endif 144 145 146 #if defined(__x86_64) || defined(_M_AMD64) || defined(_WIN64) || \ 147 defined(__LP64__) || defined(_LP64) || ( __WORDSIZE == 64 ) 148 #ifndef BM64OPT 149 # define BM64OPT 150 #endif 151 #endif 152 153 154 155 #define FULL_BLOCK_REAL_ADDR bm::all_set<true>::_block._p 156 #define FULL_BLOCK_FAKE_ADDR bm::all_set<true>::_block._p_fullp 157 #define FULL_SUB_BLOCK_REAL_ADDR bm::all_set<true>::_block._s 158 #define BLOCK_ADDR_SAN(addr) (addr == FULL_BLOCK_FAKE_ADDR) ? FULL_BLOCK_REAL_ADDR : addr 159 #define IS_VALID_ADDR(addr) bm::all_set<true>::is_valid_block_addr(addr) 160 #define IS_FULL_BLOCK(addr) bm::all_set<true>::is_full_block(addr) 161 #define IS_EMPTY_BLOCK(addr) bool(addr == 0) 162 163 #define BM_BLOCK_TYPE(addr) bm::all_set<true>::block_type(addr) 164 165 // Macro definitions to manipulate bits in pointers 166 // This trick is based on the fact that pointers allocated by malloc are 167 // aligned and bit 0 is never set. It means we are safe to use it. 168 // BM library keeps GAP flag in pointer. 169 170 171 // TODO: consider UINTPTR_MAX == 0xFFFFFFFF 172 // 173 # if ULONG_MAX != 0xffffffff || defined(_WIN64) // 64-bit 174 175 # define BMPTR_SETBIT0(ptr) ( ((bm::id64_t)ptr) | 1 ) 176 # define BMPTR_CLEARBIT0(ptr) ( ((bm::id64_t)ptr) & ~(bm::id64_t)1 ) 177 # define BMPTR_TESTBIT0(ptr) ( ((bm::id64_t)ptr) & 1 ) 178 179 # else // 32-bit 180 181 # define BMPTR_SETBIT0(ptr) ( ((bm::id_t)ptr) | 1 ) 182 # define BMPTR_CLEARBIT0(ptr) ( ((bm::id_t)ptr) & ~(bm::id_t)1 ) 183 # define BMPTR_TESTBIT0(ptr) ( ((bm::id_t)ptr) & 1 ) 184 185 # endif 186 187 # define BMGAP_PTR(ptr) ((bm::gap_word_t*)BMPTR_CLEARBIT0(ptr)) 188 # define BMSET_PTRGAP(ptr) ptr = (bm::word_t*)BMPTR_SETBIT0(ptr) 189 # define BM_IS_GAP(ptr) bool(BMPTR_TESTBIT0(ptr)!=0) 190 191 192 193 194 195 #ifdef BM_HASRESTRICT 196 # ifndef BMRESTRICT 197 # define BMRESTRICT restrict 198 # endif 199 #else 200 # ifndef BMRESTRICT 201 # define BMRESTRICT 202 # endif 203 #endif 204 205 #ifndef BMFORCEINLINE 206 #ifdef BM_HASFORCEINLINE 207 # ifndef BMFORCEINLINE 208 # define BMFORCEINLINE __forceinline 209 # endif 210 #else 211 # define BMFORCEINLINE inline 212 #endif 213 #endif 214 215 216 // -------------------------------- 217 // SSE optmization macros 218 // 219 220 #ifdef BMSSE42OPT 221 # if defined(BM64OPT) || defined(__x86_64) || defined(_M_AMD64) || defined(_WIN64) || \ 222 defined(__LP64__) || defined(_LP64) || ( __WORDSIZE == 64 ) 223 # undef BM64OPT 224 # define BM64_SSE4 225 # endif 226 # undef BMSSE2OPT 227 #endif 228 229 #ifdef BMAVX2OPT 230 # if defined(BM64OPT) || defined(__x86_64) || defined(_M_AMD64) || defined(_WIN64) || \ 231 defined(__LP64__) || defined(_LP64) 232 # undef BM64OPT 233 # undef BM64_SSE4 234 # define BM64_AVX2 235 # endif 236 # undef BMSSE2OPT 237 # undef BMSSE42OPT 238 #endif 239 240 #ifdef BMAVX512OPT 241 # if defined(BM64OPT) || defined(__x86_64) || defined(_M_AMD64) || defined(_WIN64) || \ 242 defined(__LP64__) || defined(_LP64) 243 # undef BM64OPT 244 # undef BM64_SSE4 245 # undef BM64_AVX2 246 # define BM64_AVX512 247 # endif 248 # undef BMSSE2OPT 249 # undef BMSSE42OPT 250 #endif 251 252 253 254 # ifndef BM_SET_MMX_GUARD 255 # define BM_SET_MMX_GUARD 256 # endif 257 258 259 #if (defined(BMSSE2OPT) || defined(BMSSE42OPT) || defined(BMAVX2OPT) || defined(BMAVX512OPT)) 260 261 # ifndef BM_SET_MMX_GUARD 262 # define BM_SET_MMX_GUARD sse_empty_guard bm_mmx_guard_; 263 # endif 264 265 #ifdef _MSC_VER 266 267 #ifndef BM_ALIGN16 268 # define BM_ALIGN16 __declspec(align(16)) 269 # define BM_ALIGN16ATTR 270 #endif 271 272 #ifndef BM_ALIGN32 273 # define BM_ALIGN32 __declspec(align(32)) 274 # define BM_ALIGN32ATTR 275 #endif 276 277 #ifndef BM_ALIGN64 278 # define BM_ALIGN64 __declspec(align(64)) 279 # define BM_ALIGN64ATTR 280 #endif 281 282 # else // GCC 283 284 #ifndef BM_ALIGN16 285 # define BM_ALIGN16 286 # define BM_ALIGN16ATTR __attribute__((aligned(16))) 287 #endif 288 289 #ifndef BM_ALIGN32 290 # define BM_ALIGN32 291 # define BM_ALIGN32ATTR __attribute__((aligned(32))) 292 #endif 293 294 #ifndef BM_ALIGN64 295 # define BM_ALIGN64 296 # define BM_ALIGN64ATTR __attribute__((aligned(64))) 297 #endif 298 #endif 299 300 #else 301 302 #define BM_ALIGN16 303 #define BM_ALIGN16ATTR 304 #define BM_ALIGN32 305 #define BM_ALIGN32ATTR 306 #define BM_ALIGN64 307 #define BM_ALIGN64ATTR 308 309 #endif 310 311 312 /* 313 #if !(defined(BMSSE2OPT) || defined(BMSSE42OPT) || defined(BMAVX2OPT) || defined(BMAVX512OPT)) 314 315 #define BM_ALIGN16 316 #define BM_ALIGN16ATTR 317 #define BM_ALIGN32 318 #define BM_ALIGN32ATTR 319 #define BM_ALIGN64 320 #define BM_ALIGN64ATTR 321 322 #else 323 324 # ifndef BM_SET_MMX_GUARD 325 # define BM_SET_MMX_GUARD sse_empty_guard bm_mmx_guard_; 326 # endif 327 328 #ifdef _MSC_VER 329 330 #ifndef BM_ALIGN16 331 # define BM_ALIGN16 __declspec(align(16)) 332 # define BM_ALIGN16ATTR 333 #endif 334 335 #ifndef BM_ALIGN32 336 # define BM_ALIGN32 __declspec(align(32)) 337 # define BM_ALIGN32ATTR 338 #endif 339 340 #ifndef BM_ALIGN64 341 # define BM_ALIGN64 __declspec(align(64)) 342 # define BM_ALIGN64ATTR 343 #endif 344 345 # else // GCC 346 347 #ifndef BM_ALIGN16 348 # define BM_ALIGN16 349 # define BM_ALIGN16ATTR __attribute__((aligned(16))) 350 #endif 351 352 #ifndef BM_ALIGN32 353 # define BM_ALIGN32 354 # define BM_ALIGN32ATTR __attribute__((aligned(32))) 355 #endif 356 357 #ifndef BM_ALIGN64 358 # define BM_ALIGN64 359 # define BM_ALIGN64ATTR __attribute__((aligned(64))) 360 #endif 361 #endif 362 363 #endif 364 */ 365 366 367 #if (defined(BMSSE2OPT) || defined(BMSSE42OPT)) 368 # define BM_VECT_ALIGN BM_ALIGN16 369 # define BM_VECT_ALIGN_ATTR BM_ALIGN16ATTR 370 #else 371 # if defined(BMAVX2OPT) 372 # define BM_VECT_ALIGN BM_ALIGN32 373 # define BM_VECT_ALIGN_ATTR BM_ALIGN32ATTR 374 # else 375 # if defined(BMAVX512OPT) 376 # define BM_VECT_ALIGN BM_ALIGN64 377 # define BM_VECT_ALIGN_ATTR BM_ALIGN64ATTR 378 # else 379 # define BM_VECT_ALIGN 380 # define BM_VECT_ALIGN_ATTR 381 # endif 382 # endif 383 #endif 384 385 386 387 388 /*! 389 Define calculates number of 1 bits in 32-bit word. 390 @ingroup bitfunc 391 */ 392 #ifndef BM_INCWORD_BITCOUNT 393 394 #if (defined(BMSSE42OPT) || defined(BMAVX2OPT) || defined(BMAVX512OPT)) 395 # define BM_INCWORD_BITCOUNT(cnt, w) cnt += unsigned(_mm_popcnt_u32(w)); 396 #else 397 398 # define BM_INCWORD_BITCOUNT(cnt, w) cnt += \ 399 bm::bit_count_table<true>::_count[(unsigned char)(w)] + \ 400 bm::bit_count_table<true>::_count[(unsigned char)((w) >> 8)] + \ 401 bm::bit_count_table<true>::_count[(unsigned char)((w) >> 16)] + \ 402 bm::bit_count_table<true>::_count[(unsigned char)((w) >> 24)]; 403 404 405 #endif 406 407 // throw redefinintion for compatibility with language wrappers 408 // 409 #ifndef BM_ASSERT_THROW 410 #define BM_ASSERT_THROW(x, xerrcode) 411 #endif 412 413 414 #ifndef __has_cpp_attribute 415 # define __has_cpp_attribute(x) 0 416 #endif 417 #ifndef __has_attribute 418 # define __has_attribute(x) 0 419 #endif 420 #if __has_cpp_attribute(fallthrough) && \ 421 (!defined(__clang__) || (__clang_major__ > 7 && __cplusplus >= 201703L)) 422 # define BM_FALLTHROUGH [[fallthrough]] 423 #elif __has_cpp_attribute(gcc::fallthrough) 424 # define BM_FALLTHROUGH [[gcc::fallthrough]] 425 #elif __has_cpp_attribute(clang::fallthrough) 426 # define BM_FALLTHROUGH [[clang::fallthrough]] 427 #elif __has_attribute(fallthrough) 428 # define BM_FALLTHROUGH __attribute__ ((fallthrough)) 429 #else 430 # define BM_FALLTHROUGH 431 #endif 432 433 #endif 434 435 436