1 // Copyright 2018 The Abseil Authors.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // https://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #ifndef ABSL_BASE_INTERNAL_BITS_H_
16 #define ABSL_BASE_INTERNAL_BITS_H_
17
18 // This file contains bitwise ops which are implementation details of various
19 // absl libraries.
20
21 #include <cstdint>
22
23 #include "absl/base/config.h"
24
25 // Clang on Windows has __builtin_clzll; otherwise we need to use the
26 // windows intrinsic functions.
27 #if defined(_MSC_VER) && !defined(__clang__)
28 #include <intrin.h>
29 #if defined(_M_X64)
30 #pragma intrinsic(_BitScanReverse64)
31 #pragma intrinsic(_BitScanForward64)
32 #endif
33 #pragma intrinsic(_BitScanReverse)
34 #pragma intrinsic(_BitScanForward)
35 #endif
36
37 #include "absl/base/attributes.h"
38
39 #if defined(_MSC_VER) && !defined(__clang__)
40 // We can achieve something similar to attribute((always_inline)) with MSVC by
41 // using the __forceinline keyword, however this is not perfect. MSVC is
42 // much less aggressive about inlining, and even with the __forceinline keyword.
43 #define ABSL_BASE_INTERNAL_FORCEINLINE __forceinline
44 #else
45 // Use default attribute inline.
46 #define ABSL_BASE_INTERNAL_FORCEINLINE inline ABSL_ATTRIBUTE_ALWAYS_INLINE
47 #endif
48
49
50 namespace absl {
51 ABSL_NAMESPACE_BEGIN
52 namespace base_internal {
53
CountLeadingZeros64Slow(uint64_t n)54 ABSL_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros64Slow(uint64_t n) {
55 int zeroes = 60;
56 if (n >> 32) {
57 zeroes -= 32;
58 n >>= 32;
59 }
60 if (n >> 16) {
61 zeroes -= 16;
62 n >>= 16;
63 }
64 if (n >> 8) {
65 zeroes -= 8;
66 n >>= 8;
67 }
68 if (n >> 4) {
69 zeroes -= 4;
70 n >>= 4;
71 }
72 return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes;
73 }
74
CountLeadingZeros64(uint64_t n)75 ABSL_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros64(uint64_t n) {
76 #if defined(_MSC_VER) && !defined(__clang__) && defined(_M_X64)
77 // MSVC does not have __buitin_clzll. Use _BitScanReverse64.
78 unsigned long result = 0; // NOLINT(runtime/int)
79 if (_BitScanReverse64(&result, n)) {
80 return 63 - result;
81 }
82 return 64;
83 #elif defined(_MSC_VER) && !defined(__clang__)
84 // MSVC does not have __buitin_clzll. Compose two calls to _BitScanReverse
85 unsigned long result = 0; // NOLINT(runtime/int)
86 if ((n >> 32) &&
87 _BitScanReverse(&result, static_cast<unsigned long>(n >> 32))) {
88 return 31 - result;
89 }
90 if (_BitScanReverse(&result, static_cast<unsigned long>(n))) {
91 return 63 - result;
92 }
93 return 64;
94 #elif defined(__GNUC__) || defined(__clang__)
95 // Use __builtin_clzll, which uses the following instructions:
96 // x86: bsr
97 // ARM64: clz
98 // PPC: cntlzd
99 static_assert(sizeof(unsigned long long) == sizeof(n), // NOLINT(runtime/int)
100 "__builtin_clzll does not take 64-bit arg");
101
102 // Handle 0 as a special case because __builtin_clzll(0) is undefined.
103 if (n == 0) {
104 return 64;
105 }
106 return __builtin_clzll(n);
107 #else
108 return CountLeadingZeros64Slow(n);
109 #endif
110 }
111
CountLeadingZeros32Slow(uint64_t n)112 ABSL_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros32Slow(uint64_t n) {
113 int zeroes = 28;
114 if (n >> 16) {
115 zeroes -= 16;
116 n >>= 16;
117 }
118 if (n >> 8) {
119 zeroes -= 8;
120 n >>= 8;
121 }
122 if (n >> 4) {
123 zeroes -= 4;
124 n >>= 4;
125 }
126 return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes;
127 }
128
CountLeadingZeros32(uint32_t n)129 ABSL_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros32(uint32_t n) {
130 #if defined(_MSC_VER) && !defined(__clang__)
131 unsigned long result = 0; // NOLINT(runtime/int)
132 if (_BitScanReverse(&result, n)) {
133 return 31 - result;
134 }
135 return 32;
136 #elif defined(__GNUC__) || defined(__clang__)
137 // Use __builtin_clz, which uses the following instructions:
138 // x86: bsr
139 // ARM64: clz
140 // PPC: cntlzd
141 static_assert(sizeof(int) == sizeof(n),
142 "__builtin_clz does not take 32-bit arg");
143
144 // Handle 0 as a special case because __builtin_clz(0) is undefined.
145 if (n == 0) {
146 return 32;
147 }
148 return __builtin_clz(n);
149 #else
150 return CountLeadingZeros32Slow(n);
151 #endif
152 }
153
CountTrailingZerosNonZero64Slow(uint64_t n)154 ABSL_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero64Slow(uint64_t n) {
155 int c = 63;
156 n &= ~n + 1;
157 if (n & 0x00000000FFFFFFFF) c -= 32;
158 if (n & 0x0000FFFF0000FFFF) c -= 16;
159 if (n & 0x00FF00FF00FF00FF) c -= 8;
160 if (n & 0x0F0F0F0F0F0F0F0F) c -= 4;
161 if (n & 0x3333333333333333) c -= 2;
162 if (n & 0x5555555555555555) c -= 1;
163 return c;
164 }
165
CountTrailingZerosNonZero64(uint64_t n)166 ABSL_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero64(uint64_t n) {
167 #if defined(_MSC_VER) && !defined(__clang__) && defined(_M_X64)
168 unsigned long result = 0; // NOLINT(runtime/int)
169 _BitScanForward64(&result, n);
170 return result;
171 #elif defined(_MSC_VER) && !defined(__clang__)
172 unsigned long result = 0; // NOLINT(runtime/int)
173 if (static_cast<uint32_t>(n) == 0) {
174 _BitScanForward(&result, static_cast<unsigned long>(n >> 32));
175 return result + 32;
176 }
177 _BitScanForward(&result, static_cast<unsigned long>(n));
178 return result;
179 #elif defined(__GNUC__) || defined(__clang__)
180 static_assert(sizeof(unsigned long long) == sizeof(n), // NOLINT(runtime/int)
181 "__builtin_ctzll does not take 64-bit arg");
182 return __builtin_ctzll(n);
183 #else
184 return CountTrailingZerosNonZero64Slow(n);
185 #endif
186 }
187
CountTrailingZerosNonZero32Slow(uint32_t n)188 ABSL_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero32Slow(uint32_t n) {
189 int c = 31;
190 n &= ~n + 1;
191 if (n & 0x0000FFFF) c -= 16;
192 if (n & 0x00FF00FF) c -= 8;
193 if (n & 0x0F0F0F0F) c -= 4;
194 if (n & 0x33333333) c -= 2;
195 if (n & 0x55555555) c -= 1;
196 return c;
197 }
198
CountTrailingZerosNonZero32(uint32_t n)199 ABSL_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero32(uint32_t n) {
200 #if defined(_MSC_VER) && !defined(__clang__)
201 unsigned long result = 0; // NOLINT(runtime/int)
202 _BitScanForward(&result, n);
203 return result;
204 #elif defined(__GNUC__) || defined(__clang__)
205 static_assert(sizeof(int) == sizeof(n),
206 "__builtin_ctz does not take 32-bit arg");
207 return __builtin_ctz(n);
208 #else
209 return CountTrailingZerosNonZero32Slow(n);
210 #endif
211 }
212
213 #undef ABSL_BASE_INTERNAL_FORCEINLINE
214
215 } // namespace base_internal
216 ABSL_NAMESPACE_END
217 } // namespace absl
218
219 #endif // ABSL_BASE_INTERNAL_BITS_H_
220