1 // Copyright 2016 Brian Smith.
2 //
3 // Permission to use, copy, modify, and/or distribute this software for any
4 // purpose with or without fee is hereby granted, provided that the above
5 // copyright notice and this permission notice appear in all copies.
6 //
7 // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES
8 // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
9 // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY
10 // SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
11 // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
12 // OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
13 // CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
14
15 /// A witness indicating that CPU features have been detected and cached.
16 ///
17 /// TODO: Eventually all feature detection logic should be done through
18 /// functions that accept a `Features` parameter, to guarantee that nothing
19 /// tries to read the cached values before they are written.
20 ///
21 /// This is a zero-sized type so that it can be "stored" wherever convenient.
22 #[derive(Copy, Clone)]
23 pub(crate) struct Features(());
24
25 #[inline(always)]
features() -> Features26 pub(crate) fn features() -> Features {
27 // We don't do runtime feature detection on aarch64-apple-* as all AAarch64
28 // features we use are available on every device since the first devices.
29 #[cfg(any(
30 target_arch = "x86",
31 target_arch = "x86_64",
32 all(
33 any(target_arch = "aarch64", target_arch = "arm"),
34 any(target_os = "android", target_os = "fuchsia", target_os = "freebsd", target_os = "linux")
35 )
36 ))]
37 {
38 static INIT: spin::Once<()> = spin::Once::new();
39 let () = INIT.call_once(|| {
40 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
41 {
42 extern "C" {
43 fn GFp_cpuid_setup();
44 }
45 unsafe {
46 GFp_cpuid_setup();
47 }
48 }
49
50 #[cfg(all(
51 any(target_arch = "aarch64", target_arch = "arm"),
52 any(target_os = "android", target_os = "fuchsia", target_os = "freebsd", target_os = "linux")
53 ))]
54 {
55 arm::setup();
56 }
57 });
58 }
59
60 Features(())
61 }
62
63 pub(crate) mod arm {
64 #[cfg(all(
65 any(target_os = "android", target_os = "freebsd", target_os = "linux"),
66 any(target_arch = "aarch64", target_arch = "arm")
67 ))]
setup()68 pub fn setup() {
69 use libc::c_ulong;
70 #[cfg(target_os = "freebsd")]
71 use libc::{c_int, c_void};
72 #[cfg(target_os = "freebsd")]
73 extern crate std;
74
75 // XXX: The `libc` crate doesn't provide `libc::getauxval` consistently
76 // across all Android/Linux targets, e.g. musl.
77 #[cfg(any(target_os = "android", target_os = "linux"))]
78 extern "C" {
79 fn getauxval(type_: c_ulong) -> c_ulong;
80 }
81
82 #[cfg(target_os = "freebsd")]
83 extern "C" {
84 fn elf_aux_info(aux: c_int, buf: *mut c_void, buflen: c_int) -> c_int;
85 }
86
87 #[cfg(not(target_os = "freebsd"))]
88 const AT_HWCAP: c_ulong = 16;
89
90 #[cfg(target_os = "freebsd")]
91 const AT_HWCAP: c_int = 25;
92
93 #[cfg(target_arch = "aarch64")]
94 const HWCAP_NEON: c_ulong = 1 << 1;
95
96 #[cfg(target_arch = "arm")]
97 const HWCAP_NEON: c_ulong = 1 << 12;
98
99 #[cfg(not(target_os = "freebsd"))]
100 let caps = unsafe { getauxval(AT_HWCAP) };
101
102 #[cfg(target_os = "freebsd")]
103 let caps: c_ulong = 0;
104
105 #[cfg(target_os = "freebsd")]
106 {
107 let buffer : *mut c_void = { let t: *const c_ulong = ∩︀ t} as *mut c_void;
108 unsafe {
109 let _ret = elf_aux_info(
110 AT_HWCAP,
111 buffer,
112 std::mem::size_of_val(&caps) as i32
113 );
114 }
115 }
116
117 // We assume NEON is available on AARCH64 because it is a required
118 // feature.
119 #[cfg(target_arch = "aarch64")]
120 debug_assert!(caps & HWCAP_NEON == HWCAP_NEON);
121
122 // OpenSSL and BoringSSL don't enable any other features if NEON isn't
123 // available.
124 if caps & HWCAP_NEON == HWCAP_NEON {
125 let mut features = NEON.mask;
126
127 #[cfg(target_arch = "aarch64")]
128 const OFFSET: c_ulong = 3;
129
130 #[cfg(target_arch = "arm")]
131 const OFFSET: c_ulong = 0;
132
133 #[cfg(target_os = "freebsd")]
134 let buffer : *mut c_void = { let t: *const c_ulong = ∩︀ t} as *mut c_void;
135
136 #[cfg(not(target_os = "freebsd"))]
137 let caps = {
138 const AT_HWCAP2: c_ulong = 26;
139 unsafe { getauxval(AT_HWCAP2) }
140 };
141
142 #[cfg(target_os = "freebsd")]
143 {
144 const AT_HWCAP2: c_int = 26;
145 unsafe {
146 let _ret = elf_aux_info(
147 AT_HWCAP2,
148 buffer,
149 std::mem::size_of_val(&caps) as i32
150 );
151 };
152 }
153
154 const HWCAP_AES: c_ulong = 1 << 0 + OFFSET;
155 const HWCAP_PMULL: c_ulong = 1 << 1 + OFFSET;
156 const HWCAP_SHA2: c_ulong = 1 << 3 + OFFSET;
157
158 if caps & HWCAP_AES == HWCAP_AES {
159 features |= AES.mask;
160 }
161 if caps & HWCAP_PMULL == HWCAP_PMULL {
162 features |= PMULL.mask;
163 }
164 if caps & HWCAP_SHA2 == HWCAP_SHA2 {
165 features |= SHA256.mask;
166 }
167
168 unsafe { GFp_armcap_P = features };
169 }
170 }
171
172 #[cfg(all(target_os = "fuchsia", target_arch = "aarch64"))]
setup()173 pub fn setup() {
174 type zx_status_t = i32;
175
176 #[link(name = "zircon")]
177 extern "C" {
178 fn zx_system_get_features(kind: u32, features: *mut u32) -> zx_status_t;
179 }
180
181 const ZX_OK: i32 = 0;
182 const ZX_FEATURE_KIND_CPU: u32 = 0;
183 const ZX_ARM64_FEATURE_ISA_ASIMD: u32 = 1 << 2;
184 const ZX_ARM64_FEATURE_ISA_AES: u32 = 1 << 3;
185 const ZX_ARM64_FEATURE_ISA_PMULL: u32 = 1 << 4;
186 const ZX_ARM64_FEATURE_ISA_SHA2: u32 = 1 << 6;
187
188 let mut caps = 0;
189 let rc = unsafe { zx_system_get_features(ZX_FEATURE_KIND_CPU, &mut caps) };
190
191 // OpenSSL and BoringSSL don't enable any other features if NEON isn't
192 // available.
193 if rc == ZX_OK && (caps & ZX_ARM64_FEATURE_ISA_ASIMD == ZX_ARM64_FEATURE_ISA_ASIMD) {
194 let mut features = NEON.mask;
195
196 if caps & ZX_ARM64_FEATURE_ISA_AES == ZX_ARM64_FEATURE_ISA_AES {
197 features |= AES.mask;
198 }
199 if caps & ZX_ARM64_FEATURE_ISA_PMULL == ZX_ARM64_FEATURE_ISA_PMULL {
200 features |= PMULL.mask;
201 }
202 if caps & ZX_ARM64_FEATURE_ISA_SHA2 == ZX_ARM64_FEATURE_ISA_SHA2 {
203 features |= 1 << 4;
204 }
205
206 unsafe { GFp_armcap_P = features };
207 }
208 }
209
210 macro_rules! features {
211 {
212 $(
213 $name:ident {
214 mask: $mask:expr,
215
216 /// Should we assume that the feature is always available
217 /// for aarch64-apple-* targets? The first AArch64 iOS
218 /// device used the Apple A7 chip.
219 // TODO: When we can use `if` in const expressions:
220 // ```
221 // aarch64_apple: $aarch64_apple,
222 // ```
223 aarch64_apple: true,
224 }
225 ),+
226 , // trailing comma is required.
227 } => {
228 $(
229 #[allow(dead_code)]
230 pub(crate) const $name: Feature = Feature {
231 mask: $mask,
232 };
233 )+
234
235 // TODO: When we can use `if` in const expressions, do this:
236 // ```
237 // const ARMCAP_STATIC: u32 = 0
238 // $(
239 // | ( if $aarch64_apple &&
240 // cfg!(all(target_arch = "aarch64",
241 // target_vendor = "apple")) {
242 // $name.mask
243 // } else {
244 // 0
245 // }
246 // )
247 // )+;
248 // ```
249 //
250 // TODO: Add static feature detection to other targets.
251 // TODO: Combine static feature detection with runtime feature
252 // detection.
253 #[cfg(all(target_arch = "aarch64", target_vendor = "apple"))]
254 const ARMCAP_STATIC: u32 = 0
255 $( | $name.mask
256 )+;
257 #[cfg(not(all(target_arch = "aarch64", target_vendor = "apple")))]
258 const ARMCAP_STATIC: u32 = 0;
259
260 #[cfg(all(target_arch = "aarch64", target_vendor = "apple"))]
261 #[test]
262 fn test_armcap_static_available() {
263 let features = crate::cpu::features();
264 $(
265 assert!($name.available(features));
266 )+
267 }
268 }
269 }
270
271 #[allow(dead_code)]
272 pub(crate) struct Feature {
273 mask: u32,
274 }
275
276 impl Feature {
277 #[allow(dead_code)]
278 #[inline(always)]
available(&self, _: super::Features) -> bool279 pub fn available(&self, _: super::Features) -> bool {
280 if self.mask == self.mask & ARMCAP_STATIC {
281 return true;
282 }
283
284 #[cfg(all(
285 any(target_os = "android", target_os = "fuchsia", target_os = "linux"),
286 any(target_arch = "arm", target_arch = "aarch64")
287 ))]
288 {
289 if self.mask == self.mask & unsafe { GFp_armcap_P } {
290 return true;
291 }
292 }
293
294 false
295 }
296 }
297
298 features! {
299 // Keep in sync with `ARMV7_NEON`.
300 NEON {
301 mask: 1 << 0,
302 aarch64_apple: true,
303 },
304
305 // Keep in sync with `ARMV8_AES`.
306 AES {
307 mask: 1 << 2,
308 aarch64_apple: true,
309 },
310
311 // Keep in sync with `ARMV8_SHA256`.
312 SHA256 {
313 mask: 1 << 4,
314 aarch64_apple: true,
315 },
316
317 // Keep in sync with `ARMV8_PMULL`.
318 PMULL {
319 mask: 1 << 5,
320 aarch64_apple: true,
321 },
322 }
323
324 // Some non-Rust code still checks this even when it is statically known
325 // the given feature is available, so we have to ensure that this is
326 // initialized properly. Keep this in sync with the initialization in
327 // BoringSSL's crypto.c.
328 //
329 // TODO: This should have "hidden" visibility but we don't have a way of
330 // controlling that yet: https://github.com/rust-lang/rust/issues/73958.
331 #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
332 #[no_mangle]
333 static mut GFp_armcap_P: u32 = ARMCAP_STATIC;
334
335 #[cfg(all(
336 any(target_arch = "arm", target_arch = "aarch64"),
337 target_vendor = "apple"
338 ))]
339 #[test]
test_armcap_static_matches_armcap_dynamic()340 fn test_armcap_static_matches_armcap_dynamic() {
341 assert_eq!(ARMCAP_STATIC, 1 | 4 | 16 | 32);
342 assert_eq!(ARMCAP_STATIC, unsafe { GFp_armcap_P });
343 }
344 }
345
346 #[cfg_attr(
347 not(any(target_arch = "x86", target_arch = "x86_64")),
348 allow(dead_code)
349 )]
350 pub(crate) mod intel {
351 pub(crate) struct Feature {
352 word: usize,
353 mask: u32,
354 }
355
356 impl Feature {
357 #[allow(clippy::needless_return)]
358 #[inline(always)]
available(&self, _: super::Features) -> bool359 pub fn available(&self, _: super::Features) -> bool {
360 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
361 {
362 extern "C" {
363 static mut GFp_ia32cap_P: [u32; 4];
364 }
365 return self.mask == self.mask & unsafe { GFp_ia32cap_P[self.word] };
366 }
367
368 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
369 {
370 return false;
371 }
372 }
373 }
374
375 pub(crate) const FXSR: Feature = Feature {
376 word: 0,
377 mask: 1 << 24,
378 };
379
380 pub(crate) const PCLMULQDQ: Feature = Feature {
381 word: 1,
382 mask: 1 << 1,
383 };
384
385 pub(crate) const SSSE3: Feature = Feature {
386 word: 1,
387 mask: 1 << 9,
388 };
389
390 #[cfg(target_arch = "x86_64")]
391 pub(crate) const SSE41: Feature = Feature {
392 word: 1,
393 mask: 1 << 19,
394 };
395
396 #[cfg(target_arch = "x86_64")]
397 pub(crate) const MOVBE: Feature = Feature {
398 word: 1,
399 mask: 1 << 22,
400 };
401
402 pub(crate) const AES: Feature = Feature {
403 word: 1,
404 mask: 1 << 25,
405 };
406
407 #[cfg(target_arch = "x86_64")]
408 pub(crate) const AVX: Feature = Feature {
409 word: 1,
410 mask: 1 << 28,
411 };
412
413 #[cfg(all(target_arch = "x86_64", test))]
414 mod x86_64_tests {
415 use super::*;
416
417 #[test]
test_avx_movbe_mask()418 fn test_avx_movbe_mask() {
419 // This is the OpenSSL style of testing these bits.
420 assert_eq!((AVX.mask | MOVBE.mask) >> 22, 0x41);
421 }
422 }
423 }
424