ring-0.16.15/src/cpu.rs

// Copyright 2016 Brian Smith.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES
// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY
// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

/// A witness indicating that CPU features have been detected and cached.
///
/// TODO: Eventually all feature detection logic should be done through
/// functions that accept a `Features` parameter, to guarantee that nothing
/// tries to read the cached values before they are written.
///
/// This is a zero-sized type so that it can be "stored" wherever convenient.
#[derive(Copy, Clone)]
pub(crate) struct Features(());

#[inline(always)]
pub(crate) fn features() -> Features {
    // We don't do runtime feature detection on iOS. instead some features are
    // assumed to be present; see `arm::Feature`.
    #[cfg(all(
        any(
            target_arch = "aarch64",
            target_arch = "arm",
            target_arch = "x86",
            target_arch = "x86_64"
        ),
        not(target_os = "ios")
    ))]
    {
        static INIT: spin::Once<()> = spin::Once::new();
        let () = INIT.call_once(|| {
            #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
            {
                extern "C" {
                    fn GFp_cpuid_setup();
                }
                unsafe {
                    GFp_cpuid_setup();
                }
            }

            #[cfg(all(
                any(target_os = "android", target_os = "linux"),
                any(target_arch = "aarch64", target_arch = "arm")
            ))]
            {
                arm::linux_setup();
            }

            #[cfg(all(target_os = "fuchsia", any(target_arch = "aarch64")))]
            {
                arm::fuchsia_setup();
            }

            #[cfg(all(target_os = "freebsd", any(target_arch = "aarch64", target_arch = "arm")))]
            {
                arm::freebsd_setup();
            }
        });
    }

    Features(())
}

pub(crate) mod arm {
    #[cfg(all(
        any(target_os = "android", target_os = "linux"),
        any(target_arch = "aarch64", target_arch = "arm")
    ))]
    pub fn linux_setup() {
        use libc::c_ulong;

        // XXX: The `libc` crate doesn't provide `libc::getauxval` consistently
        // across all Android/Linux targets, e.g. musl.
        extern "C" {
            fn getauxval(type_: c_ulong) -> c_ulong;
        }

        const AT_HWCAP: c_ulong = 16;

        #[cfg(target_arch = "aarch64")]
        const HWCAP_NEON: c_ulong = 1 << 1;

        #[cfg(target_arch = "arm")]
        const HWCAP_NEON: c_ulong = 1 << 12;

        let caps = unsafe { getauxval(AT_HWCAP) };

        // We assume NEON is available on AARCH64 because it is a required
        // feature.
        #[cfg(target_arch = "aarch64")]
        debug_assert!(caps & HWCAP_NEON == HWCAP_NEON);

        // OpenSSL and BoringSSL don't enable any other features if NEON isn't
        // available.
        if caps & HWCAP_NEON == HWCAP_NEON {
            let mut features = NEON.mask;

            #[cfg(target_arch = "aarch64")]
            const OFFSET: c_ulong = 3;

            #[cfg(target_arch = "arm")]
            const OFFSET: c_ulong = 0;

            #[cfg(target_arch = "arm")]
            let caps = {
                const AT_HWCAP2: c_ulong = 26;
                unsafe { getauxval(AT_HWCAP2) }
            };

            const HWCAP_AES: c_ulong = 1 << 0 + OFFSET;
            const HWCAP_PMULL: c_ulong = 1 << 1 + OFFSET;
            const HWCAP_SHA2: c_ulong = 1 << 3 + OFFSET;

            if caps & HWCAP_AES == HWCAP_AES {
                features |= AES.mask;
            }
            if caps & HWCAP_PMULL == HWCAP_PMULL {
                features |= PMULL.mask;
            }
            if caps & HWCAP_SHA2 == HWCAP_SHA2 {
                features |= 1 << 4;
            }

            unsafe { GFp_armcap_P = features };
        }
    }

    #[cfg(all(target_os = "fuchsia", any(target_arch = "aarch64")))]
    pub fn fuchsia_setup() {
        type zx_status_t = i32;

        #[link(name = "zircon")]
        extern "C" {
            fn zx_system_get_features(kind: u32, features: *mut u32) -> zx_status_t;
        }

        const ZX_OK: i32 = 0;
        const ZX_FEATURE_KIND_CPU: u32 = 0;
        const ZX_ARM64_FEATURE_ISA_ASIMD: u32 = 1 << 2;
        const ZX_ARM64_FEATURE_ISA_AES: u32 = 1 << 3;
        const ZX_ARM64_FEATURE_ISA_PMULL: u32 = 1 << 4;
        const ZX_ARM64_FEATURE_ISA_SHA2: u32 = 1 << 6;

        let mut caps = 0;
        let rc = unsafe { zx_system_get_features(ZX_FEATURE_KIND_CPU, &mut caps) };

        // OpenSSL and BoringSSL don't enable any other features if NEON isn't
        // available.
        if rc == ZX_OK && (caps & ZX_ARM64_FEATURE_ISA_ASIMD == ZX_ARM64_FEATURE_ISA_ASIMD) {
            let mut features = NEON.mask;

            if caps & ZX_ARM64_FEATURE_ISA_AES == ZX_ARM64_FEATURE_ISA_AES {
                features |= AES.mask;
            }
            if caps & ZX_ARM64_FEATURE_ISA_PMULL == ZX_ARM64_FEATURE_ISA_PMULL {
                features |= PMULL.mask;
            }
            if caps & ZX_ARM64_FEATURE_ISA_SHA2 == ZX_ARM64_FEATURE_ISA_SHA2 {
                features |= 1 << 4;
            }

            unsafe { GFp_armcap_P = features };
        }
    }

    #[cfg(all(target_os = "freebsd", any(target_arch = "aarch64", target_arch = "arm")))]
    pub fn freebsd_setup() {
        extern crate std;
        use libc::{c_int, c_ulong, c_void};

        extern "C" {
            fn elf_aux_info(aux: c_int, buf: *mut c_void, buflen: c_int) -> c_int;
        }

        const AT_HWCAP: c_int = 25;

        #[cfg(target_arch = "aarch64")]
        const HWCAP_NEON: c_ulong = 1 << 1;

        #[cfg(target_arch = "arm")]
        const HWCAP_NEON: c_ulong = 1 << 12;

        let caps: c_ulong = 0;
        let buffer : *mut c_void = { let t: *const c_ulong = &caps; t} as *mut c_void;

        unsafe {
            let _ret = elf_aux_info(
                AT_HWCAP,
                buffer,
                std::mem::size_of_val(&caps) as i32
            );
        }

        // We assume NEON is available on AARCH64 because it is a required
        // feature.
        #[cfg(target_arch = "aarch64")]
        debug_assert!(caps & HWCAP_NEON == HWCAP_NEON);

        // OpenSSL and BoringSSL don't enable any other features if NEON isn't
        // available.
        if caps & HWCAP_NEON == HWCAP_NEON {
            let mut features = NEON.mask;

            #[cfg(target_arch = "aarch64")]
            const OFFSET: c_ulong = 3;

            #[cfg(target_arch = "arm")]
            const OFFSET: c_ulong = 0;
            #[cfg(target_arch = "arm")]
            const AT_HWCAP2: c_int = 26;
            #[cfg(target_arch = "arm")]
            let caps: c_ulong = 0;
            #[cfg(target_arch = "arm")]
            let buffer : *mut c_void = { let t: *const c_ulong = &caps; t} as *mut c_void;

            #[cfg(target_arch = "arm")]
            unsafe {
                let _ret = elf_aux_info(
                    AT_HWCAP2,
                    buffer,
                    std::mem::size_of_val(&caps) as i32
                );
            };

            const HWCAP_AES: c_ulong = 1 << 0 + OFFSET;
            const HWCAP_PMULL: c_ulong = 1 << 1 + OFFSET;
            const HWCAP_SHA2: c_ulong = 1 << 3 + OFFSET;

            if caps & HWCAP_AES == HWCAP_AES {
                features |= AES.mask;
            }
            if caps & HWCAP_PMULL == HWCAP_PMULL {
                features |= PMULL.mask;
            }
            if caps & HWCAP_SHA2 == HWCAP_SHA2 {
                features |= 1 << 4;
            }

            unsafe { GFp_armcap_P = features };
        }
    }

    #[cfg(not(target_arch = "wasm32"))]
    pub(crate) struct Feature {
        #[cfg_attr(
            any(
                target_os = "ios",
                not(any(target_arch = "arm", target_arch = "aarch64"))
            ),
            allow(dead_code)
        )]
        mask: u32,

        #[cfg_attr(not(target_os = "ios"), allow(dead_code))]
        ios: bool,
    }

    #[cfg(not(target_arch = "wasm32"))]
    impl Feature {
        #[inline(always)]
        pub fn available(&self, _: super::Features) -> bool {
            #[cfg(all(target_os = "ios", any(target_arch = "arm", target_arch = "aarch64")))]
            {
                return self.ios;
            }

            #[cfg(all(
                any(target_os = "android", target_os = "linux", target_os = "fuchsia", target_os = "freebsd"),
                any(target_arch = "arm", target_arch = "aarch64")
            ))]
            {
                return self.mask == self.mask & unsafe { GFp_armcap_P };
            }

            #[cfg(not(any(target_arch = "arm", target_arch = "aarch64")))]
            {
                return false;
            }
        }
    }

    // Keep in sync with `ARMV7_NEON`.
    #[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
    pub(crate) const NEON: Feature = Feature {
        mask: 1 << 0,
        ios: true,
    };

    // Keep in sync with `ARMV8_AES`.
    #[cfg(any(
        target_arch = "aarch64",
        target_arch = "arm",
        target_arch = "x86",
        target_arch = "x86_64"
    ))]
    pub(crate) const AES: Feature = Feature {
        mask: 1 << 2,
        ios: true,
    };

    // Keep in sync with `ARMV8_PMULL`.
    #[cfg(any(
        target_arch = "aarch64",
        target_arch = "arm",
        target_arch = "x86",
        target_arch = "x86_64"
    ))]
    pub(crate) const PMULL: Feature = Feature {
        mask: 1 << 5,
        ios: true,
    };

    #[cfg(all(
        any(target_os = "android", target_os = "linux", target_os = "fuchsia", target_os = "freebsd"),
        any(target_arch = "arm", target_arch = "aarch64")
    ))]
    extern "C" {
        static mut GFp_armcap_P: u32;
    }
}

#[cfg_attr(
    not(any(target_arch = "x86", target_arch = "x86_64")),
    allow(dead_code)
)]
pub(crate) mod intel {
    pub(crate) struct Feature {
        word: usize,
        mask: u32,
    }

    impl Feature {
        #[inline(always)]
        pub fn available(&self, _: super::Features) -> bool {
            #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
            {
                extern "C" {
                    static mut GFp_ia32cap_P: [u32; 4];
                }
                return self.mask == self.mask & unsafe { GFp_ia32cap_P[self.word] };
            }

            #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
            {
                return false;
            }
        }
    }

    pub(crate) const FXSR: Feature = Feature {
        word: 0,
        mask: 1 << 24,
    };

    pub(crate) const PCLMULQDQ: Feature = Feature {
        word: 1,
        mask: 1 << 1,
    };

    pub(crate) const SSSE3: Feature = Feature {
        word: 1,
        mask: 1 << 9,
    };

    #[cfg(target_arch = "x86_64")]
    pub(crate) const MOVBE: Feature = Feature {
        word: 1,
        mask: 1 << 22,
    };

    pub(crate) const AES: Feature = Feature {
        word: 1,
        mask: 1 << 25,
    };

    #[cfg(target_arch = "x86_64")]
    pub(crate) const AVX: Feature = Feature {
        word: 1,
        mask: 1 << 28,
    };

    #[cfg(all(target_arch = "x86_64", test))]
    mod x86_64_tests {
        use super::*;

        #[test]
        fn test_avx_movbe_mask() {
            // This is the OpenSSL style of testing these bits.
            assert_eq!((AVX.mask | MOVBE.mask) >> 22, 0x41);
        }
    }
}