1 // Copyright 2015 blake2-rfc Developers
2 //
3 // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
4 // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
5 // http://opensource.org/licenses/MIT>, at your option. This file may not be
6 // copied, modified, or distributed except according to those terms.
7 
8 use crate::simd::simdty::u64x4;
9 
10 #[cfg(feature = "simd_opt")]
11 #[inline(always)]
rotate_right_const(vec: u64x4, n: u32) -> u64x412 pub fn rotate_right_const(vec: u64x4, n: u32) -> u64x4 {
13     match n {
14         32 => rotate_right_32(vec),
15         24 => rotate_right_24(vec),
16         16 => rotate_right_16(vec),
17         _ => rotate_right_any(vec, n),
18     }
19 }
20 
21 #[cfg(not(feature = "simd_opt"))]
22 #[inline(always)]
rotate_right_const(vec: u64x4, n: u32) -> u64x423 pub fn rotate_right_const(vec: u64x4, n: u32) -> u64x4 {
24     rotate_right_any(vec, n)
25 }
26 
27 #[inline(always)]
rotate_right_any(vec: u64x4, n: u32) -> u64x428 fn rotate_right_any(vec: u64x4, n: u32) -> u64x4 {
29     let r = n as u64;
30     let l = 64 - r;
31 
32     (vec >> u64x4::new(r, r, r, r)) ^ (vec << u64x4::new(l, l, l, l))
33 }
34 
35 #[cfg(feature = "simd_opt")]
36 #[inline(always)]
rotate_right_32(vec: u64x4) -> u64x437 fn rotate_right_32(vec: u64x4) -> u64x4 {
38     if cfg!(any(target_feature = "sse2", target_feature = "neon")) {
39         // 2 x pshufd (SSE2) / vpshufd (AVX2) / 2 x vrev (NEON)
40         transmute_shuffle!(u32x8, simd_shuffle8, vec, 8, [1, 0, 3, 2, 5, 4, 7, 6])
41     } else {
42         rotate_right_any(vec, 32)
43     }
44 }
45 
46 #[cfg(feature = "simd_opt")]
47 #[inline(always)]
rotate_right_24(vec: u64x4) -> u64x448 fn rotate_right_24(vec: u64x4) -> u64x4 {
49     if cfg!(all(
50         feature = "simd_asm",
51         target_feature = "neon",
52         target_arch = "arm"
53     )) {
54         // 4 x vext (NEON)
55         rotate_right_vext(vec, 3)
56     } else if cfg!(target_feature = "ssse3") {
57         // 2 x pshufb (SSSE3) / vpshufb (AVX2)
58         transmute_shuffle!(
59             u8x32,
60             simd_shuffle32,
61             vec,
62             32,
63             [
64                 3, 4, 5, 6, 7, 0, 1, 2, 11, 12, 13, 14, 15, 8, 9, 10, 19, 20, 21, 22, 23, 16, 17,
65                 18, 27, 28, 29, 30, 31, 24, 25, 26
66             ]
67         )
68     } else {
69         rotate_right_any(vec, 24)
70     }
71 }
72 
73 #[cfg(feature = "simd_opt")]
74 #[inline(always)]
rotate_right_16(vec: u64x4) -> u64x475 fn rotate_right_16(vec: u64x4) -> u64x4 {
76     if cfg!(all(
77         feature = "simd_asm",
78         target_feature = "neon",
79         target_arch = "arm"
80     )) {
81         // 4 x vext (NEON)
82         rotate_right_vext(vec, 2)
83     } else if cfg!(target_feature = "ssse3") {
84         // 2 x pshufb (SSSE3) / vpshufb (AVX2)
85         transmute_shuffle!(
86             u8x32,
87             simd_shuffle32,
88             vec,
89             32,
90             [
91                 2, 3, 4, 5, 6, 7, 0, 1, 10, 11, 12, 13, 14, 15, 8, 9, 18, 19, 20, 21, 22, 23, 16,
92                 17, 26, 27, 28, 29, 30, 31, 24, 25
93             ]
94         )
95     } else if cfg!(target_feature = "sse2") {
96         // 2 x pshuflw+pshufhw (SSE2)
97         transmute_shuffle!(
98             u16x16,
99             simd_shuffle16,
100             vec,
101             16,
102             [1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12]
103         )
104     } else {
105         rotate_right_any(vec, 16)
106     }
107 }
108 
109 #[cfg(all(feature = "simd_asm", target_feature = "neon", target_arch = "arm"))]
110 mod simd_asm_neon_arm {
111     use crate::simd::simdty::{u64x2, u64x4};
112 
113     #[inline(always)]
vext_u64(vec: u64x2, b: u8) -> u64x2114     fn vext_u64(vec: u64x2, b: u8) -> u64x2 {
115         unsafe {
116             let result: u64x2;
117             asm!("vext.8 ${0:e}, ${1:e}, ${1:e}, $2\nvext.8 ${0:f}, ${1:f}, ${1:f}, $2"
118                : "=w" (result)
119                : "w" (vec), "n" (b));
120             result
121         }
122     }
123 
124     #[inline(always)]
rotate_right_vext(vec: u64x4, b: u8) -> u64x4125     pub fn rotate_right_vext(vec: u64x4, b: u8) -> u64x4 {
126         use crate::simd::simdint::{simd_shuffle2, simd_shuffle4};
127 
128         unsafe {
129             let tmp0 = vext_u64(simd_shuffle2(vec, vec, [0, 1]), b);
130             let tmp1 = vext_u64(simd_shuffle2(vec, vec, [2, 3]), b);
131             simd_shuffle4(tmp0, tmp1, [0, 1, 2, 3])
132         }
133     }
134 }
135 
136 #[cfg(all(feature = "simd_asm", target_feature = "neon", target_arch = "arm"))]
137 use self::simd_asm_neon_arm::rotate_right_vext;
138 
139 #[cfg(feature = "simd_opt")]
140 #[cfg(not(all(feature = "simd_asm", target_feature = "neon", target_arch = "arm")))]
rotate_right_vext(_vec: u64x4, _n: u8) -> u64x4141 fn rotate_right_vext(_vec: u64x4, _n: u8) -> u64x4 {
142     unreachable!()
143 }
144