1 // Copyright 2015 blake2-rfc Developers
2 //
3 // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
4 // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
5 // http://opensource.org/licenses/MIT>, at your option. This file may not be
6 // copied, modified, or distributed except according to those terms.
7
8 use crate::simd::simdty::u64x4;
9
10 #[cfg(feature = "simd_opt")]
11 #[inline(always)]
rotate_right_const(vec: u64x4, n: u32) -> u64x412 pub fn rotate_right_const(vec: u64x4, n: u32) -> u64x4 {
13 match n {
14 32 => rotate_right_32(vec),
15 24 => rotate_right_24(vec),
16 16 => rotate_right_16(vec),
17 _ => rotate_right_any(vec, n),
18 }
19 }
20
21 #[cfg(not(feature = "simd_opt"))]
22 #[inline(always)]
rotate_right_const(vec: u64x4, n: u32) -> u64x423 pub fn rotate_right_const(vec: u64x4, n: u32) -> u64x4 {
24 rotate_right_any(vec, n)
25 }
26
27 #[inline(always)]
rotate_right_any(vec: u64x4, n: u32) -> u64x428 fn rotate_right_any(vec: u64x4, n: u32) -> u64x4 {
29 let r = n as u64;
30 let l = 64 - r;
31
32 (vec >> u64x4::new(r, r, r, r)) ^ (vec << u64x4::new(l, l, l, l))
33 }
34
35 #[cfg(feature = "simd_opt")]
36 #[inline(always)]
rotate_right_32(vec: u64x4) -> u64x437 fn rotate_right_32(vec: u64x4) -> u64x4 {
38 if cfg!(any(target_feature = "sse2", target_feature = "neon")) {
39 // 2 x pshufd (SSE2) / vpshufd (AVX2) / 2 x vrev (NEON)
40 transmute_shuffle!(u32x8, simd_shuffle8, vec, 8, [1, 0, 3, 2, 5, 4, 7, 6])
41 } else {
42 rotate_right_any(vec, 32)
43 }
44 }
45
46 #[cfg(feature = "simd_opt")]
47 #[inline(always)]
rotate_right_24(vec: u64x4) -> u64x448 fn rotate_right_24(vec: u64x4) -> u64x4 {
49 if cfg!(all(
50 feature = "simd_asm",
51 target_feature = "neon",
52 target_arch = "arm"
53 )) {
54 // 4 x vext (NEON)
55 rotate_right_vext(vec, 3)
56 } else if cfg!(target_feature = "ssse3") {
57 // 2 x pshufb (SSSE3) / vpshufb (AVX2)
58 transmute_shuffle!(
59 u8x32,
60 simd_shuffle32,
61 vec,
62 32,
63 [
64 3, 4, 5, 6, 7, 0, 1, 2, 11, 12, 13, 14, 15, 8, 9, 10, 19, 20, 21, 22, 23, 16, 17,
65 18, 27, 28, 29, 30, 31, 24, 25, 26
66 ]
67 )
68 } else {
69 rotate_right_any(vec, 24)
70 }
71 }
72
73 #[cfg(feature = "simd_opt")]
74 #[inline(always)]
rotate_right_16(vec: u64x4) -> u64x475 fn rotate_right_16(vec: u64x4) -> u64x4 {
76 if cfg!(all(
77 feature = "simd_asm",
78 target_feature = "neon",
79 target_arch = "arm"
80 )) {
81 // 4 x vext (NEON)
82 rotate_right_vext(vec, 2)
83 } else if cfg!(target_feature = "ssse3") {
84 // 2 x pshufb (SSSE3) / vpshufb (AVX2)
85 transmute_shuffle!(
86 u8x32,
87 simd_shuffle32,
88 vec,
89 32,
90 [
91 2, 3, 4, 5, 6, 7, 0, 1, 10, 11, 12, 13, 14, 15, 8, 9, 18, 19, 20, 21, 22, 23, 16,
92 17, 26, 27, 28, 29, 30, 31, 24, 25
93 ]
94 )
95 } else if cfg!(target_feature = "sse2") {
96 // 2 x pshuflw+pshufhw (SSE2)
97 transmute_shuffle!(
98 u16x16,
99 simd_shuffle16,
100 vec,
101 16,
102 [1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12]
103 )
104 } else {
105 rotate_right_any(vec, 16)
106 }
107 }
108
109 #[cfg(all(feature = "simd_asm", target_feature = "neon", target_arch = "arm"))]
110 mod simd_asm_neon_arm {
111 use crate::simd::simdty::{u64x2, u64x4};
112
113 #[inline(always)]
vext_u64(vec: u64x2, b: u8) -> u64x2114 fn vext_u64(vec: u64x2, b: u8) -> u64x2 {
115 unsafe {
116 let result: u64x2;
117 asm!("vext.8 ${0:e}, ${1:e}, ${1:e}, $2\nvext.8 ${0:f}, ${1:f}, ${1:f}, $2"
118 : "=w" (result)
119 : "w" (vec), "n" (b));
120 result
121 }
122 }
123
124 #[inline(always)]
rotate_right_vext(vec: u64x4, b: u8) -> u64x4125 pub fn rotate_right_vext(vec: u64x4, b: u8) -> u64x4 {
126 use crate::simd::simdint::{simd_shuffle2, simd_shuffle4};
127
128 unsafe {
129 let tmp0 = vext_u64(simd_shuffle2(vec, vec, [0, 1]), b);
130 let tmp1 = vext_u64(simd_shuffle2(vec, vec, [2, 3]), b);
131 simd_shuffle4(tmp0, tmp1, [0, 1, 2, 3])
132 }
133 }
134 }
135
136 #[cfg(all(feature = "simd_asm", target_feature = "neon", target_arch = "arm"))]
137 use self::simd_asm_neon_arm::rotate_right_vext;
138
139 #[cfg(feature = "simd_opt")]
140 #[cfg(not(all(feature = "simd_asm", target_feature = "neon", target_arch = "arm")))]
rotate_right_vext(_vec: u64x4, _n: u8) -> u64x4141 fn rotate_right_vext(_vec: u64x4, _n: u8) -> u64x4 {
142 unreachable!()
143 }
144