1 #[cfg(not(all(
2     httparse_simd,
3     any(
4         target_arch = "x86",
5         target_arch = "x86_64",
6     ),
7 )))]
8 mod fallback;
9 
10 #[cfg(not(all(
11     httparse_simd,
12     any(
13         target_arch = "x86",
14         target_arch = "x86_64",
15     ),
16 )))]
17 pub use self::fallback::*;
18 
19 #[cfg(all(
20     httparse_simd,
21     any(
22         target_arch = "x86",
23         target_arch = "x86_64",
24     ),
25 ))]
26 mod sse42;
27 
28 #[cfg(all(
29     httparse_simd,
30     any(
31         httparse_simd_target_feature_avx2,
32         not(httparse_simd_target_feature_sse42),
33     ),
34     any(
35         target_arch = "x86",
36         target_arch = "x86_64",
37     ),
38 ))]
39 mod avx2;
40 
41 #[cfg(all(
42     httparse_simd,
43     not(any(
44         httparse_simd_target_feature_sse42,
45         httparse_simd_target_feature_avx2,
46     )),
47     any(
48         target_arch = "x86",
49         target_arch = "x86_64",
50     ),
51 ))]
52 mod runtime {
53     //! Runtime detection of simd features. Used when the build script
54     //! doesn't notice any target features at build time.
55     //!
56     //! While `is_x86_feature_detected!` has it's own caching built-in,
57     //! at least in 1.27.0, the functions don't inline, leaving using it
58     //! actually *slower* than just using the scalar fallback.
59 
60     use core::sync::atomic::{AtomicUsize, ATOMIC_USIZE_INIT, Ordering};
61 
62     static FEATURE: AtomicUsize = ATOMIC_USIZE_INIT;
63 
64     const INIT: usize = 0;
65     const SSE_42: usize = 1;
66     const AVX_2: usize = 2;
67     const AVX_2_AND_SSE_42: usize = 3;
68     const NONE: usize = ::core::usize::MAX;
69 
detect() -> usize70     fn detect() -> usize {
71         let feat = FEATURE.load(Ordering::Relaxed);
72         if feat == INIT {
73             if cfg!(target_arch = "x86_64") && is_x86_feature_detected!("avx2") {
74                 if is_x86_feature_detected!("sse4.2") {
75                     FEATURE.store(AVX_2_AND_SSE_42, Ordering::Relaxed);
76                     return AVX_2_AND_SSE_42;
77                 } else {
78                     FEATURE.store(AVX_2, Ordering::Relaxed);
79                     return AVX_2;
80                 }
81             } else if is_x86_feature_detected!("sse4.2") {
82                 FEATURE.store(SSE_42, Ordering::Relaxed);
83                 return SSE_42;
84             } else {
85                 FEATURE.store(NONE, Ordering::Relaxed);
86             }
87         }
88         feat
89     }
90 
match_uri_vectored(bytes: &mut ::Bytes)91     pub fn match_uri_vectored(bytes: &mut ::Bytes) {
92         unsafe {
93             match detect() {
94                 SSE_42 => super::sse42::parse_uri_batch_16(bytes),
95                 AVX_2 => { super::avx2::parse_uri_batch_32(bytes); },
96                 AVX_2_AND_SSE_42 => {
97                     if let super::avx2::Scan::Found = super::avx2::parse_uri_batch_32(bytes) {
98                         return;
99                     }
100                     super::sse42::parse_uri_batch_16(bytes)
101                 },
102                 _ => ()
103             }
104         }
105 
106         // else do nothing
107     }
108 
match_header_value_vectored(bytes: &mut ::Bytes)109     pub fn match_header_value_vectored(bytes: &mut ::Bytes) {
110         unsafe {
111             match detect() {
112                 SSE_42 => super::sse42::match_header_value_batch_16(bytes),
113                 AVX_2 => { super::avx2::match_header_value_batch_32(bytes); },
114                 AVX_2_AND_SSE_42 => {
115                     if let super::avx2::Scan::Found = super::avx2::match_header_value_batch_32(bytes) {
116                         return;
117                     }
118                     super::sse42::match_header_value_batch_16(bytes)
119                 },
120                 _ => ()
121             }
122         }
123 
124         // else do nothing
125     }
126 }
127 
128 #[cfg(all(
129     httparse_simd,
130     not(any(
131         httparse_simd_target_feature_sse42,
132         httparse_simd_target_feature_avx2,
133     )),
134     any(
135         target_arch = "x86",
136         target_arch = "x86_64",
137     ),
138 ))]
139 pub use self::runtime::*;
140 
141 #[cfg(all(
142     httparse_simd,
143     httparse_simd_target_feature_sse42,
144     not(httparse_simd_target_feature_avx2),
145     any(
146         target_arch = "x86",
147         target_arch = "x86_64",
148     ),
149 ))]
150 mod sse42_compile_time {
match_uri_vectored(bytes: &mut ::Bytes)151     pub fn match_uri_vectored(bytes: &mut ::Bytes) {
152         if is_x86_feature_detected!("sse4.2") {
153             unsafe {
154                 super::sse42::parse_uri_batch_16(bytes);
155             }
156         }
157 
158         // else do nothing
159     }
160 
match_header_value_vectored(bytes: &mut ::Bytes)161     pub fn match_header_value_vectored(bytes: &mut ::Bytes) {
162         if is_x86_feature_detected!("sse4.2") {
163             unsafe {
164                 super::sse42::match_header_value_batch_16(bytes);
165             }
166         }
167 
168         // else do nothing
169     }
170 }
171 
172 #[cfg(all(
173     httparse_simd,
174     httparse_simd_target_feature_sse42,
175     not(httparse_simd_target_feature_avx2),
176     any(
177         target_arch = "x86",
178         target_arch = "x86_64",
179     ),
180 ))]
181 pub use self::sse42_compile_time::*;
182 
183 #[cfg(all(
184     httparse_simd,
185     httparse_simd_target_feature_avx2,
186     any(
187         target_arch = "x86",
188         target_arch = "x86_64",
189     ),
190 ))]
191 mod avx2_compile_time {
match_uri_vectored(bytes: &mut ::Bytes)192     pub fn match_uri_vectored(bytes: &mut ::Bytes) {
193         // do both, since avx2 only works when bytes.len() >= 32
194         if cfg!(target_arch = "x86_64") && is_x86_feature_detected!("avx2") {
195             unsafe {
196                 super::avx2::parse_uri_batch_32(bytes);
197             }
198 
199         }
200         if is_x86_feature_detected!("sse4.2") {
201             unsafe {
202                 super::sse42::parse_uri_batch_16(bytes);
203             }
204         }
205 
206         // else do nothing
207     }
208 
match_header_value_vectored(bytes: &mut ::Bytes)209     pub fn match_header_value_vectored(bytes: &mut ::Bytes) {
210         // do both, since avx2 only works when bytes.len() >= 32
211         if cfg!(target_arch = "x86_64") && is_x86_feature_detected!("avx2") {
212             let scanned = unsafe {
213                 super::avx2::match_header_value_batch_32(bytes)
214             };
215 
216             if let super::avx2::Scan::Found = scanned {
217                 return;
218             }
219         }
220         if is_x86_feature_detected!("sse4.2") {
221             unsafe {
222                 super::sse42::match_header_value_batch_16(bytes);
223             }
224         }
225 
226         // else do nothing
227     }
228 }
229 
230 #[cfg(all(
231     httparse_simd,
232     httparse_simd_target_feature_avx2,
233     any(
234         target_arch = "x86",
235         target_arch = "x86_64",
236     ),
237 ))]
238 pub use self::avx2_compile_time::*;
239