1 #[cfg(not(all( 2 httparse_simd, 3 any( 4 target_arch = "x86", 5 target_arch = "x86_64", 6 ), 7 )))] 8 mod fallback; 9 10 #[cfg(not(all( 11 httparse_simd, 12 any( 13 target_arch = "x86", 14 target_arch = "x86_64", 15 ), 16 )))] 17 pub use self::fallback::*; 18 19 #[cfg(all( 20 httparse_simd, 21 any( 22 target_arch = "x86", 23 target_arch = "x86_64", 24 ), 25 ))] 26 mod sse42; 27 28 #[cfg(all( 29 httparse_simd, 30 any( 31 httparse_simd_target_feature_avx2, 32 not(httparse_simd_target_feature_sse42), 33 ), 34 any( 35 target_arch = "x86", 36 target_arch = "x86_64", 37 ), 38 ))] 39 mod avx2; 40 41 #[cfg(all( 42 httparse_simd, 43 not(any( 44 httparse_simd_target_feature_sse42, 45 httparse_simd_target_feature_avx2, 46 )), 47 any( 48 target_arch = "x86", 49 target_arch = "x86_64", 50 ), 51 ))] 52 mod runtime { 53 //! Runtime detection of simd features. Used when the build script 54 //! doesn't notice any target features at build time. 55 //! 56 //! While `is_x86_feature_detected!` has it's own caching built-in, 57 //! at least in 1.27.0, the functions don't inline, leaving using it 58 //! actually *slower* than just using the scalar fallback. 59 60 use core::sync::atomic::{AtomicUsize, ATOMIC_USIZE_INIT, Ordering}; 61 62 static FEATURE: AtomicUsize = ATOMIC_USIZE_INIT; 63 64 const INIT: usize = 0; 65 const SSE_42: usize = 1; 66 const AVX_2: usize = 2; 67 const AVX_2_AND_SSE_42: usize = 3; 68 const NONE: usize = ::core::usize::MAX; 69 detect() -> usize70 fn detect() -> usize { 71 let feat = FEATURE.load(Ordering::Relaxed); 72 if feat == INIT { 73 if cfg!(target_arch = "x86_64") && is_x86_feature_detected!("avx2") { 74 if is_x86_feature_detected!("sse4.2") { 75 FEATURE.store(AVX_2_AND_SSE_42, Ordering::Relaxed); 76 return AVX_2_AND_SSE_42; 77 } else { 78 FEATURE.store(AVX_2, Ordering::Relaxed); 79 return AVX_2; 80 } 81 } else if is_x86_feature_detected!("sse4.2") { 82 FEATURE.store(SSE_42, Ordering::Relaxed); 83 return SSE_42; 84 } else { 85 FEATURE.store(NONE, Ordering::Relaxed); 86 } 87 } 88 feat 89 } 90 match_uri_vectored(bytes: &mut ::Bytes)91 pub fn match_uri_vectored(bytes: &mut ::Bytes) { 92 unsafe { 93 match detect() { 94 SSE_42 => super::sse42::parse_uri_batch_16(bytes), 95 AVX_2 => { super::avx2::parse_uri_batch_32(bytes); }, 96 AVX_2_AND_SSE_42 => { 97 if let super::avx2::Scan::Found = super::avx2::parse_uri_batch_32(bytes) { 98 return; 99 } 100 super::sse42::parse_uri_batch_16(bytes) 101 }, 102 _ => () 103 } 104 } 105 106 // else do nothing 107 } 108 match_header_value_vectored(bytes: &mut ::Bytes)109 pub fn match_header_value_vectored(bytes: &mut ::Bytes) { 110 unsafe { 111 match detect() { 112 SSE_42 => super::sse42::match_header_value_batch_16(bytes), 113 AVX_2 => { super::avx2::match_header_value_batch_32(bytes); }, 114 AVX_2_AND_SSE_42 => { 115 if let super::avx2::Scan::Found = super::avx2::match_header_value_batch_32(bytes) { 116 return; 117 } 118 super::sse42::match_header_value_batch_16(bytes) 119 }, 120 _ => () 121 } 122 } 123 124 // else do nothing 125 } 126 } 127 128 #[cfg(all( 129 httparse_simd, 130 not(any( 131 httparse_simd_target_feature_sse42, 132 httparse_simd_target_feature_avx2, 133 )), 134 any( 135 target_arch = "x86", 136 target_arch = "x86_64", 137 ), 138 ))] 139 pub use self::runtime::*; 140 141 #[cfg(all( 142 httparse_simd, 143 httparse_simd_target_feature_sse42, 144 not(httparse_simd_target_feature_avx2), 145 any( 146 target_arch = "x86", 147 target_arch = "x86_64", 148 ), 149 ))] 150 mod sse42_compile_time { match_uri_vectored(bytes: &mut ::Bytes)151 pub fn match_uri_vectored(bytes: &mut ::Bytes) { 152 if is_x86_feature_detected!("sse4.2") { 153 unsafe { 154 super::sse42::parse_uri_batch_16(bytes); 155 } 156 } 157 158 // else do nothing 159 } 160 match_header_value_vectored(bytes: &mut ::Bytes)161 pub fn match_header_value_vectored(bytes: &mut ::Bytes) { 162 if is_x86_feature_detected!("sse4.2") { 163 unsafe { 164 super::sse42::match_header_value_batch_16(bytes); 165 } 166 } 167 168 // else do nothing 169 } 170 } 171 172 #[cfg(all( 173 httparse_simd, 174 httparse_simd_target_feature_sse42, 175 not(httparse_simd_target_feature_avx2), 176 any( 177 target_arch = "x86", 178 target_arch = "x86_64", 179 ), 180 ))] 181 pub use self::sse42_compile_time::*; 182 183 #[cfg(all( 184 httparse_simd, 185 httparse_simd_target_feature_avx2, 186 any( 187 target_arch = "x86", 188 target_arch = "x86_64", 189 ), 190 ))] 191 mod avx2_compile_time { match_uri_vectored(bytes: &mut ::Bytes)192 pub fn match_uri_vectored(bytes: &mut ::Bytes) { 193 // do both, since avx2 only works when bytes.len() >= 32 194 if cfg!(target_arch = "x86_64") && is_x86_feature_detected!("avx2") { 195 unsafe { 196 super::avx2::parse_uri_batch_32(bytes); 197 } 198 199 } 200 if is_x86_feature_detected!("sse4.2") { 201 unsafe { 202 super::sse42::parse_uri_batch_16(bytes); 203 } 204 } 205 206 // else do nothing 207 } 208 match_header_value_vectored(bytes: &mut ::Bytes)209 pub fn match_header_value_vectored(bytes: &mut ::Bytes) { 210 // do both, since avx2 only works when bytes.len() >= 32 211 if cfg!(target_arch = "x86_64") && is_x86_feature_detected!("avx2") { 212 let scanned = unsafe { 213 super::avx2::match_header_value_batch_32(bytes) 214 }; 215 216 if let super::avx2::Scan::Found = scanned { 217 return; 218 } 219 } 220 if is_x86_feature_detected!("sse4.2") { 221 unsafe { 222 super::sse42::match_header_value_batch_16(bytes); 223 } 224 } 225 226 // else do nothing 227 } 228 } 229 230 #[cfg(all( 231 httparse_simd, 232 httparse_simd_target_feature_avx2, 233 any( 234 target_arch = "x86", 235 target_arch = "x86_64", 236 ), 237 ))] 238 pub use self::avx2_compile_time::*; 239