1 | #[cfg (not(all( |
2 | httparse_simd, |
3 | any( |
4 | target_arch = "x86" , |
5 | target_arch = "x86_64" , |
6 | ), |
7 | )))] |
8 | mod fallback; |
9 | |
10 | #[cfg (not(all( |
11 | httparse_simd, |
12 | any( |
13 | target_arch = "x86" , |
14 | target_arch = "x86_64" , |
15 | ), |
16 | )))] |
17 | pub use self::fallback::*; |
18 | |
19 | #[cfg (all( |
20 | httparse_simd, |
21 | any( |
22 | target_arch = "x86" , |
23 | target_arch = "x86_64" , |
24 | ), |
25 | ))] |
26 | mod sse42; |
27 | |
28 | #[cfg (all( |
29 | httparse_simd, |
30 | any( |
31 | httparse_simd_target_feature_avx2, |
32 | not(httparse_simd_target_feature_sse42), |
33 | ), |
34 | any( |
35 | target_arch = "x86" , |
36 | target_arch = "x86_64" , |
37 | ), |
38 | ))] |
39 | mod avx2; |
40 | |
41 | #[cfg (all( |
42 | httparse_simd, |
43 | any( |
44 | target_arch = "x86" , |
45 | target_arch = "x86_64" , |
46 | ), |
47 | ))] |
48 | pub const SSE_42: usize = 1; |
49 | #[cfg (all( |
50 | httparse_simd, |
51 | any(not(httparse_simd_target_feature_sse42), httparse_simd_target_feature_avx2), |
52 | any( |
53 | target_arch = "x86" , |
54 | target_arch = "x86_64" , |
55 | ), |
56 | ))] |
57 | pub const AVX_2: usize = 2; |
58 | #[cfg (all( |
59 | httparse_simd, |
60 | any( |
61 | not(httparse_simd_target_feature_sse42), |
62 | httparse_simd_target_feature_avx2, |
63 | test, |
64 | ), |
65 | any( |
66 | target_arch = "x86" , |
67 | target_arch = "x86_64" , |
68 | ), |
69 | ))] |
70 | pub const AVX_2_AND_SSE_42: usize = 3; |
71 | |
72 | #[cfg (all( |
73 | httparse_simd, |
74 | any( |
75 | target_arch = "x86" , |
76 | target_arch = "x86_64" , |
77 | ), |
78 | ))] |
79 | const NONE: usize = std::usize::MAX; |
80 | #[cfg (all( |
81 | httparse_simd, |
82 | not(any( |
83 | httparse_simd_target_feature_sse42, |
84 | httparse_simd_target_feature_avx2, |
85 | )), |
86 | any( |
87 | target_arch = "x86" , |
88 | target_arch = "x86_64" , |
89 | ), |
90 | ))] |
91 | mod runtime { |
92 | //! Runtime detection of simd features. Used when the build script |
93 | //! doesn't notice any target features at build time. |
94 | //! |
95 | //! While `is_x86_feature_detected!` has it's own caching built-in, |
96 | //! at least in 1.27.0, the functions don't inline, leaving using it |
97 | //! actually *slower* than just using the scalar fallback. |
98 | |
99 | use core::sync::atomic::{AtomicUsize, Ordering}; |
100 | |
101 | static FEATURE: AtomicUsize = AtomicUsize::new(0); |
102 | |
103 | const INIT: usize = 0; |
104 | |
105 | pub fn detect() -> usize { |
106 | let feat = FEATURE.load(Ordering::Relaxed); |
107 | if feat == INIT { |
108 | if cfg!(target_arch = "x86_64" ) && is_x86_feature_detected!("avx2" ) { |
109 | if is_x86_feature_detected!("sse4.2" ) { |
110 | FEATURE.store(super::AVX_2_AND_SSE_42, Ordering::Relaxed); |
111 | return super::AVX_2_AND_SSE_42; |
112 | } else { |
113 | FEATURE.store(super::AVX_2, Ordering::Relaxed); |
114 | return super::AVX_2; |
115 | } |
116 | } else if is_x86_feature_detected!("sse4.2" ) { |
117 | FEATURE.store(super::SSE_42, Ordering::Relaxed); |
118 | return super::SSE_42; |
119 | } else { |
120 | FEATURE.store(super::NONE, Ordering::Relaxed); |
121 | } |
122 | } |
123 | feat |
124 | } |
125 | |
126 | pub fn match_uri_vectored(bytes: &mut crate::iter::Bytes) { |
127 | unsafe { |
128 | match detect() { |
129 | super::SSE_42 => super::sse42::parse_uri_batch_16(bytes), |
130 | super::AVX_2 => { super::avx2::parse_uri_batch_32(bytes); }, |
131 | super::AVX_2_AND_SSE_42 => { |
132 | if let super::avx2::Scan::Found = super::avx2::parse_uri_batch_32(bytes) { |
133 | return; |
134 | } |
135 | super::sse42::parse_uri_batch_16(bytes) |
136 | }, |
137 | _ => () |
138 | } |
139 | } |
140 | |
141 | // else do nothing |
142 | } |
143 | |
144 | pub fn match_header_value_vectored(bytes: &mut crate::iter::Bytes) { |
145 | unsafe { |
146 | match detect() { |
147 | super::SSE_42 => super::sse42::match_header_value_batch_16(bytes), |
148 | super::AVX_2 => { super::avx2::match_header_value_batch_32(bytes); }, |
149 | super::AVX_2_AND_SSE_42 => { |
150 | if let super::avx2::Scan::Found = super::avx2::match_header_value_batch_32(bytes) { |
151 | return; |
152 | } |
153 | super::sse42::match_header_value_batch_16(bytes) |
154 | }, |
155 | _ => () |
156 | } |
157 | } |
158 | |
159 | // else do nothing |
160 | } |
161 | } |
162 | |
163 | #[cfg (all( |
164 | httparse_simd, |
165 | not(any( |
166 | httparse_simd_target_feature_sse42, |
167 | httparse_simd_target_feature_avx2, |
168 | )), |
169 | any( |
170 | target_arch = "x86" , |
171 | target_arch = "x86_64" , |
172 | ), |
173 | ))] |
174 | pub use self::runtime::*; |
175 | |
176 | #[cfg (all( |
177 | httparse_simd, |
178 | httparse_simd_target_feature_sse42, |
179 | not(httparse_simd_target_feature_avx2), |
180 | any( |
181 | target_arch = "x86" , |
182 | target_arch = "x86_64" , |
183 | ), |
184 | ))] |
185 | mod sse42_compile_time { |
186 | pub fn match_uri_vectored(bytes: &mut crate::iter::Bytes) { |
187 | if detect() == super::SSE_42 { |
188 | unsafe { |
189 | super::sse42::parse_uri_batch_16(bytes); |
190 | } |
191 | } |
192 | |
193 | // else do nothing |
194 | } |
195 | |
196 | pub fn match_header_value_vectored(bytes: &mut crate::iter::Bytes) { |
197 | if detect() == super::SSE_42 { |
198 | unsafe { |
199 | super::sse42::match_header_value_batch_16(bytes); |
200 | } |
201 | } |
202 | |
203 | // else do nothing |
204 | } |
205 | |
206 | pub fn detect() -> usize { |
207 | if is_x86_feature_detected!("sse4.2" ) { |
208 | super::SSE_42 |
209 | } else { |
210 | super::NONE |
211 | } |
212 | } |
213 | } |
214 | |
215 | #[cfg (all( |
216 | httparse_simd, |
217 | httparse_simd_target_feature_sse42, |
218 | not(httparse_simd_target_feature_avx2), |
219 | any( |
220 | target_arch = "x86" , |
221 | target_arch = "x86_64" , |
222 | ), |
223 | ))] |
224 | pub use self::sse42_compile_time::*; |
225 | |
226 | #[cfg (all( |
227 | httparse_simd, |
228 | httparse_simd_target_feature_avx2, |
229 | any( |
230 | target_arch = "x86" , |
231 | target_arch = "x86_64" , |
232 | ), |
233 | ))] |
234 | mod avx2_compile_time { |
235 | pub fn match_uri_vectored(bytes: &mut crate::iter::Bytes) { |
236 | // do both, since avx2 only works when bytes.len() >= 32 |
237 | if detect() == super::AVX_2_AND_SSE_42 { |
238 | unsafe { |
239 | super::avx2::parse_uri_batch_32(bytes); |
240 | } |
241 | |
242 | } |
243 | if detect() == super::SSE_42 { |
244 | unsafe { |
245 | super::sse42::parse_uri_batch_16(bytes); |
246 | } |
247 | } |
248 | |
249 | // else do nothing |
250 | } |
251 | |
252 | pub fn match_header_value_vectored(bytes: &mut crate::iter::Bytes) { |
253 | // do both, since avx2 only works when bytes.len() >= 32 |
254 | if detect() == super::AVX_2_AND_SSE_42 { |
255 | let scanned = unsafe { |
256 | super::avx2::match_header_value_batch_32(bytes) |
257 | }; |
258 | |
259 | if let super::avx2::Scan::Found = scanned { |
260 | return; |
261 | } |
262 | } |
263 | if detect() == super::SSE_42 { |
264 | unsafe { |
265 | super::sse42::match_header_value_batch_16(bytes); |
266 | } |
267 | } |
268 | |
269 | // else do nothing |
270 | } |
271 | |
272 | pub fn detect() -> usize { |
273 | if cfg!(target_arch = "x86_64" ) && is_x86_feature_detected!("avx2" ) { |
274 | super::AVX_2_AND_SSE_42 |
275 | } else if is_x86_feature_detected!("sse4.2" ) { |
276 | super::SSE_42 |
277 | } else { |
278 | super::NONE |
279 | } |
280 | } |
281 | } |
282 | |
283 | #[cfg (all( |
284 | httparse_simd, |
285 | httparse_simd_target_feature_avx2, |
286 | any( |
287 | target_arch = "x86" , |
288 | target_arch = "x86_64" , |
289 | ), |
290 | ))] |
291 | pub use self::avx2_compile_time::*; |
292 | |