1#[cfg(not(all(
2 httparse_simd,
3 any(
4 target_arch = "x86",
5 target_arch = "x86_64",
6 ),
7)))]
8mod fallback;
9
10#[cfg(not(all(
11 httparse_simd,
12 any(
13 target_arch = "x86",
14 target_arch = "x86_64",
15 ),
16)))]
17pub use self::fallback::*;
18
19#[cfg(all(
20 httparse_simd,
21 any(
22 target_arch = "x86",
23 target_arch = "x86_64",
24 ),
25))]
26mod sse42;
27
28#[cfg(all(
29 httparse_simd,
30 any(
31 httparse_simd_target_feature_avx2,
32 not(httparse_simd_target_feature_sse42),
33 ),
34 any(
35 target_arch = "x86",
36 target_arch = "x86_64",
37 ),
38))]
39mod avx2;
40
41#[cfg(all(
42 httparse_simd,
43 any(
44 target_arch = "x86",
45 target_arch = "x86_64",
46 ),
47))]
48pub const SSE_42: usize = 1;
49#[cfg(all(
50 httparse_simd,
51 any(not(httparse_simd_target_feature_sse42), httparse_simd_target_feature_avx2),
52 any(
53 target_arch = "x86",
54 target_arch = "x86_64",
55 ),
56))]
57pub const AVX_2: usize = 2;
58#[cfg(all(
59 httparse_simd,
60 any(
61 not(httparse_simd_target_feature_sse42),
62 httparse_simd_target_feature_avx2,
63 test,
64 ),
65 any(
66 target_arch = "x86",
67 target_arch = "x86_64",
68 ),
69))]
70pub const AVX_2_AND_SSE_42: usize = 3;
71
72#[cfg(all(
73 httparse_simd,
74 any(
75 target_arch = "x86",
76 target_arch = "x86_64",
77 ),
78))]
79const NONE: usize = std::usize::MAX;
80#[cfg(all(
81 httparse_simd,
82 not(any(
83 httparse_simd_target_feature_sse42,
84 httparse_simd_target_feature_avx2,
85 )),
86 any(
87 target_arch = "x86",
88 target_arch = "x86_64",
89 ),
90))]
91mod runtime {
92 //! Runtime detection of simd features. Used when the build script
93 //! doesn't notice any target features at build time.
94 //!
95 //! While `is_x86_feature_detected!` has it's own caching built-in,
96 //! at least in 1.27.0, the functions don't inline, leaving using it
97 //! actually *slower* than just using the scalar fallback.
98
99 use core::sync::atomic::{AtomicUsize, Ordering};
100
101 static FEATURE: AtomicUsize = AtomicUsize::new(0);
102
103 const INIT: usize = 0;
104
105 pub fn detect() -> usize {
106 let feat = FEATURE.load(Ordering::Relaxed);
107 if feat == INIT {
108 if cfg!(target_arch = "x86_64") && is_x86_feature_detected!("avx2") {
109 if is_x86_feature_detected!("sse4.2") {
110 FEATURE.store(super::AVX_2_AND_SSE_42, Ordering::Relaxed);
111 return super::AVX_2_AND_SSE_42;
112 } else {
113 FEATURE.store(super::AVX_2, Ordering::Relaxed);
114 return super::AVX_2;
115 }
116 } else if is_x86_feature_detected!("sse4.2") {
117 FEATURE.store(super::SSE_42, Ordering::Relaxed);
118 return super::SSE_42;
119 } else {
120 FEATURE.store(super::NONE, Ordering::Relaxed);
121 }
122 }
123 feat
124 }
125
126 pub fn match_uri_vectored(bytes: &mut crate::iter::Bytes) {
127 unsafe {
128 match detect() {
129 super::SSE_42 => super::sse42::parse_uri_batch_16(bytes),
130 super::AVX_2 => { super::avx2::parse_uri_batch_32(bytes); },
131 super::AVX_2_AND_SSE_42 => {
132 if let super::avx2::Scan::Found = super::avx2::parse_uri_batch_32(bytes) {
133 return;
134 }
135 super::sse42::parse_uri_batch_16(bytes)
136 },
137 _ => ()
138 }
139 }
140
141 // else do nothing
142 }
143
144 pub fn match_header_value_vectored(bytes: &mut crate::iter::Bytes) {
145 unsafe {
146 match detect() {
147 super::SSE_42 => super::sse42::match_header_value_batch_16(bytes),
148 super::AVX_2 => { super::avx2::match_header_value_batch_32(bytes); },
149 super::AVX_2_AND_SSE_42 => {
150 if let super::avx2::Scan::Found = super::avx2::match_header_value_batch_32(bytes) {
151 return;
152 }
153 super::sse42::match_header_value_batch_16(bytes)
154 },
155 _ => ()
156 }
157 }
158
159 // else do nothing
160 }
161}
162
163#[cfg(all(
164 httparse_simd,
165 not(any(
166 httparse_simd_target_feature_sse42,
167 httparse_simd_target_feature_avx2,
168 )),
169 any(
170 target_arch = "x86",
171 target_arch = "x86_64",
172 ),
173))]
174pub use self::runtime::*;
175
176#[cfg(all(
177 httparse_simd,
178 httparse_simd_target_feature_sse42,
179 not(httparse_simd_target_feature_avx2),
180 any(
181 target_arch = "x86",
182 target_arch = "x86_64",
183 ),
184))]
185mod sse42_compile_time {
186 pub fn match_uri_vectored(bytes: &mut crate::iter::Bytes) {
187 if detect() == super::SSE_42 {
188 unsafe {
189 super::sse42::parse_uri_batch_16(bytes);
190 }
191 }
192
193 // else do nothing
194 }
195
196 pub fn match_header_value_vectored(bytes: &mut crate::iter::Bytes) {
197 if detect() == super::SSE_42 {
198 unsafe {
199 super::sse42::match_header_value_batch_16(bytes);
200 }
201 }
202
203 // else do nothing
204 }
205
206 pub fn detect() -> usize {
207 if is_x86_feature_detected!("sse4.2") {
208 super::SSE_42
209 } else {
210 super::NONE
211 }
212 }
213}
214
215#[cfg(all(
216 httparse_simd,
217 httparse_simd_target_feature_sse42,
218 not(httparse_simd_target_feature_avx2),
219 any(
220 target_arch = "x86",
221 target_arch = "x86_64",
222 ),
223))]
224pub use self::sse42_compile_time::*;
225
226#[cfg(all(
227 httparse_simd,
228 httparse_simd_target_feature_avx2,
229 any(
230 target_arch = "x86",
231 target_arch = "x86_64",
232 ),
233))]
234mod avx2_compile_time {
235 pub fn match_uri_vectored(bytes: &mut crate::iter::Bytes) {
236 // do both, since avx2 only works when bytes.len() >= 32
237 if detect() == super::AVX_2_AND_SSE_42 {
238 unsafe {
239 super::avx2::parse_uri_batch_32(bytes);
240 }
241
242 }
243 if detect() == super::SSE_42 {
244 unsafe {
245 super::sse42::parse_uri_batch_16(bytes);
246 }
247 }
248
249 // else do nothing
250 }
251
252 pub fn match_header_value_vectored(bytes: &mut crate::iter::Bytes) {
253 // do both, since avx2 only works when bytes.len() >= 32
254 if detect() == super::AVX_2_AND_SSE_42 {
255 let scanned = unsafe {
256 super::avx2::match_header_value_batch_32(bytes)
257 };
258
259 if let super::avx2::Scan::Found = scanned {
260 return;
261 }
262 }
263 if detect() == super::SSE_42 {
264 unsafe {
265 super::sse42::match_header_value_batch_16(bytes);
266 }
267 }
268
269 // else do nothing
270 }
271
272 pub fn detect() -> usize {
273 if cfg!(target_arch = "x86_64") && is_x86_feature_detected!("avx2") {
274 super::AVX_2_AND_SSE_42
275 } else if is_x86_feature_detected!("sse4.2") {
276 super::SSE_42
277 } else {
278 super::NONE
279 }
280 }
281}
282
283#[cfg(all(
284 httparse_simd,
285 httparse_simd_target_feature_avx2,
286 any(
287 target_arch = "x86",
288 target_arch = "x86_64",
289 ),
290))]
291pub use self::avx2_compile_time::*;
292