1//! SIMD compiler intrinsics.
2//!
3//! In this module, a "vector" is any `repr(simd)` type.
4
5extern "platform-intrinsic" {
6 /// Add two simd vectors elementwise.
7 ///
8 /// `T` must be a vector of integer or floating point primitive types.
9 pub fn simd_add<T>(x: T, y: T) -> T;
10
11 /// Subtract `rhs` from `lhs` elementwise.
12 ///
13 /// `T` must be a vector of integer or floating point primitive types.
14 pub fn simd_sub<T>(lhs: T, rhs: T) -> T;
15
16 /// Multiply two simd vectors elementwise.
17 ///
18 /// `T` must be a vector of integer or floating point primitive types.
19 pub fn simd_mul<T>(x: T, y: T) -> T;
20
21 /// Divide `lhs` by `rhs` elementwise.
22 ///
23 /// `T` must be a vector of integer or floating point primitive types.
24 ///
25 /// # Safety
26 /// For integers, `rhs` must not contain any zero elements.
27 /// Additionally for signed integers, `<int>::MIN / -1` is undefined behavior.
28 pub fn simd_div<T>(lhs: T, rhs: T) -> T;
29
30 /// Remainder of two vectors elementwise
31 ///
32 /// `T` must be a vector of integer or floating point primitive types.
33 ///
34 /// # Safety
35 /// For integers, `rhs` must not contain any zero elements.
36 /// Additionally for signed integers, `<int>::MIN / -1` is undefined behavior.
37 pub fn simd_rem<T>(lhs: T, rhs: T) -> T;
38
39 /// Elementwise vector left shift, with UB on overflow.
40 ///
41 /// Shift `lhs` left by `rhs`, shifting in sign bits for signed types.
42 ///
43 /// `T` must be a vector of integer primitive types.
44 ///
45 /// # Safety
46 ///
47 /// Each element of `rhs` must be less than `<int>::BITS`.
48 pub fn simd_shl<T>(lhs: T, rhs: T) -> T;
49
50 /// Elementwise vector right shift, with UB on overflow.
51 ///
52 /// `T` must be a vector of integer primitive types.
53 ///
54 /// Shift `lhs` right by `rhs`, shifting in sign bits for signed types.
55 ///
56 /// # Safety
57 ///
58 /// Each element of `rhs` must be less than `<int>::BITS`.
59 pub fn simd_shr<T>(lhs: T, rhs: T) -> T;
60
61 /// Elementwise vector "and".
62 ///
63 /// `T` must be a vector of integer primitive types.
64 pub fn simd_and<T>(x: T, y: T) -> T;
65
66 /// Elementwise vector "or".
67 ///
68 /// `T` must be a vector of integer primitive types.
69 pub fn simd_or<T>(x: T, y: T) -> T;
70
71 /// Elementwise vector "exclusive or".
72 ///
73 /// `T` must be a vector of integer primitive types.
74 pub fn simd_xor<T>(x: T, y: T) -> T;
75
76 /// Numerically cast a vector, elementwise.
77 ///
78 /// `T` and `U` must be vectors of integer or floating point primitive types, and must have the
79 /// same length.
80 ///
81 /// When casting floats to integers, the result is truncated. Out-of-bounds result lead to UB.
82 /// When casting integers to floats, the result is rounded.
83 /// Otherwise, truncates or extends the value, maintaining the sign for signed integers.
84 ///
85 /// # Safety
86 /// Casting from integer types is always safe.
87 /// Casting between two float types is also always safe.
88 ///
89 /// Casting floats to integers truncates, following the same rules as `to_int_unchecked`.
90 /// Specifically, each element must:
91 /// * Not be `NaN`
92 /// * Not be infinite
93 /// * Be representable in the return type, after truncating off its fractional part
94 pub fn simd_cast<T, U>(x: T) -> U;
95
96 /// Numerically cast a vector, elementwise.
97 ///
98 /// `T` and `U` be a vectors of integer or floating point primitive types, and must have the
99 /// same length.
100 ///
101 /// Like `simd_cast`, but saturates float-to-integer conversions (NaN becomes 0).
102 /// This matches regular `as` and is always safe.
103 ///
104 /// When casting floats to integers, the result is truncated.
105 /// When casting integers to floats, the result is rounded.
106 /// Otherwise, truncates or extends the value, maintaining the sign for signed integers.
107 pub fn simd_as<T, U>(x: T) -> U;
108
109 /// Elementwise negation of a vector.
110 ///
111 /// `T` must be a vector of integer or floating-point primitive types.
112 ///
113 /// Rust panics for `-<int>::Min` due to overflow, but it is not UB with this intrinsic.
114 pub fn simd_neg<T>(x: T) -> T;
115
116 /// Elementwise absolute value of a vector.
117 ///
118 /// `T` must be a vector of floating-point primitive types.
119 pub fn simd_fabs<T>(x: T) -> T;
120
121 /// Elementwise minimum of a vector.
122 ///
123 /// `T` must be a vector of floating-point primitive types.
124 ///
125 /// Follows IEEE-754 `minNum` semantics.
126 pub fn simd_fmin<T>(x: T, y: T) -> T;
127
128 /// Elementwise maximum of a vector.
129 ///
130 /// `T` must be a vector of floating-point primitive types.
131 ///
132 /// Follows IEEE-754 `maxNum` semantics.
133 pub fn simd_fmax<T>(x: T, y: T) -> T;
134
135 /// Tests elementwise equality of two vectors.
136 ///
137 /// `T` must be a vector of floating-point primitive types.
138 ///
139 /// `U` must be a vector of integers with the same number of elements and element size as `T`.
140 ///
141 /// Returns `0` for false and `!0` for true.
142 pub fn simd_eq<T, U>(x: T, y: T) -> U;
143
144 /// Tests elementwise inequality equality of two vectors.
145 ///
146 /// `T` must be a vector of floating-point primitive types.
147 ///
148 /// `U` must be a vector of integers with the same number of elements and element size as `T`.
149 ///
150 /// Returns `0` for false and `!0` for true.
151 pub fn simd_ne<T, U>(x: T, y: T) -> U;
152
153 /// Tests if `x` is less than `y`, elementwise.
154 ///
155 /// `T` must be a vector of floating-point primitive types.
156 ///
157 /// `U` must be a vector of integers with the same number of elements and element size as `T`.
158 ///
159 /// Returns `0` for false and `!0` for true.
160 pub fn simd_lt<T, U>(x: T, y: T) -> U;
161
162 /// Tests if `x` is less than or equal to `y`, elementwise.
163 ///
164 /// `T` must be a vector of floating-point primitive types.
165 ///
166 /// `U` must be a vector of integers with the same number of elements and element size as `T`.
167 ///
168 /// Returns `0` for false and `!0` for true.
169 pub fn simd_le<T, U>(x: T, y: T) -> U;
170
171 /// Tests if `x` is greater than `y`, elementwise.
172 ///
173 /// `T` must be a vector of floating-point primitive types.
174 ///
175 /// `U` must be a vector of integers with the same number of elements and element size as `T`.
176 ///
177 /// Returns `0` for false and `!0` for true.
178 pub fn simd_gt<T, U>(x: T, y: T) -> U;
179
180 /// Tests if `x` is greater than or equal to `y`, elementwise.
181 ///
182 /// `T` must be a vector of floating-point primitive types.
183 ///
184 /// `U` must be a vector of integers with the same number of elements and element size as `T`.
185 ///
186 /// Returns `0` for false and `!0` for true.
187 pub fn simd_ge<T, U>(x: T, y: T) -> U;
188
189 /// Shuffle two vectors by const indices.
190 ///
191 /// `T` must be a vector.
192 ///
193 /// `U` must be a const array of `i32`s.
194 ///
195 /// `V` must be a vector with the same element type as `T` and the same length as `U`.
196 ///
197 /// Concatenates `x` and `y`, then returns a new vector such that each element is selected from
198 /// the concatenation by the matching index in `idx`.
199 pub fn simd_shuffle<T, U, V>(x: T, y: T, idx: U) -> V;
200
201 /// Read a vector of pointers.
202 ///
203 /// `T` must be a vector.
204 ///
205 /// `U` must be a vector of pointers to the element type of `T`, with the same length as `T`.
206 ///
207 /// `V` must be a vector of integers with the same length as `T` (but any element size).
208 ///
209 /// `idx` must be a constant: either naming a constant item, or an inline
210 /// `const {}` expression.
211 ///
212 /// For each pointer in `ptr`, if the corresponding value in `mask` is `!0`, read the pointer.
213 /// Otherwise if the corresponding value in `mask` is `0`, return the corresponding value from
214 /// `val`.
215 ///
216 /// # Safety
217 /// Unmasked values in `T` must be readable as if by `<ptr>::read` (e.g. aligned to the element
218 /// type).
219 ///
220 /// `mask` must only contain `0` or `!0` values.
221 pub fn simd_gather<T, U, V>(val: T, ptr: U, mask: V) -> T;
222
223 /// Write to a vector of pointers.
224 ///
225 /// `T` must be a vector.
226 ///
227 /// `U` must be a vector of pointers to the element type of `T`, with the same length as `T`.
228 ///
229 /// `V` must be a vector of integers with the same length as `T` (but any element size).
230 ///
231 /// For each pointer in `ptr`, if the corresponding value in `mask` is `!0`, write the
232 /// corresponding value in `val` to the pointer.
233 /// Otherwise if the corresponding value in `mask` is `0`, do nothing.
234 ///
235 /// # Safety
236 /// Unmasked values in `T` must be writeable as if by `<ptr>::write` (e.g. aligned to the element
237 /// type).
238 ///
239 /// `mask` must only contain `0` or `!0` values.
240 pub fn simd_scatter<T, U, V>(val: T, ptr: U, mask: V);
241
242 /// Read a vector of pointers.
243 ///
244 /// `T` must be a vector.
245 ///
246 /// `U` must be a vector of pointers to the element type of `T`, with the same length as `T`.
247 ///
248 /// `V` must be a vector of integers with the same length as `T` (but any element size).
249 ///
250 /// For each element, if the corresponding value in `mask` is `!0`, read the corresponding
251 /// pointer from `ptr`.
252 /// Otherwise if the corresponding value in `mask` is `0`, return the corresponding value from
253 /// `val`.
254 ///
255 /// # Safety
256 /// Unmasked values in `T` must be readable as if by `<ptr>::read` (e.g. aligned to the element
257 /// type).
258 ///
259 /// `mask` must only contain `0` or `!0` values.
260 pub fn simd_masked_load<V, U, T>(mask: V, ptr: U, val: T) -> T;
261
262 /// Write to a vector of pointers.
263 ///
264 /// `T` must be a vector.
265 ///
266 /// `U` must be a vector of pointers to the element type of `T`, with the same length as `T`.
267 ///
268 /// `V` must be a vector of integers with the same length as `T` (but any element size).
269 ///
270 /// For each element, if the corresponding value in `mask` is `!0`, write the corresponding
271 /// value in `val` to the pointer.
272 /// Otherwise if the corresponding value in `mask` is `0`, do nothing.
273 ///
274 /// # Safety
275 /// Unmasked values in `T` must be writeable as if by `<ptr>::write` (e.g. aligned to the element
276 /// type).
277 ///
278 /// `mask` must only contain `0` or `!0` values.
279 pub fn simd_masked_store<V, U, T>(mask: V, ptr: U, val: T);
280
281 /// Add two simd vectors elementwise, with saturation.
282 ///
283 /// `T` must be a vector of integer primitive types.
284 pub fn simd_saturating_add<T>(x: T, y: T) -> T;
285
286 /// Subtract two simd vectors elementwise, with saturation.
287 ///
288 /// `T` must be a vector of integer primitive types.
289 ///
290 /// Subtract `rhs` from `lhs`.
291 pub fn simd_saturating_sub<T>(lhs: T, rhs: T) -> T;
292
293 /// Add elements within a vector from left to right.
294 ///
295 /// `T` must be a vector of integer or floating-point primitive types.
296 ///
297 /// `U` must be the element type of `T`.
298 ///
299 /// Starting with the value `y`, add the elements of `x` and accumulate.
300 pub fn simd_reduce_add_ordered<T, U>(x: T, y: U) -> U;
301
302 /// Multiply elements within a vector from left to right.
303 ///
304 /// `T` must be a vector of integer or floating-point primitive types.
305 ///
306 /// `U` must be the element type of `T`.
307 ///
308 /// Starting with the value `y`, multiply the elements of `x` and accumulate.
309 pub fn simd_reduce_mul_ordered<T, U>(x: T, y: U) -> U;
310
311 /// Check if all mask values are true.
312 ///
313 /// `T` must be a vector of integer primitive types.
314 ///
315 /// # Safety
316 /// `x` must contain only `0` or `!0`.
317 pub fn simd_reduce_all<T>(x: T) -> bool;
318
319 /// Check if all mask values are true.
320 ///
321 /// `T` must be a vector of integer primitive types.
322 ///
323 /// # Safety
324 /// `x` must contain only `0` or `!0`.
325 pub fn simd_reduce_any<T>(x: T) -> bool;
326
327 /// Return the maximum element of a vector.
328 ///
329 /// `T` must be a vector of integer or floating-point primitive types.
330 ///
331 /// `U` must be the element type of `T`.
332 ///
333 /// For floating-point values, uses IEEE-754 `maxNum`.
334 pub fn simd_reduce_max<T, U>(x: T) -> U;
335
336 /// Return the minimum element of a vector.
337 ///
338 /// `T` must be a vector of integer or floating-point primitive types.
339 ///
340 /// `U` must be the element type of `T`.
341 ///
342 /// For floating-point values, uses IEEE-754 `minNum`.
343 pub fn simd_reduce_min<T, U>(x: T) -> U;
344
345 /// Logical "and" all elements together.
346 ///
347 /// `T` must be a vector of integer or floating-point primitive types.
348 ///
349 /// `U` must be the element type of `T`.
350 pub fn simd_reduce_and<T, U>(x: T) -> U;
351
352 /// Logical "or" all elements together.
353 ///
354 /// `T` must be a vector of integer or floating-point primitive types.
355 ///
356 /// `U` must be the element type of `T`.
357 pub fn simd_reduce_or<T, U>(x: T) -> U;
358
359 /// Logical "exclusive or" all elements together.
360 ///
361 /// `T` must be a vector of integer or floating-point primitive types.
362 ///
363 /// `U` must be the element type of `T`.
364 pub fn simd_reduce_xor<T, U>(x: T) -> U;
365
366 /// Truncate an integer vector to a bitmask.
367 ///
368 /// `T` must be an integer vector.
369 ///
370 /// `U` must be either the smallest unsigned integer with at least as many bits as the length
371 /// of `T`, or the smallest array of `u8` with as many bits as the length of `T`.
372 ///
373 /// Each element is truncated to a single bit and packed into the result.
374 ///
375 /// No matter whether the output is an array or an unsigned integer, it is treated as a single
376 /// contiguous list of bits. The bitmask is always packed on the least-significant side of the
377 /// output, and padded with 0s in the most-significant bits. The order of the bits depends on
378 /// endianess:
379 ///
380 /// * On little endian, the least significant bit corresponds to the first vector element.
381 /// * On big endian, the least significant bit corresponds to the last vector element.
382 ///
383 /// For example, `[!0, 0, !0, !0]` packs to `0b1101` on little endian and `0b1011` on big
384 /// endian.
385 ///
386 /// To consider a larger example, `[!0, 0, 0, 0, 0, 0, 0, 0, !0, !0, 0, 0, 0, 0, !0, 0]` packs
387 /// to `[0b00000001, 0b01000011]` or `0b0100001100000001` on little endian, and `[0b10000000,
388 /// 0b11000010]` or `0b1000000011000010` on big endian.
389 ///
390 /// # Safety
391 /// `x` must contain only `0` and `!0`.
392 pub fn simd_bitmask<T, U>(x: T) -> U;
393
394 /// Select elements from a mask.
395 ///
396 /// `M` must be an integer vector.
397 ///
398 /// `T` must be a vector with the same number of elements as `M`.
399 ///
400 /// For each element, if the corresponding value in `mask` is `!0`, select the element from
401 /// `if_true`. If the corresponding value in `mask` is `0`, select the element from
402 /// `if_false`.
403 ///
404 /// # Safety
405 /// `mask` must only contain `0` and `!0`.
406 pub fn simd_select<M, T>(mask: M, if_true: T, if_false: T) -> T;
407
408 /// Select elements from a bitmask.
409 ///
410 /// `M` must be an unsigned integer or array of `u8`, matching `simd_bitmask`.
411 ///
412 /// `T` must be a vector.
413 ///
414 /// For each element, if the bit in `mask` is `1`, select the element from
415 /// `if_true`. If the corresponding bit in `mask` is `0`, select the element from
416 /// `if_false`.
417 ///
418 /// The bitmask bit order matches `simd_bitmask`.
419 ///
420 /// # Safety
421 /// Padding bits must be all zero.
422 pub fn simd_select_bitmask<M, T>(m: M, yes: T, no: T) -> T;
423
424 /// Elementwise calculates the offset from a pointer vector, potentially wrapping.
425 ///
426 /// `T` must be a vector of pointers.
427 ///
428 /// `U` must be a vector of `isize` or `usize` with the same number of elements as `T`.
429 ///
430 /// Operates as if by `<ptr>::wrapping_offset`.
431 pub fn simd_arith_offset<T, U>(ptr: T, offset: U) -> T;
432
433 /// Cast a vector of pointers.
434 ///
435 /// `T` and `U` must be vectors of pointers with the same number of elements.
436 pub fn simd_cast_ptr<T, U>(ptr: T) -> U;
437
438 /// Expose a vector of pointers as a vector of addresses.
439 ///
440 /// `T` must be a vector of pointers.
441 ///
442 /// `U` must be a vector of `usize` with the same length as `T`.
443 pub fn simd_expose_addr<T, U>(ptr: T) -> U;
444
445 /// Create a vector of pointers from a vector of addresses.
446 ///
447 /// `T` must be a vector of `usize`.
448 ///
449 /// `U` must be a vector of pointers, with the same length as `T`.
450 pub fn simd_from_exposed_addr<T, U>(addr: T) -> U;
451
452 /// Swap bytes of each element.
453 ///
454 /// `T` must be a vector of integers.
455 pub fn simd_bswap<T>(x: T) -> T;
456
457 /// Reverse bits of each element.
458 ///
459 /// `T` must be a vector of integers.
460 pub fn simd_bitreverse<T>(x: T) -> T;
461
462 /// Count the leading zeros of each element.
463 ///
464 /// `T` must be a vector of integers.
465 pub fn simd_ctlz<T>(x: T) -> T;
466
467 /// Count the trailing zeros of each element.
468 ///
469 /// `T` must be a vector of integers.
470 pub fn simd_cttz<T>(x: T) -> T;
471}
472