1 | //! SIMD compiler intrinsics. |
2 | //! |
3 | //! In this module, a "vector" is any `repr(simd)` type. |
4 | |
5 | /// Inserts an element into a vector, returning the updated vector. |
6 | /// |
7 | /// `T` must be a vector with element type `U`. |
8 | /// |
9 | /// # Safety |
10 | /// |
11 | /// `idx` must be in-bounds of the vector. |
12 | #[rustc_intrinsic ] |
13 | #[rustc_nounwind ] |
14 | pub const unsafe fn simd_insert<T, U>(x: T, idx: u32, val: U) -> T; |
15 | |
16 | /// Extracts an element from a vector. |
17 | /// |
18 | /// `T` must be a vector with element type `U`. |
19 | /// |
20 | /// # Safety |
21 | /// |
22 | /// `idx` must be in-bounds of the vector. |
23 | #[rustc_intrinsic ] |
24 | #[rustc_nounwind ] |
25 | pub const unsafe fn simd_extract<T, U>(x: T, idx: u32) -> U; |
26 | |
27 | /// Adds two simd vectors elementwise. |
28 | /// |
29 | /// `T` must be a vector of integers or floats. |
30 | #[rustc_intrinsic ] |
31 | #[rustc_nounwind ] |
32 | pub unsafe fn simd_add<T>(x: T, y: T) -> T; |
33 | |
34 | /// Subtracts `rhs` from `lhs` elementwise. |
35 | /// |
36 | /// `T` must be a vector of integers or floats. |
37 | #[rustc_intrinsic ] |
38 | #[rustc_nounwind ] |
39 | pub unsafe fn simd_sub<T>(lhs: T, rhs: T) -> T; |
40 | |
41 | /// Multiplies two simd vectors elementwise. |
42 | /// |
43 | /// `T` must be a vector of integers or floats. |
44 | #[rustc_intrinsic ] |
45 | #[rustc_nounwind ] |
46 | pub unsafe fn simd_mul<T>(x: T, y: T) -> T; |
47 | |
48 | /// Divides `lhs` by `rhs` elementwise. |
49 | /// |
50 | /// `T` must be a vector of integers or floats. |
51 | /// |
52 | /// # Safety |
53 | /// For integers, `rhs` must not contain any zero elements. |
54 | /// Additionally for signed integers, `<int>::MIN / -1` is undefined behavior. |
55 | #[rustc_intrinsic ] |
56 | #[rustc_nounwind ] |
57 | pub unsafe fn simd_div<T>(lhs: T, rhs: T) -> T; |
58 | |
59 | /// Returns remainder of two vectors elementwise. |
60 | /// |
61 | /// `T` must be a vector of integers or floats. |
62 | /// |
63 | /// # Safety |
64 | /// For integers, `rhs` must not contain any zero elements. |
65 | /// Additionally for signed integers, `<int>::MIN / -1` is undefined behavior. |
66 | #[rustc_intrinsic ] |
67 | #[rustc_nounwind ] |
68 | pub unsafe fn simd_rem<T>(lhs: T, rhs: T) -> T; |
69 | |
70 | /// Shifts vector left elementwise, with UB on overflow. |
71 | /// |
72 | /// Shifts `lhs` left by `rhs`, shifting in sign bits for signed types. |
73 | /// |
74 | /// `T` must be a vector of integers. |
75 | /// |
76 | /// # Safety |
77 | /// |
78 | /// Each element of `rhs` must be less than `<int>::BITS`. |
79 | #[rustc_intrinsic ] |
80 | #[rustc_nounwind ] |
81 | pub unsafe fn simd_shl<T>(lhs: T, rhs: T) -> T; |
82 | |
83 | /// Shifts vector right elementwise, with UB on overflow. |
84 | /// |
85 | /// `T` must be a vector of integers. |
86 | /// |
87 | /// Shifts `lhs` right by `rhs`, shifting in sign bits for signed types. |
88 | /// |
89 | /// # Safety |
90 | /// |
91 | /// Each element of `rhs` must be less than `<int>::BITS`. |
92 | #[rustc_intrinsic ] |
93 | #[rustc_nounwind ] |
94 | pub unsafe fn simd_shr<T>(lhs: T, rhs: T) -> T; |
95 | |
96 | /// "Ands" vectors elementwise. |
97 | /// |
98 | /// `T` must be a vector of integers. |
99 | #[rustc_intrinsic ] |
100 | #[rustc_nounwind ] |
101 | pub unsafe fn simd_and<T>(x: T, y: T) -> T; |
102 | |
103 | /// "Ors" vectors elementwise. |
104 | /// |
105 | /// `T` must be a vector of integers. |
106 | #[rustc_intrinsic ] |
107 | #[rustc_nounwind ] |
108 | pub unsafe fn simd_or<T>(x: T, y: T) -> T; |
109 | |
110 | /// "Exclusive ors" vectors elementwise. |
111 | /// |
112 | /// `T` must be a vector of integers. |
113 | #[rustc_intrinsic ] |
114 | #[rustc_nounwind ] |
115 | pub unsafe fn simd_xor<T>(x: T, y: T) -> T; |
116 | |
117 | /// Numerically casts a vector, elementwise. |
118 | /// |
119 | /// `T` and `U` must be vectors of integers or floats, and must have the same length. |
120 | /// |
121 | /// When casting floats to integers, the result is truncated. Out-of-bounds result lead to UB. |
122 | /// When casting integers to floats, the result is rounded. |
123 | /// Otherwise, truncates or extends the value, maintaining the sign for signed integers. |
124 | /// |
125 | /// # Safety |
126 | /// Casting from integer types is always safe. |
127 | /// Casting between two float types is also always safe. |
128 | /// |
129 | /// Casting floats to integers truncates, following the same rules as `to_int_unchecked`. |
130 | /// Specifically, each element must: |
131 | /// * Not be `NaN` |
132 | /// * Not be infinite |
133 | /// * Be representable in the return type, after truncating off its fractional part |
134 | #[rustc_intrinsic ] |
135 | #[rustc_nounwind ] |
136 | pub unsafe fn simd_cast<T, U>(x: T) -> U; |
137 | |
138 | /// Numerically casts a vector, elementwise. |
139 | /// |
140 | /// `T` and `U` be a vectors of integers or floats, and must have the same length. |
141 | /// |
142 | /// Like `simd_cast`, but saturates float-to-integer conversions (NaN becomes 0). |
143 | /// This matches regular `as` and is always safe. |
144 | /// |
145 | /// When casting floats to integers, the result is truncated. |
146 | /// When casting integers to floats, the result is rounded. |
147 | /// Otherwise, truncates or extends the value, maintaining the sign for signed integers. |
148 | #[rustc_intrinsic ] |
149 | #[rustc_nounwind ] |
150 | pub unsafe fn simd_as<T, U>(x: T) -> U; |
151 | |
152 | /// Negates a vector elementwise. |
153 | /// |
154 | /// `T` must be a vector of integers or floats. |
155 | /// |
156 | /// Rust panics for `-<int>::Min` due to overflow, but it is not UB with this intrinsic. |
157 | #[rustc_intrinsic ] |
158 | #[rustc_nounwind ] |
159 | pub unsafe fn simd_neg<T>(x: T) -> T; |
160 | |
161 | /// Returns absolute value of a vector, elementwise. |
162 | /// |
163 | /// `T` must be a vector of floating-point primitive types. |
164 | #[rustc_intrinsic ] |
165 | #[rustc_nounwind ] |
166 | pub unsafe fn simd_fabs<T>(x: T) -> T; |
167 | |
168 | /// Returns the minimum of two vectors, elementwise. |
169 | /// |
170 | /// `T` must be a vector of floating-point primitive types. |
171 | /// |
172 | /// Follows IEEE-754 `minNum` semantics. |
173 | #[rustc_intrinsic ] |
174 | #[rustc_nounwind ] |
175 | pub unsafe fn simd_fmin<T>(x: T, y: T) -> T; |
176 | |
177 | /// Returns the maximum of two vectors, elementwise. |
178 | /// |
179 | /// `T` must be a vector of floating-point primitive types. |
180 | /// |
181 | /// Follows IEEE-754 `maxNum` semantics. |
182 | #[rustc_intrinsic ] |
183 | #[rustc_nounwind ] |
184 | pub unsafe fn simd_fmax<T>(x: T, y: T) -> T; |
185 | |
186 | /// Tests elementwise equality of two vectors. |
187 | /// |
188 | /// `T` must be a vector of integers or floats. |
189 | /// |
190 | /// `U` must be a vector of integers with the same number of elements and element size as `T`. |
191 | /// |
192 | /// Returns `0` for false and `!0` for true. |
193 | #[rustc_intrinsic ] |
194 | #[rustc_nounwind ] |
195 | pub unsafe fn simd_eq<T, U>(x: T, y: T) -> U; |
196 | |
197 | /// Tests elementwise inequality equality of two vectors. |
198 | /// |
199 | /// `T` must be a vector of integers or floats. |
200 | /// |
201 | /// `U` must be a vector of integers with the same number of elements and element size as `T`. |
202 | /// |
203 | /// Returns `0` for false and `!0` for true. |
204 | #[rustc_intrinsic ] |
205 | #[rustc_nounwind ] |
206 | pub unsafe fn simd_ne<T, U>(x: T, y: T) -> U; |
207 | |
208 | /// Tests if `x` is less than `y`, elementwise. |
209 | /// |
210 | /// `T` must be a vector of integers or floats. |
211 | /// |
212 | /// `U` must be a vector of integers with the same number of elements and element size as `T`. |
213 | /// |
214 | /// Returns `0` for false and `!0` for true. |
215 | #[rustc_intrinsic ] |
216 | #[rustc_nounwind ] |
217 | pub unsafe fn simd_lt<T, U>(x: T, y: T) -> U; |
218 | |
219 | /// Tests if `x` is less than or equal to `y`, elementwise. |
220 | /// |
221 | /// `T` must be a vector of integers or floats. |
222 | /// |
223 | /// `U` must be a vector of integers with the same number of elements and element size as `T`. |
224 | /// |
225 | /// Returns `0` for false and `!0` for true. |
226 | #[rustc_intrinsic ] |
227 | #[rustc_nounwind ] |
228 | pub unsafe fn simd_le<T, U>(x: T, y: T) -> U; |
229 | |
230 | /// Tests if `x` is greater than `y`, elementwise. |
231 | /// |
232 | /// `T` must be a vector of integers or floats. |
233 | /// |
234 | /// `U` must be a vector of integers with the same number of elements and element size as `T`. |
235 | /// |
236 | /// Returns `0` for false and `!0` for true. |
237 | #[rustc_intrinsic ] |
238 | #[rustc_nounwind ] |
239 | pub unsafe fn simd_gt<T, U>(x: T, y: T) -> U; |
240 | |
241 | /// Tests if `x` is greater than or equal to `y`, elementwise. |
242 | /// |
243 | /// `T` must be a vector of integers or floats. |
244 | /// |
245 | /// `U` must be a vector of integers with the same number of elements and element size as `T`. |
246 | /// |
247 | /// Returns `0` for false and `!0` for true. |
248 | #[rustc_intrinsic ] |
249 | #[rustc_nounwind ] |
250 | pub unsafe fn simd_ge<T, U>(x: T, y: T) -> U; |
251 | |
252 | /// Shuffles two vectors by const indices. |
253 | /// |
254 | /// `T` must be a vector. |
255 | /// |
256 | /// `U` must be a **const** vector of `u32`s. This means it must either refer to a named |
257 | /// const or be given as an inline const expression (`const { ... }`). |
258 | /// |
259 | /// `V` must be a vector with the same element type as `T` and the same length as `U`. |
260 | /// |
261 | /// Returns a new vector such that element `i` is selected from `xy[idx[i]]`, where `xy` |
262 | /// is the concatenation of `x` and `y`. It is a compile-time error if `idx[i]` is out-of-bounds |
263 | /// of `xy`. |
264 | #[rustc_intrinsic ] |
265 | #[rustc_nounwind ] |
266 | pub unsafe fn simd_shuffle<T, U, V>(x: T, y: T, idx: U) -> V; |
267 | |
268 | /// Reads a vector of pointers. |
269 | /// |
270 | /// `T` must be a vector. |
271 | /// |
272 | /// `U` must be a vector of pointers to the element type of `T`, with the same length as `T`. |
273 | /// |
274 | /// `V` must be a vector of signed integers with the same length as `T` (but any element size). |
275 | /// |
276 | /// For each pointer in `ptr`, if the corresponding value in `mask` is `!0`, read the pointer. |
277 | /// Otherwise if the corresponding value in `mask` is `0`, return the corresponding value from |
278 | /// `val`. |
279 | /// |
280 | /// # Safety |
281 | /// Unmasked values in `T` must be readable as if by `<ptr>::read` (e.g. aligned to the element |
282 | /// type). |
283 | /// |
284 | /// `mask` must only contain `0` or `!0` values. |
285 | #[rustc_intrinsic ] |
286 | #[rustc_nounwind ] |
287 | pub unsafe fn simd_gather<T, U, V>(val: T, ptr: U, mask: V) -> T; |
288 | |
289 | /// Writes to a vector of pointers. |
290 | /// |
291 | /// `T` must be a vector. |
292 | /// |
293 | /// `U` must be a vector of pointers to the element type of `T`, with the same length as `T`. |
294 | /// |
295 | /// `V` must be a vector of signed integers with the same length as `T` (but any element size). |
296 | /// |
297 | /// For each pointer in `ptr`, if the corresponding value in `mask` is `!0`, write the |
298 | /// corresponding value in `val` to the pointer. |
299 | /// Otherwise if the corresponding value in `mask` is `0`, do nothing. |
300 | /// |
301 | /// The stores happen in left-to-right order. |
302 | /// (This is relevant in case two of the stores overlap.) |
303 | /// |
304 | /// # Safety |
305 | /// Unmasked values in `T` must be writeable as if by `<ptr>::write` (e.g. aligned to the element |
306 | /// type). |
307 | /// |
308 | /// `mask` must only contain `0` or `!0` values. |
309 | #[rustc_intrinsic ] |
310 | #[rustc_nounwind ] |
311 | pub unsafe fn simd_scatter<T, U, V>(val: T, ptr: U, mask: V); |
312 | |
313 | /// Reads a vector of pointers. |
314 | /// |
315 | /// `T` must be a vector. |
316 | /// |
317 | /// `U` must be a pointer to the element type of `T` |
318 | /// |
319 | /// `V` must be a vector of signed integers with the same length as `T` (but any element size). |
320 | /// |
321 | /// For each element, if the corresponding value in `mask` is `!0`, read the corresponding |
322 | /// pointer offset from `ptr`. |
323 | /// The first element is loaded from `ptr`, the second from `ptr.wrapping_offset(1)` and so on. |
324 | /// Otherwise if the corresponding value in `mask` is `0`, return the corresponding value from |
325 | /// `val`. |
326 | /// |
327 | /// # Safety |
328 | /// Unmasked values in `T` must be readable as if by `<ptr>::read` (e.g. aligned to the element |
329 | /// type). |
330 | /// |
331 | /// `mask` must only contain `0` or `!0` values. |
332 | #[rustc_intrinsic ] |
333 | #[rustc_nounwind ] |
334 | pub unsafe fn simd_masked_load<V, U, T>(mask: V, ptr: U, val: T) -> T; |
335 | |
336 | /// Writes to a vector of pointers. |
337 | /// |
338 | /// `T` must be a vector. |
339 | /// |
340 | /// `U` must be a pointer to the element type of `T` |
341 | /// |
342 | /// `V` must be a vector of signed integers with the same length as `T` (but any element size). |
343 | /// |
344 | /// For each element, if the corresponding value in `mask` is `!0`, write the corresponding |
345 | /// value in `val` to the pointer offset from `ptr`. |
346 | /// The first element is written to `ptr`, the second to `ptr.wrapping_offset(1)` and so on. |
347 | /// Otherwise if the corresponding value in `mask` is `0`, do nothing. |
348 | /// |
349 | /// # Safety |
350 | /// Unmasked values in `T` must be writeable as if by `<ptr>::write` (e.g. aligned to the element |
351 | /// type). |
352 | /// |
353 | /// `mask` must only contain `0` or `!0` values. |
354 | #[rustc_intrinsic ] |
355 | #[rustc_nounwind ] |
356 | pub unsafe fn simd_masked_store<V, U, T>(mask: V, ptr: U, val: T); |
357 | |
358 | /// Adds two simd vectors elementwise, with saturation. |
359 | /// |
360 | /// `T` must be a vector of integer primitive types. |
361 | #[rustc_intrinsic ] |
362 | #[rustc_nounwind ] |
363 | pub unsafe fn simd_saturating_add<T>(x: T, y: T) -> T; |
364 | |
365 | /// Subtracts two simd vectors elementwise, with saturation. |
366 | /// |
367 | /// `T` must be a vector of integer primitive types. |
368 | /// |
369 | /// Subtract `rhs` from `lhs`. |
370 | #[rustc_intrinsic ] |
371 | #[rustc_nounwind ] |
372 | pub unsafe fn simd_saturating_sub<T>(lhs: T, rhs: T) -> T; |
373 | |
374 | /// Adds elements within a vector from left to right. |
375 | /// |
376 | /// `T` must be a vector of integers or floats. |
377 | /// |
378 | /// `U` must be the element type of `T`. |
379 | /// |
380 | /// Starting with the value `y`, add the elements of `x` and accumulate. |
381 | #[rustc_intrinsic ] |
382 | #[rustc_nounwind ] |
383 | pub unsafe fn simd_reduce_add_ordered<T, U>(x: T, y: U) -> U; |
384 | |
385 | /// Adds elements within a vector in arbitrary order. May also be re-associated with |
386 | /// unordered additions on the inputs/outputs. |
387 | /// |
388 | /// `T` must be a vector of integers or floats. |
389 | /// |
390 | /// `U` must be the element type of `T`. |
391 | #[rustc_intrinsic ] |
392 | #[rustc_nounwind ] |
393 | pub unsafe fn simd_reduce_add_unordered<T, U>(x: T) -> U; |
394 | |
395 | /// Multiplies elements within a vector from left to right. |
396 | /// |
397 | /// `T` must be a vector of integers or floats. |
398 | /// |
399 | /// `U` must be the element type of `T`. |
400 | /// |
401 | /// Starting with the value `y`, multiply the elements of `x` and accumulate. |
402 | #[rustc_intrinsic ] |
403 | #[rustc_nounwind ] |
404 | pub unsafe fn simd_reduce_mul_ordered<T, U>(x: T, y: U) -> U; |
405 | |
406 | /// Multiplies elements within a vector in arbitrary order. May also be re-associated with |
407 | /// unordered additions on the inputs/outputs. |
408 | /// |
409 | /// `T` must be a vector of integers or floats. |
410 | /// |
411 | /// `U` must be the element type of `T`. |
412 | #[rustc_intrinsic ] |
413 | #[rustc_nounwind ] |
414 | pub unsafe fn simd_reduce_mul_unordered<T, U>(x: T) -> U; |
415 | |
416 | /// Checks if all mask values are true. |
417 | /// |
418 | /// `T` must be a vector of integer primitive types. |
419 | /// |
420 | /// # Safety |
421 | /// `x` must contain only `0` or `!0`. |
422 | #[rustc_intrinsic ] |
423 | #[rustc_nounwind ] |
424 | pub unsafe fn simd_reduce_all<T>(x: T) -> bool; |
425 | |
426 | /// Checks if any mask value is true. |
427 | /// |
428 | /// `T` must be a vector of integer primitive types. |
429 | /// |
430 | /// # Safety |
431 | /// `x` must contain only `0` or `!0`. |
432 | #[rustc_intrinsic ] |
433 | #[rustc_nounwind ] |
434 | pub unsafe fn simd_reduce_any<T>(x: T) -> bool; |
435 | |
436 | /// Returns the maximum element of a vector. |
437 | /// |
438 | /// `T` must be a vector of integers or floats. |
439 | /// |
440 | /// `U` must be the element type of `T`. |
441 | /// |
442 | /// For floating-point values, uses IEEE-754 `maxNum`. |
443 | #[rustc_intrinsic ] |
444 | #[rustc_nounwind ] |
445 | pub unsafe fn simd_reduce_max<T, U>(x: T) -> U; |
446 | |
447 | /// Returns the minimum element of a vector. |
448 | /// |
449 | /// `T` must be a vector of integers or floats. |
450 | /// |
451 | /// `U` must be the element type of `T`. |
452 | /// |
453 | /// For floating-point values, uses IEEE-754 `minNum`. |
454 | #[rustc_intrinsic ] |
455 | #[rustc_nounwind ] |
456 | pub unsafe fn simd_reduce_min<T, U>(x: T) -> U; |
457 | |
458 | /// Logical "ands" all elements together. |
459 | /// |
460 | /// `T` must be a vector of integers or floats. |
461 | /// |
462 | /// `U` must be the element type of `T`. |
463 | #[rustc_intrinsic ] |
464 | #[rustc_nounwind ] |
465 | pub unsafe fn simd_reduce_and<T, U>(x: T) -> U; |
466 | |
467 | /// Logical "ors" all elements together. |
468 | /// |
469 | /// `T` must be a vector of integers or floats. |
470 | /// |
471 | /// `U` must be the element type of `T`. |
472 | #[rustc_intrinsic ] |
473 | #[rustc_nounwind ] |
474 | pub unsafe fn simd_reduce_or<T, U>(x: T) -> U; |
475 | |
476 | /// Logical "exclusive ors" all elements together. |
477 | /// |
478 | /// `T` must be a vector of integers or floats. |
479 | /// |
480 | /// `U` must be the element type of `T`. |
481 | #[rustc_intrinsic ] |
482 | #[rustc_nounwind ] |
483 | pub unsafe fn simd_reduce_xor<T, U>(x: T) -> U; |
484 | |
485 | /// Truncates an integer vector to a bitmask. |
486 | /// |
487 | /// `T` must be an integer vector. |
488 | /// |
489 | /// `U` must be either the smallest unsigned integer with at least as many bits as the length |
490 | /// of `T`, or the smallest array of `u8` with at least as many bits as the length of `T`. |
491 | /// |
492 | /// Each element is truncated to a single bit and packed into the result. |
493 | /// |
494 | /// No matter whether the output is an array or an unsigned integer, it is treated as a single |
495 | /// contiguous list of bits. The bitmask is always packed on the least-significant side of the |
496 | /// output, and padded with 0s in the most-significant bits. The order of the bits depends on |
497 | /// endianness: |
498 | /// |
499 | /// * On little endian, the least significant bit corresponds to the first vector element. |
500 | /// * On big endian, the least significant bit corresponds to the last vector element. |
501 | /// |
502 | /// For example, `[!0, 0, !0, !0]` packs to |
503 | /// - `0b1101u8` or `[0b1101]` on little endian, and |
504 | /// - `0b1011u8` or `[0b1011]` on big endian. |
505 | /// |
506 | /// To consider a larger example, |
507 | /// `[!0, 0, 0, 0, 0, 0, 0, 0, !0, !0, 0, 0, 0, 0, !0, 0]` packs to |
508 | /// - `0b0100001100000001u16` or `[0b00000001, 0b01000011]` on little endian, and |
509 | /// - `0b1000000011000010u16` or `[0b10000000, 0b11000010]` on big endian. |
510 | /// |
511 | /// And finally, a non-power-of-2 example with multiple bytes: |
512 | /// `[!0, !0, 0, !0, 0, 0, !0, 0, !0, 0]` packs to |
513 | /// - `0b0101001011u16` or `[0b01001011, 0b01]` on little endian, and |
514 | /// - `0b1101001010u16` or `[0b11, 0b01001010]` on big endian. |
515 | /// |
516 | /// # Safety |
517 | /// `x` must contain only `0` and `!0`. |
518 | #[rustc_intrinsic ] |
519 | #[rustc_nounwind ] |
520 | pub unsafe fn simd_bitmask<T, U>(x: T) -> U; |
521 | |
522 | /// Selects elements from a mask. |
523 | /// |
524 | /// `T` must be a vector. |
525 | /// |
526 | /// `M` must be a signed integer vector with the same length as `T` (but any element size). |
527 | /// |
528 | /// For each element, if the corresponding value in `mask` is `!0`, select the element from |
529 | /// `if_true`. If the corresponding value in `mask` is `0`, select the element from |
530 | /// `if_false`. |
531 | /// |
532 | /// # Safety |
533 | /// `mask` must only contain `0` and `!0`. |
534 | #[rustc_intrinsic ] |
535 | #[rustc_nounwind ] |
536 | pub unsafe fn simd_select<M, T>(mask: M, if_true: T, if_false: T) -> T; |
537 | |
538 | /// Selects elements from a bitmask. |
539 | /// |
540 | /// `M` must be an unsigned integer or array of `u8`, matching `simd_bitmask`. |
541 | /// |
542 | /// `T` must be a vector. |
543 | /// |
544 | /// For each element, if the bit in `mask` is `1`, select the element from |
545 | /// `if_true`. If the corresponding bit in `mask` is `0`, select the element from |
546 | /// `if_false`. |
547 | /// |
548 | /// The bitmask bit order matches `simd_bitmask`. |
549 | /// |
550 | /// # Safety |
551 | /// Padding bits must be all zero. |
552 | #[rustc_intrinsic ] |
553 | #[rustc_nounwind ] |
554 | pub unsafe fn simd_select_bitmask<M, T>(m: M, yes: T, no: T) -> T; |
555 | |
556 | /// Calculates the offset from a pointer vector elementwise, potentially |
557 | /// wrapping. |
558 | /// |
559 | /// `T` must be a vector of pointers. |
560 | /// |
561 | /// `U` must be a vector of `isize` or `usize` with the same number of elements as `T`. |
562 | /// |
563 | /// Operates as if by `<ptr>::wrapping_offset`. |
564 | #[rustc_intrinsic ] |
565 | #[rustc_nounwind ] |
566 | pub unsafe fn simd_arith_offset<T, U>(ptr: T, offset: U) -> T; |
567 | |
568 | /// Casts a vector of pointers. |
569 | /// |
570 | /// `T` and `U` must be vectors of pointers with the same number of elements. |
571 | #[rustc_intrinsic ] |
572 | #[rustc_nounwind ] |
573 | pub unsafe fn simd_cast_ptr<T, U>(ptr: T) -> U; |
574 | |
575 | /// Exposes a vector of pointers as a vector of addresses. |
576 | /// |
577 | /// `T` must be a vector of pointers. |
578 | /// |
579 | /// `U` must be a vector of `usize` with the same length as `T`. |
580 | #[rustc_intrinsic ] |
581 | #[rustc_nounwind ] |
582 | pub unsafe fn simd_expose_provenance<T, U>(ptr: T) -> U; |
583 | |
584 | /// Creates a vector of pointers from a vector of addresses. |
585 | /// |
586 | /// `T` must be a vector of `usize`. |
587 | /// |
588 | /// `U` must be a vector of pointers, with the same length as `T`. |
589 | #[rustc_intrinsic ] |
590 | #[rustc_nounwind ] |
591 | pub unsafe fn simd_with_exposed_provenance<T, U>(addr: T) -> U; |
592 | |
593 | /// Swaps bytes of each element. |
594 | /// |
595 | /// `T` must be a vector of integers. |
596 | #[rustc_intrinsic ] |
597 | #[rustc_nounwind ] |
598 | pub unsafe fn simd_bswap<T>(x: T) -> T; |
599 | |
600 | /// Reverses bits of each element. |
601 | /// |
602 | /// `T` must be a vector of integers. |
603 | #[rustc_intrinsic ] |
604 | #[rustc_nounwind ] |
605 | pub unsafe fn simd_bitreverse<T>(x: T) -> T; |
606 | |
607 | /// Counts the leading zeros of each element. |
608 | /// |
609 | /// `T` must be a vector of integers. |
610 | #[rustc_intrinsic ] |
611 | #[rustc_nounwind ] |
612 | pub unsafe fn simd_ctlz<T>(x: T) -> T; |
613 | |
614 | /// Counts the number of ones in each element. |
615 | /// |
616 | /// `T` must be a vector of integers. |
617 | #[rustc_intrinsic ] |
618 | #[rustc_nounwind ] |
619 | pub unsafe fn simd_ctpop<T>(x: T) -> T; |
620 | |
621 | /// Counts the trailing zeros of each element. |
622 | /// |
623 | /// `T` must be a vector of integers. |
624 | #[rustc_intrinsic ] |
625 | #[rustc_nounwind ] |
626 | pub unsafe fn simd_cttz<T>(x: T) -> T; |
627 | |
628 | /// Rounds up each element to the next highest integer-valued float. |
629 | /// |
630 | /// `T` must be a vector of floats. |
631 | #[rustc_intrinsic ] |
632 | #[rustc_nounwind ] |
633 | pub unsafe fn simd_ceil<T>(x: T) -> T; |
634 | |
635 | /// Rounds down each element to the next lowest integer-valued float. |
636 | /// |
637 | /// `T` must be a vector of floats. |
638 | #[rustc_intrinsic ] |
639 | #[rustc_nounwind ] |
640 | pub unsafe fn simd_floor<T>(x: T) -> T; |
641 | |
642 | /// Rounds each element to the closest integer-valued float. |
643 | /// Ties are resolved by rounding away from 0. |
644 | /// |
645 | /// `T` must be a vector of floats. |
646 | #[rustc_intrinsic ] |
647 | #[rustc_nounwind ] |
648 | pub unsafe fn simd_round<T>(x: T) -> T; |
649 | |
650 | /// Returns the integer part of each element as an integer-valued float. |
651 | /// In other words, non-integer values are truncated towards zero. |
652 | /// |
653 | /// `T` must be a vector of floats. |
654 | #[rustc_intrinsic ] |
655 | #[rustc_nounwind ] |
656 | pub unsafe fn simd_trunc<T>(x: T) -> T; |
657 | |
658 | /// Takes the square root of each element. |
659 | /// |
660 | /// `T` must be a vector of floats. |
661 | #[rustc_intrinsic ] |
662 | #[rustc_nounwind ] |
663 | pub unsafe fn simd_fsqrt<T>(x: T) -> T; |
664 | |
665 | /// Computes `(x*y) + z` for each element, but without any intermediate rounding. |
666 | /// |
667 | /// `T` must be a vector of floats. |
668 | #[rustc_intrinsic ] |
669 | #[rustc_nounwind ] |
670 | pub unsafe fn simd_fma<T>(x: T, y: T, z: T) -> T; |
671 | |
672 | /// Computes `(x*y) + z` for each element, non-deterministically executing either |
673 | /// a fused multiply-add or two operations with rounding of the intermediate result. |
674 | /// |
675 | /// The operation is fused if the code generator determines that target instruction |
676 | /// set has support for a fused operation, and that the fused operation is more efficient |
677 | /// than the equivalent, separate pair of mul and add instructions. It is unspecified |
678 | /// whether or not a fused operation is selected, and that may depend on optimization |
679 | /// level and context, for example. It may even be the case that some SIMD lanes get fused |
680 | /// and others do not. |
681 | /// |
682 | /// `T` must be a vector of floats. |
683 | #[rustc_intrinsic ] |
684 | #[rustc_nounwind ] |
685 | pub unsafe fn simd_relaxed_fma<T>(x: T, y: T, z: T) -> T; |
686 | |
687 | // Computes the sine of each element. |
688 | /// |
689 | /// `T` must be a vector of floats. |
690 | #[rustc_intrinsic ] |
691 | #[rustc_nounwind ] |
692 | pub unsafe fn simd_fsin<T>(a: T) -> T; |
693 | |
694 | // Computes the cosine of each element. |
695 | /// |
696 | /// `T` must be a vector of floats. |
697 | #[rustc_intrinsic ] |
698 | #[rustc_nounwind ] |
699 | pub unsafe fn simd_fcos<T>(a: T) -> T; |
700 | |
701 | // Computes the exponential function of each element. |
702 | /// |
703 | /// `T` must be a vector of floats. |
704 | #[rustc_intrinsic ] |
705 | #[rustc_nounwind ] |
706 | pub unsafe fn simd_fexp<T>(a: T) -> T; |
707 | |
708 | // Computes 2 raised to the power of each element. |
709 | /// |
710 | /// `T` must be a vector of floats. |
711 | #[rustc_intrinsic ] |
712 | #[rustc_nounwind ] |
713 | pub unsafe fn simd_fexp2<T>(a: T) -> T; |
714 | |
715 | // Computes the base 10 logarithm of each element. |
716 | /// |
717 | /// `T` must be a vector of floats. |
718 | #[rustc_intrinsic ] |
719 | #[rustc_nounwind ] |
720 | pub unsafe fn simd_flog10<T>(a: T) -> T; |
721 | |
722 | // Computes the base 2 logarithm of each element. |
723 | /// |
724 | /// `T` must be a vector of floats. |
725 | #[rustc_intrinsic ] |
726 | #[rustc_nounwind ] |
727 | pub unsafe fn simd_flog2<T>(a: T) -> T; |
728 | |
729 | // Computes the natural logarithm of each element. |
730 | /// |
731 | /// `T` must be a vector of floats. |
732 | #[rustc_intrinsic ] |
733 | #[rustc_nounwind ] |
734 | pub unsafe fn simd_flog<T>(a: T) -> T; |
735 | |