1//! This module contains the LLVM intrinsics bindings that provide the functionality for this
2//! crate.
3//!
4//! The LLVM assembly language is documented here: <https://llvm.org/docs/LangRef.html>
5//!
6//! A quick glossary of jargon that may appear in this module, mostly paraphrasing LLVM's LangRef:
7//! - poison: "undefined behavior as a value". specifically, it is like uninit memory (such as padding bytes). it is "safe" to create poison, BUT
8//! poison MUST NOT be observed from safe code, as operations on poison return poison, like NaN. unlike NaN, which has defined comparisons,
9//! poison is neither true nor false, and LLVM may also convert it to undef (at which point it is both). so, it can't be conditioned on, either.
10//! - undef: "a value that is every value". functionally like poison, insofar as Rust is concerned. poison may become this. note:
11//! this means that division by poison or undef is like division by zero, which means it inflicts...
12//! - "UB": poison and undef cover most of what people call "UB". "UB" means this operation immediately invalidates the program:
13//! LLVM is allowed to lower it to `ud2` or other opcodes that may cause an illegal instruction exception, and this is the "good end".
14//! The "bad end" is that LLVM may reverse time to the moment control flow diverged on a path towards undefined behavior,
15//! and destroy the other branch, potentially deleting safe code and violating Rust's `unsafe` contract.
16//!
17//! Note that according to LLVM, vectors are not arrays, but they are equivalent when stored to and loaded from memory.
18//!
19//! Unless stated otherwise, all intrinsics for binary operations require SIMD vectors of equal types and lengths.
20
21// These intrinsics aren't linked directly from LLVM and are mostly undocumented, however they are
22// mostly lowered to the matching LLVM instructions by the compiler in a fairly straightforward manner.
23// The associated LLVM instruction or intrinsic is documented alongside each Rust intrinsic function.
24extern "platform-intrinsic" {
25 /// add/fadd
26 pub(crate) fn simd_add<T>(x: T, y: T) -> T;
27
28 /// sub/fsub
29 pub(crate) fn simd_sub<T>(lhs: T, rhs: T) -> T;
30
31 /// mul/fmul
32 pub(crate) fn simd_mul<T>(x: T, y: T) -> T;
33
34 /// udiv/sdiv/fdiv
35 /// ints and uints: {s,u}div incur UB if division by zero occurs.
36 /// ints: sdiv is UB for int::MIN / -1.
37 /// floats: fdiv is never UB, but may create NaNs or infinities.
38 pub(crate) fn simd_div<T>(lhs: T, rhs: T) -> T;
39
40 /// urem/srem/frem
41 /// ints and uints: {s,u}rem incur UB if division by zero occurs.
42 /// ints: srem is UB for int::MIN / -1.
43 /// floats: frem is equivalent to libm::fmod in the "default" floating point environment, sans errno.
44 pub(crate) fn simd_rem<T>(lhs: T, rhs: T) -> T;
45
46 /// shl
47 /// for (u)ints. poison if rhs >= lhs::BITS
48 pub(crate) fn simd_shl<T>(lhs: T, rhs: T) -> T;
49
50 /// ints: ashr
51 /// uints: lshr
52 /// poison if rhs >= lhs::BITS
53 pub(crate) fn simd_shr<T>(lhs: T, rhs: T) -> T;
54
55 /// and
56 pub(crate) fn simd_and<T>(x: T, y: T) -> T;
57
58 /// or
59 pub(crate) fn simd_or<T>(x: T, y: T) -> T;
60
61 /// xor
62 pub(crate) fn simd_xor<T>(x: T, y: T) -> T;
63
64 /// fptoui/fptosi/uitofp/sitofp
65 /// casting floats to integers is truncating, so it is safe to convert values like e.g. 1.5
66 /// but the truncated value must fit in the target type or the result is poison.
67 /// use `simd_as` instead for a cast that performs a saturating conversion.
68 pub(crate) fn simd_cast<T, U>(x: T) -> U;
69 /// follows Rust's `T as U` semantics, including saturating float casts
70 /// which amounts to the same as `simd_cast` for many cases
71 pub(crate) fn simd_as<T, U>(x: T) -> U;
72
73 /// neg/fneg
74 /// ints: ultimately becomes a call to cg_ssa's BuilderMethods::neg. cg_llvm equates this to `simd_sub(Simd::splat(0), x)`.
75 /// floats: LLVM's fneg, which changes the floating point sign bit. Some arches have instructions for it.
76 /// Rust panics for Neg::neg(int::MIN) due to overflow, but it is not UB in LLVM without `nsw`.
77 pub(crate) fn simd_neg<T>(x: T) -> T;
78
79 /// fabs
80 pub(crate) fn simd_fabs<T>(x: T) -> T;
81
82 // minnum/maxnum
83 pub(crate) fn simd_fmin<T>(x: T, y: T) -> T;
84 pub(crate) fn simd_fmax<T>(x: T, y: T) -> T;
85
86 // these return Simd<int, N> with the same BITS size as the inputs
87 pub(crate) fn simd_eq<T, U>(x: T, y: T) -> U;
88 pub(crate) fn simd_ne<T, U>(x: T, y: T) -> U;
89 pub(crate) fn simd_lt<T, U>(x: T, y: T) -> U;
90 pub(crate) fn simd_le<T, U>(x: T, y: T) -> U;
91 pub(crate) fn simd_gt<T, U>(x: T, y: T) -> U;
92 pub(crate) fn simd_ge<T, U>(x: T, y: T) -> U;
93
94 // shufflevector
95 // idx: LLVM calls it a "shuffle mask vector constant", a vector of i32s
96 pub(crate) fn simd_shuffle<T, U, V>(x: T, y: T, idx: U) -> V;
97
98 /// llvm.masked.gather
99 /// like a loop of pointer reads
100 /// val: vector of values to select if a lane is masked
101 /// ptr: vector of pointers to read from
102 /// mask: a "wide" mask of integers, selects as if simd_select(mask, read(ptr), val)
103 /// note, the LLVM intrinsic accepts a mask vector of `<N x i1>`
104 /// FIXME: review this if/when we fix up our mask story in general?
105 pub(crate) fn simd_gather<T, U, V>(val: T, ptr: U, mask: V) -> T;
106 /// llvm.masked.scatter
107 /// like gather, but more spicy, as it writes instead of reads
108 pub(crate) fn simd_scatter<T, U, V>(val: T, ptr: U, mask: V);
109
110 // {s,u}add.sat
111 pub(crate) fn simd_saturating_add<T>(x: T, y: T) -> T;
112
113 // {s,u}sub.sat
114 pub(crate) fn simd_saturating_sub<T>(lhs: T, rhs: T) -> T;
115
116 // reductions
117 // llvm.vector.reduce.{add,fadd}
118 pub(crate) fn simd_reduce_add_ordered<T, U>(x: T, y: U) -> U;
119 // llvm.vector.reduce.{mul,fmul}
120 pub(crate) fn simd_reduce_mul_ordered<T, U>(x: T, y: U) -> U;
121 #[allow(unused)]
122 pub(crate) fn simd_reduce_all<T>(x: T) -> bool;
123 #[allow(unused)]
124 pub(crate) fn simd_reduce_any<T>(x: T) -> bool;
125 pub(crate) fn simd_reduce_max<T, U>(x: T) -> U;
126 pub(crate) fn simd_reduce_min<T, U>(x: T) -> U;
127 pub(crate) fn simd_reduce_and<T, U>(x: T) -> U;
128 pub(crate) fn simd_reduce_or<T, U>(x: T) -> U;
129 pub(crate) fn simd_reduce_xor<T, U>(x: T) -> U;
130
131 // truncate integer vector to bitmask
132 // `fn simd_bitmask(vector) -> unsigned integer` takes a vector of integers and
133 // returns either an unsigned integer or array of `u8`.
134 // Every element in the vector becomes a single bit in the returned bitmask.
135 // If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits.
136 // The bit order of the result depends on the byte endianness. LSB-first for little
137 // endian and MSB-first for big endian.
138 //
139 // UB if called on a vector with values other than 0 and -1.
140 #[allow(unused)]
141 pub(crate) fn simd_bitmask<T, U>(x: T) -> U;
142
143 // select
144 // first argument is a vector of integers, -1 (all bits 1) is "true"
145 // logically equivalent to (yes & m) | (no & (m^-1),
146 // but you can use it on floats.
147 pub(crate) fn simd_select<M, T>(m: M, yes: T, no: T) -> T;
148 #[allow(unused)]
149 pub(crate) fn simd_select_bitmask<M, T>(m: M, yes: T, no: T) -> T;
150
151 /// getelementptr (without inbounds)
152 /// equivalent to wrapping_offset
153 pub(crate) fn simd_arith_offset<T, U>(ptr: T, offset: U) -> T;
154
155 /// equivalent to `T as U` semantics, specifically for pointers
156 pub(crate) fn simd_cast_ptr<T, U>(ptr: T) -> U;
157
158 /// expose a pointer as an address
159 pub(crate) fn simd_expose_addr<T, U>(ptr: T) -> U;
160
161 /// convert an exposed address back to a pointer
162 pub(crate) fn simd_from_exposed_addr<T, U>(addr: T) -> U;
163
164 // Integer operations
165 pub(crate) fn simd_bswap<T>(x: T) -> T;
166 pub(crate) fn simd_bitreverse<T>(x: T) -> T;
167 pub(crate) fn simd_ctlz<T>(x: T) -> T;
168 pub(crate) fn simd_cttz<T>(x: T) -> T;
169}
170