1macro_rules! force_eval {
2 ($e:expr) => {
3 unsafe { ::core::ptr::read_volatile(&$e) }
4 };
5}
6
7#[cfg(not(debug_assertions))]
8macro_rules! i {
9 ($array:expr, $index:expr) => {
10 unsafe { *$array.get_unchecked($index) }
11 };
12 ($array:expr, $index:expr, = , $rhs:expr) => {
13 unsafe {
14 *$array.get_unchecked_mut($index) = $rhs;
15 }
16 };
17 ($array:expr, $index:expr, += , $rhs:expr) => {
18 unsafe {
19 *$array.get_unchecked_mut($index) += $rhs;
20 }
21 };
22 ($array:expr, $index:expr, -= , $rhs:expr) => {
23 unsafe {
24 *$array.get_unchecked_mut($index) -= $rhs;
25 }
26 };
27 ($array:expr, $index:expr, &= , $rhs:expr) => {
28 unsafe {
29 *$array.get_unchecked_mut($index) &= $rhs;
30 }
31 };
32 ($array:expr, $index:expr, == , $rhs:expr) => {
33 unsafe { *$array.get_unchecked_mut($index) == $rhs }
34 };
35}
36
37#[cfg(debug_assertions)]
38macro_rules! i {
39 ($array:expr, $index:expr) => {
40 *$array.get($index).unwrap()
41 };
42 ($array:expr, $index:expr, = , $rhs:expr) => {
43 *$array.get_mut($index).unwrap() = $rhs;
44 };
45 ($array:expr, $index:expr, -= , $rhs:expr) => {
46 *$array.get_mut($index).unwrap() -= $rhs;
47 };
48 ($array:expr, $index:expr, += , $rhs:expr) => {
49 *$array.get_mut($index).unwrap() += $rhs;
50 };
51 ($array:expr, $index:expr, &= , $rhs:expr) => {
52 *$array.get_mut($index).unwrap() &= $rhs;
53 };
54 ($array:expr, $index:expr, == , $rhs:expr) => {
55 *$array.get_mut($index).unwrap() == $rhs
56 };
57}
58
59// Temporary macro to avoid panic codegen for division (in debug mode too). At
60// the time of this writing this is only used in a few places, and once
61// rust-lang/rust#72751 is fixed then this macro will no longer be necessary and
62// the native `/` operator can be used and panics won't be codegen'd.
63#[cfg(any(debug_assertions, not(intrinsics_enabled)))]
64macro_rules! div {
65 ($a:expr, $b:expr) => {
66 $a / $b
67 };
68}
69
70#[cfg(all(not(debug_assertions), intrinsics_enabled))]
71macro_rules! div {
72 ($a:expr, $b:expr) => {
73 unsafe { core::intrinsics::unchecked_div($a, $b) }
74 };
75}
76
77// `support` may be public for testing
78#[macro_use]
79#[cfg(feature = "unstable-public-internals")]
80pub mod support;
81
82#[macro_use]
83#[cfg(not(feature = "unstable-public-internals"))]
84pub(crate) mod support;
85
86cfg_if! {
87 if #[cfg(feature = "unstable-public-internals")] {
88 pub mod generic;
89 } else {
90 mod generic;
91 }
92}
93
94// Private modules
95mod arch;
96mod expo2;
97mod k_cos;
98mod k_cosf;
99mod k_expo2;
100mod k_expo2f;
101mod k_sin;
102mod k_sinf;
103mod k_tan;
104mod k_tanf;
105mod rem_pio2;
106mod rem_pio2_large;
107mod rem_pio2f;
108
109// Private re-imports
110use self::expo2::expo2;
111use self::k_cos::k_cos;
112use self::k_cosf::k_cosf;
113use self::k_expo2::k_expo2;
114use self::k_expo2f::k_expo2f;
115use self::k_sin::k_sin;
116use self::k_sinf::k_sinf;
117use self::k_tan::k_tan;
118use self::k_tanf::k_tanf;
119use self::rem_pio2::rem_pio2;
120use self::rem_pio2_large::rem_pio2_large;
121use self::rem_pio2f::rem_pio2f;
122#[allow(unused_imports)]
123use self::support::{CastFrom, CastInto, DFloat, DInt, Float, HFloat, HInt, Int, IntTy, MinInt};
124
125// Public modules
126mod acos;
127mod acosf;
128mod acosh;
129mod acoshf;
130mod asin;
131mod asinf;
132mod asinh;
133mod asinhf;
134mod atan;
135mod atan2;
136mod atan2f;
137mod atanf;
138mod atanh;
139mod atanhf;
140mod cbrt;
141mod cbrtf;
142mod ceil;
143mod copysign;
144mod cos;
145mod cosf;
146mod cosh;
147mod coshf;
148mod erf;
149mod erff;
150mod exp;
151mod exp10;
152mod exp10f;
153mod exp2;
154mod exp2f;
155mod expf;
156mod expm1;
157mod expm1f;
158mod fabs;
159mod fdim;
160mod floor;
161mod fma;
162mod fma_wide;
163mod fmin_fmax;
164mod fminimum_fmaximum;
165mod fminimum_fmaximum_num;
166mod fmod;
167mod frexp;
168mod frexpf;
169mod hypot;
170mod hypotf;
171mod ilogb;
172mod ilogbf;
173mod j0;
174mod j0f;
175mod j1;
176mod j1f;
177mod jn;
178mod jnf;
179mod ldexp;
180mod lgamma;
181mod lgamma_r;
182mod lgammaf;
183mod lgammaf_r;
184mod log;
185mod log10;
186mod log10f;
187mod log1p;
188mod log1pf;
189mod log2;
190mod log2f;
191mod logf;
192mod modf;
193mod modff;
194mod nextafter;
195mod nextafterf;
196mod pow;
197mod powf;
198mod remainder;
199mod remainderf;
200mod remquo;
201mod remquof;
202mod rint;
203mod round;
204mod roundeven;
205mod scalbn;
206mod sin;
207mod sincos;
208mod sincosf;
209mod sinf;
210mod sinh;
211mod sinhf;
212mod sqrt;
213mod tan;
214mod tanf;
215mod tanh;
216mod tanhf;
217mod tgamma;
218mod tgammaf;
219mod trunc;
220
221// Use separated imports instead of {}-grouped imports for easier merging.
222pub use self::acos::acos;
223pub use self::acosf::acosf;
224pub use self::acosh::acosh;
225pub use self::acoshf::acoshf;
226pub use self::asin::asin;
227pub use self::asinf::asinf;
228pub use self::asinh::asinh;
229pub use self::asinhf::asinhf;
230pub use self::atan::atan;
231pub use self::atan2::atan2;
232pub use self::atan2f::atan2f;
233pub use self::atanf::atanf;
234pub use self::atanh::atanh;
235pub use self::atanhf::atanhf;
236pub use self::cbrt::cbrt;
237pub use self::cbrtf::cbrtf;
238pub use self::ceil::{ceil, ceilf};
239pub use self::copysign::{copysign, copysignf};
240pub use self::cos::cos;
241pub use self::cosf::cosf;
242pub use self::cosh::cosh;
243pub use self::coshf::coshf;
244pub use self::erf::{erf, erfc};
245pub use self::erff::{erfcf, erff};
246pub use self::exp::exp;
247pub use self::exp2::exp2;
248pub use self::exp2f::exp2f;
249pub use self::exp10::exp10;
250pub use self::exp10f::exp10f;
251pub use self::expf::expf;
252pub use self::expm1::expm1;
253pub use self::expm1f::expm1f;
254pub use self::fabs::{fabs, fabsf};
255pub use self::fdim::{fdim, fdimf};
256pub use self::floor::{floor, floorf};
257pub use self::fma::fma;
258pub use self::fma_wide::fmaf;
259pub use self::fmin_fmax::{fmax, fmaxf, fmin, fminf};
260pub use self::fminimum_fmaximum::{fmaximum, fmaximumf, fminimum, fminimumf};
261pub use self::fminimum_fmaximum_num::{fmaximum_num, fmaximum_numf, fminimum_num, fminimum_numf};
262pub use self::fmod::{fmod, fmodf};
263pub use self::frexp::frexp;
264pub use self::frexpf::frexpf;
265pub use self::hypot::hypot;
266pub use self::hypotf::hypotf;
267pub use self::ilogb::ilogb;
268pub use self::ilogbf::ilogbf;
269pub use self::j0::{j0, y0};
270pub use self::j0f::{j0f, y0f};
271pub use self::j1::{j1, y1};
272pub use self::j1f::{j1f, y1f};
273pub use self::jn::{jn, yn};
274pub use self::jnf::{jnf, ynf};
275pub use self::ldexp::{ldexp, ldexpf};
276pub use self::lgamma::lgamma;
277pub use self::lgamma_r::lgamma_r;
278pub use self::lgammaf::lgammaf;
279pub use self::lgammaf_r::lgammaf_r;
280pub use self::log::log;
281pub use self::log1p::log1p;
282pub use self::log1pf::log1pf;
283pub use self::log2::log2;
284pub use self::log2f::log2f;
285pub use self::log10::log10;
286pub use self::log10f::log10f;
287pub use self::logf::logf;
288pub use self::modf::modf;
289pub use self::modff::modff;
290pub use self::nextafter::nextafter;
291pub use self::nextafterf::nextafterf;
292pub use self::pow::pow;
293pub use self::powf::powf;
294pub use self::remainder::remainder;
295pub use self::remainderf::remainderf;
296pub use self::remquo::remquo;
297pub use self::remquof::remquof;
298pub use self::rint::{rint, rintf};
299pub use self::round::{round, roundf};
300pub use self::roundeven::{roundeven, roundevenf};
301pub use self::scalbn::{scalbn, scalbnf};
302pub use self::sin::sin;
303pub use self::sincos::sincos;
304pub use self::sincosf::sincosf;
305pub use self::sinf::sinf;
306pub use self::sinh::sinh;
307pub use self::sinhf::sinhf;
308pub use self::sqrt::{sqrt, sqrtf};
309pub use self::tan::tan;
310pub use self::tanf::tanf;
311pub use self::tanh::tanh;
312pub use self::tanhf::tanhf;
313pub use self::tgamma::tgamma;
314pub use self::tgammaf::tgammaf;
315pub use self::trunc::{trunc, truncf};
316
317cfg_if! {
318 if #[cfg(f16_enabled)] {
319 // verify-sorted-start
320 pub use self::ceil::ceilf16;
321 pub use self::copysign::copysignf16;
322 pub use self::fabs::fabsf16;
323 pub use self::fdim::fdimf16;
324 pub use self::floor::floorf16;
325 pub use self::fmin_fmax::{fmaxf16, fminf16};
326 pub use self::fminimum_fmaximum::{fmaximumf16, fminimumf16};
327 pub use self::fminimum_fmaximum_num::{fmaximum_numf16, fminimum_numf16};
328 pub use self::fmod::fmodf16;
329 pub use self::ldexp::ldexpf16;
330 pub use self::rint::rintf16;
331 pub use self::round::roundf16;
332 pub use self::roundeven::roundevenf16;
333 pub use self::scalbn::scalbnf16;
334 pub use self::sqrt::sqrtf16;
335 pub use self::trunc::truncf16;
336 // verify-sorted-end
337
338 #[allow(unused_imports)]
339 pub(crate) use self::fma_wide::fmaf16;
340 }
341}
342
343cfg_if! {
344 if #[cfg(f128_enabled)] {
345 // verify-sorted-start
346 pub use self::ceil::ceilf128;
347 pub use self::copysign::copysignf128;
348 pub use self::fabs::fabsf128;
349 pub use self::fdim::fdimf128;
350 pub use self::floor::floorf128;
351 pub use self::fma::fmaf128;
352 pub use self::fmin_fmax::{fmaxf128, fminf128};
353 pub use self::fminimum_fmaximum::{fmaximumf128, fminimumf128};
354 pub use self::fminimum_fmaximum_num::{fmaximum_numf128, fminimum_numf128};
355 pub use self::fmod::fmodf128;
356 pub use self::ldexp::ldexpf128;
357 pub use self::rint::rintf128;
358 pub use self::round::roundf128;
359 pub use self::roundeven::roundevenf128;
360 pub use self::scalbn::scalbnf128;
361 pub use self::sqrt::sqrtf128;
362 pub use self::trunc::truncf128;
363 // verify-sorted-end
364 }
365}
366
367#[inline]
368fn get_high_word(x: f64) -> u32 {
369 (x.to_bits() >> 32) as u32
370}
371
372#[inline]
373fn get_low_word(x: f64) -> u32 {
374 x.to_bits() as u32
375}
376
377#[inline]
378fn with_set_high_word(f: f64, hi: u32) -> f64 {
379 let mut tmp: u64 = f.to_bits();
380 tmp &= 0x00000000_ffffffff;
381 tmp |= (hi as u64) << 32;
382 f64::from_bits(tmp)
383}
384
385#[inline]
386fn with_set_low_word(f: f64, lo: u32) -> f64 {
387 let mut tmp: u64 = f.to_bits();
388 tmp &= 0xffffffff_00000000;
389 tmp |= lo as u64;
390 f64::from_bits(tmp)
391}
392
393#[inline]
394fn combine_words(hi: u32, lo: u32) -> f64 {
395 f64::from_bits(((hi as u64) << 32) | lo as u64)
396}
397