1macro_rules! force_eval {
2 ($e:expr) => {
3 unsafe { ::core::ptr::read_volatile(&$e) }
4 };
5}
6
7#[cfg(not(debug_assertions))]
8macro_rules! i {
9 ($array:expr, $index:expr) => {
10 unsafe { *$array.get_unchecked($index) }
11 };
12 ($array:expr, $index:expr, = , $rhs:expr) => {
13 unsafe {
14 *$array.get_unchecked_mut($index) = $rhs;
15 }
16 };
17 ($array:expr, $index:expr, += , $rhs:expr) => {
18 unsafe {
19 *$array.get_unchecked_mut($index) += $rhs;
20 }
21 };
22 ($array:expr, $index:expr, -= , $rhs:expr) => {
23 unsafe {
24 *$array.get_unchecked_mut($index) -= $rhs;
25 }
26 };
27 ($array:expr, $index:expr, &= , $rhs:expr) => {
28 unsafe {
29 *$array.get_unchecked_mut($index) &= $rhs;
30 }
31 };
32 ($array:expr, $index:expr, == , $rhs:expr) => {
33 unsafe { *$array.get_unchecked_mut($index) == $rhs }
34 };
35}
36
37#[cfg(debug_assertions)]
38macro_rules! i {
39 ($array:expr, $index:expr) => {
40 *$array.get($index).unwrap()
41 };
42 ($array:expr, $index:expr, = , $rhs:expr) => {
43 *$array.get_mut($index).unwrap() = $rhs;
44 };
45 ($array:expr, $index:expr, -= , $rhs:expr) => {
46 *$array.get_mut($index).unwrap() -= $rhs;
47 };
48 ($array:expr, $index:expr, += , $rhs:expr) => {
49 *$array.get_mut($index).unwrap() += $rhs;
50 };
51 ($array:expr, $index:expr, &= , $rhs:expr) => {
52 *$array.get_mut($index).unwrap() &= $rhs;
53 };
54 ($array:expr, $index:expr, == , $rhs:expr) => {
55 *$array.get_mut($index).unwrap() == $rhs
56 };
57}
58
59// Temporary macro to avoid panic codegen for division (in debug mode too). At
60// the time of this writing this is only used in a few places, and once
61// rust-lang/rust#72751 is fixed then this macro will no longer be necessary and
62// the native `/` operator can be used and panics won't be codegen'd.
63#[cfg(any(debug_assertions, not(intrinsics_enabled)))]
64macro_rules! div {
65 ($a:expr, $b:expr) => {
66 $a / $b
67 };
68}
69
70#[cfg(all(not(debug_assertions), intrinsics_enabled))]
71macro_rules! div {
72 ($a:expr, $b:expr) => {
73 unsafe { core::intrinsics::unchecked_div($a, $b) }
74 };
75}
76
77// `support` may be public for testing
78#[macro_use]
79#[cfg(feature = "unstable-public-internals")]
80pub mod support;
81
82#[macro_use]
83#[cfg(not(feature = "unstable-public-internals"))]
84pub(crate) mod support;
85
86cfg_if! {
87 if #[cfg(feature = "unstable-public-internals")] {
88 pub mod generic;
89 } else {
90 mod generic;
91 }
92}
93
94// Private modules
95mod arch;
96mod expo2;
97mod k_cos;
98mod k_cosf;
99mod k_expo2;
100mod k_expo2f;
101mod k_sin;
102mod k_sinf;
103mod k_tan;
104mod k_tanf;
105mod rem_pio2;
106mod rem_pio2_large;
107mod rem_pio2f;
108
109// Private re-imports
110use self::expo2::expo2;
111use self::k_cos::k_cos;
112use self::k_cosf::k_cosf;
113use self::k_expo2::k_expo2;
114use self::k_expo2f::k_expo2f;
115use self::k_sin::k_sin;
116use self::k_sinf::k_sinf;
117use self::k_tan::k_tan;
118use self::k_tanf::k_tanf;
119use self::rem_pio2::rem_pio2;
120use self::rem_pio2_large::rem_pio2_large;
121use self::rem_pio2f::rem_pio2f;
122#[allow(unused_imports)]
123use self::support::{CastFrom, CastInto, DFloat, DInt, Float, HFloat, HInt, Int, IntTy, MinInt};
124
125// Public modules
126mod acos;
127mod acosf;
128mod acosh;
129mod acoshf;
130mod asin;
131mod asinf;
132mod asinh;
133mod asinhf;
134mod atan;
135mod atan2;
136mod atan2f;
137mod atanf;
138mod atanh;
139mod atanhf;
140mod cbrt;
141mod cbrtf;
142mod ceil;
143mod ceilf;
144mod copysign;
145mod copysignf;
146mod cos;
147mod cosf;
148mod cosh;
149mod coshf;
150mod erf;
151mod erff;
152mod exp;
153mod exp10;
154mod exp10f;
155mod exp2;
156mod exp2f;
157mod expf;
158mod expm1;
159mod expm1f;
160mod fabs;
161mod fabsf;
162mod fdim;
163mod fdimf;
164mod floor;
165mod floorf;
166mod fma;
167mod fma_wide;
168mod fmin_fmax;
169mod fminimum_fmaximum;
170mod fminimum_fmaximum_num;
171mod fmod;
172mod fmodf;
173mod frexp;
174mod frexpf;
175mod hypot;
176mod hypotf;
177mod ilogb;
178mod ilogbf;
179mod j0;
180mod j0f;
181mod j1;
182mod j1f;
183mod jn;
184mod jnf;
185mod ldexp;
186mod ldexpf;
187mod lgamma;
188mod lgamma_r;
189mod lgammaf;
190mod lgammaf_r;
191mod log;
192mod log10;
193mod log10f;
194mod log1p;
195mod log1pf;
196mod log2;
197mod log2f;
198mod logf;
199mod modf;
200mod modff;
201mod nextafter;
202mod nextafterf;
203mod pow;
204mod powf;
205mod remainder;
206mod remainderf;
207mod remquo;
208mod remquof;
209mod rint;
210mod round;
211mod roundeven;
212mod roundf;
213mod scalbn;
214mod scalbnf;
215mod sin;
216mod sincos;
217mod sincosf;
218mod sinf;
219mod sinh;
220mod sinhf;
221mod sqrt;
222mod sqrtf;
223mod tan;
224mod tanf;
225mod tanh;
226mod tanhf;
227mod tgamma;
228mod tgammaf;
229mod trunc;
230mod truncf;
231
232// Use separated imports instead of {}-grouped imports for easier merging.
233pub use self::acos::acos;
234pub use self::acosf::acosf;
235pub use self::acosh::acosh;
236pub use self::acoshf::acoshf;
237pub use self::asin::asin;
238pub use self::asinf::asinf;
239pub use self::asinh::asinh;
240pub use self::asinhf::asinhf;
241pub use self::atan::atan;
242pub use self::atan2::atan2;
243pub use self::atan2f::atan2f;
244pub use self::atanf::atanf;
245pub use self::atanh::atanh;
246pub use self::atanhf::atanhf;
247pub use self::cbrt::cbrt;
248pub use self::cbrtf::cbrtf;
249pub use self::ceil::ceil;
250pub use self::ceilf::ceilf;
251pub use self::copysign::copysign;
252pub use self::copysignf::copysignf;
253pub use self::cos::cos;
254pub use self::cosf::cosf;
255pub use self::cosh::cosh;
256pub use self::coshf::coshf;
257pub use self::erf::{erf, erfc};
258pub use self::erff::{erfcf, erff};
259pub use self::exp::exp;
260pub use self::exp2::exp2;
261pub use self::exp2f::exp2f;
262pub use self::exp10::exp10;
263pub use self::exp10f::exp10f;
264pub use self::expf::expf;
265pub use self::expm1::expm1;
266pub use self::expm1f::expm1f;
267pub use self::fabs::fabs;
268pub use self::fabsf::fabsf;
269pub use self::fdim::fdim;
270pub use self::fdimf::fdimf;
271pub use self::floor::floor;
272pub use self::floorf::floorf;
273pub use self::fma::fma;
274pub use self::fma_wide::fmaf;
275pub use self::fmin_fmax::{fmax, fmaxf, fmin, fminf};
276pub use self::fminimum_fmaximum::{fmaximum, fmaximumf, fminimum, fminimumf};
277pub use self::fminimum_fmaximum_num::{fmaximum_num, fmaximum_numf, fminimum_num, fminimum_numf};
278pub use self::fmod::fmod;
279pub use self::fmodf::fmodf;
280pub use self::frexp::frexp;
281pub use self::frexpf::frexpf;
282pub use self::hypot::hypot;
283pub use self::hypotf::hypotf;
284pub use self::ilogb::ilogb;
285pub use self::ilogbf::ilogbf;
286pub use self::j0::{j0, y0};
287pub use self::j0f::{j0f, y0f};
288pub use self::j1::{j1, y1};
289pub use self::j1f::{j1f, y1f};
290pub use self::jn::{jn, yn};
291pub use self::jnf::{jnf, ynf};
292pub use self::ldexp::ldexp;
293pub use self::ldexpf::ldexpf;
294pub use self::lgamma::lgamma;
295pub use self::lgamma_r::lgamma_r;
296pub use self::lgammaf::lgammaf;
297pub use self::lgammaf_r::lgammaf_r;
298pub use self::log::log;
299pub use self::log1p::log1p;
300pub use self::log1pf::log1pf;
301pub use self::log2::log2;
302pub use self::log2f::log2f;
303pub use self::log10::log10;
304pub use self::log10f::log10f;
305pub use self::logf::logf;
306pub use self::modf::modf;
307pub use self::modff::modff;
308pub use self::nextafter::nextafter;
309pub use self::nextafterf::nextafterf;
310pub use self::pow::pow;
311pub use self::powf::powf;
312pub use self::remainder::remainder;
313pub use self::remainderf::remainderf;
314pub use self::remquo::remquo;
315pub use self::remquof::remquof;
316pub use self::rint::{rint, rintf};
317pub use self::round::round;
318pub use self::roundeven::{roundeven, roundevenf};
319pub use self::roundf::roundf;
320pub use self::scalbn::scalbn;
321pub use self::scalbnf::scalbnf;
322pub use self::sin::sin;
323pub use self::sincos::sincos;
324pub use self::sincosf::sincosf;
325pub use self::sinf::sinf;
326pub use self::sinh::sinh;
327pub use self::sinhf::sinhf;
328pub use self::sqrt::sqrt;
329pub use self::sqrtf::sqrtf;
330pub use self::tan::tan;
331pub use self::tanf::tanf;
332pub use self::tanh::tanh;
333pub use self::tanhf::tanhf;
334pub use self::tgamma::tgamma;
335pub use self::tgammaf::tgammaf;
336pub use self::trunc::trunc;
337pub use self::truncf::truncf;
338
339cfg_if! {
340 if #[cfg(f16_enabled)] {
341 // verify-sorted-start
342 mod ceilf16;
343 mod copysignf16;
344 mod fabsf16;
345 mod fdimf16;
346 mod floorf16;
347 mod fmodf16;
348 mod ldexpf16;
349 mod roundf16;
350 mod scalbnf16;
351 mod sqrtf16;
352 mod truncf16;
353 // verify-sorted-end
354
355 // verify-sorted-start
356 pub use self::ceilf16::ceilf16;
357 pub use self::copysignf16::copysignf16;
358 pub use self::fabsf16::fabsf16;
359 pub use self::fdimf16::fdimf16;
360 pub use self::floorf16::floorf16;
361 pub use self::fmin_fmax::{fmaxf16, fminf16};
362 pub use self::fminimum_fmaximum::{fmaximumf16, fminimumf16};
363 pub use self::fminimum_fmaximum_num::{fmaximum_numf16, fminimum_numf16};
364 pub use self::fmodf16::fmodf16;
365 pub use self::ldexpf16::ldexpf16;
366 pub use self::rint::rintf16;
367 pub use self::roundeven::roundevenf16;
368 pub use self::roundf16::roundf16;
369 pub use self::scalbnf16::scalbnf16;
370 pub use self::sqrtf16::sqrtf16;
371 pub use self::truncf16::truncf16;
372 // verify-sorted-end
373
374 #[allow(unused_imports)]
375 pub(crate) use self::fma_wide::fmaf16;
376 }
377}
378
379cfg_if! {
380 if #[cfg(f128_enabled)] {
381 // verify-sorted-start
382 mod ceilf128;
383 mod copysignf128;
384 mod fabsf128;
385 mod fdimf128;
386 mod floorf128;
387 mod fmodf128;
388 mod ldexpf128;
389 mod roundf128;
390 mod scalbnf128;
391 mod sqrtf128;
392 mod truncf128;
393 // verify-sorted-end
394
395 // verify-sorted-start
396 pub use self::ceilf128::ceilf128;
397 pub use self::copysignf128::copysignf128;
398 pub use self::fabsf128::fabsf128;
399 pub use self::fdimf128::fdimf128;
400 pub use self::floorf128::floorf128;
401 pub use self::fma::fmaf128;
402 pub use self::fmin_fmax::{fmaxf128, fminf128};
403 pub use self::fminimum_fmaximum::{fmaximumf128, fminimumf128};
404 pub use self::fminimum_fmaximum_num::{fmaximum_numf128, fminimum_numf128};
405 pub use self::fmodf128::fmodf128;
406 pub use self::ldexpf128::ldexpf128;
407 pub use self::rint::rintf128;
408 pub use self::roundeven::roundevenf128;
409 pub use self::roundf128::roundf128;
410 pub use self::scalbnf128::scalbnf128;
411 pub use self::sqrtf128::sqrtf128;
412 pub use self::truncf128::truncf128;
413 // verify-sorted-end
414 }
415}
416
417#[inline]
418fn get_high_word(x: f64) -> u32 {
419 (x.to_bits() >> 32) as u32
420}
421
422#[inline]
423fn get_low_word(x: f64) -> u32 {
424 x.to_bits() as u32
425}
426
427#[inline]
428fn with_set_high_word(f: f64, hi: u32) -> f64 {
429 let mut tmp: u64 = f.to_bits();
430 tmp &= 0x00000000_ffffffff;
431 tmp |= (hi as u64) << 32;
432 f64::from_bits(tmp)
433}
434
435#[inline]
436fn with_set_low_word(f: f64, lo: u32) -> f64 {
437 let mut tmp: u64 = f.to_bits();
438 tmp &= 0xffffffff_00000000;
439 tmp |= lo as u64;
440 f64::from_bits(tmp)
441}
442
443#[inline]
444fn combine_words(hi: u32, lo: u32) -> f64 {
445 f64::from_bits(((hi as u64) << 32) | lo as u64)
446}
447