1 | // Copyright 2009-2021 Intel Corporation |
2 | // SPDX-License-Identifier: Apache-2.0 |
3 | |
4 | #pragma once |
5 | |
6 | #include "../sys/platform.h" |
7 | #include "../sys/intrinsics.h" |
8 | #include "constants.h" |
9 | #include <cmath> |
10 | |
11 | #if defined(__ARM_NEON) |
12 | #include "../simd/arm/emulation.h" |
13 | #else |
14 | #include <emmintrin.h> |
15 | #include <xmmintrin.h> |
16 | #include <immintrin.h> |
17 | #endif |
18 | |
19 | #if defined(__WIN32__) |
20 | #if defined(_MSC_VER) && (_MSC_VER <= 1700) |
21 | namespace std |
22 | { |
23 | __forceinline bool isinf ( const float x ) { return _finite(x) == 0; } |
24 | __forceinline bool isnan ( const float x ) { return _isnan(x) != 0; } |
25 | __forceinline bool isfinite (const float x) { return _finite(x) != 0; } |
26 | } |
27 | #endif |
28 | #endif |
29 | |
30 | namespace embree |
31 | { |
32 | __forceinline bool isvalid ( const float& v ) { |
33 | return (v > -FLT_LARGE) & (v < +FLT_LARGE); |
34 | } |
35 | |
36 | __forceinline int cast_f2i(float f) { |
37 | union { float f; int i; } v; v.f = f; return v.i; |
38 | } |
39 | |
40 | __forceinline float cast_i2f(int i) { |
41 | union { float f; int i; } v; v.i = i; return v.f; |
42 | } |
43 | |
44 | __forceinline int toInt (const float& a) { return int(a); } |
45 | __forceinline float toFloat(const int& a) { return float(a); } |
46 | |
47 | #if defined(__WIN32__) |
48 | __forceinline bool finite ( const float x ) { return _finite(x) != 0; } |
49 | #endif |
50 | |
51 | __forceinline float sign ( const float x ) { return x<0?-1.0f:1.0f; } |
52 | __forceinline float sqr ( const float x ) { return x*x; } |
53 | |
54 | __forceinline float rcp ( const float x ) |
55 | { |
56 | const __m128 a = _mm_set_ss(w: x); |
57 | |
58 | #if defined(__AVX512VL__) |
59 | const __m128 r = _mm_rcp14_ss(_mm_set_ss(0.0f),a); |
60 | #else |
61 | const __m128 r = _mm_rcp_ss(a: a); |
62 | #endif |
63 | |
64 | #if defined(__AVX2__) |
65 | return _mm_cvtss_f32(_mm_mul_ss(r,_mm_fnmadd_ss(r, a, _mm_set_ss(2.0f)))); |
66 | #else |
67 | return _mm_cvtss_f32(a: _mm_mul_ss(a: r,b: _mm_sub_ss(a: _mm_set_ss(w: 2.0f), b: _mm_mul_ss(a: r, b: a)))); |
68 | #endif |
69 | } |
70 | |
71 | __forceinline float signmsk ( const float x ) { |
72 | return _mm_cvtss_f32(a: _mm_and_ps(a: _mm_set_ss(w: x),b: _mm_castsi128_ps(a: _mm_set1_epi32(i: 0x80000000)))); |
73 | } |
74 | __forceinline float xorf( const float x, const float y ) { |
75 | return _mm_cvtss_f32(a: _mm_xor_ps(a: _mm_set_ss(w: x),b: _mm_set_ss(w: y))); |
76 | } |
77 | __forceinline float andf( const float x, const unsigned y ) { |
78 | return _mm_cvtss_f32(a: _mm_and_ps(a: _mm_set_ss(w: x),b: _mm_castsi128_ps(a: _mm_set1_epi32(i: y)))); |
79 | } |
80 | __forceinline float rsqrt( const float x ) |
81 | { |
82 | const __m128 a = _mm_set_ss(w: x); |
83 | #if defined(__AVX512VL__) |
84 | __m128 r = _mm_rsqrt14_ss(_mm_set_ss(0.0f),a); |
85 | #else |
86 | __m128 r = _mm_rsqrt_ss(a: a); |
87 | #endif |
88 | r = _mm_add_ss(a: _mm_mul_ss(a: _mm_set_ss(w: 1.5f), b: r), b: _mm_mul_ss(a: _mm_mul_ss(a: _mm_mul_ss(a: a, b: _mm_set_ss(w: -0.5f)), b: r), b: _mm_mul_ss(a: r, b: r))); |
89 | #if defined(__ARM_NEON) |
90 | r = _mm_add_ss(_mm_mul_ss(_mm_set_ss(1.5f), r), _mm_mul_ss(_mm_mul_ss(_mm_mul_ss(a, _mm_set_ss(-0.5f)), r), _mm_mul_ss(r, r))); |
91 | #endif |
92 | return _mm_cvtss_f32(a: r); |
93 | } |
94 | |
95 | #if defined(__WIN32__) && defined(_MSC_VER) && (_MSC_VER <= 1700) |
96 | __forceinline float nextafter(float x, float y) { if ((x<y) == (x>0)) return x*(1.1f+float(ulp)); else return x*(0.9f-float(ulp)); } |
97 | __forceinline double nextafter(double x, double y) { return _nextafter(x, y); } |
98 | __forceinline int roundf(float f) { return (int)(f + 0.5f); } |
99 | #else |
100 | __forceinline float nextafter(float x, float y) { return ::nextafterf(x: x, y: y); } |
101 | __forceinline double nextafter(double x, double y) { return ::nextafter(x: x, y: y); } |
102 | #endif |
103 | |
104 | __forceinline float abs ( const float x ) { return ::fabsf(x: x); } |
105 | __forceinline float acos ( const float x ) { return ::acosf (x: x); } |
106 | __forceinline float asin ( const float x ) { return ::asinf (x: x); } |
107 | __forceinline float atan ( const float x ) { return ::atanf (x: x); } |
108 | __forceinline float atan2( const float y, const float x ) { return ::atan2f(y: y, x: x); } |
109 | __forceinline float cos ( const float x ) { return ::cosf (x: x); } |
110 | __forceinline float cosh ( const float x ) { return ::coshf (x: x); } |
111 | __forceinline float exp ( const float x ) { return ::expf (x: x); } |
112 | __forceinline float fmod ( const float x, const float y ) { return ::fmodf (x: x, y: y); } |
113 | __forceinline float log ( const float x ) { return ::logf (x: x); } |
114 | __forceinline float log10( const float x ) { return ::log10f(x: x); } |
115 | __forceinline float pow ( const float x, const float y ) { return ::powf (x: x, y: y); } |
116 | __forceinline float sin ( const float x ) { return ::sinf (x: x); } |
117 | __forceinline float sinh ( const float x ) { return ::sinhf (x: x); } |
118 | __forceinline float sqrt ( const float x ) { return ::sqrtf (x: x); } |
119 | __forceinline float tan ( const float x ) { return ::tanf (x: x); } |
120 | __forceinline float tanh ( const float x ) { return ::tanhf (x: x); } |
121 | __forceinline float floor( const float x ) { return ::floorf (x: x); } |
122 | __forceinline float ceil ( const float x ) { return ::ceilf (x: x); } |
123 | __forceinline float frac ( const float x ) { return x-floor(x); } |
124 | |
125 | __forceinline double abs ( const double x ) { return ::fabs(x: x); } |
126 | __forceinline double sign ( const double x ) { return x<0?-1.0:1.0; } |
127 | __forceinline double acos ( const double x ) { return ::acos (x: x); } |
128 | __forceinline double asin ( const double x ) { return ::asin (x: x); } |
129 | __forceinline double atan ( const double x ) { return ::atan (x: x); } |
130 | __forceinline double atan2( const double y, const double x ) { return ::atan2(y: y, x: x); } |
131 | __forceinline double cos ( const double x ) { return ::cos (x: x); } |
132 | __forceinline double cosh ( const double x ) { return ::cosh (x: x); } |
133 | __forceinline double exp ( const double x ) { return ::exp (x: x); } |
134 | __forceinline double fmod ( const double x, const double y ) { return ::fmod (x: x, y: y); } |
135 | __forceinline double log ( const double x ) { return ::log (x: x); } |
136 | __forceinline double log10( const double x ) { return ::log10(x: x); } |
137 | __forceinline double pow ( const double x, const double y ) { return ::pow (x: x, y: y); } |
138 | __forceinline double rcp ( const double x ) { return 1.0/x; } |
139 | __forceinline double rsqrt( const double x ) { return 1.0/::sqrt(x: x); } |
140 | __forceinline double sin ( const double x ) { return ::sin (x: x); } |
141 | __forceinline double sinh ( const double x ) { return ::sinh (x: x); } |
142 | __forceinline double sqr ( const double x ) { return x*x; } |
143 | __forceinline double sqrt ( const double x ) { return ::sqrt (x: x); } |
144 | __forceinline double tan ( const double x ) { return ::tan (x: x); } |
145 | __forceinline double tanh ( const double x ) { return ::tanh (x: x); } |
146 | __forceinline double floor( const double x ) { return ::floor (x: x); } |
147 | __forceinline double ceil ( const double x ) { return ::ceil (x: x); } |
148 | |
149 | #if defined(__SSE4_1__) |
150 | __forceinline float mini(float a, float b) { |
151 | const __m128i ai = _mm_castps_si128(_mm_set_ss(a)); |
152 | const __m128i bi = _mm_castps_si128(_mm_set_ss(b)); |
153 | const __m128i ci = _mm_min_epi32(ai,bi); |
154 | return _mm_cvtss_f32(_mm_castsi128_ps(ci)); |
155 | } |
156 | #endif |
157 | |
158 | #if defined(__SSE4_1__) |
159 | __forceinline float maxi(float a, float b) { |
160 | const __m128i ai = _mm_castps_si128(_mm_set_ss(a)); |
161 | const __m128i bi = _mm_castps_si128(_mm_set_ss(b)); |
162 | const __m128i ci = _mm_max_epi32(ai,bi); |
163 | return _mm_cvtss_f32(_mm_castsi128_ps(ci)); |
164 | } |
165 | #endif |
166 | |
167 | template<typename T> |
168 | __forceinline T twice(const T& a) { return a+a; } |
169 | |
170 | __forceinline int min(int a, int b) { return a<b ? a:b; } |
171 | __forceinline unsigned min(unsigned a, unsigned b) { return a<b ? a:b; } |
172 | __forceinline int64_t min(int64_t a, int64_t b) { return a<b ? a:b; } |
173 | __forceinline float min(float a, float b) { return a<b ? a:b; } |
174 | __forceinline double min(double a, double b) { return a<b ? a:b; } |
175 | #if defined(__64BIT__) || defined(__EMSCRIPTEN__) |
176 | __forceinline size_t min(size_t a, size_t b) { return a<b ? a:b; } |
177 | #endif |
178 | #if defined(__EMSCRIPTEN__) |
179 | __forceinline long min(long a, long b) { return a<b ? a:b; } |
180 | #endif |
181 | |
182 | template<typename T> __forceinline T min(const T& a, const T& b, const T& c) { return min(min(a,b),c); } |
183 | template<typename T> __forceinline T min(const T& a, const T& b, const T& c, const T& d) { return min(min(a,b),min(c,d)); } |
184 | template<typename T> __forceinline T min(const T& a, const T& b, const T& c, const T& d, const T& e) { return min(min(min(a,b),min(c,d)),e); } |
185 | |
186 | template<typename T> __forceinline T mini(const T& a, const T& b, const T& c) { return mini(mini(a,b),c); } |
187 | template<typename T> __forceinline T mini(const T& a, const T& b, const T& c, const T& d) { return mini(mini(a,b),mini(c,d)); } |
188 | template<typename T> __forceinline T mini(const T& a, const T& b, const T& c, const T& d, const T& e) { return mini(mini(mini(a,b),mini(c,d)),e); } |
189 | |
190 | __forceinline int max(int a, int b) { return a<b ? b:a; } |
191 | __forceinline unsigned max(unsigned a, unsigned b) { return a<b ? b:a; } |
192 | __forceinline int64_t max(int64_t a, int64_t b) { return a<b ? b:a; } |
193 | __forceinline float max(float a, float b) { return a<b ? b:a; } |
194 | __forceinline double max(double a, double b) { return a<b ? b:a; } |
195 | #if defined(__64BIT__) || defined(__EMSCRIPTEN__) |
196 | __forceinline size_t max(size_t a, size_t b) { return a<b ? b:a; } |
197 | #endif |
198 | #if defined(__EMSCRIPTEN__) |
199 | __forceinline long max(long a, long b) { return a<b ? b:a; } |
200 | #endif |
201 | |
202 | template<typename T> __forceinline T max(const T& a, const T& b, const T& c) { return max(max(a,b),c); } |
203 | template<typename T> __forceinline T max(const T& a, const T& b, const T& c, const T& d) { return max(max(a,b),max(c,d)); } |
204 | template<typename T> __forceinline T max(const T& a, const T& b, const T& c, const T& d, const T& e) { return max(max(max(a,b),max(c,d)),e); } |
205 | |
206 | template<typename T> __forceinline T maxi(const T& a, const T& b, const T& c) { return maxi(maxi(a,b),c); } |
207 | template<typename T> __forceinline T maxi(const T& a, const T& b, const T& c, const T& d) { return maxi(maxi(a,b),maxi(c,d)); } |
208 | template<typename T> __forceinline T maxi(const T& a, const T& b, const T& c, const T& d, const T& e) { return maxi(maxi(maxi(a,b),maxi(c,d)),e); } |
209 | |
210 | #if defined(__MACOSX__) |
211 | __forceinline ssize_t min(ssize_t a, ssize_t b) { return a<b ? a:b; } |
212 | __forceinline ssize_t max(ssize_t a, ssize_t b) { return a<b ? b:a; } |
213 | #endif |
214 | |
215 | #if defined(__MACOSX__) && !defined(__INTEL_COMPILER) |
216 | __forceinline void sincosf(float x, float *sin, float *cos) { |
217 | __sincosf(x,sin,cos); |
218 | } |
219 | #endif |
220 | |
221 | #if defined(__WIN32__) || defined(__FreeBSD__) |
222 | __forceinline void sincosf(float x, float *s, float *c) { |
223 | *s = sinf(x); *c = cosf(x); |
224 | } |
225 | #endif |
226 | |
227 | template<typename T> __forceinline T clamp(const T& x, const T& lower = T(zero), const T& upper = T(one)) { return max(min(x,upper),lower); } |
228 | template<typename T> __forceinline T clampz(const T& x, const T& upper) { return max(T(zero), min(x,upper)); } |
229 | |
230 | template<typename T> __forceinline T deg2rad ( const T& x ) { return x * T(1.74532925199432957692e-2f); } |
231 | template<typename T> __forceinline T rad2deg ( const T& x ) { return x * T(5.72957795130823208768e1f); } |
232 | template<typename T> __forceinline T sin2cos ( const T& x ) { return sqrt(max(T(zero),T(one)-x*x)); } |
233 | template<typename T> __forceinline T cos2sin ( const T& x ) { return sin2cos(x); } |
234 | |
235 | #if defined(__AVX2__) |
236 | __forceinline float madd ( const float a, const float b, const float c) { return _mm_cvtss_f32(_mm_fmadd_ss(_mm_set_ss(a),_mm_set_ss(b),_mm_set_ss(c))); } |
237 | __forceinline float msub ( const float a, const float b, const float c) { return _mm_cvtss_f32(_mm_fmsub_ss(_mm_set_ss(a),_mm_set_ss(b),_mm_set_ss(c))); } |
238 | __forceinline float nmadd ( const float a, const float b, const float c) { return _mm_cvtss_f32(_mm_fnmadd_ss(_mm_set_ss(a),_mm_set_ss(b),_mm_set_ss(c))); } |
239 | __forceinline float nmsub ( const float a, const float b, const float c) { return _mm_cvtss_f32(_mm_fnmsub_ss(_mm_set_ss(a),_mm_set_ss(b),_mm_set_ss(c))); } |
240 | #else |
241 | __forceinline float madd ( const float a, const float b, const float c) { return a*b+c; } |
242 | __forceinline float msub ( const float a, const float b, const float c) { return a*b-c; } |
243 | __forceinline float nmadd ( const float a, const float b, const float c) { return -a*b+c;} |
244 | __forceinline float nmsub ( const float a, const float b, const float c) { return -a*b-c; } |
245 | #endif |
246 | |
247 | /*! random functions */ |
248 | template<typename T> T random() { return T(0); } |
249 | #if defined(_WIN32) |
250 | template<> __forceinline int random() { return int(rand()) ^ (int(rand()) << 8) ^ (int(rand()) << 16); } |
251 | template<> __forceinline uint32_t random() { return uint32_t(rand()) ^ (uint32_t(rand()) << 8) ^ (uint32_t(rand()) << 16); } |
252 | #else |
253 | template<> __forceinline int random() { return int(rand()); } |
254 | template<> __forceinline uint32_t random() { return uint32_t(rand()) ^ (uint32_t(rand()) << 16); } |
255 | #endif |
256 | template<> __forceinline float random() { return rand()/float(RAND_MAX); } |
257 | template<> __forceinline double random() { return rand()/double(RAND_MAX); } |
258 | |
259 | #if _WIN32 |
260 | __forceinline double drand48() { |
261 | return double(rand())/double(RAND_MAX); |
262 | } |
263 | |
264 | __forceinline void srand48(long seed) { |
265 | return srand(seed); |
266 | } |
267 | #endif |
268 | |
269 | /*! selects */ |
270 | __forceinline bool select(bool s, bool t , bool f) { return s ? t : f; } |
271 | __forceinline int select(bool s, int t, int f) { return s ? t : f; } |
272 | __forceinline float select(bool s, float t, float f) { return s ? t : f; } |
273 | |
274 | __forceinline bool all(bool s) { return s; } |
275 | |
276 | __forceinline float lerp(const float v0, const float v1, const float t) { |
277 | return madd(a: 1.0f-t,b: v0,c: t*v1); |
278 | } |
279 | |
280 | template<typename T> |
281 | __forceinline T lerp2(const float x0, const float x1, const float x2, const float x3, const T& u, const T& v) { |
282 | return madd((1.0f-u),madd((1.0f-v),T(x0),v*T(x2)),u*madd((1.0f-v),T(x1),v*T(x3))); |
283 | } |
284 | |
285 | /*! exchange */ |
286 | template<typename T> __forceinline void xchg ( T& a, T& b ) { const T tmp = a; a = b; b = tmp; } |
287 | |
288 | /* load/store */ |
289 | template<typename Ty> struct mem; |
290 | |
291 | template<> struct mem<float> { |
292 | static __forceinline float load (bool mask, const void* ptr) { return mask ? *(float*)ptr : 0.0f; } |
293 | static __forceinline float loadu(bool mask, const void* ptr) { return mask ? *(float*)ptr : 0.0f; } |
294 | |
295 | static __forceinline void store (bool mask, void* ptr, const float v) { if (mask) *(float*)ptr = v; } |
296 | static __forceinline void storeu(bool mask, void* ptr, const float v) { if (mask) *(float*)ptr = v; } |
297 | }; |
298 | |
299 | /*! bit reverse operation */ |
300 | template<class T> |
301 | __forceinline T bitReverse(const T& vin) |
302 | { |
303 | T v = vin; |
304 | v = ((v >> 1) & 0x55555555) | ((v & 0x55555555) << 1); |
305 | v = ((v >> 2) & 0x33333333) | ((v & 0x33333333) << 2); |
306 | v = ((v >> 4) & 0x0F0F0F0F) | ((v & 0x0F0F0F0F) << 4); |
307 | v = ((v >> 8) & 0x00FF00FF) | ((v & 0x00FF00FF) << 8); |
308 | v = ( v >> 16 ) | ( v << 16); |
309 | return v; |
310 | } |
311 | |
312 | /*! bit interleave operation */ |
313 | template<class T> |
314 | __forceinline T bitInterleave(const T& xin, const T& yin, const T& zin) |
315 | { |
316 | T x = xin, y = yin, z = zin; |
317 | x = (x | (x << 16)) & 0x030000FF; |
318 | x = (x | (x << 8)) & 0x0300F00F; |
319 | x = (x | (x << 4)) & 0x030C30C3; |
320 | x = (x | (x << 2)) & 0x09249249; |
321 | |
322 | y = (y | (y << 16)) & 0x030000FF; |
323 | y = (y | (y << 8)) & 0x0300F00F; |
324 | y = (y | (y << 4)) & 0x030C30C3; |
325 | y = (y | (y << 2)) & 0x09249249; |
326 | |
327 | z = (z | (z << 16)) & 0x030000FF; |
328 | z = (z | (z << 8)) & 0x0300F00F; |
329 | z = (z | (z << 4)) & 0x030C30C3; |
330 | z = (z | (z << 2)) & 0x09249249; |
331 | |
332 | return x | (y << 1) | (z << 2); |
333 | } |
334 | |
335 | #if defined(__AVX2__) |
336 | |
337 | template<> |
338 | __forceinline unsigned int bitInterleave(const unsigned int &xi, const unsigned int& yi, const unsigned int& zi) |
339 | { |
340 | const unsigned int xx = pdep(xi,0x49249249 /* 0b01001001001001001001001001001001 */ ); |
341 | const unsigned int yy = pdep(yi,0x92492492 /* 0b10010010010010010010010010010010 */); |
342 | const unsigned int zz = pdep(zi,0x24924924 /* 0b00100100100100100100100100100100 */); |
343 | return xx | yy | zz; |
344 | } |
345 | |
346 | #endif |
347 | |
348 | /*! bit interleave operation for 64bit data types*/ |
349 | template<class T> |
350 | __forceinline T bitInterleave64(const T& xin, const T& yin, const T& zin){ |
351 | T x = xin & 0x1fffff; |
352 | T y = yin & 0x1fffff; |
353 | T z = zin & 0x1fffff; |
354 | |
355 | x = (x | x << 32) & 0x1f00000000ffff; |
356 | x = (x | x << 16) & 0x1f0000ff0000ff; |
357 | x = (x | x << 8) & 0x100f00f00f00f00f; |
358 | x = (x | x << 4) & 0x10c30c30c30c30c3; |
359 | x = (x | x << 2) & 0x1249249249249249; |
360 | |
361 | y = (y | y << 32) & 0x1f00000000ffff; |
362 | y = (y | y << 16) & 0x1f0000ff0000ff; |
363 | y = (y | y << 8) & 0x100f00f00f00f00f; |
364 | y = (y | y << 4) & 0x10c30c30c30c30c3; |
365 | y = (y | y << 2) & 0x1249249249249249; |
366 | |
367 | z = (z | z << 32) & 0x1f00000000ffff; |
368 | z = (z | z << 16) & 0x1f0000ff0000ff; |
369 | z = (z | z << 8) & 0x100f00f00f00f00f; |
370 | z = (z | z << 4) & 0x10c30c30c30c30c3; |
371 | z = (z | z << 2) & 0x1249249249249249; |
372 | |
373 | return x | (y << 1) | (z << 2); |
374 | } |
375 | } |
376 | |